feat: add letu detail parsing
This commit is contained in:
@@ -90,6 +90,51 @@ class TestLeTuSpider(unittest.TestCase):
|
||||
self.assertEqual(result["list"][0]["vod_id"], "detail/search-demo")
|
||||
self.assertNotIn("pagecount", result)
|
||||
|
||||
def test_parse_detail_page_extracts_metadata_and_playlists(self):
|
||||
html = """
|
||||
<h1>详情标题</h1>
|
||||
<img src="/poster.jpg" />
|
||||
<div class="scroll no-margin">
|
||||
<a>电影</a>
|
||||
<a>演员甲</a>
|
||||
</div>
|
||||
<div class="no-space no-margin m l">导演甲</div>
|
||||
<div class="no-margin m l">中国</div>
|
||||
<div class="responsive"><p>第一段</p><p>一段剧情简介</p></div>
|
||||
<div class="tabs left-align">
|
||||
<a>线路A</a>
|
||||
<a>线路B</a>
|
||||
</div>
|
||||
<div class="playno">
|
||||
<a href="/play/123-1-1.html">第1集</a>
|
||||
<a href="/play/123-1-2.html">第2集</a>
|
||||
</div>
|
||||
<div class="playno">
|
||||
<a href="/play/123-2-1.html">正片</a>
|
||||
</div>
|
||||
"""
|
||||
result = self.spider._parse_detail_page(html, "detail/demo")
|
||||
vod = result["list"][0]
|
||||
self.assertEqual(vod["vod_id"], "detail/demo")
|
||||
self.assertEqual(vod["vod_name"], "详情标题")
|
||||
self.assertEqual(vod["vod_pic"], "https://www.letu.me/poster.jpg")
|
||||
self.assertEqual(vod["type_name"], "电影")
|
||||
self.assertEqual(vod["vod_actor"], "演员甲")
|
||||
self.assertEqual(vod["vod_director"], "导演甲")
|
||||
self.assertEqual(vod["vod_area"], "中国")
|
||||
self.assertEqual(vod["vod_content"], "一段剧情简介")
|
||||
self.assertEqual(vod["vod_play_from"], "线路A$$$线路B")
|
||||
self.assertEqual(
|
||||
vod["vod_play_url"],
|
||||
"第1集$play/123-1-1#第2集$play/123-1-2$$$正片$play/123-2-1",
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_detail_content_decodes_compact_vod_id(self, mock_request_html):
|
||||
mock_request_html.return_value = "<h1>详情标题</h1>"
|
||||
self.spider.detailContent(["detail/demo"])
|
||||
self.assertEqual(mock_request_html.call_args.args[0], "https://www.letu.me/detail/demo.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -134,3 +134,72 @@ class Spider(BaseSpider):
|
||||
}
|
||||
)
|
||||
return self._page_result(items, pg)
|
||||
|
||||
def _parse_detail_page(self, html, vod_id):
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return {"list": []}
|
||||
info_nodes = root.xpath(
|
||||
"//*[contains(concat(' ', normalize-space(@class), ' '), ' no-margin ') and "
|
||||
"contains(concat(' ', normalize-space(@class), ' '), ' m ') and "
|
||||
"contains(concat(' ', normalize-space(@class), ' '), ' l ')]"
|
||||
)
|
||||
director = ""
|
||||
area = ""
|
||||
for node in info_nodes:
|
||||
classes = " ".join(node.xpath("./@class"))
|
||||
text = self._clean_text("".join(node.xpath(".//text()")))
|
||||
if not text:
|
||||
continue
|
||||
if "no-space" in classes and not director:
|
||||
director = text
|
||||
elif "no-space" not in classes and not area:
|
||||
area = text
|
||||
play_from = []
|
||||
play_urls = []
|
||||
tabs = [
|
||||
self._clean_text("".join(node.xpath(".//text()")))
|
||||
for node in root.xpath("//*[contains(@class,'tabs') and contains(@class,'left-align')]//a")
|
||||
]
|
||||
groups = root.xpath("//*[contains(@class,'playno')]")
|
||||
for index, group in enumerate(groups):
|
||||
episodes = []
|
||||
for anchor in group.xpath(".//a[@href]"):
|
||||
play_id = self._encode_play_id("".join(anchor.xpath("./@href")))
|
||||
ep_name = self._clean_text("".join(anchor.xpath(".//text()")))
|
||||
if play_id and ep_name:
|
||||
episodes.append(f"{ep_name}${play_id}")
|
||||
if episodes:
|
||||
play_from.append(tabs[index] if index < len(tabs) and tabs[index] else f"线路{index + 1}")
|
||||
play_urls.append("#".join(episodes))
|
||||
return {
|
||||
"list": [
|
||||
{
|
||||
"vod_id": vod_id,
|
||||
"vod_name": self._clean_text("".join(root.xpath("//h1[1]//text()"))),
|
||||
"vod_pic": self._build_url("".join(root.xpath("(//img/@src | //img/@data-src)[1]"))),
|
||||
"type_name": self._clean_text(
|
||||
"".join(root.xpath("(//*[contains(@class,'scroll') and contains(@class,'no-margin')]//a[1]//text())[1]"))
|
||||
),
|
||||
"vod_actor": self._clean_text(
|
||||
"".join(root.xpath("(//*[contains(@class,'scroll') and contains(@class,'no-margin')]//a[2]//text())[1]"))
|
||||
),
|
||||
"vod_director": director,
|
||||
"vod_area": area,
|
||||
"vod_content": self._clean_text("".join(root.xpath("(//*[contains(@class,'responsive')]//p[last()]//text())[1]"))),
|
||||
"vod_play_from": "$$$".join(play_from),
|
||||
"vod_play_url": "$$$".join(play_urls),
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def detailContent(self, ids):
|
||||
result = {"list": []}
|
||||
for raw_id in ids:
|
||||
vod_id = str(raw_id or "").strip()
|
||||
detail_url = self._decode_vod_id(vod_id) or self._build_url(vod_id)
|
||||
if not detail_url:
|
||||
continue
|
||||
parsed = self._parse_detail_page(self._request_html(detail_url), vod_id)
|
||||
result["list"].extend(parsed.get("list", []))
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user