feat: add letu detail parsing

This commit is contained in:
Harold
2026-04-20 09:34:50 +08:00
parent 8eb9cc6f23
commit 5aa57f5100
2 changed files with 114 additions and 0 deletions
+45
View File
@@ -90,6 +90,51 @@ class TestLeTuSpider(unittest.TestCase):
self.assertEqual(result["list"][0]["vod_id"], "detail/search-demo")
self.assertNotIn("pagecount", result)
def test_parse_detail_page_extracts_metadata_and_playlists(self):
html = """
<h1>详情标题</h1>
<img src="/poster.jpg" />
<div class="scroll no-margin">
<a>电影</a>
<a>演员甲</a>
</div>
<div class="no-space no-margin m l">导演甲</div>
<div class="no-margin m l">中国</div>
<div class="responsive"><p>第一段</p><p>一段剧情简介</p></div>
<div class="tabs left-align">
<a>线路A</a>
<a>线路B</a>
</div>
<div class="playno">
<a href="/play/123-1-1.html">第1集</a>
<a href="/play/123-1-2.html">第2集</a>
</div>
<div class="playno">
<a href="/play/123-2-1.html">正片</a>
</div>
"""
result = self.spider._parse_detail_page(html, "detail/demo")
vod = result["list"][0]
self.assertEqual(vod["vod_id"], "detail/demo")
self.assertEqual(vod["vod_name"], "详情标题")
self.assertEqual(vod["vod_pic"], "https://www.letu.me/poster.jpg")
self.assertEqual(vod["type_name"], "电影")
self.assertEqual(vod["vod_actor"], "演员甲")
self.assertEqual(vod["vod_director"], "导演甲")
self.assertEqual(vod["vod_area"], "中国")
self.assertEqual(vod["vod_content"], "一段剧情简介")
self.assertEqual(vod["vod_play_from"], "线路A$$$线路B")
self.assertEqual(
vod["vod_play_url"],
"第1集$play/123-1-1#第2集$play/123-1-2$$$正片$play/123-2-1",
)
@patch.object(Spider, "_request_html")
def test_detail_content_decodes_compact_vod_id(self, mock_request_html):
mock_request_html.return_value = "<h1>详情标题</h1>"
self.spider.detailContent(["detail/demo"])
self.assertEqual(mock_request_html.call_args.args[0], "https://www.letu.me/detail/demo.html")
if __name__ == "__main__":
unittest.main()
+69
View File
@@ -134,3 +134,72 @@ class Spider(BaseSpider):
}
)
return self._page_result(items, pg)
def _parse_detail_page(self, html, vod_id):
root = self.html(html)
if root is None:
return {"list": []}
info_nodes = root.xpath(
"//*[contains(concat(' ', normalize-space(@class), ' '), ' no-margin ') and "
"contains(concat(' ', normalize-space(@class), ' '), ' m ') and "
"contains(concat(' ', normalize-space(@class), ' '), ' l ')]"
)
director = ""
area = ""
for node in info_nodes:
classes = " ".join(node.xpath("./@class"))
text = self._clean_text("".join(node.xpath(".//text()")))
if not text:
continue
if "no-space" in classes and not director:
director = text
elif "no-space" not in classes and not area:
area = text
play_from = []
play_urls = []
tabs = [
self._clean_text("".join(node.xpath(".//text()")))
for node in root.xpath("//*[contains(@class,'tabs') and contains(@class,'left-align')]//a")
]
groups = root.xpath("//*[contains(@class,'playno')]")
for index, group in enumerate(groups):
episodes = []
for anchor in group.xpath(".//a[@href]"):
play_id = self._encode_play_id("".join(anchor.xpath("./@href")))
ep_name = self._clean_text("".join(anchor.xpath(".//text()")))
if play_id and ep_name:
episodes.append(f"{ep_name}${play_id}")
if episodes:
play_from.append(tabs[index] if index < len(tabs) and tabs[index] else f"线路{index + 1}")
play_urls.append("#".join(episodes))
return {
"list": [
{
"vod_id": vod_id,
"vod_name": self._clean_text("".join(root.xpath("//h1[1]//text()"))),
"vod_pic": self._build_url("".join(root.xpath("(//img/@src | //img/@data-src)[1]"))),
"type_name": self._clean_text(
"".join(root.xpath("(//*[contains(@class,'scroll') and contains(@class,'no-margin')]//a[1]//text())[1]"))
),
"vod_actor": self._clean_text(
"".join(root.xpath("(//*[contains(@class,'scroll') and contains(@class,'no-margin')]//a[2]//text())[1]"))
),
"vod_director": director,
"vod_area": area,
"vod_content": self._clean_text("".join(root.xpath("(//*[contains(@class,'responsive')]//p[last()]//text())[1]"))),
"vod_play_from": "$$$".join(play_from),
"vod_play_url": "$$$".join(play_urls),
}
]
}
def detailContent(self, ids):
result = {"list": []}
for raw_id in ids:
vod_id = str(raw_id or "").strip()
detail_url = self._decode_vod_id(vod_id) or self._build_url(vod_id)
if not detail_url:
continue
parsed = self._parse_detail_page(self._request_html(detail_url), vod_id)
result["list"].extend(parsed.get("list", []))
return result