feat: add juquanquan detail parsing
This commit is contained in:
@@ -96,6 +96,54 @@ class TestJuQuanQuanSpider(unittest.TestCase):
|
|||||||
self.assertEqual(result["list"][0]["vod_id"], "vod/777")
|
self.assertEqual(result["list"][0]["vod_id"], "vod/777")
|
||||||
self.assertEqual(result["pagecount"], 1)
|
self.assertEqual(result["pagecount"], 1)
|
||||||
|
|
||||||
|
def test_parse_detail_page_extracts_metadata_and_playlists(self):
|
||||||
|
html = """
|
||||||
|
<div class="module-info-heading"><h1>详情标题</h1></div>
|
||||||
|
<div class="module-info-poster"><img data-original="/poster.jpg" /></div>
|
||||||
|
<div class="module-info-item">
|
||||||
|
<div class="module-info-item-title">导演</div>
|
||||||
|
<div class="module-info-item-content"><a>导演甲</a></div>
|
||||||
|
</div>
|
||||||
|
<div class="module-info-item">
|
||||||
|
<div class="module-info-item-title">主演</div>
|
||||||
|
<div class="module-info-item-content"><a>演员甲</a><a>演员乙</a></div>
|
||||||
|
</div>
|
||||||
|
<div class="module-info-item">
|
||||||
|
<div class="module-info-item-title">备注</div>
|
||||||
|
<div class="module-info-item-content">更新至3集</div>
|
||||||
|
</div>
|
||||||
|
<div class="module-info-introduction-content">一段剧情简介</div>
|
||||||
|
<div class="module-info-tag-link"><a>古装</a><a>剧情</a></div>
|
||||||
|
<div id="y-playList">
|
||||||
|
<div class="module-tab-item" data-dropdown-value="线路A"></div>
|
||||||
|
<div class="module-tab-item" data-dropdown-value="线路B"></div>
|
||||||
|
</div>
|
||||||
|
<div class="his-tab-list">
|
||||||
|
<a class="module-play-list-link" href="/play/123-1-1.html"><span>第1集</span></a>
|
||||||
|
<a class="module-play-list-link" href="/play/123-1-2.html"><span>第2集</span></a>
|
||||||
|
</div>
|
||||||
|
<div class="his-tab-list">
|
||||||
|
<a class="module-play-list-link" href="/play/123-2-1.html"><span>正片</span></a>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
result = self.spider._parse_detail_page(html, "vod/123")
|
||||||
|
vod = result["list"][0]
|
||||||
|
self.assertEqual(vod["vod_id"], "vod/123")
|
||||||
|
self.assertEqual(vod["vod_name"], "详情标题")
|
||||||
|
self.assertEqual(vod["vod_pic"], "https://www.jqqzx.cc/poster.jpg")
|
||||||
|
self.assertEqual(vod["type_name"], "古装 / 剧情")
|
||||||
|
self.assertEqual(vod["vod_director"], "导演甲")
|
||||||
|
self.assertEqual(vod["vod_actor"], "演员甲 / 演员乙")
|
||||||
|
self.assertEqual(vod["vod_content"], "一段剧情简介")
|
||||||
|
self.assertEqual(vod["vod_play_from"], "线路A$$$线路B")
|
||||||
|
self.assertEqual(vod["vod_play_url"], "第1集$play/123-1-1#第2集$play/123-1-2$$$正片$play/123-2-1")
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_detail_content_decodes_compact_vod_id(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = '<div class="module-info-heading"><h1>详情标题</h1></div>'
|
||||||
|
self.spider.detailContent(["vod/321"])
|
||||||
|
self.assertEqual(mock_request_html.call_args.args[0], "https://www.jqqzx.cc/vod/321.html")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -144,3 +144,103 @@ class Spider(BaseSpider):
|
|||||||
self._request_html(self._build_url(f"/index.php/ajax/suggest?mid=1&wd={quote(keyword)}"))
|
self._request_html(self._build_url(f"/index.php/ajax/suggest?mid=1&wd={quote(keyword)}"))
|
||||||
)
|
)
|
||||||
return {"page": page, "pagecount": 1, "total": len(items), "list": items}
|
return {"page": page, "pagecount": 1, "total": len(items), "list": items}
|
||||||
|
|
||||||
|
def _parse_info_items(self, root):
|
||||||
|
info = {}
|
||||||
|
for item in root.xpath("//*[contains(@class,'module-info-item')]"):
|
||||||
|
title = self._clean_text(
|
||||||
|
"".join(item.xpath(".//*[contains(@class,'module-info-item-title')][1]//text()"))
|
||||||
|
).rstrip("::")
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
links = [
|
||||||
|
self._clean_text("".join(node.xpath(".//text()")))
|
||||||
|
for node in item.xpath(".//*[contains(@class,'module-info-item-content')][1]//a")
|
||||||
|
]
|
||||||
|
links = [value for value in links if value]
|
||||||
|
if links:
|
||||||
|
info[title] = " / ".join(links)
|
||||||
|
continue
|
||||||
|
info[title] = self._clean_text(
|
||||||
|
"".join(item.xpath(".//*[contains(@class,'module-info-item-content')][1]//text()"))
|
||||||
|
)
|
||||||
|
return info
|
||||||
|
|
||||||
|
def _parse_detail_page(self, html, vod_id):
|
||||||
|
root = self.html(html)
|
||||||
|
if root is None:
|
||||||
|
return {"list": []}
|
||||||
|
info = self._parse_info_items(root)
|
||||||
|
tab_names = []
|
||||||
|
for node in root.xpath("//*[@id='y-playList']//*[contains(@class,'module-tab-item')]"):
|
||||||
|
tab_names.append(
|
||||||
|
self._clean_text(
|
||||||
|
(node.xpath("./@data-dropdown-value") or [""])[0]
|
||||||
|
or "".join(node.xpath(".//span[1]//text()"))
|
||||||
|
or "".join(node.xpath(".//text()"))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
groups = []
|
||||||
|
for index, box in enumerate(root.xpath("//*[contains(@class,'his-tab-list')]")):
|
||||||
|
episodes = []
|
||||||
|
for anchor in box.xpath(".//a[contains(@class,'module-play-list-link') and @href]"):
|
||||||
|
play_id = self._encode_play_id((anchor.xpath("./@href") or [""])[0])
|
||||||
|
ep_name = self._clean_text(
|
||||||
|
"".join(anchor.xpath(".//span[1]//text()")) or "".join(anchor.xpath(".//text()"))
|
||||||
|
)
|
||||||
|
if play_id and ep_name:
|
||||||
|
episodes.append(f"{ep_name}${play_id}")
|
||||||
|
if episodes:
|
||||||
|
groups.append(
|
||||||
|
{
|
||||||
|
"from": tab_names[index] if index < len(tab_names) and tab_names[index] else f"线路{index + 1}",
|
||||||
|
"urls": "#".join(episodes),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
type_name = " / ".join(
|
||||||
|
[
|
||||||
|
self._clean_text("".join(node.xpath(".//text()")))
|
||||||
|
for node in root.xpath("//*[contains(@class,'module-info-tag-link')]//a")
|
||||||
|
if self._clean_text("".join(node.xpath(".//text()")))
|
||||||
|
]
|
||||||
|
)
|
||||||
|
pic = (
|
||||||
|
root.xpath(
|
||||||
|
"(//*[contains(@class,'module-item-pic')]//img/@data-original | "
|
||||||
|
"//*[contains(@class,'module-info-poster')]//img/@data-original | "
|
||||||
|
"//*[contains(@class,'module-item-pic')]//img/@src | "
|
||||||
|
"//*[contains(@class,'module-info-poster')]//img/@src)[1]"
|
||||||
|
)
|
||||||
|
or [""]
|
||||||
|
)[0]
|
||||||
|
return {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": self._clean_text(
|
||||||
|
"".join(root.xpath("//*[contains(@class,'module-info-heading')]//h1[1]//text()"))
|
||||||
|
),
|
||||||
|
"vod_pic": self._build_url(pic),
|
||||||
|
"type_name": type_name,
|
||||||
|
"vod_remarks": info.get("备注") or info.get("状态", ""),
|
||||||
|
"vod_actor": info.get("主演", ""),
|
||||||
|
"vod_director": info.get("导演", ""),
|
||||||
|
"vod_content": self._clean_text(
|
||||||
|
"".join(root.xpath("//*[contains(@class,'module-info-introduction-content')][1]//text()"))
|
||||||
|
),
|
||||||
|
"vod_play_from": "$$$".join(group["from"] for group in groups),
|
||||||
|
"vod_play_url": "$$$".join(group["urls"] for group in groups),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
def detailContent(self, ids):
|
||||||
|
result = {"list": []}
|
||||||
|
for raw_id in ids:
|
||||||
|
vod_id = str(raw_id or "").strip()
|
||||||
|
detail_url = self._decode_vod_id(vod_id)
|
||||||
|
if not detail_url:
|
||||||
|
continue
|
||||||
|
parsed = self._parse_detail_page(self._request_html(detail_url), vod_id)
|
||||||
|
result["list"].extend(parsed.get("list", []))
|
||||||
|
return result
|
||||||
|
|||||||
Reference in New Issue
Block a user