feat: add ouge detail and player support
This commit is contained in:
@@ -135,6 +135,80 @@ class TestOuGeSpider(unittest.TestCase):
|
||||
self.assertEqual(result, {"page": 1, "total": 0, "list": []})
|
||||
mock_request_html.assert_not_called()
|
||||
|
||||
def test_parse_detail_page_extracts_metadata_and_deduplicated_pan_groups(self):
|
||||
html = """
|
||||
<div class="page-title">欧歌示例</div>
|
||||
<div class="mobile-play">
|
||||
<img class="lazyload" data-src="/poster-detail.jpg" />
|
||||
</div>
|
||||
<div class="module-item-caption"><span>2025</span></div>
|
||||
<div class="video-info-itemtitle">导演</div>
|
||||
<div><a>导演甲</a><a>导演乙</a></div>
|
||||
<div class="video-info-itemtitle">主演</div>
|
||||
<div><a>演员甲</a><a>演员乙</a></div>
|
||||
<div class="video-info-itemtitle">剧情</div>
|
||||
<div><p>这是一段剧情简介。</p></div>
|
||||
<div class="module-row-info"><p>https://pan.quark.cn/s/q-demo</p></div>
|
||||
<div class="module-row-info"><p>百度合集 https://pan.baidu.com/s/b-demo</p></div>
|
||||
<div class="module-row-info"><p>https://pan.quark.cn/s/q-demo</p></div>
|
||||
"""
|
||||
vod = self.spider._parse_detail_page(html, "/index.php/vod/detail/id/999.html")
|
||||
self.assertEqual(vod["vod_id"], "/index.php/vod/detail/id/999.html")
|
||||
self.assertEqual(vod["vod_name"], "欧歌示例")
|
||||
self.assertEqual(vod["vod_pic"], "https://woog.nxog.eu.org/poster-detail.jpg")
|
||||
self.assertEqual(vod["vod_year"], "2025")
|
||||
self.assertEqual(vod["vod_director"], "导演甲,导演乙")
|
||||
self.assertEqual(vod["vod_actor"], "演员甲,演员乙")
|
||||
self.assertEqual(vod["vod_content"], "这是一段剧情简介。")
|
||||
self.assertEqual(vod["vod_play_from"], "baidu$$$quark")
|
||||
self.assertEqual(
|
||||
vod["vod_play_url"],
|
||||
"百度资源$https://pan.baidu.com/s/b-demo$$$夸克资源$https://pan.quark.cn/s/q-demo",
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_detail_content_reads_detail_page_and_returns_single_vod(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<div class="page-title">详情标题</div>
|
||||
<div class="module-row-info"><p>https://pan.quark.cn/s/detail-demo</p></div>
|
||||
"""
|
||||
result = self.spider.detailContent(["/index.php/vod/detail/id/1000.html"])
|
||||
self.assertEqual(
|
||||
mock_request_html.call_args.args[0],
|
||||
"https://woog.nxog.eu.org/index.php/vod/detail/id/1000.html",
|
||||
)
|
||||
self.assertEqual(result["list"][0]["vod_id"], "/index.php/vod/detail/id/1000.html")
|
||||
self.assertEqual(result["list"][0]["vod_name"], "详情标题")
|
||||
self.assertEqual(result["list"][0]["vod_play_from"], "quark")
|
||||
self.assertEqual(result["list"][0]["vod_play_url"], "夸克资源$https://pan.quark.cn/s/detail-demo")
|
||||
|
||||
def test_parse_detail_page_returns_empty_shell_for_blank_html(self):
|
||||
self.assertEqual(
|
||||
self.spider._parse_detail_page("", "/index.php/vod/detail/id/404.html"),
|
||||
{
|
||||
"vod_id": "/index.php/vod/detail/id/404.html",
|
||||
"vod_name": "",
|
||||
"vod_pic": "",
|
||||
"vod_year": "",
|
||||
"vod_director": "",
|
||||
"vod_actor": "",
|
||||
"vod_content": "",
|
||||
"vod_play_from": "",
|
||||
"vod_play_url": "",
|
||||
},
|
||||
)
|
||||
|
||||
def test_player_content_transparently_returns_supported_pan_link(self):
|
||||
result = self.spider.playerContent("quark", "https://pan.quark.cn/s/demo", {})
|
||||
self.assertEqual(result["parse"], 0)
|
||||
self.assertEqual(result["jx"], 0)
|
||||
self.assertEqual(result["url"], "https://pan.quark.cn/s/demo")
|
||||
self.assertEqual(result["header"], {})
|
||||
|
||||
def test_player_content_returns_empty_url_for_unknown_link(self):
|
||||
result = self.spider.playerContent("unknown", "/index.php/vod/play/id/1.html", {})
|
||||
self.assertEqual(result, {"parse": 0, "jx": 0, "playUrl": "", "url": "", "header": {}})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -179,3 +179,97 @@ class Spider(BaseSpider):
|
||||
return {"page": page, "total": 0, "list": []}
|
||||
items = self._parse_search_cards(self._request_html(self._build_search_url(keyword, pg)))
|
||||
return {"page": page, "total": len(items), "list": items}
|
||||
|
||||
def _blank_detail(self, vod_id):
|
||||
return {
|
||||
"vod_id": vod_id,
|
||||
"vod_name": "",
|
||||
"vod_pic": "",
|
||||
"vod_year": "",
|
||||
"vod_director": "",
|
||||
"vod_actor": "",
|
||||
"vod_content": "",
|
||||
"vod_play_from": "",
|
||||
"vod_play_url": "",
|
||||
}
|
||||
|
||||
def _extract_share_url(self, text):
|
||||
matched = re.search(r"https?://[^\s'\"<>]+", self._stringify(text), re.I)
|
||||
return matched.group(0).strip() if matched else ""
|
||||
|
||||
def _join_people(self, node):
|
||||
values = [self._clean_text(text) for text in node.xpath(".//a//text()")]
|
||||
if not any(values):
|
||||
values = [self._clean_text(text) for text in node.xpath(".//text()")]
|
||||
values = [value for value in values if value]
|
||||
return ",".join(values)
|
||||
|
||||
def _extract_pan_groups(self, root):
|
||||
groups = []
|
||||
seen = set()
|
||||
for node in root.xpath("//*[contains(@class,'module-row-info')]//p"):
|
||||
text = self._clean_text("".join(node.xpath(".//text()")))
|
||||
share_url = self._extract_share_url(text)
|
||||
pan_type, title = self._detect_pan_type(share_url)
|
||||
if not share_url or not pan_type or share_url in seen:
|
||||
continue
|
||||
seen.add(share_url)
|
||||
groups.append((pan_type, f"{title}${share_url}"))
|
||||
groups.sort(key=lambda item: self.pan_priority.get(item[0], 999))
|
||||
return groups
|
||||
|
||||
def _parse_detail_page(self, html, vod_id):
|
||||
if not self._stringify(html).strip():
|
||||
return self._blank_detail(vod_id)
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return self._blank_detail(vod_id)
|
||||
|
||||
vod = self._blank_detail(vod_id)
|
||||
vod["vod_name"] = self._clean_text("".join(root.xpath("(//*[contains(@class,'page-title')])[1]//text()")))
|
||||
pic = (
|
||||
((root.xpath("//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src") or [""])[0]).strip()
|
||||
or ((root.xpath("//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src") or [""])[0]).strip()
|
||||
)
|
||||
vod["vod_pic"] = self._fix_img_url(pic)
|
||||
vod["vod_year"] = self._clean_text(
|
||||
((root.xpath("(//*[contains(@class,'module-item-caption')]//span[1]/text())") or [""])[0])
|
||||
)
|
||||
|
||||
for label_node in root.xpath("//*[contains(@class,'video-info-itemtitle')]"):
|
||||
label = self._clean_text("".join(label_node.xpath(".//text()")))
|
||||
siblings = label_node.xpath("./following-sibling::*[1]")
|
||||
if not siblings:
|
||||
continue
|
||||
block = siblings[0]
|
||||
if "导演" in label:
|
||||
vod["vod_director"] = self._join_people(block)
|
||||
elif "主演" in label:
|
||||
vod["vod_actor"] = self._join_people(block)
|
||||
elif "剧情" in label:
|
||||
texts = block.xpath(".//p//text()") or block.xpath(".//text()")
|
||||
vod["vod_content"] = self._clean_text("".join(texts))
|
||||
|
||||
pan_groups = self._extract_pan_groups(root)
|
||||
vod["vod_play_from"] = "$$$".join(item[0] for item in pan_groups)
|
||||
vod["vod_play_url"] = "$$$".join(item[1] for item in pan_groups)
|
||||
return vod
|
||||
|
||||
def detailContent(self, ids):
|
||||
result = {"list": []}
|
||||
for raw_id in ids:
|
||||
vod_id = self._stringify(raw_id).strip()
|
||||
if not vod_id:
|
||||
continue
|
||||
html = self._request_html(self._build_url(vod_id))
|
||||
result["list"].append(self._parse_detail_page(html, vod_id))
|
||||
return result
|
||||
|
||||
def _is_supported_pan_link(self, url):
|
||||
return bool(self._detect_pan_type(url)[0])
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
target = self._stringify(id).strip()
|
||||
if target and self._is_supported_pan_link(target):
|
||||
return {"parse": 0, "jx": 0, "playUrl": "", "url": target, "header": {}}
|
||||
return {"parse": 0, "jx": 0, "playUrl": "", "url": "", "header": {}}
|
||||
|
||||
Reference in New Issue
Block a user