yeyinly/tvboxzt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

玩偶聚合

Browse Source
This commit is contained in:
Harold
2026-04-21 19:52:30 +08:00
parent 552243cccd
commit 7aa6f4f8df
4 changed files with 492 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/tests/test_二小.py
+61 -6
View File
@@ -33,7 +33,7 @@ class TestErXiaoSpider(unittest.TestCase):
def test_build_url_and_detect_pan_type(self): def test_build_url_and_detect_pan_type(self):
self.assertEqual( self.assertEqual(
self.spider._build_url("/index.php/vod/detail/id/1.html"), self.spider._build_url("/index.php/vod/detail/id/1.html"),
"http://2xiaopan.fun/index.php/vod/detail/id/1.html", f"{self.spider.host}/index.php/vod/detail/id/1.html",
) )
self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源")) self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源"))
self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源")) self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源"))
@@ -58,13 +58,55 @@ class TestErXiaoSpider(unittest.TestCase):
{ {
"vod_id": "/index.php/vod/detail/id/123.html", "vod_id": "/index.php/vod/detail/id/123.html",
"vod_name": "示例影片", "vod_name": "示例影片",
"vod_pic": "http://2xiaopan.fun/poster.jpg", "vod_pic": f"{self.spider.host}/poster.jpg",
"vod_remarks": "HD", "vod_remarks": "HD",
"vod_year": "2025", "vod_year": "2025",
} }
], ],
) )
def test_parse_cards_ignores_module_items_wrapper_and_keeps_each_card_separate(self):
html = """
<div id="main">
<div class="module-items">
<div class="module-item">
<div class="module-item-pic">
<a href="/index.php/vod/detail/id/111.html"></a>
<img data-src="/poster-1.jpg" alt="影片一" />
</div>
<div class="module-item-text">HD</div>
</div>
<div class="module-item">
<div class="module-item-pic">
<a href="/index.php/vod/detail/id/222.html"></a>
<img data-src="/poster-2.jpg" alt="影片二" />
</div>
<div class="module-item-text">更新至2集</div>
</div>
</div>
</div>
"""
cards = self.spider._parse_cards(html)
self.assertEqual(
cards,
[
{
"vod_id": "/index.php/vod/detail/id/111.html",
"vod_name": "影片一",
"vod_pic": f"{self.spider.host}/poster-1.jpg",
"vod_remarks": "HD",
"vod_year": "",
},
{
"vod_id": "/index.php/vod/detail/id/222.html",
"vod_name": "影片二",
"vod_pic": f"{self.spider.host}/poster-2.jpg",
"vod_remarks": "更新至2集",
"vod_year": "",
},
],
)
@patch.object(Spider, "_request_html") @patch.object(Spider, "_request_html")
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
mock_request_html.return_value = """ mock_request_html.return_value = """
@@ -82,7 +124,7 @@ class TestErXiaoSpider(unittest.TestCase):
result = self.spider.categoryContent("2", "3", False, {}) result = self.spider.categoryContent("2", "3", False, {})
self.assertEqual( self.assertEqual(
mock_request_html.call_args.args[0], mock_request_html.call_args.args[0],
"http://2xiaopan.fun/index.php/vod/show/id/2/page/3.html", f"{self.spider.host}/index.php/vod/show/id/2/page/3.html",
) )
self.assertEqual(result["page"], 3) self.assertEqual(result["page"], 3)
self.assertEqual(result["limit"], 1) self.assertEqual(result["limit"], 1)
@@ -102,14 +144,14 @@ class TestErXiaoSpider(unittest.TestCase):
result = self.spider.searchContent("繁花", False, "2") result = self.spider.searchContent("繁花", False, "2")
self.assertEqual( self.assertEqual(
mock_request_html.call_args.args[0], mock_request_html.call_args.args[0],
"http://2xiaopan.fun/index.php/vod/search/page/2/wd/%E7%B9%81%E8%8A%B1.html", f"{self.spider.host}/index.php/vod/search/page/2/wd/%E7%B9%81%E8%8A%B1.html",
) )
self.assertEqual( self.assertEqual(
result["list"][0], result["list"][0],
{ {
"vod_id": "/index.php/vod/detail/id/789.html", "vod_id": "/index.php/vod/detail/id/789.html",
"vod_name": "搜索影片", "vod_name": "搜索影片",
"vod_pic": "http://2xiaopan.fun/search.jpg", "vod_pic": f"{self.spider.host}/search.jpg",
"vod_remarks": "抢先版", "vod_remarks": "抢先版",
}, },
) )
@@ -149,13 +191,26 @@ class TestErXiaoSpider(unittest.TestCase):
""" """
detail = self.spider._parse_detail_page("/index.php/vod/detail/id/123.html", html) detail = self.spider._parse_detail_page("/index.php/vod/detail/id/123.html", html)
self.assertEqual(detail["vod_name"], "示例剧") self.assertEqual(detail["vod_name"], "示例剧")
self.assertEqual(detail["vod_pic"], "http://2xiaopan.fun/poster.jpg") self.assertEqual(detail["vod_pic"], f"{self.spider.host}/poster.jpg")
self.assertEqual(detail["vod_year"], "2024") self.assertEqual(detail["vod_year"], "2024")
self.assertEqual(detail["vod_director"], "导演甲") self.assertEqual(detail["vod_director"], "导演甲")
self.assertEqual(detail["vod_actor"], "演员甲,演员乙") self.assertEqual(detail["vod_actor"], "演员甲,演员乙")
self.assertEqual(detail["vod_content"], "一段剧情简介") self.assertEqual(detail["vod_content"], "一段剧情简介")
self.assertEqual(detail["pan_urls"], ["https://pan.quark.cn/s/q1", "https://pan.baidu.com/s/b1"]) self.assertEqual(detail["pan_urls"], ["https://pan.quark.cn/s/q1", "https://pan.baidu.com/s/b1"])
def test_parse_detail_page_uses_only_first_mobile_play_poster(self):
html = """
<div class="page-title">示例剧</div>
<div class="mobile-play"><img class="lazyload" data-src="/poster-main.jpg" /></div>
<div class="mobile-play"><img class="lazyload" data-src="https://img.example.com/side.webp" /></div>
<div class="mobile-play"><img class="lazyload" data-src="https://img.example.com/recommend.jpg" /></div>
<div class="module-row-info">
<p>https://pan.baidu.com/s/b1</p>
</div>
"""
detail = self.spider._parse_detail_page("/index.php/vod/detail/id/123.html", html)
self.assertEqual(detail["vod_pic"], f"{self.spider.host}/poster-main.jpg")
@patch.object(Spider, "_request_html") @patch.object(Spider, "_request_html")
def test_detail_content_builds_pan_play_fields(self, mock_request_html): def test_detail_content_builds_pan_play_fields(self, mock_request_html):
mock_request_html.return_value = """ mock_request_html.return_value = """
py/tests/test_玩偶聚合.py
+157 -39
View File
@@ -18,15 +18,23 @@ class TestWanouAggregateSpider(unittest.TestCase):
self.spider = Spider() self.spider = Spider()
self.spider.init() self.spider.init()
def test_home_content_exposes_site_classes_and_category_filter(self): def test_home_content_exposes_recommend_and_site_classes_in_priority_order(self):
content = self.spider.homeContent(False) content = self.spider.homeContent(False)
self.assertEqual( self.assertEqual(
[item["type_id"] for item in content["class"][:3]], [item["type_id"] for item in content["class"][:6]],
["site_wanou", "site_zhizhen", "site_shandian"], [
"site_recommend",
"site_wanou",
"site_muou",
"site_labi",
"site_zhizhen",
"site_erxiao",
],
) )
self.assertEqual(content["class"][0]["type_name"], "玩偶") self.assertEqual(content["class"][0]["type_name"], "推荐")
self.assertEqual(content["filters"]["site_wanou"][0]["key"], "categoryId") self.assertEqual(content["filters"]["site_wanou"][0]["key"], "categoryId")
self.assertEqual(content["filters"]["site_wanou"][0]["value"][1], {"n": "电影", "v": "1"}) self.assertEqual(content["filters"]["site_wanou"][0]["value"][1], {"n": "臻彩", "v": "44"})
self.assertEqual(content["filters"]["site_recommend"][0]["value"][0], {"n": "全部", "v": "all"})
def test_home_content_exposes_zhizhen_site_and_categories(self): def test_home_content_exposes_zhizhen_site_and_categories(self):
content = self.spider.homeContent(False) content = self.spider.homeContent(False)
@@ -35,40 +43,58 @@ class TestWanouAggregateSpider(unittest.TestCase):
self.assertEqual( self.assertEqual(
content["filters"]["site_zhizhen"][0]["value"][1:], content["filters"]["site_zhizhen"][0]["value"][1:],
[ [
{"n": "臻彩", "v": "26"},
{"n": "电影", "v": "1"}, {"n": "电影", "v": "1"},
{"n": "剧集", "v": "2"}, {"n": "电视剧", "v": "2"},
{"n": "动漫", "v": "3"}, {"n": "动漫", "v": "3"},
{"n": "综艺", "v": "4"}, {"n": "综艺", "v": "4"},
{"n": "短剧", "v": "5"}, {"n": "短剧", "v": "5"},
{"n": "老剧", "v": "24"}, {"n": "老剧", "v": "24"},
{"n": "严选", "v": "26"},
], ],
) )
def test_home_content_exposes_shandian_site_and_categories(self): def test_home_content_exposes_muou_site_and_categories(self):
content = self.spider.homeContent(False) content = self.spider.homeContent(False)
type_ids = [item["type_id"] for item in content["class"]] type_ids = [item["type_id"] for item in content["class"]]
self.assertIn("site_shandian", type_ids) self.assertIn("site_muou", type_ids)
self.assertEqual( self.assertEqual(
content["filters"]["site_shandian"][0]["value"][1:], content["filters"]["site_muou"][0]["value"][1:],
[ [
{"n": "臻选", "v": "25"},
{"n": "电影", "v": "1"}, {"n": "电影", "v": "1"},
{"n": "剧集", "v": "2"}, {"n": "电视剧", "v": "2"},
{"n": "动漫", "v": "3"},
{"n": "纪录片", "v": "4"},
{"n": "综艺", "v": "29"},
{"n": "原盘", "v": "30"},
],
)
def test_home_content_exposes_kuaiying_and_ouge_site_categories(self):
content = self.spider.homeContent(False)
type_ids = [item["type_id"] for item in content["class"]]
self.assertIn("site_kuaiying", type_ids)
self.assertIn("site_ouge", type_ids)
self.assertEqual(
content["filters"]["site_kuaiying"][0]["value"][1:],
[
{"n": "臻彩", "v": "5"},
{"n": "电影", "v": "1"},
{"n": "电视剧", "v": "2"},
{"n": "综艺", "v": "3"}, {"n": "综艺", "v": "3"},
{"n": "动漫", "v": "4"}, {"n": "动漫", "v": "4"},
{"n": "短剧", "v": "30"}, {"n": "短剧", "v": "6"},
{"n": "115", "v": "30"},
{"n": "123", "v": "35"},
{"n": "天移迅", "v": "36"},
], ],
) )
self.assertEqual(content["class"][-1]["type_name"], "欧哥")
def test_home_content_exposes_ouge_site_and_categories(self):
content = self.spider.homeContent(False)
type_ids = [item["type_id"] for item in content["class"]]
self.assertIn("site_ouge", type_ids)
self.assertEqual( self.assertEqual(
content["filters"]["site_ouge"][0]["value"][1:], content["filters"]["site_ouge"][0]["value"][1:],
[ [
{"n": "电影", "v": "1"}, {"n": "电影", "v": "1"},
{"n": "剧集", "v": "2"}, {"n": "电视剧", "v": "2"},
{"n": "动漫", "v": "3"}, {"n": "动漫", "v": "3"},
{"n": "综艺", "v": "4"}, {"n": "综艺", "v": "4"},
{"n": "短剧", "v": "5"}, {"n": "短剧", "v": "5"},
@@ -182,6 +208,53 @@ class TestWanouAggregateSpider(unittest.TestCase):
], ],
) )
def test_parse_cards_ignores_module_items_wrapper(self):
site = {"id": "erxiao", "domains": ["https://www.2xiaopan.top"], "list_xpath": "//*[contains(@class,'module-item')]"}
html = """
<div id="main">
<div class="module-items">
<div class="module-item">
<div class="module-item-pic">
<a href="/index.php/vod/detail/id/111.html"></a>
<img data-src="/poster-1.jpg" alt="影片一" />
</div>
<div class="module-item-text">HD</div>
</div>
<div class="module-item">
<div class="module-item-pic">
<a href="/index.php/vod/detail/id/222.html"></a>
<img data-src="/poster-2.jpg" alt="影片二" />
</div>
<div class="module-item-text">更新至2集</div>
</div>
</div>
</div>
"""
cards = self.spider._parse_cards(site, html)
self.assertEqual(
cards,
[
{
"vod_id": "site:erxiao:/index.php/vod/detail/id/111.html",
"vod_name": "影片一",
"vod_pic": "https://www.2xiaopan.top/poster-1.jpg",
"vod_remarks": "HD",
"vod_year": "",
"_site": "erxiao",
"_detail_path": "/index.php/vod/detail/id/111.html",
},
{
"vod_id": "site:erxiao:/index.php/vod/detail/id/222.html",
"vod_name": "影片二",
"vod_pic": "https://www.2xiaopan.top/poster-2.jpg",
"vod_remarks": "更新至2集",
"vod_year": "",
"_site": "erxiao",
"_detail_path": "/index.php/vod/detail/id/222.html",
},
],
)
@patch.object(Spider, "_request_with_failover") @patch.object(Spider, "_request_with_failover")
def test_category_content_uses_default_category_when_extend_missing(self, mock_request_with_failover): def test_category_content_uses_default_category_when_extend_missing(self, mock_request_with_failover):
mock_request_with_failover.return_value = """ mock_request_with_failover.return_value = """
@@ -197,6 +270,37 @@ class TestWanouAggregateSpider(unittest.TestCase):
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["list"][0]["vod_name"], "分类影片") self.assertEqual(result["list"][0]["vod_name"], "分类影片")
@patch.object(Spider, "_fetch_site_home_recommend")
def test_category_content_returns_recommend_results_with_site_tags(self, mock_fetch_site_home_recommend):
mock_fetch_site_home_recommend.side_effect = lambda site, limit=30: {
"wanou": [
{
"vod_id": "site:wanou:/voddetail/1.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/w.jpg",
"vod_remarks": "玩偶版",
"vod_year": "2024",
"_site": "wanou",
"_detail_path": "/voddetail/1.html",
}
],
"muou": [
{
"vod_id": "site:muou:/voddetail/2.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/m.jpg",
"vod_remarks": "木偶版",
"vod_year": "2024",
"_site": "muou",
"_detail_path": "/voddetail/2.html",
}
],
}.get(site["id"], [])
result = self.spider.categoryContent("site_recommend", "1", False, {"recommendSite": "all"})
self.assertEqual(result["total"], 1)
self.assertEqual(result["list"][0]["vod_name"], "繁花")
self.assertEqual(result["list"][0]["vod_remarks"], "[玩偶] 玩偶版")
def test_aggregate_search_results_merges_same_title_and_keeps_highest_priority_source(self): def test_aggregate_search_results_merges_same_title_and_keeps_highest_priority_source(self):
raw_results = [ raw_results = [
{ {
@@ -247,8 +351,8 @@ class TestWanouAggregateSpider(unittest.TestCase):
@patch.object(Spider, "_fetch_site_search") @patch.object(Spider, "_fetch_site_search")
def test_search_content_queries_sites_and_returns_aggregated_items(self, mock_fetch_site_search): def test_search_content_queries_sites_and_returns_aggregated_items(self, mock_fetch_site_search):
mock_fetch_site_search.side_effect = [ results_by_site = {
[ "wanou": [
{ {
"vod_id": "site:wanou:/voddetail/1.html", "vod_id": "site:wanou:/voddetail/1.html",
"vod_name": "繁花", "vod_name": "繁花",
@@ -259,7 +363,7 @@ class TestWanouAggregateSpider(unittest.TestCase):
"_detail_path": "/voddetail/1.html", "_detail_path": "/voddetail/1.html",
} }
], ],
[ "muou": [
{ {
"vod_id": "site:muou:/voddetail/2.html", "vod_id": "site:muou:/voddetail/2.html",
"vod_name": "繁花", "vod_name": "繁花",
@@ -270,8 +374,8 @@ class TestWanouAggregateSpider(unittest.TestCase):
"_detail_path": "/voddetail/2.html", "_detail_path": "/voddetail/2.html",
} }
], ],
[], }
] mock_fetch_site_search.side_effect = lambda site, keyword, page: results_by_site.get(site["id"], [])
result = self.spider.searchContent("繁花", False, "1") result = self.spider.searchContent("繁花", False, "1")
self.assertEqual(len(result["list"]), 1) self.assertEqual(len(result["list"]), 1)
self.assertEqual(result["list"][0]["vod_name"], "繁花") self.assertEqual(result["list"][0]["vod_name"], "繁花")
@@ -345,10 +449,22 @@ class TestWanouAggregateSpider(unittest.TestCase):
self.spider.playerContent("quark#玩偶", "https://pan.quark.cn/s/demo", {}), self.spider.playerContent("quark#玩偶", "https://pan.quark.cn/s/demo", {}),
{"parse": 0, "playUrl": "", "url": "https://pan.quark.cn/s/demo"}, {"parse": 0, "playUrl": "", "url": "https://pan.quark.cn/s/demo"},
) )
self.assertEqual(
self.spider.playerContent("quark#玩偶", "https://www.quark.cn/s/demo", {}),
{"parse": 0, "playUrl": "", "url": "https://www.quark.cn/s/demo"},
)
self.assertEqual( self.assertEqual(
self.spider.playerContent("baidu#玩偶", "https://pan.baidu.com/s/demo", {}), self.spider.playerContent("baidu#玩偶", "https://pan.baidu.com/s/demo", {}),
{"parse": 0, "playUrl": "", "url": "https://pan.baidu.com/s/demo"}, {"parse": 0, "playUrl": "", "url": "https://pan.baidu.com/s/demo"},
) )
self.assertEqual(
self.spider.playerContent("a189#玩偶", "https://cloud.189.cn/t/demo", {}),
{"parse": 0, "playUrl": "", "url": "https://cloud.189.cn/t/demo"},
)
self.assertEqual(
self.spider.playerContent("a139#玩偶", "https://yun.139.com/share/demo", {}),
{"parse": 0, "playUrl": "", "url": "https://yun.139.com/share/demo"},
)
def test_player_content_rejects_non_pan_url(self): def test_player_content_rejects_non_pan_url(self):
self.assertEqual( self.assertEqual(
@@ -460,22 +576,24 @@ class TestWanouAggregateSpider(unittest.TestCase):
@patch.object(Spider, "_fetch_site_search") @patch.object(Spider, "_fetch_site_search")
def test_search_content_skips_site_errors(self, mock_fetch_site_search): def test_search_content_skips_site_errors(self, mock_fetch_site_search):
mock_fetch_site_search.side_effect = [ def fake_fetch(site, keyword, page):
RuntimeError("boom"), if site["id"] == "wanou":
[ raise RuntimeError("boom")
{ if site["id"] == "muou":
"vod_id": "site:muou:/voddetail/2.html", return [
"vod_name": "繁花", {
"vod_pic": "", "vod_id": "site:muou:/voddetail/2.html",
"vod_remarks": "", "vod_name": "繁花",
"vod_year": "2024", "vod_pic": "",
"_site": "muou", "vod_remarks": "",
"_detail_path": "/voddetail/2.html", "vod_year": "2024",
} "_site": "muou",
], "_detail_path": "/voddetail/2.html",
[], }
[], ]
] return []
mock_fetch_site_search.side_effect = fake_fetch
result = self.spider.searchContent("繁花", False, "1") result = self.spider.searchContent("繁花", False, "1")
self.assertEqual(result["total"], 1) self.assertEqual(result["total"], 1)
self.assertEqual(result["list"][0]["vod_name"], "繁花") self.assertEqual(result["list"][0]["vod_name"], "繁花")
py/二小.py
+42 -25
View File
@@ -67,6 +67,9 @@ class Spider(BaseSpider):
def _clean_text(self, text): def _clean_text(self, text):
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip() return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
def _class_xpath(self, class_name):
return f"contains(concat(' ', normalize-space(@class), ' '), ' {class_name} ')"
def _detect_pan_type(self, url): def _detect_pan_type(self, url):
raw = str(url or "").strip() raw = str(url or "").strip()
for pan_type, title, pattern in self.pan_patterns: for pan_type, title, pattern in self.pan_patterns:
@@ -92,15 +95,19 @@ class Spider(BaseSpider):
items = [] items = []
seen = set() seen = set()
for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): item_xpath = self._class_xpath("module-item")
href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() pic_box_xpath = self._class_xpath("module-item-pic")
title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() text_xpath = self._class_xpath("module-item-text")
caption_xpath = self._class_xpath("module-item-caption")
for node in root.xpath(f"//*[@id='main']//*[{item_xpath}]"):
href = "".join(node.xpath(f".//*[{pic_box_xpath}]//a[1]/@href")).strip()
title = "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@alt")).strip()
pic = ( pic = (
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src")).strip()
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() or "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@src")).strip()
) )
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) remarks = self._clean_text("".join(node.xpath(f".//*[{text_xpath}][1]//text()")))
year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()"))) year = self._clean_text("".join(node.xpath(f".//*[{caption_xpath}][1]//span[1]//text()")))
if not href or not title or href in seen: if not href or not title or href in seen:
continue continue
seen.add(href) seen.add(href)
@@ -131,16 +138,20 @@ class Spider(BaseSpider):
return {"page": page, "total": 0, "list": []} return {"page": page, "total": 0, "list": []}
items = [] items = []
for node in root.xpath("//*[contains(@class,'module-search-item')]"): search_item_xpath = self._class_xpath("module-search-item")
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() video_serial_xpath = self._class_xpath("video-serial")
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() pic_box_xpath = self._class_xpath("module-item-pic")
text_xpath = self._class_xpath("module-item-text")
for node in root.xpath(f"//*[{search_item_xpath}]"):
href = "".join(node.xpath(f".//*[{video_serial_xpath}][1]/@href")).strip()
title = "".join(node.xpath(f".//*[{video_serial_xpath}][1]/@title")).strip()
pic = ( pic = (
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src")).strip()
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() or "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@src")).strip()
) )
remarks = self._clean_text( remarks = self._clean_text(
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) "".join(node.xpath(f".//*[{video_serial_xpath}][1]//text()"))
or "".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")) or "".join(node.xpath(f".//*[{text_xpath}][1]//text()"))
) )
if not href or not title: if not href or not title:
continue continue
@@ -168,17 +179,23 @@ class Spider(BaseSpider):
"pan_urls": [], "pan_urls": [],
} }
mobile_play_xpath = self._class_xpath("mobile-play")
lazyload_xpath = self._class_xpath("lazyload")
page_title_xpath = self._class_xpath("page-title")
info_title_xpath = self._class_xpath("video-info-itemtitle")
row_info_xpath = self._class_xpath("module-row-info")
pic_candidates = root.xpath(
"("
f"//*[{mobile_play_xpath}]//*[{lazyload_xpath}]/@data-src | "
f"//*[{mobile_play_xpath}]//*[{lazyload_xpath}]/@src"
")[1]"
)
detail = { detail = {
"vod_id": vod_id, "vod_id": vod_id,
"vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))), "vod_name": self._clean_text("".join(root.xpath(f"//*[{page_title_xpath}][1]//text()"))),
"vod_pic": self._build_url( "vod_pic": self._build_url((pic_candidates[0] if pic_candidates else "").strip()),
"".join(
root.xpath(
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | "
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src"
)
).strip()
),
"vod_year": "", "vod_year": "",
"vod_director": "", "vod_director": "",
"vod_actor": "", "vod_actor": "",
@@ -186,7 +203,7 @@ class Spider(BaseSpider):
"pan_urls": [], "pan_urls": [],
} }
for label_node in root.xpath("//*[contains(@class,'video-info-itemtitle')]"): for label_node in root.xpath(f"//*[{info_title_xpath}]"):
key = self._clean_text("".join(label_node.xpath(".//text()"))) key = self._clean_text("".join(label_node.xpath(".//text()")))
sibling = label_node.getnext() sibling = label_node.getnext()
if sibling is None: if sibling is None:
@@ -203,7 +220,7 @@ class Spider(BaseSpider):
elif "剧情" in key: elif "剧情" in key:
detail["vod_content"] = text_value detail["vod_content"] = text_value
for node in root.xpath("//*[contains(@class,'module-row-info')]//p"): for node in root.xpath(f"//*[{row_info_xpath}]//p"):
text = self._clean_text("".join(node.xpath(".//text()"))) text = self._clean_text("".join(node.xpath(".//text()")))
if text: if text:
detail["pan_urls"].append(text) detail["pan_urls"].append(text)
py/玩偶聚合.py
+232 -61
View File
@@ -15,6 +15,7 @@ class Spider(BaseSpider):
def __init__(self): def __init__(self):
self.name = "玩偶聚合" self.name = "玩偶聚合"
self.filter_root = os.path.join(os.path.dirname(__file__), "../筛选") self.filter_root = os.path.join(os.path.dirname(__file__), "../筛选")
self.recommend_page_size = 20
self.headers = { self.headers = {
"User-Agent": ( "User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -48,7 +49,7 @@ class Spider(BaseSpider):
{ {
"id": "wanou", "id": "wanou",
"name": "玩偶", "name": "玩偶",
"domains": ["https://wogg.xxooo.cf"], "domains": ["https://wogg.xxooo.cf", "https://www.wogg.net"],
"filter_files": ["wogg.json"], "filter_files": ["wogg.json"],
"list_xpath": "//*[contains(@class,'module-item')]", "list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]",
@@ -56,65 +57,140 @@ class Spider(BaseSpider):
"category_url": "/vodshow/{categoryId}--------{page}---.html", "category_url": "/vodshow/{categoryId}--------{page}---.html",
"category_url_with_filters": "/vodshow/{categoryId}-{area}-{by}-{class}-----{page}---{year}.html", "category_url_with_filters": "/vodshow/{categoryId}-{area}-{by}-{class}-----{page}---{year}.html",
"search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
"default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")], "default_categories": [
("44", "臻彩"),
("1", "电影"),
("2", "电视剧"),
("3", "动漫"),
("4", "综艺"),
("5", "音乐"),
("6", "短剧"),
("46", "纪录片"),
],
}, },
# {
# "id": "muou",
# "name": "木偶",
# "domains": ["https://www.muou.site", "http://123.666291.xyz"],
# "filter_files": ["mogg.json"],
# "list_xpath": "//*[contains(@class,'module-item')]",
# "search_xpath": "//*[contains(@class,'module-search-item')]",
# "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
# "category_url": "/vodshow/{categoryId}--------{page}---.html",
# "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
# "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("29", "综艺")],
# },
# {
# "id": "labi",
# "name": "蜡笔",
# "domains": ["http://xiaocge.fun"],
# "filter_files": ["labi.json"],
# "list_xpath": "//*[contains(@class,'module-item')]",
# "search_xpath": "//*[contains(@class,'module-search-item')]",
# "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
# "category_url": "/vodshow/{categoryId}--------{page}---.html",
# "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
# "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")],
# },
{ {
"id": "zhizhen", "id": "muou",
"name": "至臻", "name": "木偶",
"domains": ["http://www.miqk.cc"], "domains": ["https://www.muou.site", "https://www.muou.asia", "https://666.666291.xyz"],
"filter_files": [], "filter_files": ["mogg.json"],
"list_xpath": "//*[contains(@class,'module-item')]", "list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html",
"search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html",
"default_categories": [ "default_categories": [
("25", "臻选"),
("1", "电影"), ("1", "电影"),
("2", "剧集"), ("2", "电视剧"),
("3", "动漫"),
("4", "纪录片"),
("29", "综艺"),
("30", "原盘"),
],
},
{
"id": "labi",
"name": "蜡笔",
"domains": ["http://xiaocgege.shop", "http://fmao.shop"],
"filter_files": ["labi.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"default_categories": [
("29", "臻彩"),
("1", "电影"),
("2", "电视剧"),
("3", "动漫"),
("4", "综艺"),
("5", "短剧"),
("24", "蜡笔4K"),
],
},
{
"id": "zhizhen",
"name": "至臻",
"domains": ["http://www.miqk.cc", "https://www.mihdr.top", "https://mihdr.top"],
"filter_files": ["zhizhen.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"default_categories": [
("26", "臻彩"),
("1", "电影"),
("2", "电视剧"),
("3", "动漫"), ("3", "动漫"),
("4", "综艺"), ("4", "综艺"),
("5", "短剧"), ("5", "短剧"),
("24", "老剧"), ("24", "老剧"),
("26", "严选"),
], ],
}, },
{
"id": "erxiao",
"name": "二小",
"domains": [
"https://www.2xiaopan.top",
"https://www.erxiaozhan.top",
"https://www.wexwp.cc",
],
"filter_files": ["erxiao.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"default_categories": [
("4", "臻彩"),
("1", "电影"),
("2", "电视剧"),
("3", "动漫"),
("21", "综艺"),
],
},
# {
# "id": "huban",
# "name": "虎斑",
# "domains": ["http://154.222.27.33:20720", "http://xhban.xyz:20720", "http://103.45.162.207:20720"],
# "filter_files": ["huban.json"],
# "list_xpath": "//*[contains(@class,'module-item')]",
# "search_xpath": "//*[contains(@class,'module-search-item')]",
# "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
# "default_categories": [
# ("6", "臻彩"),
# ("1", "电影"),
# ("2", "电视剧"),
# ("3", "综艺"),
# ("4", "动漫"),
# ("5", "短剧"),
# ("30", "115网盘"),
# ],
# },
# {
# "id": "kuaiying",
# "name": "快映",
# "domains": ["http://154.201.83.50:12512", "http://xsayang.fun:12512"],
# "filter_files": ["xiaoban.json"],
# "list_xpath": "//*[contains(@class,'module-item')]",
# "search_xpath": "//*[contains(@class,'module-search-item')]",
# "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
# "default_categories": [
# ("5", "臻彩"),
# ("1", "电影"),
# ("2", "电视剧"),
# ("3", "综艺"),
# ("4", "动漫"),
# ("6", "短剧"),
# ("30", "115"),
# ("35", "123"),
# ("36", "天移迅"),
# ],
# },
{ {
"id": "shandian", "id": "shandian",
"name": "闪电", "name": "闪电",
"domains": ["https://sd.sduc.site"], "domains": ["https://sd.sduc.site"],
"filter_files": [], "filter_files": ["shandian.json"],
"list_xpath": "//*[contains(@class,'module-item')]", "list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html",
"search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html",
"default_categories": [ "default_categories": [
("1", "电影"), ("1", "电影"),
("2", "剧集"), ("2", "电视剧"),
("3", "综艺"), ("3", "综艺"),
("4", "动漫"), ("4", "动漫"),
("30", "短剧"), ("30", "短剧"),
@@ -122,17 +198,15 @@ class Spider(BaseSpider):
}, },
{ {
"id": "ouge", "id": "ouge",
"name": "欧歌", "name": "欧哥",
"domains": ["https://woog.nxog.eu.org"], "domains": ["https://woog.nxog.eu.org"],
"filter_files": [], "filter_files": ["ouge.json"],
"list_xpath": "//*[contains(@class,'module-item')]", "list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]",
"detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p",
"category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html",
"search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html",
"default_categories": [ "default_categories": [
("1", "电影"), ("1", "电影"),
("2", "剧集"), ("2", "电视剧"),
("3", "动漫"), ("3", "动漫"),
("4", "综艺"), ("4", "综艺"),
("5", "短剧"), ("5", "短剧"),
@@ -151,7 +225,47 @@ class Spider(BaseSpider):
return {"list": []} return {"list": []}
def _load_local_filter_groups(self, site): def _load_local_filter_groups(self, site):
return [] groups = []
for filename in site.get("filter_files", []):
if not filename:
continue
file_path = os.path.join(self.filter_root, filename)
if not os.path.exists(file_path):
continue
try:
with open(file_path, "r", encoding="utf-8") as handle:
raw = json.load(handle)
except Exception:
continue
merged = {}
for category_groups in raw.values() if isinstance(raw, dict) else []:
if not isinstance(category_groups, list):
continue
for group in category_groups:
key = str(group.get("key") or "").strip()
if not key or key == "categoryId":
continue
bucket = merged.setdefault(
key,
{
"key": key,
"name": str(group.get("name") or key),
"init": str(group.get("init") or ""),
"value": [],
},
)
for item in group.get("value", []):
value = str(item.get("v") if isinstance(item, dict) else "")
name = str(item.get("n") if isinstance(item, dict) else "")
if not name:
continue
if not any(existing.get("v") == value for existing in bucket["value"]):
bucket["value"].append({"n": name, "v": value})
for group in merged.values():
if not any(item.get("v") == "" for item in group["value"]):
group["value"].insert(0, {"n": "全部", "v": ""})
groups.append(group)
return groups
def _build_site_filters(self, site): def _build_site_filters(self, site):
groups = [ groups = [
@@ -167,8 +281,20 @@ class Spider(BaseSpider):
return groups return groups
def homeContent(self, filter): def homeContent(self, filter):
classes = [{"type_id": f"site_{site['id']}", "type_name": site["name"]} for site in self.sites] classes = [{"type_id": "site_recommend", "type_name": "推荐"}]
filters = {f"site_{site['id']}": self._build_site_filters(site) for site in self.sites} classes.extend({"type_id": f"site_{site['id']}", "type_name": site["name"]} for site in self.sites)
filters = {
"site_recommend": [
{
"key": "recommendSite",
"name": "站点",
"init": "all",
"value": [{"n": "全部", "v": "all"}]
+ [{"n": site["name"], "v": site["id"]} for site in self.sites],
}
]
}
filters.update({f"site_{site['id']}": self._build_site_filters(site) for site in self.sites})
return {"class": classes, "filters": filters} return {"class": classes, "filters": filters}
def _encode_site_vod_id(self, site_id, path): def _encode_site_vod_id(self, site_id, path):
@@ -218,6 +344,13 @@ class Spider(BaseSpider):
return "https:" + raw return "https:" + raw
return str(base).rstrip("/") + "/" + raw.lstrip("/") return str(base).rstrip("/") + "/" + raw.lstrip("/")
def _class_xpath(self, class_name):
return f"contains(concat(' ', normalize-space(@class), ' '), ' {class_name} ')"
def _has_class(self, node, class_name):
classes = f" {str(node.get('class') or '').strip()} "
return f" {class_name} " in classes
def _build_category_url(self, site, category_id, pg, extend): def _build_category_url(self, site, category_id, pg, extend):
values = dict(extend or {}) values = dict(extend or {})
values.setdefault("categoryId", category_id) values.setdefault("categoryId", category_id)
@@ -237,7 +370,8 @@ class Spider(BaseSpider):
} }
) )
else: else:
path = site["category_url"].format(categoryId=values["categoryId"], page=int(pg)) template = site.get("category_url") or "/index.php/vod/show/id/{categoryId}/page/{page}.html"
path = template.format(categoryId=values["categoryId"], page=int(pg))
return self._build_absolute_url(site["domains"][0], path) return self._build_absolute_url(site["domains"][0], path)
def _request_with_failover(self, site, path_or_url, referer=None): def _request_with_failover(self, site, path_or_url, referer=None):
@@ -263,17 +397,21 @@ class Spider(BaseSpider):
items = [] items = []
seen = set() seen = set()
pic_box_xpath = self._class_xpath("module-item-pic")
text_xpath = self._class_xpath("module-item-text")
for card in root.xpath(site["list_xpath"]): for card in root.xpath(site["list_xpath"]):
if not self._has_class(card, "module-item"):
continue
href = ((card.xpath(".//a[@href][1]/@href") or [""])[0]).strip() href = ((card.xpath(".//a[@href][1]/@href") or [""])[0]).strip()
title = ( title = (
((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip() ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip()
or ((card.xpath(".//a[@title][1]/@title") or [""])[0]).strip() or ((card.xpath(".//a[@title][1]/@title") or [""])[0]).strip()
) )
pic = ( pic = (
((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip() ((card.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src") or [""])[0]).strip()
or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip() or ((card.xpath(f".//*[{pic_box_xpath}]//img[1]/@src") or [""])[0]).strip()
) )
remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip() remarks = "".join(card.xpath(f".//*[{text_xpath}][1]//text()")).strip()
if not href or not title or href in seen: if not href or not title or href in seen:
continue continue
seen.add(href) seen.add(href)
@@ -292,8 +430,31 @@ class Spider(BaseSpider):
def categoryContent(self, tid, pg, filter, extend): def categoryContent(self, tid, pg, filter, extend):
site_id = str(tid).replace("site_", "", 1) site_id = str(tid).replace("site_", "", 1)
site = self._get_site(site_id)
values = extend if isinstance(extend, dict) else {} values = extend if isinstance(extend, dict) else {}
if site_id == "recommend":
recommend_site_id = str(values.get("recommendSite") or "all")
candidates = self.sites if recommend_site_id == "all" else [site for site in self.sites if site["id"] == recommend_site_id]
merged = []
for site in candidates:
for item in self._fetch_site_home_recommend(site):
matched = None
for existing in merged:
if self._is_same_title(existing, item):
matched = existing
break
if matched is None:
source_name = site["name"]
remarks = str(item.get("vod_remarks") or "").strip()
tagged = dict(item)
tagged["vod_remarks"] = f"[{source_name}] {remarks}".strip()
merged.append(tagged)
page = int(pg)
start = max(page - 1, 0) * self.recommend_page_size
end = start + self.recommend_page_size
paged = merged[start:end]
return {"list": paged, "page": page, "limit": len(paged), "total": len(merged)}
site = self._get_site(site_id)
category_id = values.get("categoryId") or site["default_categories"][0][0] category_id = values.get("categoryId") or site["default_categories"][0][0]
html = self._request_with_failover(site, self._build_category_url(site, category_id, pg, values)) html = self._request_with_failover(site, self._build_category_url(site, category_id, pg, values))
items = self._parse_cards(site, html) items = self._parse_cards(site, html)
@@ -313,13 +474,17 @@ class Spider(BaseSpider):
items = [] items = []
seen = set() seen = set()
xpath = site.get("search_xpath") or site["list_xpath"] xpath = site.get("search_xpath") or site["list_xpath"]
video_serial_xpath = self._class_xpath("video-serial")
text_xpath = self._class_xpath("module-item-text")
for card in root.xpath(xpath): for card in root.xpath(xpath):
if "module-search-item" in xpath and not self._has_class(card, "module-search-item"):
continue
href = ( href = (
((card.xpath(".//*[contains(@class,'video-serial')][1]/@href") or [""])[0]).strip() ((card.xpath(f".//*[{video_serial_xpath}][1]/@href") or [""])[0]).strip()
or ((card.xpath(".//*[@href][1]/@href") or [""])[0]).strip() or ((card.xpath(".//*[@href][1]/@href") or [""])[0]).strip()
) )
title = ( title = (
((card.xpath(".//*[contains(@class,'video-serial')][1]/@title") or [""])[0]).strip() ((card.xpath(f".//*[{video_serial_xpath}][1]/@title") or [""])[0]).strip()
or ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip() or ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip()
or ((card.xpath(".//*[@title][1]/@title") or [""])[0]).strip() or ((card.xpath(".//*[@title][1]/@title") or [""])[0]).strip()
) )
@@ -327,7 +492,7 @@ class Spider(BaseSpider):
((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip() ((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip()
or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip() or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip()
) )
remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip() remarks = "".join(card.xpath(f".//*[{text_xpath}][1]//text()")).strip()
if not href or not title or href in seen: if not href or not title or href in seen:
continue continue
seen.add(href) seen.add(href)
@@ -345,10 +510,16 @@ class Spider(BaseSpider):
return items return items
def _fetch_site_search(self, site, keyword, pg): def _fetch_site_search(self, site, keyword, pg):
search_path = site["search_url"].format(keyword=quote(str(keyword)), page=int(pg)) template = site.get("search_url") or "/index.php/vod/search/page/{page}/wd/{keyword}.html"
search_path = template.format(keyword=quote(str(keyword)), page=int(pg))
html = self._request_with_failover(site, search_path) html = self._request_with_failover(site, search_path)
return self._parse_search_cards(site, html) return self._parse_search_cards(site, html)
def _fetch_site_home_recommend(self, site, limit=30):
html = self._request_with_failover(site, "/")
items = self._parse_cards(site, html)
return items[:limit]
def _aggregate_search_results(self, items): def _aggregate_search_results(self, items):
groups = [] groups = []
for item in items: for item in items:
@@ -406,21 +577,21 @@ class Spider(BaseSpider):
value = str(url or "").lower() value = str(url or "").lower()
if "pan.baidu.com" in value: if "pan.baidu.com" in value:
return "baidu", "百度资源" return "baidu", "百度资源"
if "pan.quark.cn" in value: if "quark.cn" in value:
return "quark", "夸克资源" return "quark", "夸克资源"
if "drive.uc.cn" in value: if "uc.cn" in value:
return "uc", "UC资源" return "uc", "UC资源"
if "alipan.com" in value or "aliyundrive.com" in value: if "alipan.com" in value or "aliyundrive.com" in value:
return "aliyun", "阿里资源" return "aliyun", "阿里资源"
if "pan.xunlei.com" in value: if "xunlei.com" in value:
return "xunlei", "迅雷资源" return "xunlei", "迅雷资源"
if "123pan.com" in value: if "123pan.com" in value:
return "a123", "123资源" return "a123", "123资源"
if "115.com" in value: if "115.com" in value or "115cdn.com" in value:
return "a115", "115资源" return "a115", "115资源"
if "189.cn" in value: if "cloud.189.cn" in value or "189.cn" in value:
return "a189", "天翼资源" return "a189", "天翼资源"
if "139.com" in value: if "yun.139.com" in value or "139.com" in value:
return "a139", "移动云资源" return "a139", "移动云资源"
return "", "" return "", ""