diff --git a/py/tests/test_二小.py b/py/tests/test_二小.py index 07d44e8..85e30d2 100644 --- a/py/tests/test_二小.py +++ b/py/tests/test_二小.py @@ -33,7 +33,7 @@ class TestErXiaoSpider(unittest.TestCase): def test_build_url_and_detect_pan_type(self): self.assertEqual( self.spider._build_url("/index.php/vod/detail/id/1.html"), - "http://2xiaopan.fun/index.php/vod/detail/id/1.html", + f"{self.spider.host}/index.php/vod/detail/id/1.html", ) self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源")) self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源")) @@ -58,13 +58,55 @@ class TestErXiaoSpider(unittest.TestCase): { "vod_id": "/index.php/vod/detail/id/123.html", "vod_name": "示例影片", - "vod_pic": "http://2xiaopan.fun/poster.jpg", + "vod_pic": f"{self.spider.host}/poster.jpg", "vod_remarks": "HD", "vod_year": "2025", } ], ) + def test_parse_cards_ignores_module_items_wrapper_and_keeps_each_card_separate(self): + html = """ +
+
+
+
+ + 影片一 +
+
HD
+
+
+
+ + 影片二 +
+
更新至2集
+
+
+
+ """ + cards = self.spider._parse_cards(html) + self.assertEqual( + cards, + [ + { + "vod_id": "/index.php/vod/detail/id/111.html", + "vod_name": "影片一", + "vod_pic": f"{self.spider.host}/poster-1.jpg", + "vod_remarks": "HD", + "vod_year": "", + }, + { + "vod_id": "/index.php/vod/detail/id/222.html", + "vod_name": "影片二", + "vod_pic": f"{self.spider.host}/poster-2.jpg", + "vod_remarks": "更新至2集", + "vod_year": "", + }, + ], + ) + @patch.object(Spider, "_request_html") def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): mock_request_html.return_value = """ @@ -82,7 +124,7 @@ class TestErXiaoSpider(unittest.TestCase): result = self.spider.categoryContent("2", "3", False, {}) self.assertEqual( mock_request_html.call_args.args[0], - "http://2xiaopan.fun/index.php/vod/show/id/2/page/3.html", + f"{self.spider.host}/index.php/vod/show/id/2/page/3.html", ) self.assertEqual(result["page"], 3) self.assertEqual(result["limit"], 1) @@ -102,14 +144,14 @@ class TestErXiaoSpider(unittest.TestCase): result = self.spider.searchContent("繁花", False, "2") self.assertEqual( mock_request_html.call_args.args[0], - "http://2xiaopan.fun/index.php/vod/search/page/2/wd/%E7%B9%81%E8%8A%B1.html", + f"{self.spider.host}/index.php/vod/search/page/2/wd/%E7%B9%81%E8%8A%B1.html", ) self.assertEqual( result["list"][0], { "vod_id": "/index.php/vod/detail/id/789.html", "vod_name": "搜索影片", - "vod_pic": "http://2xiaopan.fun/search.jpg", + "vod_pic": f"{self.spider.host}/search.jpg", "vod_remarks": "抢先版", }, ) @@ -149,13 +191,26 @@ class TestErXiaoSpider(unittest.TestCase): """ detail = self.spider._parse_detail_page("/index.php/vod/detail/id/123.html", html) self.assertEqual(detail["vod_name"], "示例剧") - self.assertEqual(detail["vod_pic"], "http://2xiaopan.fun/poster.jpg") + self.assertEqual(detail["vod_pic"], f"{self.spider.host}/poster.jpg") self.assertEqual(detail["vod_year"], "2024") self.assertEqual(detail["vod_director"], "导演甲") self.assertEqual(detail["vod_actor"], "演员甲,演员乙") self.assertEqual(detail["vod_content"], "一段剧情简介") self.assertEqual(detail["pan_urls"], ["https://pan.quark.cn/s/q1", "https://pan.baidu.com/s/b1"]) + def test_parse_detail_page_uses_only_first_mobile_play_poster(self): + html = """ +
示例剧
+
+
+
+
+

https://pan.baidu.com/s/b1

+
+ """ + detail = self.spider._parse_detail_page("/index.php/vod/detail/id/123.html", html) + self.assertEqual(detail["vod_pic"], f"{self.spider.host}/poster-main.jpg") + @patch.object(Spider, "_request_html") def test_detail_content_builds_pan_play_fields(self, mock_request_html): mock_request_html.return_value = """ diff --git a/py/tests/test_玩偶聚合.py b/py/tests/test_玩偶聚合.py index 5f5d9d0..e7dc21e 100644 --- a/py/tests/test_玩偶聚合.py +++ b/py/tests/test_玩偶聚合.py @@ -18,15 +18,23 @@ class TestWanouAggregateSpider(unittest.TestCase): self.spider = Spider() self.spider.init() - def test_home_content_exposes_site_classes_and_category_filter(self): + def test_home_content_exposes_recommend_and_site_classes_in_priority_order(self): content = self.spider.homeContent(False) self.assertEqual( - [item["type_id"] for item in content["class"][:3]], - ["site_wanou", "site_zhizhen", "site_shandian"], + [item["type_id"] for item in content["class"][:6]], + [ + "site_recommend", + "site_wanou", + "site_muou", + "site_labi", + "site_zhizhen", + "site_erxiao", + ], ) - self.assertEqual(content["class"][0]["type_name"], "玩偶") + self.assertEqual(content["class"][0]["type_name"], "推荐") self.assertEqual(content["filters"]["site_wanou"][0]["key"], "categoryId") - self.assertEqual(content["filters"]["site_wanou"][0]["value"][1], {"n": "电影", "v": "1"}) + self.assertEqual(content["filters"]["site_wanou"][0]["value"][1], {"n": "臻彩", "v": "44"}) + self.assertEqual(content["filters"]["site_recommend"][0]["value"][0], {"n": "全部", "v": "all"}) def test_home_content_exposes_zhizhen_site_and_categories(self): content = self.spider.homeContent(False) @@ -35,40 +43,58 @@ class TestWanouAggregateSpider(unittest.TestCase): self.assertEqual( content["filters"]["site_zhizhen"][0]["value"][1:], [ + {"n": "臻彩", "v": "26"}, {"n": "电影", "v": "1"}, - {"n": "剧集", "v": "2"}, + {"n": "电视剧", "v": "2"}, {"n": "动漫", "v": "3"}, {"n": "综艺", "v": "4"}, {"n": "短剧", "v": "5"}, {"n": "老剧", "v": "24"}, - {"n": "严选", "v": "26"}, ], ) - def test_home_content_exposes_shandian_site_and_categories(self): + def test_home_content_exposes_muou_site_and_categories(self): content = self.spider.homeContent(False) type_ids = [item["type_id"] for item in content["class"]] - self.assertIn("site_shandian", type_ids) + self.assertIn("site_muou", type_ids) self.assertEqual( - content["filters"]["site_shandian"][0]["value"][1:], + content["filters"]["site_muou"][0]["value"][1:], [ + {"n": "臻选", "v": "25"}, {"n": "电影", "v": "1"}, - {"n": "剧集", "v": "2"}, + {"n": "电视剧", "v": "2"}, + {"n": "动漫", "v": "3"}, + {"n": "纪录片", "v": "4"}, + {"n": "综艺", "v": "29"}, + {"n": "原盘", "v": "30"}, + ], + ) + + def test_home_content_exposes_kuaiying_and_ouge_site_categories(self): + content = self.spider.homeContent(False) + type_ids = [item["type_id"] for item in content["class"]] + self.assertIn("site_kuaiying", type_ids) + self.assertIn("site_ouge", type_ids) + self.assertEqual( + content["filters"]["site_kuaiying"][0]["value"][1:], + [ + {"n": "臻彩", "v": "5"}, + {"n": "电影", "v": "1"}, + {"n": "电视剧", "v": "2"}, {"n": "综艺", "v": "3"}, {"n": "动漫", "v": "4"}, - {"n": "短剧", "v": "30"}, + {"n": "短剧", "v": "6"}, + {"n": "115", "v": "30"}, + {"n": "123", "v": "35"}, + {"n": "天移迅", "v": "36"}, ], ) - - def test_home_content_exposes_ouge_site_and_categories(self): - content = self.spider.homeContent(False) - type_ids = [item["type_id"] for item in content["class"]] - self.assertIn("site_ouge", type_ids) + self.assertEqual(content["class"][-1]["type_name"], "欧哥") self.assertEqual( content["filters"]["site_ouge"][0]["value"][1:], [ {"n": "电影", "v": "1"}, - {"n": "剧集", "v": "2"}, + {"n": "电视剧", "v": "2"}, {"n": "动漫", "v": "3"}, {"n": "综艺", "v": "4"}, {"n": "短剧", "v": "5"}, @@ -182,6 +208,53 @@ class TestWanouAggregateSpider(unittest.TestCase): ], ) + def test_parse_cards_ignores_module_items_wrapper(self): + site = {"id": "erxiao", "domains": ["https://www.2xiaopan.top"], "list_xpath": "//*[contains(@class,'module-item')]"} + html = """ +
+
+
+
+ + 影片一 +
+
HD
+
+
+
+ + 影片二 +
+
更新至2集
+
+
+
+ """ + cards = self.spider._parse_cards(site, html) + self.assertEqual( + cards, + [ + { + "vod_id": "site:erxiao:/index.php/vod/detail/id/111.html", + "vod_name": "影片一", + "vod_pic": "https://www.2xiaopan.top/poster-1.jpg", + "vod_remarks": "HD", + "vod_year": "", + "_site": "erxiao", + "_detail_path": "/index.php/vod/detail/id/111.html", + }, + { + "vod_id": "site:erxiao:/index.php/vod/detail/id/222.html", + "vod_name": "影片二", + "vod_pic": "https://www.2xiaopan.top/poster-2.jpg", + "vod_remarks": "更新至2集", + "vod_year": "", + "_site": "erxiao", + "_detail_path": "/index.php/vod/detail/id/222.html", + }, + ], + ) + @patch.object(Spider, "_request_with_failover") def test_category_content_uses_default_category_when_extend_missing(self, mock_request_with_failover): mock_request_with_failover.return_value = """ @@ -197,6 +270,37 @@ class TestWanouAggregateSpider(unittest.TestCase): self.assertEqual(result["page"], 2) self.assertEqual(result["list"][0]["vod_name"], "分类影片") + @patch.object(Spider, "_fetch_site_home_recommend") + def test_category_content_returns_recommend_results_with_site_tags(self, mock_fetch_site_home_recommend): + mock_fetch_site_home_recommend.side_effect = lambda site, limit=30: { + "wanou": [ + { + "vod_id": "site:wanou:/voddetail/1.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/w.jpg", + "vod_remarks": "玩偶版", + "vod_year": "2024", + "_site": "wanou", + "_detail_path": "/voddetail/1.html", + } + ], + "muou": [ + { + "vod_id": "site:muou:/voddetail/2.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/m.jpg", + "vod_remarks": "木偶版", + "vod_year": "2024", + "_site": "muou", + "_detail_path": "/voddetail/2.html", + } + ], + }.get(site["id"], []) + result = self.spider.categoryContent("site_recommend", "1", False, {"recommendSite": "all"}) + self.assertEqual(result["total"], 1) + self.assertEqual(result["list"][0]["vod_name"], "繁花") + self.assertEqual(result["list"][0]["vod_remarks"], "[玩偶] 玩偶版") + def test_aggregate_search_results_merges_same_title_and_keeps_highest_priority_source(self): raw_results = [ { @@ -247,8 +351,8 @@ class TestWanouAggregateSpider(unittest.TestCase): @patch.object(Spider, "_fetch_site_search") def test_search_content_queries_sites_and_returns_aggregated_items(self, mock_fetch_site_search): - mock_fetch_site_search.side_effect = [ - [ + results_by_site = { + "wanou": [ { "vod_id": "site:wanou:/voddetail/1.html", "vod_name": "繁花", @@ -259,7 +363,7 @@ class TestWanouAggregateSpider(unittest.TestCase): "_detail_path": "/voddetail/1.html", } ], - [ + "muou": [ { "vod_id": "site:muou:/voddetail/2.html", "vod_name": "繁花", @@ -270,8 +374,8 @@ class TestWanouAggregateSpider(unittest.TestCase): "_detail_path": "/voddetail/2.html", } ], - [], - ] + } + mock_fetch_site_search.side_effect = lambda site, keyword, page: results_by_site.get(site["id"], []) result = self.spider.searchContent("繁花", False, "1") self.assertEqual(len(result["list"]), 1) self.assertEqual(result["list"][0]["vod_name"], "繁花") @@ -345,10 +449,22 @@ class TestWanouAggregateSpider(unittest.TestCase): self.spider.playerContent("quark#玩偶", "https://pan.quark.cn/s/demo", {}), {"parse": 0, "playUrl": "", "url": "https://pan.quark.cn/s/demo"}, ) + self.assertEqual( + self.spider.playerContent("quark#玩偶", "https://www.quark.cn/s/demo", {}), + {"parse": 0, "playUrl": "", "url": "https://www.quark.cn/s/demo"}, + ) self.assertEqual( self.spider.playerContent("baidu#玩偶", "https://pan.baidu.com/s/demo", {}), {"parse": 0, "playUrl": "", "url": "https://pan.baidu.com/s/demo"}, ) + self.assertEqual( + self.spider.playerContent("a189#玩偶", "https://cloud.189.cn/t/demo", {}), + {"parse": 0, "playUrl": "", "url": "https://cloud.189.cn/t/demo"}, + ) + self.assertEqual( + self.spider.playerContent("a139#玩偶", "https://yun.139.com/share/demo", {}), + {"parse": 0, "playUrl": "", "url": "https://yun.139.com/share/demo"}, + ) def test_player_content_rejects_non_pan_url(self): self.assertEqual( @@ -460,22 +576,24 @@ class TestWanouAggregateSpider(unittest.TestCase): @patch.object(Spider, "_fetch_site_search") def test_search_content_skips_site_errors(self, mock_fetch_site_search): - mock_fetch_site_search.side_effect = [ - RuntimeError("boom"), - [ - { - "vod_id": "site:muou:/voddetail/2.html", - "vod_name": "繁花", - "vod_pic": "", - "vod_remarks": "", - "vod_year": "2024", - "_site": "muou", - "_detail_path": "/voddetail/2.html", - } - ], - [], - [], - ] + def fake_fetch(site, keyword, page): + if site["id"] == "wanou": + raise RuntimeError("boom") + if site["id"] == "muou": + return [ + { + "vod_id": "site:muou:/voddetail/2.html", + "vod_name": "繁花", + "vod_pic": "", + "vod_remarks": "", + "vod_year": "2024", + "_site": "muou", + "_detail_path": "/voddetail/2.html", + } + ] + return [] + + mock_fetch_site_search.side_effect = fake_fetch result = self.spider.searchContent("繁花", False, "1") self.assertEqual(result["total"], 1) self.assertEqual(result["list"][0]["vod_name"], "繁花") diff --git a/py/二小.py b/py/二小.py index 29bc6f3..67cf4e9 100644 --- a/py/二小.py +++ b/py/二小.py @@ -67,6 +67,9 @@ class Spider(BaseSpider): def _clean_text(self, text): return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip() + def _class_xpath(self, class_name): + return f"contains(concat(' ', normalize-space(@class), ' '), ' {class_name} ')" + def _detect_pan_type(self, url): raw = str(url or "").strip() for pan_type, title, pattern in self.pan_patterns: @@ -92,15 +95,19 @@ class Spider(BaseSpider): items = [] seen = set() - for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): - href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() - title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() + item_xpath = self._class_xpath("module-item") + pic_box_xpath = self._class_xpath("module-item-pic") + text_xpath = self._class_xpath("module-item-text") + caption_xpath = self._class_xpath("module-item-caption") + for node in root.xpath(f"//*[@id='main']//*[{item_xpath}]"): + href = "".join(node.xpath(f".//*[{pic_box_xpath}]//a[1]/@href")).strip() + title = "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@alt")).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src")).strip() + or "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@src")).strip() ) - remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) - year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()"))) + remarks = self._clean_text("".join(node.xpath(f".//*[{text_xpath}][1]//text()"))) + year = self._clean_text("".join(node.xpath(f".//*[{caption_xpath}][1]//span[1]//text()"))) if not href or not title or href in seen: continue seen.add(href) @@ -131,16 +138,20 @@ class Spider(BaseSpider): return {"page": page, "total": 0, "list": []} items = [] - for node in root.xpath("//*[contains(@class,'module-search-item')]"): - href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() - title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() + search_item_xpath = self._class_xpath("module-search-item") + video_serial_xpath = self._class_xpath("video-serial") + pic_box_xpath = self._class_xpath("module-item-pic") + text_xpath = self._class_xpath("module-item-text") + for node in root.xpath(f"//*[{search_item_xpath}]"): + href = "".join(node.xpath(f".//*[{video_serial_xpath}][1]/@href")).strip() + title = "".join(node.xpath(f".//*[{video_serial_xpath}][1]/@title")).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src")).strip() + or "".join(node.xpath(f".//*[{pic_box_xpath}]//img[1]/@src")).strip() ) remarks = self._clean_text( - "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) - or "".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")) + "".join(node.xpath(f".//*[{video_serial_xpath}][1]//text()")) + or "".join(node.xpath(f".//*[{text_xpath}][1]//text()")) ) if not href or not title: continue @@ -168,17 +179,23 @@ class Spider(BaseSpider): "pan_urls": [], } + mobile_play_xpath = self._class_xpath("mobile-play") + lazyload_xpath = self._class_xpath("lazyload") + page_title_xpath = self._class_xpath("page-title") + info_title_xpath = self._class_xpath("video-info-itemtitle") + row_info_xpath = self._class_xpath("module-row-info") + + pic_candidates = root.xpath( + "(" + f"//*[{mobile_play_xpath}]//*[{lazyload_xpath}]/@data-src | " + f"//*[{mobile_play_xpath}]//*[{lazyload_xpath}]/@src" + ")[1]" + ) + detail = { "vod_id": vod_id, - "vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))), - "vod_pic": self._build_url( - "".join( - root.xpath( - "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | " - "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src" - ) - ).strip() - ), + "vod_name": self._clean_text("".join(root.xpath(f"//*[{page_title_xpath}][1]//text()"))), + "vod_pic": self._build_url((pic_candidates[0] if pic_candidates else "").strip()), "vod_year": "", "vod_director": "", "vod_actor": "", @@ -186,7 +203,7 @@ class Spider(BaseSpider): "pan_urls": [], } - for label_node in root.xpath("//*[contains(@class,'video-info-itemtitle')]"): + for label_node in root.xpath(f"//*[{info_title_xpath}]"): key = self._clean_text("".join(label_node.xpath(".//text()"))) sibling = label_node.getnext() if sibling is None: @@ -203,7 +220,7 @@ class Spider(BaseSpider): elif "剧情" in key: detail["vod_content"] = text_value - for node in root.xpath("//*[contains(@class,'module-row-info')]//p"): + for node in root.xpath(f"//*[{row_info_xpath}]//p"): text = self._clean_text("".join(node.xpath(".//text()"))) if text: detail["pan_urls"].append(text) diff --git a/py/玩偶聚合.py b/py/玩偶聚合.py index 16082f2..9f686a5 100644 --- a/py/玩偶聚合.py +++ b/py/玩偶聚合.py @@ -15,6 +15,7 @@ class Spider(BaseSpider): def __init__(self): self.name = "玩偶聚合" self.filter_root = os.path.join(os.path.dirname(__file__), "../筛选") + self.recommend_page_size = 20 self.headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " @@ -48,7 +49,7 @@ class Spider(BaseSpider): { "id": "wanou", "name": "玩偶", - "domains": ["https://wogg.xxooo.cf"], + "domains": ["https://wogg.xxooo.cf", "https://www.wogg.net"], "filter_files": ["wogg.json"], "list_xpath": "//*[contains(@class,'module-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]", @@ -56,65 +57,140 @@ class Spider(BaseSpider): "category_url": "/vodshow/{categoryId}--------{page}---.html", "category_url_with_filters": "/vodshow/{categoryId}-{area}-{by}-{class}-----{page}---{year}.html", "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", - "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")], + "default_categories": [ + ("44", "臻彩"), + ("1", "电影"), + ("2", "电视剧"), + ("3", "动漫"), + ("4", "综艺"), + ("5", "音乐"), + ("6", "短剧"), + ("46", "纪录片"), + ], }, - # { - # "id": "muou", - # "name": "木偶", - # "domains": ["https://www.muou.site", "http://123.666291.xyz"], - # "filter_files": ["mogg.json"], - # "list_xpath": "//*[contains(@class,'module-item')]", - # "search_xpath": "//*[contains(@class,'module-search-item')]", - # "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", - # "category_url": "/vodshow/{categoryId}--------{page}---.html", - # "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", - # "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("29", "综艺")], - # }, - # { - # "id": "labi", - # "name": "蜡笔", - # "domains": ["http://xiaocge.fun"], - # "filter_files": ["labi.json"], - # "list_xpath": "//*[contains(@class,'module-item')]", - # "search_xpath": "//*[contains(@class,'module-search-item')]", - # "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", - # "category_url": "/vodshow/{categoryId}--------{page}---.html", - # "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", - # "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")], - # }, { - "id": "zhizhen", - "name": "至臻", - "domains": ["http://www.miqk.cc"], - "filter_files": [], + "id": "muou", + "name": "木偶", + "domains": ["https://www.muou.site", "https://www.muou.asia", "https://666.666291.xyz"], + "filter_files": ["mogg.json"], "list_xpath": "//*[contains(@class,'module-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", - "category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html", - "search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html", "default_categories": [ + ("25", "臻选"), ("1", "电影"), - ("2", "剧集"), + ("2", "电视剧"), + ("3", "动漫"), + ("4", "纪录片"), + ("29", "综艺"), + ("30", "原盘"), + ], + }, + { + "id": "labi", + "name": "蜡笔", + "domains": ["http://xiaocgege.shop", "http://fmao.shop"], + "filter_files": ["labi.json"], + "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", + "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", + "default_categories": [ + ("29", "臻彩"), + ("1", "电影"), + ("2", "电视剧"), + ("3", "动漫"), + ("4", "综艺"), + ("5", "短剧"), + ("24", "蜡笔4K"), + ], + }, + { + "id": "zhizhen", + "name": "至臻", + "domains": ["http://www.miqk.cc", "https://www.mihdr.top", "https://mihdr.top"], + "filter_files": ["zhizhen.json"], + "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", + "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", + "default_categories": [ + ("26", "臻彩"), + ("1", "电影"), + ("2", "电视剧"), ("3", "动漫"), ("4", "综艺"), ("5", "短剧"), ("24", "老剧"), - ("26", "严选"), ], }, + { + "id": "erxiao", + "name": "二小", + "domains": [ + "https://www.2xiaopan.top", + "https://www.erxiaozhan.top", + "https://www.wexwp.cc", + ], + "filter_files": ["erxiao.json"], + "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", + "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", + "default_categories": [ + ("4", "臻彩"), + ("1", "电影"), + ("2", "电视剧"), + ("3", "动漫"), + ("21", "综艺"), + ], + }, + # { + # "id": "huban", + # "name": "虎斑", + # "domains": ["http://154.222.27.33:20720", "http://xhban.xyz:20720", "http://103.45.162.207:20720"], + # "filter_files": ["huban.json"], + # "list_xpath": "//*[contains(@class,'module-item')]", + # "search_xpath": "//*[contains(@class,'module-search-item')]", + # "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", + # "default_categories": [ + # ("6", "臻彩"), + # ("1", "电影"), + # ("2", "电视剧"), + # ("3", "综艺"), + # ("4", "动漫"), + # ("5", "短剧"), + # ("30", "115网盘"), + # ], + # }, + # { + # "id": "kuaiying", + # "name": "快映", + # "domains": ["http://154.201.83.50:12512", "http://xsayang.fun:12512"], + # "filter_files": ["xiaoban.json"], + # "list_xpath": "//*[contains(@class,'module-item')]", + # "search_xpath": "//*[contains(@class,'module-search-item')]", + # "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", + # "default_categories": [ + # ("5", "臻彩"), + # ("1", "电影"), + # ("2", "电视剧"), + # ("3", "综艺"), + # ("4", "动漫"), + # ("6", "短剧"), + # ("30", "115"), + # ("35", "123"), + # ("36", "天移迅"), + # ], + # }, { "id": "shandian", "name": "闪电", "domains": ["https://sd.sduc.site"], - "filter_files": [], + "filter_files": ["shandian.json"], "list_xpath": "//*[contains(@class,'module-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", - "category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html", - "search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html", "default_categories": [ ("1", "电影"), - ("2", "剧集"), + ("2", "电视剧"), ("3", "综艺"), ("4", "动漫"), ("30", "短剧"), @@ -122,17 +198,15 @@ class Spider(BaseSpider): }, { "id": "ouge", - "name": "欧歌", + "name": "欧哥", "domains": ["https://woog.nxog.eu.org"], - "filter_files": [], + "filter_files": ["ouge.json"], "list_xpath": "//*[contains(@class,'module-item')]", "search_xpath": "//*[contains(@class,'module-search-item')]", "detail_pan_xpath": "//*[contains(@class,'module-row-info')]//p", - "category_url": "/index.php/vod/show/id/{categoryId}/page/{page}.html", - "search_url": "/index.php/vod/search/page/{page}/wd/{keyword}.html", "default_categories": [ ("1", "电影"), - ("2", "剧集"), + ("2", "电视剧"), ("3", "动漫"), ("4", "综艺"), ("5", "短剧"), @@ -151,7 +225,47 @@ class Spider(BaseSpider): return {"list": []} def _load_local_filter_groups(self, site): - return [] + groups = [] + for filename in site.get("filter_files", []): + if not filename: + continue + file_path = os.path.join(self.filter_root, filename) + if not os.path.exists(file_path): + continue + try: + with open(file_path, "r", encoding="utf-8") as handle: + raw = json.load(handle) + except Exception: + continue + merged = {} + for category_groups in raw.values() if isinstance(raw, dict) else []: + if not isinstance(category_groups, list): + continue + for group in category_groups: + key = str(group.get("key") or "").strip() + if not key or key == "categoryId": + continue + bucket = merged.setdefault( + key, + { + "key": key, + "name": str(group.get("name") or key), + "init": str(group.get("init") or ""), + "value": [], + }, + ) + for item in group.get("value", []): + value = str(item.get("v") if isinstance(item, dict) else "") + name = str(item.get("n") if isinstance(item, dict) else "") + if not name: + continue + if not any(existing.get("v") == value for existing in bucket["value"]): + bucket["value"].append({"n": name, "v": value}) + for group in merged.values(): + if not any(item.get("v") == "" for item in group["value"]): + group["value"].insert(0, {"n": "全部", "v": ""}) + groups.append(group) + return groups def _build_site_filters(self, site): groups = [ @@ -167,8 +281,20 @@ class Spider(BaseSpider): return groups def homeContent(self, filter): - classes = [{"type_id": f"site_{site['id']}", "type_name": site["name"]} for site in self.sites] - filters = {f"site_{site['id']}": self._build_site_filters(site) for site in self.sites} + classes = [{"type_id": "site_recommend", "type_name": "推荐"}] + classes.extend({"type_id": f"site_{site['id']}", "type_name": site["name"]} for site in self.sites) + filters = { + "site_recommend": [ + { + "key": "recommendSite", + "name": "站点", + "init": "all", + "value": [{"n": "全部", "v": "all"}] + + [{"n": site["name"], "v": site["id"]} for site in self.sites], + } + ] + } + filters.update({f"site_{site['id']}": self._build_site_filters(site) for site in self.sites}) return {"class": classes, "filters": filters} def _encode_site_vod_id(self, site_id, path): @@ -218,6 +344,13 @@ class Spider(BaseSpider): return "https:" + raw return str(base).rstrip("/") + "/" + raw.lstrip("/") + def _class_xpath(self, class_name): + return f"contains(concat(' ', normalize-space(@class), ' '), ' {class_name} ')" + + def _has_class(self, node, class_name): + classes = f" {str(node.get('class') or '').strip()} " + return f" {class_name} " in classes + def _build_category_url(self, site, category_id, pg, extend): values = dict(extend or {}) values.setdefault("categoryId", category_id) @@ -237,7 +370,8 @@ class Spider(BaseSpider): } ) else: - path = site["category_url"].format(categoryId=values["categoryId"], page=int(pg)) + template = site.get("category_url") or "/index.php/vod/show/id/{categoryId}/page/{page}.html" + path = template.format(categoryId=values["categoryId"], page=int(pg)) return self._build_absolute_url(site["domains"][0], path) def _request_with_failover(self, site, path_or_url, referer=None): @@ -263,17 +397,21 @@ class Spider(BaseSpider): items = [] seen = set() + pic_box_xpath = self._class_xpath("module-item-pic") + text_xpath = self._class_xpath("module-item-text") for card in root.xpath(site["list_xpath"]): + if not self._has_class(card, "module-item"): + continue href = ((card.xpath(".//a[@href][1]/@href") or [""])[0]).strip() title = ( ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip() or ((card.xpath(".//a[@title][1]/@title") or [""])[0]).strip() ) pic = ( - ((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip() - or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip() + ((card.xpath(f".//*[{pic_box_xpath}]//img[1]/@data-src") or [""])[0]).strip() + or ((card.xpath(f".//*[{pic_box_xpath}]//img[1]/@src") or [""])[0]).strip() ) - remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip() + remarks = "".join(card.xpath(f".//*[{text_xpath}][1]//text()")).strip() if not href or not title or href in seen: continue seen.add(href) @@ -292,8 +430,31 @@ class Spider(BaseSpider): def categoryContent(self, tid, pg, filter, extend): site_id = str(tid).replace("site_", "", 1) - site = self._get_site(site_id) values = extend if isinstance(extend, dict) else {} + if site_id == "recommend": + recommend_site_id = str(values.get("recommendSite") or "all") + candidates = self.sites if recommend_site_id == "all" else [site for site in self.sites if site["id"] == recommend_site_id] + merged = [] + for site in candidates: + for item in self._fetch_site_home_recommend(site): + matched = None + for existing in merged: + if self._is_same_title(existing, item): + matched = existing + break + if matched is None: + source_name = site["name"] + remarks = str(item.get("vod_remarks") or "").strip() + tagged = dict(item) + tagged["vod_remarks"] = f"[{source_name}] {remarks}".strip() + merged.append(tagged) + page = int(pg) + start = max(page - 1, 0) * self.recommend_page_size + end = start + self.recommend_page_size + paged = merged[start:end] + return {"list": paged, "page": page, "limit": len(paged), "total": len(merged)} + + site = self._get_site(site_id) category_id = values.get("categoryId") or site["default_categories"][0][0] html = self._request_with_failover(site, self._build_category_url(site, category_id, pg, values)) items = self._parse_cards(site, html) @@ -313,13 +474,17 @@ class Spider(BaseSpider): items = [] seen = set() xpath = site.get("search_xpath") or site["list_xpath"] + video_serial_xpath = self._class_xpath("video-serial") + text_xpath = self._class_xpath("module-item-text") for card in root.xpath(xpath): + if "module-search-item" in xpath and not self._has_class(card, "module-search-item"): + continue href = ( - ((card.xpath(".//*[contains(@class,'video-serial')][1]/@href") or [""])[0]).strip() + ((card.xpath(f".//*[{video_serial_xpath}][1]/@href") or [""])[0]).strip() or ((card.xpath(".//*[@href][1]/@href") or [""])[0]).strip() ) title = ( - ((card.xpath(".//*[contains(@class,'video-serial')][1]/@title") or [""])[0]).strip() + ((card.xpath(f".//*[{video_serial_xpath}][1]/@title") or [""])[0]).strip() or ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip() or ((card.xpath(".//*[@title][1]/@title") or [""])[0]).strip() ) @@ -327,7 +492,7 @@ class Spider(BaseSpider): ((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip() or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip() ) - remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip() + remarks = "".join(card.xpath(f".//*[{text_xpath}][1]//text()")).strip() if not href or not title or href in seen: continue seen.add(href) @@ -345,10 +510,16 @@ class Spider(BaseSpider): return items def _fetch_site_search(self, site, keyword, pg): - search_path = site["search_url"].format(keyword=quote(str(keyword)), page=int(pg)) + template = site.get("search_url") or "/index.php/vod/search/page/{page}/wd/{keyword}.html" + search_path = template.format(keyword=quote(str(keyword)), page=int(pg)) html = self._request_with_failover(site, search_path) return self._parse_search_cards(site, html) + def _fetch_site_home_recommend(self, site, limit=30): + html = self._request_with_failover(site, "/") + items = self._parse_cards(site, html) + return items[:limit] + def _aggregate_search_results(self, items): groups = [] for item in items: @@ -406,21 +577,21 @@ class Spider(BaseSpider): value = str(url or "").lower() if "pan.baidu.com" in value: return "baidu", "百度资源" - if "pan.quark.cn" in value: + if "quark.cn" in value: return "quark", "夸克资源" - if "drive.uc.cn" in value: + if "uc.cn" in value: return "uc", "UC资源" if "alipan.com" in value or "aliyundrive.com" in value: return "aliyun", "阿里资源" - if "pan.xunlei.com" in value: + if "xunlei.com" in value: return "xunlei", "迅雷资源" if "123pan.com" in value: return "a123", "123资源" - if "115.com" in value: + if "115.com" in value or "115cdn.com" in value: return "a115", "115资源" - if "189.cn" in value: + if "cloud.189.cn" in value or "189.cn" in value: return "a189", "天翼资源" - if "139.com" in value: + if "yun.139.com" in value or "139.com" in value: return "a139", "移动云资源" return "", ""