diff --git a/py/tests/test_玩偶聚合.py b/py/tests/test_玩偶聚合.py index 25fe08e..19e30bb 100644 --- a/py/tests/test_玩偶聚合.py +++ b/py/tests/test_玩偶聚合.py @@ -148,3 +148,83 @@ class TestWanouAggregateSpider(unittest.TestCase): result = self.spider.categoryContent("site_wanou", "2", False, {}) self.assertEqual(result["page"], 2) self.assertEqual(result["list"][0]["vod_name"], "分类影片") + + def test_aggregate_search_results_merges_same_title_and_keeps_highest_priority_source(self): + raw_results = [ + { + "vod_id": "site:muou:/voddetail/2.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/m.jpg", + "vod_remarks": "木偶版", + "vod_year": "2024", + "_site": "muou", + "_detail_path": "/voddetail/2.html", + }, + { + "vod_id": "site:wanou:/voddetail/1.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/w.jpg", + "vod_remarks": "玩偶版", + "vod_year": "2024", + "_site": "wanou", + "_detail_path": "/voddetail/1.html", + }, + ] + aggregated = self.spider._aggregate_search_results(raw_results) + self.assertEqual(len(aggregated), 1) + self.assertEqual(aggregated[0]["vod_name"], "繁花") + self.assertEqual(aggregated[0]["vod_pic"], "https://img.example/w.jpg") + self.assertEqual(aggregated[0]["vod_remarks"], "玩偶版") + self.assertTrue(aggregated[0]["vod_id"].startswith("agg:")) + + def test_aggregate_search_results_keeps_year_conflict_as_two_items(self): + raw_results = [ + { + "vod_id": "site:wanou:/voddetail/1.html", + "vod_name": "倚天屠龙记", + "vod_year": "2019", + "_site": "wanou", + "_detail_path": "/voddetail/1.html", + }, + { + "vod_id": "site:muou:/voddetail/2.html", + "vod_name": "倚天屠龙记", + "vod_year": "2022", + "_site": "muou", + "_detail_path": "/voddetail/2.html", + }, + ] + aggregated = self.spider._aggregate_search_results(raw_results) + self.assertEqual(len(aggregated), 2) + + @patch.object(Spider, "_fetch_site_search") + def test_search_content_queries_sites_and_returns_aggregated_items(self, mock_fetch_site_search): + mock_fetch_site_search.side_effect = [ + [ + { + "vod_id": "site:wanou:/voddetail/1.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/w.jpg", + "vod_remarks": "玩偶版", + "vod_year": "2024", + "_site": "wanou", + "_detail_path": "/voddetail/1.html", + } + ], + [ + { + "vod_id": "site:muou:/voddetail/2.html", + "vod_name": "繁花", + "vod_pic": "https://img.example/m.jpg", + "vod_remarks": "木偶版", + "vod_year": "2024", + "_site": "muou", + "_detail_path": "/voddetail/2.html", + } + ], + [], + ] + result = self.spider.searchContent("繁花", False, "1") + self.assertEqual(len(result["list"]), 1) + self.assertEqual(result["list"][0]["vod_name"], "繁花") + self.assertNotIn("pagecount", result) diff --git a/py/玩偶聚合.py b/py/玩偶聚合.py index ccbfb0c..ee176d8 100644 --- a/py/玩偶聚合.py +++ b/py/玩偶聚合.py @@ -40,8 +40,10 @@ class Spider(BaseSpider): "domains": ["https://www.wogg.net"], "filter_files": ["wogg.json"], "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", "category_url": "/vodshow/{categoryId}--------{page}---.html", "category_url_with_filters": "/vodshow/{categoryId}-{area}-{by}-{class}-----{page}---{year}.html", + "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")], }, { @@ -50,7 +52,9 @@ class Spider(BaseSpider): "domains": ["https://www.muou.site"], "filter_files": ["mogg.json"], "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", "category_url": "/vodshow/{categoryId}--------{page}---.html", + "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("29", "综艺")], }, { @@ -59,7 +63,9 @@ class Spider(BaseSpider): "domains": ["http://xiaocge.fun"], "filter_files": ["labi.json"], "list_xpath": "//*[contains(@class,'module-item')]", + "search_xpath": "//*[contains(@class,'module-search-item')]", "category_url": "/vodshow/{categoryId}--------{page}---.html", + "search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}", "default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")], }, ] @@ -221,3 +227,106 @@ class Spider(BaseSpider): html = self._request_with_failover(site, self._build_category_url(site, category_id, pg, values)) items = self._parse_cards(site, html) return {"list": items, "page": int(pg), "limit": len(items), "total": int(pg) * 20 + len(items)} + + def _site_rank(self, site_id): + try: + return self.site_priority.index(site_id) + except ValueError: + return 999 + + def _parse_search_cards(self, site, html): + root = self.html(html) + if root is None: + return [] + + items = [] + seen = set() + xpath = site.get("search_xpath") or site["list_xpath"] + for card in root.xpath(xpath): + href = ( + ((card.xpath(".//*[contains(@class,'video-serial')][1]/@href") or [""])[0]).strip() + or ((card.xpath(".//*[@href][1]/@href") or [""])[0]).strip() + ) + title = ( + ((card.xpath(".//*[contains(@class,'video-serial')][1]/@title") or [""])[0]).strip() + or ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip() + or ((card.xpath(".//*[@title][1]/@title") or [""])[0]).strip() + ) + pic = ( + ((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip() + or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip() + ) + remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip() + if not href or not title or href in seen: + continue + seen.add(href) + items.append( + { + "vod_id": self._encode_site_vod_id(site["id"], href), + "vod_name": title, + "vod_pic": self._build_absolute_url(site["domains"][0], pic), + "vod_remarks": remarks, + "vod_year": "", + "_site": site["id"], + "_detail_path": href, + } + ) + return items + + def _fetch_site_search(self, site, keyword, pg): + search_path = site["search_url"].format(keyword=quote(str(keyword)), page=int(pg)) + html = self._request_with_failover(site, search_path) + return self._parse_search_cards(site, html) + + def _aggregate_search_results(self, items): + groups = [] + for item in items: + matched_group = None + for group in groups: + if self._is_same_title(group[0], item): + matched_group = group + break + if matched_group is None: + groups.append([item]) + else: + matched_group.append(item) + + result = [] + for group in groups: + group.sort(key=lambda item: self._site_rank(item["_site"])) + primary = group[0] + payload = [ + { + "site": item["_site"], + "path": item["_detail_path"], + "name": item.get("vod_name", ""), + "year": item.get("vod_year", ""), + } + for item in group + ] + result.append( + { + "vod_id": self._encode_aggregate_vod_id(payload), + "vod_name": primary["vod_name"], + "vod_pic": primary.get("vod_pic", ""), + "vod_remarks": primary.get("vod_remarks", ""), + "vod_year": primary.get("vod_year", ""), + } + ) + return result + + def searchContent(self, key, quick, pg="1"): + page = int(pg) + keyword = str(key or "").strip() + if not keyword: + return {"page": page, "total": 0, "list": []} + + all_items = [] + for site in self.sites: + try: + all_items.extend(self._fetch_site_search(site, keyword, page)) + except Exception: + continue + + merged = self._aggregate_search_results(all_items) + return {"page": page, "total": len(merged), "list": merged}