yeyinly/tvboxzt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add wanou aggregate search merging

Browse Source
This commit is contained in:
Harold
2026-04-20 14:08:51 +08:00
parent cd12177a3b
commit 94d7554981
2 changed files with 189 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/tests/test_玩偶聚合.py
+80
View File
@@ -148,3 +148,83 @@ class TestWanouAggregateSpider(unittest.TestCase):
result = self.spider.categoryContent("site_wanou", "2", False, {})
self.assertEqual(result["page"], 2)
self.assertEqual(result["list"][0]["vod_name"], "分类影片")
def test_aggregate_search_results_merges_same_title_and_keeps_highest_priority_source(self):
raw_results = [
{
"vod_id": "site:muou:/voddetail/2.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/m.jpg",
"vod_remarks": "木偶版",
"vod_year": "2024",
"_site": "muou",
"_detail_path": "/voddetail/2.html",
},
{
"vod_id": "site:wanou:/voddetail/1.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/w.jpg",
"vod_remarks": "玩偶版",
"vod_year": "2024",
"_site": "wanou",
"_detail_path": "/voddetail/1.html",
},
]
aggregated = self.spider._aggregate_search_results(raw_results)
self.assertEqual(len(aggregated), 1)
self.assertEqual(aggregated[0]["vod_name"], "繁花")
self.assertEqual(aggregated[0]["vod_pic"], "https://img.example/w.jpg")
self.assertEqual(aggregated[0]["vod_remarks"], "玩偶版")
self.assertTrue(aggregated[0]["vod_id"].startswith("agg:"))
def test_aggregate_search_results_keeps_year_conflict_as_two_items(self):
raw_results = [
{
"vod_id": "site:wanou:/voddetail/1.html",
"vod_name": "倚天屠龙记",
"vod_year": "2019",
"_site": "wanou",
"_detail_path": "/voddetail/1.html",
},
{
"vod_id": "site:muou:/voddetail/2.html",
"vod_name": "倚天屠龙记",
"vod_year": "2022",
"_site": "muou",
"_detail_path": "/voddetail/2.html",
},
]
aggregated = self.spider._aggregate_search_results(raw_results)
self.assertEqual(len(aggregated), 2)
@patch.object(Spider, "_fetch_site_search")
def test_search_content_queries_sites_and_returns_aggregated_items(self, mock_fetch_site_search):
mock_fetch_site_search.side_effect = [
[
{
"vod_id": "site:wanou:/voddetail/1.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/w.jpg",
"vod_remarks": "玩偶版",
"vod_year": "2024",
"_site": "wanou",
"_detail_path": "/voddetail/1.html",
}
],
[
{
"vod_id": "site:muou:/voddetail/2.html",
"vod_name": "繁花",
"vod_pic": "https://img.example/m.jpg",
"vod_remarks": "木偶版",
"vod_year": "2024",
"_site": "muou",
"_detail_path": "/voddetail/2.html",
}
],
[],
]
result = self.spider.searchContent("繁花", False, "1")
self.assertEqual(len(result["list"]), 1)
self.assertEqual(result["list"][0]["vod_name"], "繁花")
self.assertNotIn("pagecount", result)
py/玩偶聚合.py
+109
View File
@@ -40,8 +40,10 @@ class Spider(BaseSpider):
"domains": ["https://www.wogg.net"],
"filter_files": ["wogg.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"category_url": "/vodshow/{categoryId}--------{page}---.html",
"category_url_with_filters": "/vodshow/{categoryId}-{area}-{by}-{class}-----{page}---{year}.html",
"search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
"default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")],
},
{
@@ -50,7 +52,9 @@ class Spider(BaseSpider):
"domains": ["https://www.muou.site"],
"filter_files": ["mogg.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"category_url": "/vodshow/{categoryId}--------{page}---.html",
"search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
"default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("29", "综艺")],
},
{
@@ -59,7 +63,9 @@ class Spider(BaseSpider):
"domains": ["http://xiaocge.fun"],
"filter_files": ["labi.json"],
"list_xpath": "//*[contains(@class,'module-item')]",
"search_xpath": "//*[contains(@class,'module-search-item')]",
"category_url": "/vodshow/{categoryId}--------{page}---.html",
"search_url": "/vodsearch/-------------.html?wd={keyword}&page={page}",
"default_categories": [("1", "电影"), ("2", "电视剧"), ("3", "动漫"), ("4", "综艺")],
},
]
@@ -221,3 +227,106 @@ class Spider(BaseSpider):
html = self._request_with_failover(site, self._build_category_url(site, category_id, pg, values))
items = self._parse_cards(site, html)
return {"list": items, "page": int(pg), "limit": len(items), "total": int(pg) * 20 + len(items)}
def _site_rank(self, site_id):
try:
return self.site_priority.index(site_id)
except ValueError:
return 999
def _parse_search_cards(self, site, html):
root = self.html(html)
if root is None:
return []
items = []
seen = set()
xpath = site.get("search_xpath") or site["list_xpath"]
for card in root.xpath(xpath):
href = (
((card.xpath(".//*[contains(@class,'video-serial')][1]/@href") or [""])[0]).strip()
or ((card.xpath(".//*[@href][1]/@href") or [""])[0]).strip()
)
title = (
((card.xpath(".//*[contains(@class,'video-serial')][1]/@title") or [""])[0]).strip()
or ((card.xpath(".//img[@alt][1]/@alt") or [""])[0]).strip()
or ((card.xpath(".//*[@title][1]/@title") or [""])[0]).strip()
)
pic = (
((card.xpath(".//img[@data-src][1]/@data-src") or [""])[0]).strip()
or ((card.xpath(".//img[@src][1]/@src") or [""])[0]).strip()
)
remarks = "".join(card.xpath(".//*[contains(@class,'module-item-text')][1]//text()")).strip()
if not href or not title or href in seen:
continue
seen.add(href)
items.append(
{
"vod_id": self._encode_site_vod_id(site["id"], href),
"vod_name": title,
"vod_pic": self._build_absolute_url(site["domains"][0], pic),
"vod_remarks": remarks,
"vod_year": "",
"_site": site["id"],
"_detail_path": href,
}
)
return items
def _fetch_site_search(self, site, keyword, pg):
search_path = site["search_url"].format(keyword=quote(str(keyword)), page=int(pg))
html = self._request_with_failover(site, search_path)
return self._parse_search_cards(site, html)
def _aggregate_search_results(self, items):
groups = []
for item in items:
matched_group = None
for group in groups:
if self._is_same_title(group[0], item):
matched_group = group
break
if matched_group is None:
groups.append([item])
else:
matched_group.append(item)
result = []
for group in groups:
group.sort(key=lambda item: self._site_rank(item["_site"]))
primary = group[0]
payload = [
{
"site": item["_site"],
"path": item["_detail_path"],
"name": item.get("vod_name", ""),
"year": item.get("vod_year", ""),
}
for item in group
]
result.append(
{
"vod_id": self._encode_aggregate_vod_id(payload),
"vod_name": primary["vod_name"],
"vod_pic": primary.get("vod_pic", ""),
"vod_remarks": primary.get("vod_remarks", ""),
"vod_year": primary.get("vod_year", ""),
}
)
return result
def searchContent(self, key, quick, pg="1"):
page = int(pg)
keyword = str(key or "").strip()
if not keyword:
return {"page": page, "total": 0, "list": []}
all_items = []
for site in self.sites:
try:
all_items.extend(self._fetch_site_search(site, keyword, page))
except Exception:
continue
merged = self._aggregate_search_results(all_items)
return {"page": page, "total": len(merged), "list": merged}