From fc1e30725d7ef3028230530fcdbe8c8b3af9c6c8 Mon Sep 17 00:00:00 2001 From: Harold <8866033@gmail.com> Date: Mon, 20 Apr 2026 14:36:55 +0800 Subject: [PATCH] feat: add wanougege list and search parsing --- py/tests/test_玩偶哥哥.py | 76 +++++++++++++++++++++++++++++++++++ py/玩偶哥哥.py | 83 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/py/tests/test_玩偶哥哥.py b/py/tests/test_玩偶哥哥.py index 034f7fd..1f2fa83 100644 --- a/py/tests/test_玩偶哥哥.py +++ b/py/tests/test_玩偶哥哥.py @@ -1,6 +1,7 @@ import unittest from importlib.machinery import SourceFileLoader from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] @@ -48,3 +49,78 @@ class TestWanOuGeGeSpider(unittest.TestCase): self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源")) self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源")) self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), ("", "")) + + def test_parse_cards_extracts_short_path_ids(self): + html = """ +
+
+
+ + 示例影片 +
+
HD
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/voddetail/123.html", + "vod_name": "示例影片", + "vod_pic": "http://wogg.xxooo.cf/poster.jpg", + "vod_remarks": "HD", + } + ], + ) + + @patch.object(Spider, "_request_html") + def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): + mock_request_html.return_value = """ +
+
+
+ + 分类影片 +
+
更新至10集
+
+
+ """ + result = self.spider.categoryContent("2", "3", False, {}) + self.assertEqual( + mock_request_html.call_args.args[0], + "http://wogg.xxooo.cf/vodshow/2--------3---.html", + ) + self.assertEqual(result["page"], 3) + self.assertEqual(result["limit"], 1) + self.assertEqual(result["list"][0]["vod_name"], "分类影片") + self.assertNotIn("pagecount", result) + + @patch.object(Spider, "_request_html") + def test_search_content_builds_reference_search_url_and_parses_results(self, mock_request_html): + mock_request_html.return_value = """ +
+ 抢先版 +
+ 搜索影片 +
+
+ """ + result = self.spider.searchContent("繁花", False, "2") + self.assertEqual( + mock_request_html.call_args.args[0], + "http://wogg.xxooo.cf/vodsearch/-------------.html?wd=%E7%B9%81%E8%8A%B1&page=2", + ) + self.assertEqual( + result["list"][0], + { + "vod_id": "/voddetail/789.html", + "vod_name": "搜索影片", + "vod_pic": "http://wogg.xxooo.cf/search.jpg", + "vod_remarks": "抢先版", + }, + ) + + def test_search_content_returns_empty_list_for_blank_keyword(self): + self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []}) diff --git a/py/玩偶哥哥.py b/py/玩偶哥哥.py index c75ba81..8ab4960 100644 --- a/py/玩偶哥哥.py +++ b/py/玩偶哥哥.py @@ -1,7 +1,7 @@ # coding=utf-8 import re import sys -from urllib.parse import urljoin +from urllib.parse import quote, urljoin from base.spider import Spider as BaseSpider @@ -73,3 +73,84 @@ class Spider(BaseSpider): if re.search(pattern, raw, re.I): return pan_type, title return "", "" + + def _clean_text(self, text): + return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip() + + def _request_html(self, path_or_url): + target = path_or_url if str(path_or_url).startswith("http") else self._build_url(path_or_url) + response = self.fetch(target, headers=dict(self.headers), timeout=10) + if response.status_code != 200: + return "" + return response.text or "" + + def _page_result(self, items, pg): + page = int(pg) + return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items} + + def _parse_cards(self, html): + root = self.html(html) + if root is None: + return [] + + items = [] + seen = set() + for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): + href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() + title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() + pic = ( + "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() + or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ) + remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) + if not href or not title or href in seen: + continue + seen.add(href) + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(self._fix_img_url(pic)), + "vod_remarks": remarks, + } + ) + return items + + def categoryContent(self, tid, pg, filter, extend): + url = self._build_url(f"/vodshow/{tid}--------{int(pg)}---.html") + return self._page_result(self._parse_cards(self._request_html(url)), pg) + + def searchContent(self, key, quick, pg="1"): + keyword = self._clean_text(key) + page = int(pg) + if not keyword: + return {"page": page, "total": 0, "list": []} + + url = self._build_url(f"/vodsearch/-------------.html?wd={quote(keyword)}&page={page}") + root = self.html(self._request_html(url)) + if root is None: + return {"page": page, "total": 0, "list": []} + + items = [] + for node in root.xpath("//*[contains(@class,'module-search-item')]"): + href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() + title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() + pic = ( + "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() + or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ) + remarks = self._clean_text( + "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) + or "".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")) + ) + if not href or not title: + continue + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(self._fix_img_url(pic)), + "vod_remarks": remarks, + } + ) + return {"page": page, "total": len(items), "list": items}