feat: add shuangxing list and search parsing

2026-04-29 15:31:47 +08:00
parent d436b699a0
commit 39cdce34c9
2 changed files with 84 additions and 0 deletions
@@ -1,6 +1,7 @@
 import unittest
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
+from urllib.parse import quote
 from unittest.mock import patch


@@ -62,6 +63,52 @@ class TestShuangXingSpider(unittest.TestCase):
        self.assertEqual(self.spider._detect_pan_type("https://www.alipan.com/s/demo"), "ali")
        self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), "")

+    @patch.object(Spider, "_get_html")
+    def test_category_content_builds_reference_url_and_parses_cards(self, mock_get_html):
+        mock_get_html.return_value = """
+        <body>
+          <div><div><main><div><ul>
+            <li><div class="a"><a href="/post/alpha">示例国剧</a></div></li>
+            <li><div class="a"><a href="/post/beta">示例综艺</a></div></li>
+          </ul></div></main></div></div>
+        </body>
+        """
+        result = self.spider.categoryContent("ju", "3", False, {})
+        self.assertEqual(mock_get_html.call_args.args[0], "https://1.star2.cn/ju_3/")
+        self.assertEqual(result["page"], 3)
+        self.assertEqual(result["limit"], 15)
+        self.assertEqual(result["total"], 32)
+        self.assertEqual(
+            result["list"],
+            [
+                {"vod_id": "/post/alpha", "vod_name": "示例国剧", "vod_pic": "", "vod_remarks": ""},
+                {"vod_id": "/post/beta", "vod_name": "示例综艺", "vod_pic": "", "vod_remarks": ""},
+            ],
+        )
+
+    @patch.object(Spider, "_get_html")
+    def test_search_content_builds_reference_url_and_parses_results(self, mock_get_html):
+        mock_get_html.return_value = """
+        <body>
+          <div><div><main><div><ul>
+            <li><div class="a"><a href="/post/search">搜索结果</a></div></li>
+          </ul></div></main></div></div>
+        </body>
+        """
+        result = self.spider.searchContent("繁花", False, "2")
+        self.assertEqual(
+            mock_get_html.call_args.args[0],
+            f"https://1.star2.cn/search/?keyword={quote('繁花')}&page=2",
+        )
+        self.assertEqual(result["page"], 2)
+        self.assertEqual(
+            result["list"],
+            [{"vod_id": "/post/search", "vod_name": "搜索结果", "vod_pic": "", "vod_remarks": ""}],
+        )
+
+    def test_search_content_short_circuits_blank_keyword(self):
+        self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
+

 if __name__ == "__main__":
    unittest.main()
@@ -1,5 +1,7 @@
 # coding=utf-8
+import re
 import sys
+from urllib.parse import quote

 from base.spider import Spider as BaseSpider

@@ -75,3 +77,38 @@ class Spider(BaseSpider):
        if "aliyundrive" in text or "alipan" in text:
            return "ali"
        return ""
+
+    def _clean_text(self, text):
+        return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
+
+    def _get_html(self, url):
+        response = self.fetch(url, headers=self._headers(), timeout=15)
+        if response.status_code != 200:
+            return ""
+        return response.text or ""
+
+    def _parse_cards(self, html):
+        root = self.html(html)
+        if root is None:
+            return []
+        items = []
+        for node in root.xpath("/html/body/div/div/main/div/ul/li"):
+            href = "".join(node.xpath(".//div[contains(@class,'a')]//a[1]/@href")).strip()
+            title = self._clean_text("".join(node.xpath(".//div[contains(@class,'a')]//a[1]//text()")))
+            if not href or not title:
+                continue
+            items.append({"vod_id": href, "vod_name": title, "vod_pic": "", "vod_remarks": ""})
+        return items
+
+    def categoryContent(self, tid, pg, filter, extend):
+        page = int(pg)
+        items = self._parse_cards(self._get_html(f"{self.BASE_URL}/{str(tid).strip()}_{page}/"))
+        return {"page": page, "limit": 15, "total": (page - 1) * 15 + len(items), "list": items}
+
+    def searchContent(self, key, quick, pg="1"):
+        page = int(pg)
+        keyword = self._clean_text(key)
+        if not keyword:
+            return {"page": page, "total": 0, "list": []}
+        items = self._parse_cards(self._get_html(f"{self.BASE_URL}/search/?keyword={quote(keyword)}&page={page}"))
+        return {"page": page, "total": len(items), "list": items}