feat: add shuangxing list and search parsing
This commit is contained in:
Harold
@@ -1,6 +1,7 @@
|
|||||||
import unittest
|
import unittest
|
||||||
from importlib.machinery import SourceFileLoader
|
from importlib.machinery import SourceFileLoader
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
@@ -62,6 +63,52 @@ class TestShuangXingSpider(unittest.TestCase):
|
|||||||
self.assertEqual(self.spider._detect_pan_type("https://www.alipan.com/s/demo"), "ali")
|
self.assertEqual(self.spider._detect_pan_type("https://www.alipan.com/s/demo"), "ali")
|
||||||
self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), "")
|
self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), "")
|
||||||
|
|
||||||
|
@patch.object(Spider, "_get_html")
|
||||||
|
def test_category_content_builds_reference_url_and_parses_cards(self, mock_get_html):
|
||||||
|
mock_get_html.return_value = """
|
||||||
|
<body>
|
||||||
|
<div><div><main><div><ul>
|
||||||
|
<li><div class="a"><a href="/post/alpha">示例国剧</a></div></li>
|
||||||
|
<li><div class="a"><a href="/post/beta">示例综艺</a></div></li>
|
||||||
|
</ul></div></main></div></div>
|
||||||
|
</body>
|
||||||
|
"""
|
||||||
|
result = self.spider.categoryContent("ju", "3", False, {})
|
||||||
|
self.assertEqual(mock_get_html.call_args.args[0], "https://1.star2.cn/ju_3/")
|
||||||
|
self.assertEqual(result["page"], 3)
|
||||||
|
self.assertEqual(result["limit"], 15)
|
||||||
|
self.assertEqual(result["total"], 32)
|
||||||
|
self.assertEqual(
|
||||||
|
result["list"],
|
||||||
|
[
|
||||||
|
{"vod_id": "/post/alpha", "vod_name": "示例国剧", "vod_pic": "", "vod_remarks": ""},
|
||||||
|
{"vod_id": "/post/beta", "vod_name": "示例综艺", "vod_pic": "", "vod_remarks": ""},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch.object(Spider, "_get_html")
|
||||||
|
def test_search_content_builds_reference_url_and_parses_results(self, mock_get_html):
|
||||||
|
mock_get_html.return_value = """
|
||||||
|
<body>
|
||||||
|
<div><div><main><div><ul>
|
||||||
|
<li><div class="a"><a href="/post/search">搜索结果</a></div></li>
|
||||||
|
</ul></div></main></div></div>
|
||||||
|
</body>
|
||||||
|
"""
|
||||||
|
result = self.spider.searchContent("繁花", False, "2")
|
||||||
|
self.assertEqual(
|
||||||
|
mock_get_html.call_args.args[0],
|
||||||
|
f"https://1.star2.cn/search/?keyword={quote('繁花')}&page=2",
|
||||||
|
)
|
||||||
|
self.assertEqual(result["page"], 2)
|
||||||
|
self.assertEqual(
|
||||||
|
result["list"],
|
||||||
|
[{"vod_id": "/post/search", "vod_name": "搜索结果", "vod_pic": "", "vod_remarks": ""}],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_search_content_short_circuits_blank_keyword(self):
|
||||||
|
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
from base.spider import Spider as BaseSpider
|
from base.spider import Spider as BaseSpider
|
||||||
|
|
||||||
@@ -75,3 +77,38 @@ class Spider(BaseSpider):
|
|||||||
if "aliyundrive" in text or "alipan" in text:
|
if "aliyundrive" in text or "alipan" in text:
|
||||||
return "ali"
|
return "ali"
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def _clean_text(self, text):
|
||||||
|
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
||||||
|
|
||||||
|
def _get_html(self, url):
|
||||||
|
response = self.fetch(url, headers=self._headers(), timeout=15)
|
||||||
|
if response.status_code != 200:
|
||||||
|
return ""
|
||||||
|
return response.text or ""
|
||||||
|
|
||||||
|
def _parse_cards(self, html):
|
||||||
|
root = self.html(html)
|
||||||
|
if root is None:
|
||||||
|
return []
|
||||||
|
items = []
|
||||||
|
for node in root.xpath("/html/body/div/div/main/div/ul/li"):
|
||||||
|
href = "".join(node.xpath(".//div[contains(@class,'a')]//a[1]/@href")).strip()
|
||||||
|
title = self._clean_text("".join(node.xpath(".//div[contains(@class,'a')]//a[1]//text()")))
|
||||||
|
if not href or not title:
|
||||||
|
continue
|
||||||
|
items.append({"vod_id": href, "vod_name": title, "vod_pic": "", "vod_remarks": ""})
|
||||||
|
return items
|
||||||
|
|
||||||
|
def categoryContent(self, tid, pg, filter, extend):
|
||||||
|
page = int(pg)
|
||||||
|
items = self._parse_cards(self._get_html(f"{self.BASE_URL}/{str(tid).strip()}_{page}/"))
|
||||||
|
return {"page": page, "limit": 15, "total": (page - 1) * 15 + len(items), "list": items}
|
||||||
|
|
||||||
|
def searchContent(self, key, quick, pg="1"):
|
||||||
|
page = int(pg)
|
||||||
|
keyword = self._clean_text(key)
|
||||||
|
if not keyword:
|
||||||
|
return {"page": page, "total": 0, "list": []}
|
||||||
|
items = self._parse_cards(self._get_html(f"{self.BASE_URL}/search/?keyword={quote(keyword)}&page={page}"))
|
||||||
|
return {"page": page, "total": len(items), "list": items}
|
||||||
|
|||||||
Reference in New Issue
Block a user