This commit is contained in:
Harold
2026-04-19 19:45:56 +08:00
parent 9d01039a35
commit 0848dbf05b
13 changed files with 25 additions and 34 deletions
+2 -2
View File
@@ -170,14 +170,14 @@ class TestWooyunSpider(unittest.TestCase):
self.assertEqual(kwargs["data"]["pageIndex"], "2") self.assertEqual(kwargs["data"]["pageIndex"], "2")
self.assertEqual(kwargs["data"]["sortCode"], "latest") self.assertEqual(kwargs["data"]["sortCode"], "latest")
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 2) self.assertNotIn("pagecount", result)
self.assertEqual(result["limit"], 24) self.assertEqual(result["limit"], 24)
self.assertEqual(result["total"], 30) self.assertEqual(result["total"], 30)
@patch.object(Spider, "_request_json") @patch.object(Spider, "_request_json")
def test_search_content_returns_empty_result_for_blank_keyword(self, mock_request_json): def test_search_content_returns_empty_result_for_blank_keyword(self, mock_request_json):
result = self.spider.searchContent("", False, "1") result = self.spider.searchContent("", False, "1")
self.assertEqual(result, {"page": 1, "pagecount": 0, "total": 0, "list": []}) self.assertEqual(result, {"page": 1, "total": 0, "list": []})
mock_request_json.assert_not_called() mock_request_json.assert_not_called()
@patch.object(Spider, "_request_json") @patch.object(Spider, "_request_json")
+2 -2
View File
@@ -75,7 +75,7 @@ class TestDDYSSpider(unittest.TestCase):
result = self.spider.categoryContent("series", "2", False, {"area": "/region/japan"}) result = self.spider.categoryContent("series", "2", False, {"area": "/region/japan"})
self.assertEqual(mock_request_html.call_args.args[0], "https://ddys.io/series/region/japan/page/2") self.assertEqual(mock_request_html.call_args.args[0], "https://ddys.io/series/region/japan/page/2")
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 3) self.assertNotIn("pagecount", result)
self.assertEqual(result["limit"], 24) self.assertEqual(result["limit"], 24)
self.assertEqual(result["list"][0]["vod_name"], "示例剧") self.assertEqual(result["list"][0]["vod_name"], "示例剧")
@@ -138,7 +138,7 @@ class TestDDYSSpider(unittest.TestCase):
self.assertEqual(kwargs["method"], "POST") self.assertEqual(kwargs["method"], "POST")
self.assertEqual(kwargs["data"], "q=%E7%B9%81%E8%8A%B1") self.assertEqual(kwargs["data"], "q=%E7%B9%81%E8%8A%B1")
self.assertEqual(result["list"][0]["vod_id"], "anime/result") self.assertEqual(result["list"][0]["vod_id"], "anime/result")
self.assertEqual(result["pagecount"], 2) self.assertNotIn("pagecount", result)
def test_parse_detail_page_merges_direct_and_pan_sources(self): def test_parse_detail_page_merges_direct_and_pan_sources(self):
html = """ html = """
+2 -2
View File
@@ -83,7 +83,7 @@ class TestJuQuanQuanSpider(unittest.TestCase):
result = self.spider.categoryContent("juji", "2", False, {}) result = self.spider.categoryContent("juji", "2", False, {})
self.assertEqual(mock_request_html.call_args.args[0], "https://www.jqqzx.cc/type/juji/page/2.html") self.assertEqual(mock_request_html.call_args.args[0], "https://www.jqqzx.cc/type/juji/page/2.html")
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 3) self.assertNotIn("pagecount", result)
self.assertEqual(result["list"][0]["vod_id"], "vod/456") self.assertEqual(result["list"][0]["vod_id"], "vod/456")
@patch.object(Spider, "_request_html") @patch.object(Spider, "_request_html")
@@ -95,7 +95,7 @@ class TestJuQuanQuanSpider(unittest.TestCase):
"https://www.jqqzx.cc/index.php/ajax/suggest?mid=1&wd=%E7%B9%81%E8%8A%B1", "https://www.jqqzx.cc/index.php/ajax/suggest?mid=1&wd=%E7%B9%81%E8%8A%B1",
) )
self.assertEqual(result["list"][0]["vod_id"], "vod/777") self.assertEqual(result["list"][0]["vod_id"], "vod/777")
self.assertEqual(result["pagecount"], 1) self.assertNotIn("pagecount", result)
def test_parse_detail_page_extracts_metadata_and_playlists(self): def test_parse_detail_page_extracts_metadata_and_playlists(self):
html = """ html = """
+2 -2
View File
@@ -78,7 +78,7 @@ class TestGimySpider(unittest.TestCase):
result = self.spider.categoryContent("2", "2", False, {"by": "hits"}) result = self.spider.categoryContent("2", "2", False, {"by": "hits"})
self.assertEqual(mock_request_html.call_args.args[0], "https://gimyai.tw/genre/2.html?page=2&by=hits") self.assertEqual(mock_request_html.call_args.args[0], "https://gimyai.tw/genre/2.html?page=2&by=hits")
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 2) self.assertNotIn("pagecount", result)
self.assertEqual(result["list"][0]["vod_name"], "分类影片") self.assertEqual(result["list"][0]["vod_name"], "分类影片")
def test_parse_detail_page_extracts_meta_and_playlists(self): def test_parse_detail_page_extracts_meta_and_playlists(self):
@@ -161,7 +161,7 @@ class TestGimySpider(unittest.TestCase):
result = self.spider.searchContent("斗罗大陆", False, "1") result = self.spider.searchContent("斗罗大陆", False, "1")
self.assertEqual([item["vod_id"] for item in result["list"]], ["11"]) self.assertEqual([item["vod_id"] for item in result["list"]], ["11"])
self.assertEqual(result["page"], 1) self.assertEqual(result["page"], 1)
self.assertEqual(result["pagecount"], 1) self.assertNotIn("pagecount", result)
def test_extract_player_data_reads_json_block(self): def test_extract_player_data_reads_json_block(self):
html = '<script>var player_data = {"url":"https://video.example/direct.m3u8","encrypt":"0","from":"gimy"};</script>' html = '<script>var player_data = {"url":"https://video.example/direct.m3u8","encrypt":"0","from":"gimy"};</script>'
+2 -2
View File
@@ -148,7 +148,7 @@ class TestJuZiTVSpider(unittest.TestCase):
self.spider.init() self.spider.init()
result = self.spider.categoryContent("7", "2", False, {}) result = self.spider.categoryContent("7", "2", False, {})
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 3) self.assertNotIn("pagecount", result)
self.assertEqual(result["list"][0]["vod_id"], "21") self.assertEqual(result["list"][0]["vod_id"], "21")
self.assertEqual(result["list"][0]["vod_content"], "分类简介") self.assertEqual(result["list"][0]["vod_content"], "分类简介")
payload = mock_post_api.call_args.args[1] payload = mock_post_api.call_args.args[1]
@@ -186,7 +186,7 @@ class TestJuZiTVSpider(unittest.TestCase):
self.spider.init() self.spider.init()
result = self.spider.searchContent("繁花", False, "1") result = self.spider.searchContent("繁花", False, "1")
self.assertEqual(result["page"], 1) self.assertEqual(result["page"], 1)
self.assertEqual(result["pagecount"], 1) self.assertNotIn("pagecount", result)
self.assertEqual(result["list"][0]["vod_id"], "31") self.assertEqual(result["list"][0]["vod_id"], "31")
self.assertEqual(result["list"][0]["vod_remarks"], "评分:8.5") self.assertEqual(result["list"][0]["vod_remarks"], "评分:8.5")
payload = mock_post_api.call_args.args[1] payload = mock_post_api.call_args.args[1]
+2 -2
View File
@@ -74,7 +74,7 @@ class TestDidaSpider(unittest.TestCase):
result = self.spider.categoryContent("1", "2", False, {"area": "香港", "sort": "score"}) result = self.spider.categoryContent("1", "2", False, {"area": "香港", "sort": "score"})
self.assertEqual(mock_request_html.call_args.args[0], "https://www.didahd.pro/show/1-香港-score------2---") self.assertEqual(mock_request_html.call_args.args[0], "https://www.didahd.pro/show/1-香港-score------2---")
self.assertEqual(result["page"], 2) self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 3) self.assertNotIn("pagecount", result)
self.assertEqual(result["limit"], 12) self.assertEqual(result["limit"], 12)
self.assertEqual(result["list"][0]["vod_name"], "分类影片") self.assertEqual(result["list"][0]["vod_name"], "分类影片")
@@ -93,7 +93,7 @@ class TestDidaSpider(unittest.TestCase):
"https://www.didahd.pro/search/-------------.html?wd=%E7%B9%81%E8%8A%B1", "https://www.didahd.pro/search/-------------.html?wd=%E7%B9%81%E8%8A%B1",
) )
self.assertEqual(result["list"][0]["vod_id"], "detail/321") self.assertEqual(result["list"][0]["vod_id"], "detail/321")
self.assertEqual(result["pagecount"], 2) self.assertNotIn("pagecount", result)
def test_extract_netdisk_groups_deduplicates_links_and_sorts_by_priority(self): def test_extract_netdisk_groups_deduplicates_links_and_sorts_by_priority(self):
html = """ html = """
+3 -6
View File
@@ -1,7 +1,6 @@
# coding=utf-8 # coding=utf-8
import base64 import base64
import json import json
import math
import sys import sys
from base.spider import Spider as BaseSpider from base.spider import Spider as BaseSpider
@@ -259,14 +258,13 @@ class Spider(BaseSpider):
) or {} ) or {}
records = [self._map_vod(item) for item in self._ensure_list(data.get("records"))] records = [self._map_vod(item) for item in self._ensure_list(data.get("records"))]
total = int(data.get("total") or 0) total = int(data.get("total") or 0)
pagecount = int(data.get("pages") or (math.ceil(total / self.page_size) if total else (page if records else 0))) return {"page": page, "limit": self.page_size, "total": total, "list": records}
return {"page": page, "pagecount": pagecount, "limit": self.page_size, "total": total, "list": records}
def searchContent(self, key, quick, pg="1"): def searchContent(self, key, quick, pg="1"):
page = int(pg) page = int(pg)
keyword = self._stringify(key).strip() keyword = self._stringify(key).strip()
if not keyword: if not keyword:
return {"page": page, "pagecount": 0, "total": 0, "list": []} return {"page": page, "total": 0, "list": []}
data = self._request_json( data = self._request_json(
"/movie/media/search", "/movie/media/search",
@@ -283,8 +281,7 @@ class Spider(BaseSpider):
) or {} ) or {}
records = [self._map_vod(item) for item in self._ensure_list(data.get("records"))] records = [self._map_vod(item) for item in self._ensure_list(data.get("records"))]
total = int(data.get("total") or 0) total = int(data.get("total") or 0)
pagecount = int(data.get("pages") or (math.ceil(total / self.page_size) if total else (page if records else 0))) return {"page": page, "total": total, "list": records}
return {"page": page, "pagecount": pagecount, "total": total, "list": records}
def _encode_play_payload(self, payload): def _encode_play_payload(self, payload):
raw = json.dumps(payload, ensure_ascii=False).encode("utf-8") raw = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+2 -2
View File
@@ -240,7 +240,7 @@ class Spider(BaseSpider):
page = int(pg) page = int(pg)
keyword = self._stringify(key).strip() keyword = self._stringify(key).strip()
if not keyword: if not keyword:
return {"page": page, "pagecount": 0, "total": 0, "list": []} return {"page": page, "total": 0, "list": []}
html = self._request_html( html = self._request_html(
self.host + "/search", self.host + "/search",
method="POST", method="POST",
@@ -253,7 +253,7 @@ class Spider(BaseSpider):
) )
if not items: if not items:
items = self._parse_movie_cards(html) items = self._parse_movie_cards(html)
return {"page": page, "pagecount": page + 1 if items else page, "total": len(items), "list": items} return {"page": page, "total": len(items), "list": items}
def _extract_switch_sources(self, html): def _extract_switch_sources(self, html):
root = self.html(html) root = self.html(html)
+2 -3
View File
@@ -133,7 +133,6 @@ class Spider(BaseSpider):
items = self._parse_cards(self._request_html(self._build_url(f"/type/{tid}/page/{page}.html"))) items = self._parse_cards(self._request_html(self._build_url(f"/type/{tid}/page/{page}.html")))
return { return {
"page": page, "page": page,
"pagecount": page + 1 if items else page,
"total": page * len(items) + (1 if items else 0), "total": page * len(items) + (1 if items else 0),
"list": items, "list": items,
} }
@@ -142,11 +141,11 @@ class Spider(BaseSpider):
page = int(pg) page = int(pg)
keyword = self._clean_text(key) keyword = self._clean_text(key)
if not keyword: if not keyword:
return {"page": page, "pagecount": 1, "total": 0, "list": []} return {"page": page, "total": 0, "list": []}
items = self._parse_search_list( items = self._parse_search_list(
self._request_html(self._build_url(f"/index.php/ajax/suggest?mid=1&wd={quote(keyword)}")) self._request_html(self._build_url(f"/index.php/ajax/suggest?mid=1&wd={quote(keyword)}"))
) )
return {"page": page, "pagecount": 1, "total": len(items), "list": items} return {"page": page, "total": len(items), "list": items}
def _parse_info_items(self, root): def _parse_info_items(self, root):
info = {} info = {}
-2
View File
@@ -317,11 +317,9 @@ class Spider(BaseSpider):
raw_items = self._extract_cards(html) raw_items = self._extract_cards(html)
items = self._refine_search_results(raw_items, key) items = self._refine_search_results(raw_items, key)
page = int(pg) page = int(pg)
pagecount = page + 1 if len(raw_items) >= 20 else page
return { return {
"list": items, "list": items,
"page": page, "page": page,
"pagecount": pagecount,
"limit": len(items), "limit": len(items),
"total": len(items), "total": len(items),
} }
+3 -4
View File
@@ -175,10 +175,9 @@ class Spider(BaseSpider):
response = self._post_api("/v1/api/search/queryNow", payload) response = self._post_api("/v1/api/search/queryNow", payload)
data = response.get("data", {}) data = response.get("data", {})
nextVal = data.get("nextVal", '') nextVal = data.get("nextVal", '')
if len(self.next) == page + 1: while len(self.next) <= page + 1:
self.next.append(nextVal) self.next.append("")
else: self.next[page + 1] = nextVal
self.next[page + 1] = nextVal
items = [self._map_video_item(item) for item in data.get("items", []) if isinstance(item, dict)] items = [self._map_video_item(item) for item in data.get("items", []) if isinstance(item, dict)]
return self._page_result(items, pg, data.get("hasNext") == 1) return self._page_result(items, pg, data.get("hasNext") == 1)
+2 -2
View File
@@ -186,10 +186,10 @@ class Spider(BaseSpider):
page = int(pg) page = int(pg)
keyword = self._stringify(key).strip() keyword = self._stringify(key).strip()
if not keyword: if not keyword:
return {"page": page, "pagecount": 0, "total": 0, "list": []} return {"page": page, "total": 0, "list": []}
url = f"{self.host}/search/-------------.html?wd={quote(keyword)}" url = f"{self.host}/search/-------------.html?wd={quote(keyword)}"
items = self._parse_cards(self._request_html(url)) items = self._parse_cards(self._request_html(url))
return {"page": page, "pagecount": page + 1 if items else page, "total": len(items), "list": items} return {"page": page, "total": len(items), "list": items}
def _extract_detail_field(self, root, label, joiner=""): def _extract_detail_field(self, root, label, joiner=""):
if root is None: if root is None:
-2
View File
@@ -142,7 +142,6 @@ class Spider(Spider):
return { return {
'list': videos, 'list': videos,
'page': int(pg), 'page': int(pg),
'pagecount': 9999,
'limit': 30, 'limit': 30,
'total': 999999 'total': 999999
} }
@@ -248,7 +247,6 @@ class Spider(Spider):
return { return {
'list': videos, 'list': videos,
'page': int(pg), 'page': int(pg),
'pagecount': 9999,
'limit': 30, 'limit': 30,
'total': 999999 'total': 999999
} }