feat: add ddys list and search flows

This commit is contained in:
Harold
2026-04-19 16:41:29 +08:00
parent b6a44611f1
commit 9b68b6ba43
2 changed files with 181 additions and 0 deletions
+104
View File
@@ -1,6 +1,8 @@
import unittest
from importlib.machinery import SourceFileLoader
from pathlib import Path
from urllib.parse import quote
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
@@ -36,6 +38,108 @@ class TestDDYSSpider(unittest.TestCase):
url = self.spider._build_category_url("series", "1", {})
self.assertEqual(url, "https://ddys.io/series/page/1")
def test_parse_movie_cards_extracts_expected_fields(self):
html = """
<div class="movie-card">
<a href="/movie/test-title/">
<img src="/poster.jpg" />
<span class="poster-badge">HD</span>
<h3>测试电影</h3>
</a>
</div>
"""
cards = self.spider._parse_movie_cards(html)
self.assertEqual(
cards,
[
{
"vod_id": "https://ddys.io/movie/test-title/",
"vod_name": "测试电影",
"vod_pic": "https://ddys.io/poster.jpg",
"vod_remarks": "HD",
}
],
)
@patch.object(Spider, "_request_html")
def test_category_content_uses_built_url_and_returns_page_result(self, mock_request_html):
mock_request_html.return_value = """
<div class="movie-card">
<a href="/series/demo/">
<img src="/series.jpg" />
<span class="poster-badge">更新至3集</span>
<h3>示例剧</h3>
</a>
</div>
"""
result = self.spider.categoryContent("series", "2", False, {"area": "/region/japan"})
self.assertEqual(mock_request_html.call_args.args[0], "https://ddys.io/series/region/japan/page/2")
self.assertEqual(result["page"], 2)
self.assertEqual(result["pagecount"], 3)
self.assertEqual(result["limit"], 24)
self.assertEqual(result["list"][0]["vod_name"], "示例剧")
def test_request_html_posts_form_data_for_search(self):
calls = {}
class FakeResponse:
def __init__(self):
self.status_code = 200
self.text = (
'<div class="movie-card"><a href="/movie/hit/">'
'<img src="/search.jpg" /><h3>命中结果</h3></a></div>'
)
self.encoding = "utf-8"
def fake_post(
url,
params=None,
data=None,
json=None,
cookies=None,
headers=None,
timeout=5,
verify=True,
stream=False,
allow_redirects=True,
):
calls["url"] = url
calls["data"] = data
calls["headers"] = headers
return FakeResponse()
self.spider.post = fake_post
html = self.spider._request_html(
self.spider.host + "/search",
method="POST",
data=f"q={quote('繁花')}",
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
self.assertIn("命中结果", html)
self.assertEqual(calls["url"], "https://ddys.io/search")
self.assertEqual(calls["data"], "q=%E7%B9%81%E8%8A%B1")
self.assertEqual(calls["headers"]["Content-Type"], "application/x-www-form-urlencoded")
@patch.object(Spider, "_request_html")
def test_search_content_posts_keyword_and_parses_cards(self, mock_request_html):
mock_request_html.return_value = """
<div class="mb-12">
<div class="movie-card">
<a href="/anime/result/">
<img src="/anime.jpg" alt="搜索动漫" />
<span class="poster-badge">全集</span>
<h3>搜索动漫</h3>
</a>
</div>
</div>
"""
result = self.spider.searchContent("繁花", False, "1")
kwargs = mock_request_html.call_args.kwargs
self.assertEqual(kwargs["method"], "POST")
self.assertEqual(kwargs["data"], "q=%E7%B9%81%E8%8A%B1")
self.assertEqual(result["list"][0]["vod_id"], "https://ddys.io/anime/result/")
self.assertEqual(result["pagecount"], 2)
if __name__ == "__main__":
unittest.main()
+77
View File
@@ -1,6 +1,8 @@
# coding=utf-8
import json
import re
import sys
from urllib.parse import quote
from base.spider import Spider as BaseSpider
@@ -99,3 +101,78 @@ class Spider(BaseSpider):
f"/page/{int(pg)}"
)
return self._build_url(path)
def _request_html(self, path_or_url, method="GET", data=None, headers=None, referer=None):
target = path_or_url if str(path_or_url).startswith("http") else self._build_url(path_or_url)
merged_headers = dict(self.headers)
if headers:
merged_headers.update(headers)
merged_headers["Referer"] = referer or self.headers["Referer"]
if method == "POST":
response = self.post(target, data=data, headers=merged_headers, timeout=10)
else:
response = self.fetch(target, headers=merged_headers, timeout=10)
if response.status_code != 200:
return ""
return response.text or ""
def _clean_text(self, text):
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
def _parse_movie_cards(self, html, root_xpath="//*[contains(@class,'movie-card')]"):
root = self.html(html)
if root is None:
return []
items = []
seen = set()
for node in root.xpath(root_xpath):
href = ((node.xpath(".//a[@href][1]/@href") or [""])[0]).strip()
title = self._clean_text("".join(node.xpath(".//h3[1]//text()")))
pic = (
((node.xpath(".//img[1]/@src") or [""])[0]).strip()
or ((node.xpath(".//img[1]/@data-src") or [""])[0]).strip()
)
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'poster-badge')][1]//text()")))
vod_id = self._build_url(href)
if not vod_id or not title or vod_id in seen:
continue
seen.add(vod_id)
items.append(
{
"vod_id": vod_id,
"vod_name": title,
"vod_pic": self._build_url(pic),
"vod_remarks": remarks,
}
)
return items
def categoryContent(self, tid, pg, filter, extend):
page = int(pg)
items = self._parse_movie_cards(self._request_html(self._build_category_url(tid, pg, extend)))
return {
"list": items,
"page": page,
"pagecount": page + 1 if items else page,
"limit": 24,
"total": page * 24 + len(items),
}
def searchContent(self, key, quick, pg="1"):
page = int(pg)
keyword = self._stringify(key).strip()
if not keyword:
return {"page": page, "pagecount": 0, "total": 0, "list": []}
html = self._request_html(
self.host + "/search",
method="POST",
data=f"q={quote(keyword)}",
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
items = self._parse_movie_cards(
html,
root_xpath="(//*[contains(@class,'mb-12')])[1]//*[contains(@class,'movie-card')]",
)
if not items:
items = self._parse_movie_cards(html)
return {"page": page, "pagecount": page + 1 if items else page, "total": len(items), "list": items}