feat: add ddys list and search flows
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import unittest
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -36,6 +38,108 @@ class TestDDYSSpider(unittest.TestCase):
|
||||
url = self.spider._build_category_url("series", "1", {})
|
||||
self.assertEqual(url, "https://ddys.io/series/page/1")
|
||||
|
||||
def test_parse_movie_cards_extracts_expected_fields(self):
|
||||
html = """
|
||||
<div class="movie-card">
|
||||
<a href="/movie/test-title/">
|
||||
<img src="/poster.jpg" />
|
||||
<span class="poster-badge">HD</span>
|
||||
<h3>测试电影</h3>
|
||||
</a>
|
||||
</div>
|
||||
"""
|
||||
cards = self.spider._parse_movie_cards(html)
|
||||
self.assertEqual(
|
||||
cards,
|
||||
[
|
||||
{
|
||||
"vod_id": "https://ddys.io/movie/test-title/",
|
||||
"vod_name": "测试电影",
|
||||
"vod_pic": "https://ddys.io/poster.jpg",
|
||||
"vod_remarks": "HD",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_category_content_uses_built_url_and_returns_page_result(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<div class="movie-card">
|
||||
<a href="/series/demo/">
|
||||
<img src="/series.jpg" />
|
||||
<span class="poster-badge">更新至3集</span>
|
||||
<h3>示例剧</h3>
|
||||
</a>
|
||||
</div>
|
||||
"""
|
||||
result = self.spider.categoryContent("series", "2", False, {"area": "/region/japan"})
|
||||
self.assertEqual(mock_request_html.call_args.args[0], "https://ddys.io/series/region/japan/page/2")
|
||||
self.assertEqual(result["page"], 2)
|
||||
self.assertEqual(result["pagecount"], 3)
|
||||
self.assertEqual(result["limit"], 24)
|
||||
self.assertEqual(result["list"][0]["vod_name"], "示例剧")
|
||||
|
||||
def test_request_html_posts_form_data_for_search(self):
|
||||
calls = {}
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self):
|
||||
self.status_code = 200
|
||||
self.text = (
|
||||
'<div class="movie-card"><a href="/movie/hit/">'
|
||||
'<img src="/search.jpg" /><h3>命中结果</h3></a></div>'
|
||||
)
|
||||
self.encoding = "utf-8"
|
||||
|
||||
def fake_post(
|
||||
url,
|
||||
params=None,
|
||||
data=None,
|
||||
json=None,
|
||||
cookies=None,
|
||||
headers=None,
|
||||
timeout=5,
|
||||
verify=True,
|
||||
stream=False,
|
||||
allow_redirects=True,
|
||||
):
|
||||
calls["url"] = url
|
||||
calls["data"] = data
|
||||
calls["headers"] = headers
|
||||
return FakeResponse()
|
||||
|
||||
self.spider.post = fake_post
|
||||
html = self.spider._request_html(
|
||||
self.spider.host + "/search",
|
||||
method="POST",
|
||||
data=f"q={quote('繁花')}",
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
self.assertIn("命中结果", html)
|
||||
self.assertEqual(calls["url"], "https://ddys.io/search")
|
||||
self.assertEqual(calls["data"], "q=%E7%B9%81%E8%8A%B1")
|
||||
self.assertEqual(calls["headers"]["Content-Type"], "application/x-www-form-urlencoded")
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_search_content_posts_keyword_and_parses_cards(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<div class="mb-12">
|
||||
<div class="movie-card">
|
||||
<a href="/anime/result/">
|
||||
<img src="/anime.jpg" alt="搜索动漫" />
|
||||
<span class="poster-badge">全集</span>
|
||||
<h3>搜索动漫</h3>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
result = self.spider.searchContent("繁花", False, "1")
|
||||
kwargs = mock_request_html.call_args.kwargs
|
||||
self.assertEqual(kwargs["method"], "POST")
|
||||
self.assertEqual(kwargs["data"], "q=%E7%B9%81%E8%8A%B1")
|
||||
self.assertEqual(result["list"][0]["vod_id"], "https://ddys.io/anime/result/")
|
||||
self.assertEqual(result["pagecount"], 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+77
@@ -1,6 +1,8 @@
|
||||
# coding=utf-8
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
|
||||
from base.spider import Spider as BaseSpider
|
||||
|
||||
@@ -99,3 +101,78 @@ class Spider(BaseSpider):
|
||||
f"/page/{int(pg)}"
|
||||
)
|
||||
return self._build_url(path)
|
||||
|
||||
def _request_html(self, path_or_url, method="GET", data=None, headers=None, referer=None):
|
||||
target = path_or_url if str(path_or_url).startswith("http") else self._build_url(path_or_url)
|
||||
merged_headers = dict(self.headers)
|
||||
if headers:
|
||||
merged_headers.update(headers)
|
||||
merged_headers["Referer"] = referer or self.headers["Referer"]
|
||||
if method == "POST":
|
||||
response = self.post(target, data=data, headers=merged_headers, timeout=10)
|
||||
else:
|
||||
response = self.fetch(target, headers=merged_headers, timeout=10)
|
||||
if response.status_code != 200:
|
||||
return ""
|
||||
return response.text or ""
|
||||
|
||||
def _clean_text(self, text):
|
||||
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
||||
|
||||
def _parse_movie_cards(self, html, root_xpath="//*[contains(@class,'movie-card')]"):
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return []
|
||||
items = []
|
||||
seen = set()
|
||||
for node in root.xpath(root_xpath):
|
||||
href = ((node.xpath(".//a[@href][1]/@href") or [""])[0]).strip()
|
||||
title = self._clean_text("".join(node.xpath(".//h3[1]//text()")))
|
||||
pic = (
|
||||
((node.xpath(".//img[1]/@src") or [""])[0]).strip()
|
||||
or ((node.xpath(".//img[1]/@data-src") or [""])[0]).strip()
|
||||
)
|
||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'poster-badge')][1]//text()")))
|
||||
vod_id = self._build_url(href)
|
||||
if not vod_id or not title or vod_id in seen:
|
||||
continue
|
||||
seen.add(vod_id)
|
||||
items.append(
|
||||
{
|
||||
"vod_id": vod_id,
|
||||
"vod_name": title,
|
||||
"vod_pic": self._build_url(pic),
|
||||
"vod_remarks": remarks,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
page = int(pg)
|
||||
items = self._parse_movie_cards(self._request_html(self._build_category_url(tid, pg, extend)))
|
||||
return {
|
||||
"list": items,
|
||||
"page": page,
|
||||
"pagecount": page + 1 if items else page,
|
||||
"limit": 24,
|
||||
"total": page * 24 + len(items),
|
||||
}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
page = int(pg)
|
||||
keyword = self._stringify(key).strip()
|
||||
if not keyword:
|
||||
return {"page": page, "pagecount": 0, "total": 0, "list": []}
|
||||
html = self._request_html(
|
||||
self.host + "/search",
|
||||
method="POST",
|
||||
data=f"q={quote(keyword)}",
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
items = self._parse_movie_cards(
|
||||
html,
|
||||
root_xpath="(//*[contains(@class,'mb-12')])[1]//*[contains(@class,'movie-card')]",
|
||||
)
|
||||
if not items:
|
||||
items = self._parse_movie_cards(html)
|
||||
return {"page": page, "pagecount": page + 1 if items else page, "total": len(items), "list": items}
|
||||
|
||||
Reference in New Issue
Block a user