feat: add juquanquan list and search flows
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -41,6 +42,60 @@ class TestJuQuanQuanSpider(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_parse_cards_extracts_compact_vod_ids(self):
|
||||
html = """
|
||||
<a class="module-poster-item module-item" href="/vod/123.html">
|
||||
<img data-original="/cover.jpg" />
|
||||
<div class="module-poster-item-title">示例影片</div>
|
||||
<div class="module-item-note">更新至1集</div>
|
||||
</a>
|
||||
"""
|
||||
self.assertEqual(
|
||||
self.spider._parse_cards(html),
|
||||
[
|
||||
{
|
||||
"vod_id": "vod/123",
|
||||
"vod_name": "示例影片",
|
||||
"vod_pic": "https://www.jqqzx.cc/cover.jpg",
|
||||
"vod_remarks": "更新至1集",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_home_video_content_limits_recommendations(self, mock_request_html):
|
||||
mock_request_html.return_value = "".join(
|
||||
f'<a class="module-poster-item module-item" href="/vod/{index}.html"><div class="module-poster-item-title">影片{index}</div></a>'
|
||||
for index in range(1, 45)
|
||||
)
|
||||
result = self.spider.homeVideoContent()
|
||||
self.assertEqual(len(result["list"]), 40)
|
||||
self.assertEqual(result["list"][0]["vod_id"], "vod/1")
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_category_content_builds_page_result(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<a class="module-poster-item module-item" href="/vod/456.html">
|
||||
<div class="module-poster-item-title">分类影片</div>
|
||||
</a>
|
||||
"""
|
||||
result = self.spider.categoryContent("juji", "2", False, {})
|
||||
self.assertEqual(mock_request_html.call_args.args[0], "https://www.jqqzx.cc/type/juji/page/2.html")
|
||||
self.assertEqual(result["page"], 2)
|
||||
self.assertEqual(result["pagecount"], 3)
|
||||
self.assertEqual(result["list"][0]["vod_id"], "vod/456")
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_search_content_uses_suggest_api(self, mock_request_html):
|
||||
mock_request_html.return_value = '{"list":[{"id":"777","name":"搜索结果","pic":"/pic.jpg"}]}'
|
||||
result = self.spider.searchContent("繁花", False, "1")
|
||||
self.assertEqual(
|
||||
mock_request_html.call_args.args[0],
|
||||
"https://www.jqqzx.cc/index.php/ajax/suggest?mid=1&wd=%E7%B9%81%E8%8A%B1",
|
||||
)
|
||||
self.assertEqual(result["list"][0]["vod_id"], "vod/777")
|
||||
self.assertEqual(result["pagecount"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
from base.spider import Spider as BaseSpider
|
||||
|
||||
@@ -77,3 +77,70 @@ class Spider(BaseSpider):
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
def _request_html(self, path_or_url, headers=None):
|
||||
target = path_or_url if str(path_or_url).startswith("http") else self._build_url(path_or_url)
|
||||
request_headers = dict(self.headers)
|
||||
if headers:
|
||||
request_headers.update(headers)
|
||||
response = self.fetch(target, headers=request_headers, timeout=10)
|
||||
if response.status_code != 200:
|
||||
return ""
|
||||
return response.text or ""
|
||||
|
||||
def _parse_cards(self, html):
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return []
|
||||
items = []
|
||||
seen = set()
|
||||
for anchor in root.xpath("//a[contains(@class,'module-poster-item') and contains(@class,'module-item')]"):
|
||||
vod_id = self._encode_vod_id((anchor.xpath("./@href") or [""])[0])
|
||||
if not vod_id or vod_id in seen:
|
||||
continue
|
||||
title = self._clean_text(
|
||||
"".join(anchor.xpath(".//*[contains(@class,'module-poster-item-title')][1]//text()"))
|
||||
or (anchor.xpath("./@title") or [""])[0]
|
||||
or (anchor.xpath(".//img[1]/@alt") or [""])[0]
|
||||
)
|
||||
if not title:
|
||||
continue
|
||||
seen.add(vod_id)
|
||||
pic = (
|
||||
anchor.xpath(".//img[1]/@data-original")
|
||||
or anchor.xpath(".//img[1]/@src")
|
||||
or [""]
|
||||
)[0]
|
||||
note = self._clean_text("".join(anchor.xpath(".//*[contains(@class,'module-item-note')][1]//text()")))
|
||||
items.append(
|
||||
{
|
||||
"vod_id": vod_id,
|
||||
"vod_name": title,
|
||||
"vod_pic": self._build_url(pic),
|
||||
"vod_remarks": note,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
def homeVideoContent(self):
|
||||
return {"list": self._parse_cards(self._request_html(self.host))[:40]}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
page = int(pg)
|
||||
items = self._parse_cards(self._request_html(self._build_url(f"/type/{tid}/page/{page}.html")))
|
||||
return {
|
||||
"page": page,
|
||||
"pagecount": page + 1 if items else page,
|
||||
"total": page * len(items) + (1 if items else 0),
|
||||
"list": items,
|
||||
}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
page = int(pg)
|
||||
keyword = self._clean_text(key)
|
||||
if not keyword:
|
||||
return {"page": page, "pagecount": 1, "total": 0, "list": []}
|
||||
items = self._parse_search_list(
|
||||
self._request_html(self._build_url(f"/index.php/ajax/suggest?mid=1&wd={quote(keyword)}"))
|
||||
)
|
||||
return {"page": page, "pagecount": 1, "total": len(items), "list": items}
|
||||
|
||||
Reference in New Issue
Block a user