feat: add rrdy search parsing
This commit is contained in:
@@ -93,3 +93,34 @@ class TestRenRenDianYingSpider(unittest.TestCase):
|
||||
self.assertEqual(result["limit"], 1)
|
||||
self.assertEqual(result["list"][0]["vod_name"], "分类影片")
|
||||
self.assertNotIn("pagecount", result)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_search_content_builds_reference_url_and_cleans_highlight_title(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<ul id="movielist">
|
||||
<li class="pure-g shadow">
|
||||
<div class="pure-u-5-24"><img data-original="/search.jpg" /></div>
|
||||
<div class="intro">
|
||||
<h2><a href="/movie/789.html" title="<font color='red'>剑来</font> 第二季">抢先看</a></h2>
|
||||
</div>
|
||||
<div class="dou"><b>9.2</b></div>
|
||||
</li>
|
||||
</ul>
|
||||
"""
|
||||
result = self.spider.searchContent("剑来", False, "2")
|
||||
self.assertEqual(
|
||||
mock_request_html.call_args.args[0],
|
||||
"https://www.rrdynb.com/plus/search.php?q=%E5%89%91%E6%9D%A5&pagesize=10&submit=&PageNo=2",
|
||||
)
|
||||
self.assertEqual(
|
||||
result["list"][0],
|
||||
{
|
||||
"vod_id": "/movie/789.html",
|
||||
"vod_name": "剑来 第二季",
|
||||
"vod_pic": "https://www.rrdynb.com/search.jpg",
|
||||
"vod_remarks": "9.2",
|
||||
},
|
||||
)
|
||||
|
||||
def test_search_content_returns_empty_list_for_blank_keyword(self):
|
||||
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
||||
|
||||
+16
-1
@@ -1,7 +1,7 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
from base.spider import Spider as BaseSpider
|
||||
|
||||
@@ -107,6 +107,8 @@ class Spider(BaseSpider):
|
||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')][1]/@src")).strip()
|
||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@data-original")).strip()
|
||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@src")).strip()
|
||||
or "".join(node.xpath(".//*[contains(@class,'pure-u-5-24')]//img[1]/@data-original")).strip()
|
||||
or "".join(node.xpath(".//*[contains(@class,'pure-u-5-24')]//img[1]/@src")).strip()
|
||||
)
|
||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'dou')][1]//text()")))
|
||||
if not href or not title or href in seen:
|
||||
@@ -128,3 +130,16 @@ class Spider(BaseSpider):
|
||||
url = self._build_url(f"{class_path}_{page}.html")
|
||||
items = self._parse_cards(self._request_html(url))
|
||||
return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
page = int(pg)
|
||||
keyword = self._clean_text(key)
|
||||
if not keyword:
|
||||
return {"page": page, "total": 0, "list": []}
|
||||
|
||||
url = self._build_url(f"/plus/search.php?q={quote(keyword)}&pagesize=10&submit=")
|
||||
if page > 1:
|
||||
url += f"&PageNo={page}"
|
||||
|
||||
items = self._parse_cards(self._request_html(url))
|
||||
return {"page": page, "total": len(items), "list": items}
|
||||
|
||||
Reference in New Issue
Block a user