feat: add rrdy search parsing
This commit is contained in:
@@ -93,3 +93,34 @@ class TestRenRenDianYingSpider(unittest.TestCase):
|
|||||||
self.assertEqual(result["limit"], 1)
|
self.assertEqual(result["limit"], 1)
|
||||||
self.assertEqual(result["list"][0]["vod_name"], "分类影片")
|
self.assertEqual(result["list"][0]["vod_name"], "分类影片")
|
||||||
self.assertNotIn("pagecount", result)
|
self.assertNotIn("pagecount", result)
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_search_content_builds_reference_url_and_cleans_highlight_title(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = """
|
||||||
|
<ul id="movielist">
|
||||||
|
<li class="pure-g shadow">
|
||||||
|
<div class="pure-u-5-24"><img data-original="/search.jpg" /></div>
|
||||||
|
<div class="intro">
|
||||||
|
<h2><a href="/movie/789.html" title="<font color='red'>剑来</font> 第二季">抢先看</a></h2>
|
||||||
|
</div>
|
||||||
|
<div class="dou"><b>9.2</b></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
"""
|
||||||
|
result = self.spider.searchContent("剑来", False, "2")
|
||||||
|
self.assertEqual(
|
||||||
|
mock_request_html.call_args.args[0],
|
||||||
|
"https://www.rrdynb.com/plus/search.php?q=%E5%89%91%E6%9D%A5&pagesize=10&submit=&PageNo=2",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
result["list"][0],
|
||||||
|
{
|
||||||
|
"vod_id": "/movie/789.html",
|
||||||
|
"vod_name": "剑来 第二季",
|
||||||
|
"vod_pic": "https://www.rrdynb.com/search.jpg",
|
||||||
|
"vod_remarks": "9.2",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_search_content_returns_empty_list_for_blank_keyword(self):
|
||||||
|
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
||||||
|
|||||||
+16
-1
@@ -1,7 +1,7 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import quote, urljoin
|
||||||
|
|
||||||
from base.spider import Spider as BaseSpider
|
from base.spider import Spider as BaseSpider
|
||||||
|
|
||||||
@@ -107,6 +107,8 @@ class Spider(BaseSpider):
|
|||||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')][1]/@src")).strip()
|
or "".join(node.xpath(".//*[contains(@class,'pure-img')][1]/@src")).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@data-original")).strip()
|
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@data-original")).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@src")).strip()
|
or "".join(node.xpath(".//*[contains(@class,'pure-img')]//img[1]/@src")).strip()
|
||||||
|
or "".join(node.xpath(".//*[contains(@class,'pure-u-5-24')]//img[1]/@data-original")).strip()
|
||||||
|
or "".join(node.xpath(".//*[contains(@class,'pure-u-5-24')]//img[1]/@src")).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'dou')][1]//text()")))
|
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'dou')][1]//text()")))
|
||||||
if not href or not title or href in seen:
|
if not href or not title or href in seen:
|
||||||
@@ -128,3 +130,16 @@ class Spider(BaseSpider):
|
|||||||
url = self._build_url(f"{class_path}_{page}.html")
|
url = self._build_url(f"{class_path}_{page}.html")
|
||||||
items = self._parse_cards(self._request_html(url))
|
items = self._parse_cards(self._request_html(url))
|
||||||
return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items}
|
return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items}
|
||||||
|
|
||||||
|
def searchContent(self, key, quick, pg="1"):
|
||||||
|
page = int(pg)
|
||||||
|
keyword = self._clean_text(key)
|
||||||
|
if not keyword:
|
||||||
|
return {"page": page, "total": 0, "list": []}
|
||||||
|
|
||||||
|
url = self._build_url(f"/plus/search.php?q={quote(keyword)}&pagesize=10&submit=")
|
||||||
|
if page > 1:
|
||||||
|
url += f"&PageNo={page}"
|
||||||
|
|
||||||
|
items = self._parse_cards(self._request_html(url))
|
||||||
|
return {"page": page, "total": len(items), "list": items}
|
||||||
|
|||||||
Reference in New Issue
Block a user