feat: add rrdy detail pan parsing
This commit is contained in:
@@ -124,3 +124,45 @@ class TestRenRenDianYingSpider(unittest.TestCase):
|
||||
|
||||
def test_search_content_returns_empty_list_for_blank_keyword(self):
|
||||
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
||||
|
||||
def test_extract_pan_links_filters_xunlei_and_aliyun(self):
|
||||
html = """
|
||||
<div class="movie-txt">
|
||||
<a href="https://pan.baidu.com/s/b1">百度网盘</a>
|
||||
<a href="https://pan.quark.cn/s/q1">夸克资源</a>
|
||||
<a href="https://pan.xunlei.com/s/x1">迅雷资源</a>
|
||||
<a href="https://www.alipan.com/s/a1">阿里资源</a>
|
||||
</div>
|
||||
"""
|
||||
self.assertEqual(
|
||||
self.spider._extract_pan_links(html),
|
||||
[
|
||||
("百度网盘", "https://pan.baidu.com/s/b1"),
|
||||
("夸克资源", "https://pan.quark.cn/s/q1"),
|
||||
],
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_detail_content_extracts_meta_and_builds_single_pan_line(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<div class="movie-des"><h1>人人示例片</h1></div>
|
||||
<div class="movie-img"><img src="/poster-detail.jpg" /></div>
|
||||
<div class="movie-txt">
|
||||
<p>一段简介</p>
|
||||
<a href="https://pan.baidu.com/s/b1">百度网盘</a>
|
||||
<a href="https://pan.quark.cn/s/q1">夸克资源</a>
|
||||
<a href="https://pan.xunlei.com/s/x1">迅雷资源</a>
|
||||
</div>
|
||||
"""
|
||||
result = self.spider.detailContent(["/movie/123.html"])
|
||||
self.assertEqual(mock_request_html.call_args.args[0], "https://www.rrdynb.com/movie/123.html")
|
||||
vod = result["list"][0]
|
||||
self.assertEqual(vod["vod_id"], "/movie/123.html")
|
||||
self.assertEqual(vod["vod_name"], "人人示例片")
|
||||
self.assertEqual(vod["vod_pic"], "https://www.rrdynb.com/poster-detail.jpg")
|
||||
self.assertIn("一段简介", vod["vod_content"])
|
||||
self.assertEqual(vod["vod_play_from"], "网盘")
|
||||
self.assertEqual(
|
||||
vod["vod_play_url"],
|
||||
"百度网盘$https://pan.baidu.com/s/b1#夸克资源$https://pan.quark.cn/s/q1",
|
||||
)
|
||||
|
||||
+50
@@ -143,3 +143,53 @@ class Spider(BaseSpider):
|
||||
|
||||
items = self._parse_cards(self._request_html(url))
|
||||
return {"page": page, "total": len(items), "list": items}
|
||||
|
||||
def _extract_pan_links(self, html):
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return []
|
||||
|
||||
items = []
|
||||
seen = set()
|
||||
for node in root.xpath("//*[contains(@class,'movie-txt')]//a[@href]"):
|
||||
href = "".join(node.xpath("./@href")).strip()
|
||||
title = self._clean_text("".join(node.xpath(".//text()"))) or "网盘资源"
|
||||
if not self._is_supported_pan_url(href) or href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
items.append((title, href))
|
||||
return items
|
||||
|
||||
def _parse_detail(self, vod_id, html):
|
||||
root = self.html(html)
|
||||
if root is None:
|
||||
return {
|
||||
"vod_id": vod_id,
|
||||
"vod_name": "",
|
||||
"vod_pic": "",
|
||||
"vod_content": "",
|
||||
"vod_play_from": "",
|
||||
"vod_play_url": "",
|
||||
}
|
||||
|
||||
vod_name = self._clean_text("".join(root.xpath("//*[contains(@class,'movie-des')]//h1[1]//text()")))
|
||||
vod_pic = self._build_url("".join(root.xpath("//*[contains(@class,'movie-img')]//img[1]/@src")).strip())
|
||||
vod_content = self._clean_text("".join(root.xpath("//*[contains(@class,'movie-txt')][1]//text()")))
|
||||
pan_links = self._extract_pan_links(html)
|
||||
play_url = "#".join([f"{title}${url}" for title, url in pan_links])
|
||||
return {
|
||||
"vod_id": vod_id,
|
||||
"vod_name": vod_name,
|
||||
"vod_pic": vod_pic,
|
||||
"vod_content": vod_content,
|
||||
"vod_play_from": "网盘" if play_url else "",
|
||||
"vod_play_url": play_url,
|
||||
}
|
||||
|
||||
def detailContent(self, ids):
|
||||
result = {"list": []}
|
||||
for raw_id in ids:
|
||||
vod_id = str(raw_id or "").strip()
|
||||
detail = self._parse_detail(vod_id, self._request_html(self._build_url(vod_id)))
|
||||
result["list"].append(detail)
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user