feat: add rrdy detail pan parsing
This commit is contained in:
@@ -124,3 +124,45 @@ class TestRenRenDianYingSpider(unittest.TestCase):
|
|||||||
|
|
||||||
def test_search_content_returns_empty_list_for_blank_keyword(self):
|
def test_search_content_returns_empty_list_for_blank_keyword(self):
|
||||||
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []})
|
||||||
|
|
||||||
|
def test_extract_pan_links_filters_xunlei_and_aliyun(self):
|
||||||
|
html = """
|
||||||
|
<div class="movie-txt">
|
||||||
|
<a href="https://pan.baidu.com/s/b1">百度网盘</a>
|
||||||
|
<a href="https://pan.quark.cn/s/q1">夸克资源</a>
|
||||||
|
<a href="https://pan.xunlei.com/s/x1">迅雷资源</a>
|
||||||
|
<a href="https://www.alipan.com/s/a1">阿里资源</a>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
self.spider._extract_pan_links(html),
|
||||||
|
[
|
||||||
|
("百度网盘", "https://pan.baidu.com/s/b1"),
|
||||||
|
("夸克资源", "https://pan.quark.cn/s/q1"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_detail_content_extracts_meta_and_builds_single_pan_line(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = """
|
||||||
|
<div class="movie-des"><h1>人人示例片</h1></div>
|
||||||
|
<div class="movie-img"><img src="/poster-detail.jpg" /></div>
|
||||||
|
<div class="movie-txt">
|
||||||
|
<p>一段简介</p>
|
||||||
|
<a href="https://pan.baidu.com/s/b1">百度网盘</a>
|
||||||
|
<a href="https://pan.quark.cn/s/q1">夸克资源</a>
|
||||||
|
<a href="https://pan.xunlei.com/s/x1">迅雷资源</a>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
result = self.spider.detailContent(["/movie/123.html"])
|
||||||
|
self.assertEqual(mock_request_html.call_args.args[0], "https://www.rrdynb.com/movie/123.html")
|
||||||
|
vod = result["list"][0]
|
||||||
|
self.assertEqual(vod["vod_id"], "/movie/123.html")
|
||||||
|
self.assertEqual(vod["vod_name"], "人人示例片")
|
||||||
|
self.assertEqual(vod["vod_pic"], "https://www.rrdynb.com/poster-detail.jpg")
|
||||||
|
self.assertIn("一段简介", vod["vod_content"])
|
||||||
|
self.assertEqual(vod["vod_play_from"], "网盘")
|
||||||
|
self.assertEqual(
|
||||||
|
vod["vod_play_url"],
|
||||||
|
"百度网盘$https://pan.baidu.com/s/b1#夸克资源$https://pan.quark.cn/s/q1",
|
||||||
|
)
|
||||||
|
|||||||
+50
@@ -143,3 +143,53 @@ class Spider(BaseSpider):
|
|||||||
|
|
||||||
items = self._parse_cards(self._request_html(url))
|
items = self._parse_cards(self._request_html(url))
|
||||||
return {"page": page, "total": len(items), "list": items}
|
return {"page": page, "total": len(items), "list": items}
|
||||||
|
|
||||||
|
def _extract_pan_links(self, html):
|
||||||
|
root = self.html(html)
|
||||||
|
if root is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
items = []
|
||||||
|
seen = set()
|
||||||
|
for node in root.xpath("//*[contains(@class,'movie-txt')]//a[@href]"):
|
||||||
|
href = "".join(node.xpath("./@href")).strip()
|
||||||
|
title = self._clean_text("".join(node.xpath(".//text()"))) or "网盘资源"
|
||||||
|
if not self._is_supported_pan_url(href) or href in seen:
|
||||||
|
continue
|
||||||
|
seen.add(href)
|
||||||
|
items.append((title, href))
|
||||||
|
return items
|
||||||
|
|
||||||
|
def _parse_detail(self, vod_id, html):
|
||||||
|
root = self.html(html)
|
||||||
|
if root is None:
|
||||||
|
return {
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": "",
|
||||||
|
"vod_pic": "",
|
||||||
|
"vod_content": "",
|
||||||
|
"vod_play_from": "",
|
||||||
|
"vod_play_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
vod_name = self._clean_text("".join(root.xpath("//*[contains(@class,'movie-des')]//h1[1]//text()")))
|
||||||
|
vod_pic = self._build_url("".join(root.xpath("//*[contains(@class,'movie-img')]//img[1]/@src")).strip())
|
||||||
|
vod_content = self._clean_text("".join(root.xpath("//*[contains(@class,'movie-txt')][1]//text()")))
|
||||||
|
pan_links = self._extract_pan_links(html)
|
||||||
|
play_url = "#".join([f"{title}${url}" for title, url in pan_links])
|
||||||
|
return {
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": vod_name,
|
||||||
|
"vod_pic": vod_pic,
|
||||||
|
"vod_content": vod_content,
|
||||||
|
"vod_play_from": "网盘" if play_url else "",
|
||||||
|
"vod_play_url": play_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
def detailContent(self, ids):
|
||||||
|
result = {"list": []}
|
||||||
|
for raw_id in ids:
|
||||||
|
vod_id = str(raw_id or "").strip()
|
||||||
|
detail = self._parse_detail(vod_id, self._request_html(self._build_url(vod_id)))
|
||||||
|
result["list"].append(detail)
|
||||||
|
return result
|
||||||
|
|||||||
Reference in New Issue
Block a user