feat: add 两个BT detail parsing
This commit is contained in:
Harold
@@ -134,6 +134,41 @@ class TestLiangGeBTSpider(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_encode_and_decode_play_id_round_trip(self):
|
||||||
|
payload = self.spider._decode_play_id(self.spider._encode_play_id("play-1", "900", "第1集"))
|
||||||
|
self.assertEqual(payload, {"pid": "play-1", "sid": "900", "name": "第1集"})
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_detail_content_extracts_meta_and_playlist(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = """
|
||||||
|
<html>
|
||||||
|
<head><title>两个BT详情页</title></head>
|
||||||
|
<body>
|
||||||
|
<h1>示例详情</h1>
|
||||||
|
<div class="poster"><img src="/detail.jpg" /></div>
|
||||||
|
<div class="description">这里是剧情简介</div>
|
||||||
|
<li>主演：演员甲 / 演员乙</li>
|
||||||
|
<li>导演：导演甲</li>
|
||||||
|
<a href="/v_play/play-1.html">第1集</a>
|
||||||
|
<a href="/v_play/play-2.html">第2集</a>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
result = self.spider.detailContent(["900"])
|
||||||
|
vod = result["list"][0]
|
||||||
|
first_name, first_id = vod["vod_play_url"].split("#")[0].split("$", 1)
|
||||||
|
self.assertEqual(mock_request_html.call_args.args[0], "https://www.bttwoo.com/movie/900.html")
|
||||||
|
self.assertEqual(vod["vod_id"], "900")
|
||||||
|
self.assertEqual(vod["vod_name"], "示例详情")
|
||||||
|
self.assertEqual(vod["vod_pic"], "https://www.bttwoo.com/detail.jpg")
|
||||||
|
self.assertEqual(vod["vod_content"], "这里是剧情简介")
|
||||||
|
self.assertEqual(vod["vod_actor"], "演员甲 / 演员乙")
|
||||||
|
self.assertEqual(vod["vod_director"], "导演甲")
|
||||||
|
self.assertEqual(vod["vod_play_from"], "两个BT")
|
||||||
|
self.assertEqual(first_name, "第1集")
|
||||||
|
self.assertEqual(self.spider._decode_play_id(first_id)["pid"], "play-1")
|
||||||
|
self.assertEqual(self.spider._decode_play_id(first_id)["sid"], "900")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+88
-1
@@ -1,4 +1,6 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from urllib.parse import quote, urljoin
|
from urllib.parse import quote, urljoin
|
||||||
@@ -69,7 +71,12 @@ class Spider(BaseSpider):
|
|||||||
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
||||||
|
|
||||||
def detailContent(self, ids):
|
def detailContent(self, ids):
|
||||||
return {"list": []}
|
vod_id = str(ids[0] if isinstance(ids, list) and ids else ids or "").strip()
|
||||||
|
if not vod_id:
|
||||||
|
return {"list": []}
|
||||||
|
html = self._request_html(self.host + f"/movie/{vod_id}.html")
|
||||||
|
detail = self._parse_detail(html, vod_id)
|
||||||
|
return {"list": [detail]} if detail else {"list": []}
|
||||||
|
|
||||||
def playerContent(self, flag, id, vipFlags):
|
def playerContent(self, flag, id, vipFlags):
|
||||||
return {"parse": 1, "jx": 1, "playUrl": "", "url": "", "header": {}}
|
return {"parse": 1, "jx": 1, "playUrl": "", "url": "", "header": {}}
|
||||||
@@ -151,6 +158,86 @@ class Spider(BaseSpider):
|
|||||||
matched = re.search(r"/movie/(\d+)\.html", str(href or "").strip())
|
matched = re.search(r"/movie/(\d+)\.html", str(href or "").strip())
|
||||||
return matched.group(1) if matched else ""
|
return matched.group(1) if matched else ""
|
||||||
|
|
||||||
|
def _extract_play_pid(self, href):
|
||||||
|
matched = re.search(r"/v_play/([^.]+)\.html", str(href or "").strip())
|
||||||
|
return matched.group(1) if matched else ""
|
||||||
|
|
||||||
|
def _parse_detail(self, html, vod_id):
|
||||||
|
root = self._parse_html(html)
|
||||||
|
if root is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
vod_name = (
|
||||||
|
self._first_text(root, "//h1[1]")
|
||||||
|
or self._first_text(root, "//h2[1]")
|
||||||
|
or self._extract_title_text(html)
|
||||||
|
)
|
||||||
|
vod_pic = (
|
||||||
|
self._first_attr(root, "//img[contains(@class,'poster')][1]", "src")
|
||||||
|
or self._first_attr(root, "//*[contains(@class,'poster')]//img[1]", "src")
|
||||||
|
or self._first_attr(root, "//img[1]", "src")
|
||||||
|
)
|
||||||
|
vod_content = (
|
||||||
|
self._first_text(root, "//*[contains(@class,'intro')][1]")
|
||||||
|
or self._first_text(root, "//*[contains(@class,'description')][1]")
|
||||||
|
or self._first_text(root, "//*[contains(@class,'desc')][1]")
|
||||||
|
)
|
||||||
|
vod_actor = self._extract_meta_text(root, "主演")
|
||||||
|
vod_director = self._extract_meta_text(root, "导演")
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
seen = set()
|
||||||
|
for index, node in enumerate(root.xpath("//a[contains(@href,'/v_play/')]")):
|
||||||
|
href = str(node.get("href") or "").strip()
|
||||||
|
pid = self._extract_play_pid(href)
|
||||||
|
name = self._clean_text(node.text_content()) or f"第{index + 1}集"
|
||||||
|
if not pid or pid in seen:
|
||||||
|
continue
|
||||||
|
seen.add(pid)
|
||||||
|
episodes.append(f"{name}${self._encode_play_id(pid, vod_id, name)}")
|
||||||
|
|
||||||
|
if not episodes:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": vod_name or "未知标题",
|
||||||
|
"vod_pic": self._abs_url(vod_pic),
|
||||||
|
"vod_content": vod_content,
|
||||||
|
"vod_actor": vod_actor,
|
||||||
|
"vod_director": vod_director,
|
||||||
|
"vod_play_from": "两个BT",
|
||||||
|
"vod_play_url": "#".join(episodes),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _encode_play_id(self, pid, sid, name):
|
||||||
|
raw = json.dumps(
|
||||||
|
{"pid": str(pid or ""), "sid": str(sid or ""), "name": str(name or "")},
|
||||||
|
ensure_ascii=False,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
return base64.b64encode(raw.encode("utf-8")).decode("utf-8")
|
||||||
|
|
||||||
|
def _decode_play_id(self, value):
|
||||||
|
try:
|
||||||
|
raw = base64.b64decode(str(value or "").encode("utf-8")).decode("utf-8")
|
||||||
|
data = json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return {"pid": "", "sid": "", "name": ""}
|
||||||
|
return {
|
||||||
|
"pid": str(data.get("pid") or ""),
|
||||||
|
"sid": str(data.get("sid") or ""),
|
||||||
|
"name": str(data.get("name") or ""),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _extract_title_text(self, html):
|
||||||
|
matched = re.search(r"<title>(.*?)</title>", str(html or ""), re.I | re.S)
|
||||||
|
return self._clean_text(matched.group(1)) if matched else ""
|
||||||
|
|
||||||
|
def _extract_meta_text(self, root, label):
|
||||||
|
text = self._first_text(root, f"//*[contains(text(),'{label}')][1]")
|
||||||
|
return re.sub(rf"^{label}[:：]?", "", text).strip()
|
||||||
|
|
||||||
def _abs_url(self, value):
|
def _abs_url(self, value):
|
||||||
raw = str(value or "").strip()
|
raw = str(value or "").strip()
|
||||||
if not raw:
|
if not raw:
|
||||||
|
|||||||
Reference in New Issue
Block a user