feat: add dbku detail parsing
This commit is contained in:
@@ -95,6 +95,39 @@ class TestDBKUSpider(unittest.TestCase):
|
|||||||
self.assertEqual(result["list"][0]["vod_id"], "https://www.dbku.tv/voddetail/321.html")
|
self.assertEqual(result["list"][0]["vod_id"], "https://www.dbku.tv/voddetail/321.html")
|
||||||
self.assertEqual(result["list"][0]["vod_name"], "搜索影片")
|
self.assertEqual(result["list"][0]["vod_name"], "搜索影片")
|
||||||
|
|
||||||
|
def test_parse_detail_page_extracts_meta_and_episodes(self):
|
||||||
|
html = """
|
||||||
|
<div class="myui-content__thumb">
|
||||||
|
<img data-original="/poster.jpg" />
|
||||||
|
</div>
|
||||||
|
<div class="myui-content__detail">
|
||||||
|
<h1 class="title">独播剧</h1>
|
||||||
|
<p>年份:2025</p>
|
||||||
|
<p>地区:大陆</p>
|
||||||
|
<p>导演:张三</p>
|
||||||
|
<p>主演:李四</p>
|
||||||
|
</div>
|
||||||
|
<span class="data">一段剧情简介</span>
|
||||||
|
<a href="/vodplay/100-1-1.html">第1集</a>
|
||||||
|
<a href="/vodplay/100-1-2.html">第2集</a>
|
||||||
|
"""
|
||||||
|
result = self.spider._parse_detail_page(html, "https://www.dbku.tv/voddetail/100.html")
|
||||||
|
vod = result["list"][0]
|
||||||
|
self.assertEqual(vod["vod_name"], "独播剧")
|
||||||
|
self.assertEqual(vod["vod_year"], "2025")
|
||||||
|
self.assertEqual(vod["vod_play_from"], "独播库")
|
||||||
|
self.assertIn("第1集$https://www.dbku.tv/vodplay/100-1-1.html", vod["vod_play_url"])
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_detail_content_reads_from_vod_id_url(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = """
|
||||||
|
<h1 class="title">详情影片</h1>
|
||||||
|
<a href="/vodplay/200-1-1.html">第1集</a>
|
||||||
|
"""
|
||||||
|
result = self.spider.detailContent(["https://www.dbku.tv/voddetail/200.html"])
|
||||||
|
self.assertEqual(result["list"][0]["vod_id"], "https://www.dbku.tv/voddetail/200.html")
|
||||||
|
self.assertEqual(result["list"][0]["vod_name"], "详情影片")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
@@ -160,3 +161,54 @@ class Spider(BaseSpider):
|
|||||||
path = "/vodsearch/-------------.html?wd={0}&submit=".format(quote(key))
|
path = "/vodsearch/-------------.html?wd={0}&submit=".format(quote(key))
|
||||||
html = self._request_html(path, expect_xpath="//*[@id='searchList']|//*[contains(@class,'myui-vodlist__box')]")
|
html = self._request_html(path, expect_xpath="//*[@id='searchList']|//*[contains(@class,'myui-vodlist__box')]")
|
||||||
return self._page_result(self._parse_search_cards(html), pg)
|
return self._page_result(self._parse_search_cards(html), pg)
|
||||||
|
|
||||||
|
def _extract_text_by_prefix(self, html, prefixes):
|
||||||
|
texts = re.findall(r">([^<>]+)<", html)
|
||||||
|
for text in texts:
|
||||||
|
clean = text.strip()
|
||||||
|
for prefix in prefixes:
|
||||||
|
if clean.startswith(prefix):
|
||||||
|
return clean.split(":", 1)[-1].strip()
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def _parse_detail_page(self, html, vod_id):
|
||||||
|
root = self.html(html)
|
||||||
|
title = ((root.xpath("//*[contains(@class,'title')][1]//text()") or [""])[0]).strip()
|
||||||
|
pic = (
|
||||||
|
(root.xpath("//*[contains(@class,'myui-content__thumb')]//img/@data-original") or [""])[0].strip()
|
||||||
|
or (root.xpath("//*[contains(@class,'myui-content__thumb')]//img/@src") or [""])[0].strip()
|
||||||
|
)
|
||||||
|
content = "".join(root.xpath("//*[contains(@class,'data')][1]//text()")).strip()
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
seen = set()
|
||||||
|
for href, label in re.findall(
|
||||||
|
r'<a[^>]+href=["\']([^"\']*/vodplay/\d+-\d+-\d+\.html[^"\']*)["\'][^>]*>([\s\S]*?)</a>',
|
||||||
|
html,
|
||||||
|
re.I,
|
||||||
|
):
|
||||||
|
url = self._build_url(href)
|
||||||
|
name = re.sub(r"<[^>]*>", "", label).strip()
|
||||||
|
if not url or not name or "立即播放" in name or url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
episodes.append(f"{name}${url}")
|
||||||
|
|
||||||
|
vod = {
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": title,
|
||||||
|
"vod_pic": self._build_url(pic),
|
||||||
|
"vod_year": self._extract_text_by_prefix(html, ["年份:"]),
|
||||||
|
"vod_area": self._extract_text_by_prefix(html, ["地区:"]),
|
||||||
|
"vod_actor": self._extract_text_by_prefix(html, ["主演:"]),
|
||||||
|
"vod_director": self._extract_text_by_prefix(html, ["导演:"]),
|
||||||
|
"vod_content": content,
|
||||||
|
"vod_play_from": "独播库",
|
||||||
|
"vod_play_url": "#".join(episodes),
|
||||||
|
}
|
||||||
|
return {"list": [vod]}
|
||||||
|
|
||||||
|
def detailContent(self, ids):
|
||||||
|
vod_id = ids[0]
|
||||||
|
html = self._request_html(vod_id, expect_xpath="//*[contains(@class,'title')]|//a[contains(@href,'/vodplay/')]")
|
||||||
|
return self._parse_detail_page(html, vod_id)
|
||||||
|
|||||||
Reference in New Issue
Block a user