fix: compress dida detail ids
This commit is contained in:
@@ -54,7 +54,7 @@ class TestDidaSpider(unittest.TestCase):
|
|||||||
cards,
|
cards,
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"vod_id": "https://www.didahd.pro/detail/888.html",
|
"vod_id": "detail/888",
|
||||||
"vod_name": "示例影片",
|
"vod_name": "示例影片",
|
||||||
"vod_pic": "https://www.didahd.pro/cover.jpg",
|
"vod_pic": "https://www.didahd.pro/cover.jpg",
|
||||||
"vod_remarks": "HD",
|
"vod_remarks": "HD",
|
||||||
@@ -92,7 +92,7 @@ class TestDidaSpider(unittest.TestCase):
|
|||||||
mock_request_html.call_args.args[0],
|
mock_request_html.call_args.args[0],
|
||||||
"https://www.didahd.pro/search/-------------.html?wd=%E7%B9%81%E8%8A%B1",
|
"https://www.didahd.pro/search/-------------.html?wd=%E7%B9%81%E8%8A%B1",
|
||||||
)
|
)
|
||||||
self.assertEqual(result["list"][0]["vod_id"], "https://www.didahd.pro/detail/321.html")
|
self.assertEqual(result["list"][0]["vod_id"], "detail/321")
|
||||||
self.assertEqual(result["pagecount"], 2)
|
self.assertEqual(result["pagecount"], 2)
|
||||||
|
|
||||||
def test_extract_netdisk_groups_deduplicates_links_and_sorts_by_priority(self):
|
def test_extract_netdisk_groups_deduplicates_links_and_sorts_by_priority(self):
|
||||||
@@ -138,8 +138,9 @@ class TestDidaSpider(unittest.TestCase):
|
|||||||
</div>
|
</div>
|
||||||
<ul class="myui-content__list"><li><a href="/play/111-1-1.html">正片</a></li></ul>
|
<ul class="myui-content__list"><li><a href="/play/111-1-1.html">正片</a></li></ul>
|
||||||
"""
|
"""
|
||||||
result = self.spider.detailContent(["https://www.didahd.pro/detail/111.html"])
|
result = self.spider.detailContent(["detail/111"])
|
||||||
self.assertEqual(result["list"][0]["vod_id"], "https://www.didahd.pro/detail/111.html")
|
self.assertEqual(mock_request_html.call_args.args[0], "https://www.didahd.pro/detail/111.html")
|
||||||
|
self.assertEqual(result["list"][0]["vod_id"], "detail/111")
|
||||||
self.assertEqual(result["list"][0]["vod_play_from"], "uc")
|
self.assertEqual(result["list"][0]["vod_play_from"], "uc")
|
||||||
self.assertEqual(result["list"][0]["vod_play_url"], "合集$https://drive.uc.cn/s/u1")
|
self.assertEqual(result["list"][0]["vod_play_url"], "合集$https://drive.uc.cn/s/u1")
|
||||||
|
|
||||||
|
|||||||
+12
-2
@@ -109,6 +109,14 @@ class Spider(BaseSpider):
|
|||||||
return self.host + raw
|
return self.host + raw
|
||||||
return self.host + "/" + raw
|
return self.host + "/" + raw
|
||||||
|
|
||||||
|
def _extract_site_path_id(self, href):
|
||||||
|
matched = re.search(r"/(detail/[^/?#]+)\.html", self._build_url(href))
|
||||||
|
return matched.group(1) if matched else ""
|
||||||
|
|
||||||
|
def _build_detail_request_url(self, vod_id):
|
||||||
|
value = self._stringify(vod_id).strip().strip("/")
|
||||||
|
return self._build_url(f"/{value}.html") if value else ""
|
||||||
|
|
||||||
def _build_category_url(self, tid, pg, extend):
|
def _build_category_url(self, tid, pg, extend):
|
||||||
values = dict(self.filter_def.get(str(tid), {"cateId": str(tid), "sort": "time"}))
|
values = dict(self.filter_def.get(str(tid), {"cateId": str(tid), "sort": "time"}))
|
||||||
values.update(self._normalize_ext(extend))
|
values.update(self._normalize_ext(extend))
|
||||||
@@ -150,7 +158,7 @@ class Spider(BaseSpider):
|
|||||||
or ((card.xpath(".//img[1]/@src") or [""])[0]).strip()
|
or ((card.xpath(".//img[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text("".join(card.xpath(".//*[contains(@class,'pic-text')][1]//text()")))
|
remarks = self._clean_text("".join(card.xpath(".//*[contains(@class,'pic-text')][1]//text()")))
|
||||||
vod_id = self._build_url(href)
|
vod_id = self._extract_site_path_id(href)
|
||||||
if not vod_id or not title or vod_id in seen:
|
if not vod_id or not title or vod_id in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(vod_id)
|
seen.add(vod_id)
|
||||||
@@ -287,7 +295,9 @@ class Spider(BaseSpider):
|
|||||||
vod_id = self._stringify(raw).strip()
|
vod_id = self._stringify(raw).strip()
|
||||||
if not vod_id:
|
if not vod_id:
|
||||||
continue
|
continue
|
||||||
result["list"].append(self._parse_detail_page(self._request_html(vod_id), vod_id))
|
result["list"].append(
|
||||||
|
self._parse_detail_page(self._request_html(self._build_detail_request_url(vod_id)), vod_id)
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def _is_netdisk_url(self, value):
|
def _is_netdisk_url(self, value):
|
||||||
|
|||||||
Reference in New Issue
Block a user