修复封面
This commit is contained in:
@@ -111,6 +111,35 @@ class TestZhiZhenSpider(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_cards_normalizes_nested_absolute_cover_url(self):
|
||||||
|
html = """
|
||||||
|
<div id="main">
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/index.php/vod/detail/id/123.html"></a>
|
||||||
|
<img
|
||||||
|
data-src="https://pic1.imgyzzy.com/https://img.ffzy888.com/https://img.ffzy888.com/upload/vod/2024-06-13/17182715511.jpg"
|
||||||
|
alt="嵌套封面"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">HD</div>
|
||||||
|
<div class="module-item-caption"><span>2024</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
self.spider._parse_cards(html),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"vod_id": "/index.php/vod/detail/id/123.html",
|
||||||
|
"vod_name": "嵌套封面",
|
||||||
|
"vod_pic": "https://img.ffzy888.com/upload/vod/2024-06-13/17182715511.jpg",
|
||||||
|
"vod_remarks": "HD",
|
||||||
|
"vod_year": "2024",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
@patch.object(Spider, "_request_html")
|
@patch.object(Spider, "_request_html")
|
||||||
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
||||||
mock_request_html.return_value = """
|
mock_request_html.return_value = """
|
||||||
|
|||||||
@@ -70,6 +70,15 @@ class Spider(BaseSpider):
|
|||||||
def _clean_text(self, text):
|
def _clean_text(self, text):
|
||||||
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
||||||
|
|
||||||
|
def _normalize_img_url(self, img_url):
|
||||||
|
raw = str(img_url or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return ""
|
||||||
|
matches = list(re.finditer(r"https?://", raw, re.I))
|
||||||
|
if len(matches) > 1:
|
||||||
|
raw = raw[matches[-1].start() :]
|
||||||
|
return raw
|
||||||
|
|
||||||
def _detect_pan_type(self, url):
|
def _detect_pan_type(self, url):
|
||||||
raw = str(url or "").strip()
|
raw = str(url or "").strip()
|
||||||
for pan_type, title, pattern in self.pan_patterns:
|
for pan_type, title, pattern in self.pan_patterns:
|
||||||
@@ -113,7 +122,7 @@ class Spider(BaseSpider):
|
|||||||
{
|
{
|
||||||
"vod_id": href,
|
"vod_id": href,
|
||||||
"vod_name": title,
|
"vod_name": title,
|
||||||
"vod_pic": self._build_url(pic),
|
"vod_pic": self._build_url(self._normalize_img_url(pic)),
|
||||||
"vod_remarks": remarks,
|
"vod_remarks": remarks,
|
||||||
"vod_year": year,
|
"vod_year": year,
|
||||||
}
|
}
|
||||||
@@ -153,7 +162,7 @@ class Spider(BaseSpider):
|
|||||||
{
|
{
|
||||||
"vod_id": href,
|
"vod_id": href,
|
||||||
"vod_name": title,
|
"vod_name": title,
|
||||||
"vod_pic": self._build_url(pic),
|
"vod_pic": self._build_url(self._normalize_img_url(pic)),
|
||||||
"vod_remarks": remarks,
|
"vod_remarks": remarks,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -177,12 +186,14 @@ class Spider(BaseSpider):
|
|||||||
"vod_id": vod_id,
|
"vod_id": vod_id,
|
||||||
"vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))),
|
"vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))),
|
||||||
"vod_pic": self._build_url(
|
"vod_pic": self._build_url(
|
||||||
"".join(
|
self._normalize_img_url(
|
||||||
root.xpath(
|
"".join(
|
||||||
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | "
|
root.xpath(
|
||||||
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src"
|
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | "
|
||||||
)
|
"//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src"
|
||||||
).strip()
|
)
|
||||||
|
).strip()
|
||||||
|
)
|
||||||
),
|
),
|
||||||
"vod_year": "",
|
"vod_year": "",
|
||||||
"vod_director": "",
|
"vod_director": "",
|
||||||
|
|||||||
Reference in New Issue
Block a user