修复封面
This commit is contained in:
@@ -74,6 +74,45 @@ class TestWanOuGeGeSpider(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self):
|
||||||
|
html = """
|
||||||
|
<div id="main">
|
||||||
|
<div class="module-items">
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/voddetail/123.html"></a>
|
||||||
|
<img data-src="/poster-a.jpg" alt="示例影片A" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">HD</div>
|
||||||
|
</div>
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/voddetail/456.html"></a>
|
||||||
|
<img data-src="/poster-b.jpg" alt="示例影片B" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">更新至10集</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
self.spider._parse_cards(html),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"vod_id": "/voddetail/123.html",
|
||||||
|
"vod_name": "示例影片A",
|
||||||
|
"vod_pic": "http://wogg.xxooo.cf/poster-a.jpg",
|
||||||
|
"vod_remarks": "HD",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"vod_id": "/voddetail/456.html",
|
||||||
|
"vod_name": "示例影片B",
|
||||||
|
"vod_pic": "http://wogg.xxooo.cf/poster-b.jpg",
|
||||||
|
"vod_remarks": "更新至10集",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
@patch.object(Spider, "_request_html")
|
@patch.object(Spider, "_request_html")
|
||||||
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
||||||
mock_request_html.return_value = """
|
mock_request_html.return_value = """
|
||||||
|
|||||||
@@ -68,6 +68,49 @@ class TestZhiZhenSpider(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self):
|
||||||
|
html = """
|
||||||
|
<div id="main">
|
||||||
|
<div class="module-items">
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/index.php/vod/detail/id/123.html"></a>
|
||||||
|
<img data-src="/poster-a.jpg" alt="示例影片A" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">HD</div>
|
||||||
|
<div class="module-item-caption"><span>2025</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/index.php/vod/detail/id/456.html"></a>
|
||||||
|
<img data-src="/poster-b.jpg" alt="示例影片B" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">更新至10集</div>
|
||||||
|
<div class="module-item-caption"><span>2024</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
self.spider._parse_cards(html),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"vod_id": "/index.php/vod/detail/id/123.html",
|
||||||
|
"vod_name": "示例影片A",
|
||||||
|
"vod_pic": "http://www.miqk.cc/poster-a.jpg",
|
||||||
|
"vod_remarks": "HD",
|
||||||
|
"vod_year": "2025",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"vod_id": "/index.php/vod/detail/id/456.html",
|
||||||
|
"vod_name": "示例影片B",
|
||||||
|
"vod_pic": "http://www.miqk.cc/poster-b.jpg",
|
||||||
|
"vod_remarks": "更新至10集",
|
||||||
|
"vod_year": "2024",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
@patch.object(Spider, "_request_html")
|
@patch.object(Spider, "_request_html")
|
||||||
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
||||||
mock_request_html.return_value = """
|
mock_request_html.return_value = """
|
||||||
|
|||||||
@@ -66,6 +66,49 @@ class TestShanDianSpider(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self):
|
||||||
|
html = """
|
||||||
|
<div id="main">
|
||||||
|
<div class="module-items">
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/index.php/vod/detail/id/123.html"></a>
|
||||||
|
<img data-src="/poster-a.jpg" alt="示例影片A" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">HD</div>
|
||||||
|
<div class="module-item-caption"><span>2025</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="module-item">
|
||||||
|
<div class="module-item-pic">
|
||||||
|
<a href="/index.php/vod/detail/id/456.html"></a>
|
||||||
|
<img data-src="/poster-b.jpg" alt="示例影片B" />
|
||||||
|
</div>
|
||||||
|
<div class="module-item-text">更新至10集</div>
|
||||||
|
<div class="module-item-caption"><span>2024</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
self.spider._parse_cards(html),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"vod_id": "/index.php/vod/detail/id/123.html",
|
||||||
|
"vod_name": "示例影片A",
|
||||||
|
"vod_pic": "https://sd.sduc.site/poster-a.jpg",
|
||||||
|
"vod_remarks": "HD",
|
||||||
|
"vod_year": "2025",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"vod_id": "/index.php/vod/detail/id/456.html",
|
||||||
|
"vod_name": "示例影片B",
|
||||||
|
"vod_pic": "https://sd.sduc.site/poster-b.jpg",
|
||||||
|
"vod_remarks": "更新至10集",
|
||||||
|
"vod_year": "2024",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
@patch.object(Spider, "_request_html")
|
@patch.object(Spider, "_request_html")
|
||||||
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html):
|
||||||
mock_request_html.return_value = """
|
mock_request_html.return_value = """
|
||||||
|
|||||||
+10
-8
@@ -106,14 +106,16 @@ class Spider(BaseSpider):
|
|||||||
|
|
||||||
items = []
|
items = []
|
||||||
seen = set()
|
seen = set()
|
||||||
for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"):
|
for node in root.xpath(
|
||||||
href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip()
|
"//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]"
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip()
|
):
|
||||||
|
href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip()
|
||||||
|
title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")))
|
remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()")))
|
||||||
if not href or not title or href in seen:
|
if not href or not title or href in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(href)
|
seen.add(href)
|
||||||
@@ -147,8 +149,8 @@ class Spider(BaseSpider):
|
|||||||
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text(
|
remarks = self._clean_text(
|
||||||
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
||||||
|
|||||||
@@ -95,15 +95,17 @@ class Spider(BaseSpider):
|
|||||||
|
|
||||||
items = []
|
items = []
|
||||||
seen = set()
|
seen = set()
|
||||||
for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"):
|
for node in root.xpath(
|
||||||
href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip()
|
"//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]"
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip()
|
):
|
||||||
|
href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip()
|
||||||
|
title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")))
|
remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()")))
|
||||||
year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()")))
|
year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()")))
|
||||||
if not href or not title or href in seen:
|
if not href or not title or href in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(href)
|
seen.add(href)
|
||||||
@@ -138,8 +140,8 @@ class Spider(BaseSpider):
|
|||||||
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text(
|
remarks = self._clean_text(
|
||||||
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
||||||
|
|||||||
@@ -93,15 +93,17 @@ class Spider(BaseSpider):
|
|||||||
|
|
||||||
items = []
|
items = []
|
||||||
seen = set()
|
seen = set()
|
||||||
for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"):
|
for node in root.xpath(
|
||||||
href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip()
|
"//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]"
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip()
|
):
|
||||||
|
href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip()
|
||||||
|
title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")))
|
remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()")))
|
||||||
year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()")))
|
year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()")))
|
||||||
if not href or not title or href in seen:
|
if not href or not title or href in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(href)
|
seen.add(href)
|
||||||
@@ -136,8 +138,8 @@ class Spider(BaseSpider):
|
|||||||
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip()
|
||||||
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip()
|
||||||
pic = (
|
pic = (
|
||||||
"".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip()
|
((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip()
|
||||||
or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip()
|
or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip()
|
||||||
)
|
)
|
||||||
remarks = self._clean_text(
|
remarks = self._clean_text(
|
||||||
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
"".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))
|
||||||
|
|||||||
Reference in New Issue
Block a user