From 1d4fbff724bfade0c7c60096f0a23daf4af13b9c Mon Sep 17 00:00:00 2001 From: Harold <8866033@gmail.com> Date: Mon, 20 Apr 2026 16:24:07 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=81=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- py/tests/test_玩偶哥哥.py | 39 +++++++++++++++++++++++++++++++++++ py/tests/test_至臻.py | 43 +++++++++++++++++++++++++++++++++++++++ py/tests/test_闪电.py | 43 +++++++++++++++++++++++++++++++++++++++ py/玩偶哥哥.py | 18 ++++++++-------- py/至臻.py | 20 ++++++++++-------- py/闪电.py | 20 ++++++++++-------- 6 files changed, 157 insertions(+), 26 deletions(-) diff --git a/py/tests/test_玩偶哥哥.py b/py/tests/test_玩偶哥哥.py index 2c9cca7..5347775 100644 --- a/py/tests/test_玩偶哥哥.py +++ b/py/tests/test_玩偶哥哥.py @@ -74,6 +74,45 @@ class TestWanOuGeGeSpider(unittest.TestCase): ], ) + def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self): + html = """ +
+
+
+
+ + 示例影片A +
+
HD
+
+
+
+ + 示例影片B +
+
更新至10集
+
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/voddetail/123.html", + "vod_name": "示例影片A", + "vod_pic": "http://wogg.xxooo.cf/poster-a.jpg", + "vod_remarks": "HD", + }, + { + "vod_id": "/voddetail/456.html", + "vod_name": "示例影片B", + "vod_pic": "http://wogg.xxooo.cf/poster-b.jpg", + "vod_remarks": "更新至10集", + }, + ], + ) + @patch.object(Spider, "_request_html") def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): mock_request_html.return_value = """ diff --git a/py/tests/test_至臻.py b/py/tests/test_至臻.py index 09628d2..94cf604 100644 --- a/py/tests/test_至臻.py +++ b/py/tests/test_至臻.py @@ -68,6 +68,49 @@ class TestZhiZhenSpider(unittest.TestCase): ], ) + def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self): + html = """ +
+
+
+
+ + 示例影片A +
+
HD
+
2025
+
+
+
+ + 示例影片B +
+
更新至10集
+
2024
+
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/index.php/vod/detail/id/123.html", + "vod_name": "示例影片A", + "vod_pic": "http://www.miqk.cc/poster-a.jpg", + "vod_remarks": "HD", + "vod_year": "2025", + }, + { + "vod_id": "/index.php/vod/detail/id/456.html", + "vod_name": "示例影片B", + "vod_pic": "http://www.miqk.cc/poster-b.jpg", + "vod_remarks": "更新至10集", + "vod_year": "2024", + }, + ], + ) + @patch.object(Spider, "_request_html") def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): mock_request_html.return_value = """ diff --git a/py/tests/test_闪电.py b/py/tests/test_闪电.py index df74fcd..5cc2b79 100644 --- a/py/tests/test_闪电.py +++ b/py/tests/test_闪电.py @@ -66,6 +66,49 @@ class TestShanDianSpider(unittest.TestCase): ], ) + def test_parse_cards_ignores_module_items_container_and_uses_first_card_cover(self): + html = """ +
+
+
+
+ + 示例影片A +
+
HD
+
2025
+
+
+
+ + 示例影片B +
+
更新至10集
+
2024
+
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/index.php/vod/detail/id/123.html", + "vod_name": "示例影片A", + "vod_pic": "https://sd.sduc.site/poster-a.jpg", + "vod_remarks": "HD", + "vod_year": "2025", + }, + { + "vod_id": "/index.php/vod/detail/id/456.html", + "vod_name": "示例影片B", + "vod_pic": "https://sd.sduc.site/poster-b.jpg", + "vod_remarks": "更新至10集", + "vod_year": "2024", + }, + ], + ) + @patch.object(Spider, "_request_html") def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): mock_request_html.return_value = """ diff --git a/py/玩偶哥哥.py b/py/玩偶哥哥.py index cf00770..fa5f941 100644 --- a/py/玩偶哥哥.py +++ b/py/玩偶哥哥.py @@ -106,14 +106,16 @@ class Spider(BaseSpider): items = [] seen = set() - for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): - href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() - title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() + for node in root.xpath( + "//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]" + ): + href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip() + title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) - remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) + remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()"))) if not href or not title or href in seen: continue seen.add(href) @@ -147,8 +149,8 @@ class Spider(BaseSpider): href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) remarks = self._clean_text( "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) diff --git a/py/至臻.py b/py/至臻.py index 0d33569..b0156d1 100644 --- a/py/至臻.py +++ b/py/至臻.py @@ -95,15 +95,17 @@ class Spider(BaseSpider): items = [] seen = set() - for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): - href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() - title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() + for node in root.xpath( + "//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]" + ): + href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip() + title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) - remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) - year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()"))) + remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()"))) + year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()"))) if not href or not title or href in seen: continue seen.add(href) @@ -138,8 +140,8 @@ class Spider(BaseSpider): href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) remarks = self._clean_text( "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) diff --git a/py/闪电.py b/py/闪电.py index e60d9ed..880f608 100644 --- a/py/闪电.py +++ b/py/闪电.py @@ -93,15 +93,17 @@ class Spider(BaseSpider): items = [] seen = set() - for node in root.xpath("//*[@id='main']//*[contains(@class,'module-item')]"): - href = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//a[1]/@href")).strip() - title = "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@alt")).strip() + for node in root.xpath( + "//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]" + ): + href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip() + title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) - remarks = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()"))) - year = self._clean_text("".join(node.xpath(".//*[contains(@class,'module-item-caption')][1]//span[1]//text()"))) + remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()"))) + year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()"))) if not href or not title or href in seen: continue seen.add(href) @@ -136,8 +138,8 @@ class Spider(BaseSpider): href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() pic = ( - "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@data-src")).strip() - or "".join(node.xpath(".//*[contains(@class,'module-item-pic')]//img[1]/@src")).strip() + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() ) remarks = self._clean_text( "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()"))