fix: correct youknow category page paths
This commit is contained in:
@@ -58,8 +58,20 @@ class TestYouKnowSpider(unittest.TestCase):
|
|||||||
"""
|
"""
|
||||||
result = self.spider.categoryContent("movie", "2", False, {})
|
result = self.spider.categoryContent("movie", "2", False, {})
|
||||||
self.assertEqual(result["page"], 2)
|
self.assertEqual(result["page"], 2)
|
||||||
|
self.assertEqual(mock_request_html.call_args.args[0], "/show/2--------2---/")
|
||||||
self.assertEqual(result["list"][0]["vod_id"], "222")
|
self.assertEqual(result["list"][0]["vod_id"], "222")
|
||||||
|
|
||||||
|
@patch.object(Spider, "_request_html")
|
||||||
|
def test_category_content_uses_page1_path_without_page_number(self, mock_request_html):
|
||||||
|
mock_request_html.return_value = """
|
||||||
|
<a class="module-poster-item" href="/v/555.html" title="分类第一页" data-original="/page1.jpg">
|
||||||
|
<div class="module-item-note">HD</div>
|
||||||
|
</a>
|
||||||
|
"""
|
||||||
|
result = self.spider.categoryContent("movie", "1", False, {})
|
||||||
|
self.assertEqual(mock_request_html.call_args.args[0], "/show/2-----------/")
|
||||||
|
self.assertEqual(result["list"][0]["vod_id"], "555")
|
||||||
|
|
||||||
@patch.object(Spider, "_request_html")
|
@patch.object(Spider, "_request_html")
|
||||||
def test_search_content_reuses_card_parser(self, mock_request_html):
|
def test_search_content_reuses_card_parser(self, mock_request_html):
|
||||||
mock_request_html.return_value = """
|
mock_request_html.return_value = """
|
||||||
|
|||||||
+18
-1
@@ -144,6 +144,23 @@ class Spider(BaseSpider):
|
|||||||
"total": pagecount * max(len(items), 1),
|
"total": pagecount * max(len(items), 1),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _build_category_path(self, tid, pg):
|
||||||
|
page = int(pg)
|
||||||
|
if tid == "index":
|
||||||
|
return self.category_paths["index"]
|
||||||
|
type_map = {
|
||||||
|
"drama": "1",
|
||||||
|
"movie": "2",
|
||||||
|
"variety": "3",
|
||||||
|
"anime": "4",
|
||||||
|
"short": "55",
|
||||||
|
"doc": "5",
|
||||||
|
}
|
||||||
|
type_id = type_map.get(tid, "2")
|
||||||
|
if page <= 1:
|
||||||
|
return f"/show/{type_id}-----------/"
|
||||||
|
return f"/show/{type_id}--------{page}---/"
|
||||||
|
|
||||||
def _clean_text(self, text):
|
def _clean_text(self, text):
|
||||||
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
||||||
|
|
||||||
@@ -249,7 +266,7 @@ class Spider(BaseSpider):
|
|||||||
return {"list": self._parse_list_cards(html)}
|
return {"list": self._parse_list_cards(html)}
|
||||||
|
|
||||||
def categoryContent(self, tid, pg, filter, extend):
|
def categoryContent(self, tid, pg, filter, extend):
|
||||||
path = self.category_paths.get(tid, self.category_paths["movie"]).format(pg=pg)
|
path = self._build_category_path(tid, pg)
|
||||||
html = self._request_html(
|
html = self._request_html(
|
||||||
path,
|
path,
|
||||||
expect_xpath="//*[contains(@class,'module-poster-item') or contains(@class,'module-card-item-poster')]",
|
expect_xpath="//*[contains(@class,'module-poster-item') or contains(@class,'module-card-item-poster')]",
|
||||||
|
|||||||
Reference in New Issue
Block a user