feat: add sjmusic category and search parsing
This commit is contained in:
Harold
@@ -30,6 +30,50 @@ HOME_HTML = """
|
||||
</html>
|
||||
"""
|
||||
|
||||
PLAYLIST_HTML = """
|
||||
<html><body>
|
||||
<ul class="video_list">
|
||||
<li>
|
||||
<div class="name"><a href="/playlist/gedan001.html">华语经典</a></div>
|
||||
<div class="pic"><img src="/img/list.jpg"></div>
|
||||
</li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
SINGER_HTML = """
|
||||
<html><body>
|
||||
<ul class="singer_list">
|
||||
<li>
|
||||
<div class="pic"><a href="/singer/zhoujielun.html"><img src="/img/singer.jpg"></a></div>
|
||||
<div class="name"><a>周杰伦</a></div>
|
||||
</li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
MV_HTML = """
|
||||
<html><body>
|
||||
<ul class="video_list">
|
||||
<li>
|
||||
<div class="name"><a href="/mp4/999.html">晴天MV</a></div>
|
||||
<div class="pic"><img src="/img/mv2.jpg"></div>
|
||||
</li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
SEARCH_HTML = """
|
||||
<html><body>
|
||||
<ul class="play_list">
|
||||
<li><div class="name"><a href="/mp3/777.html">七里香</a></div><img src="/img/1.jpg"></li>
|
||||
<li><div class="name"><a href="/mp4/778.html">七里香MV</a></div><img src="/img/2.jpg"></li>
|
||||
<li><div class="name"><a href="/playlist/top777.html">周董精选</a></div><img src="/img/3.jpg"></li>
|
||||
<li><div class="name"><a href="/singer/jay.html">周杰伦</a></div><img src="/img/4.jpg"></li>
|
||||
</ul>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
class TestSJMusicSpider(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -56,6 +100,51 @@ class TestSJMusicSpider(unittest.TestCase):
|
||||
result = self.spider.homeVideoContent()
|
||||
self.assertEqual([item["vod_id"] for item in result["list"]], ["song:123", "mv:456"])
|
||||
|
||||
@patch.object(Spider, "fetch")
|
||||
def test_category_content_supports_rank_playlist_singer_and_mv(self, mock_fetch):
|
||||
mock_fetch.side_effect = [
|
||||
SimpleNamespace(status_code=200, text=PLAYLIST_HTML),
|
||||
SimpleNamespace(status_code=200, text=SINGER_HTML),
|
||||
SimpleNamespace(status_code=200, text=MV_HTML),
|
||||
]
|
||||
rank_result = self.spider.categoryContent("rank_list", "1", False, {})
|
||||
playlist_result = self.spider.categoryContent("playlist", "1", False, {"lang": "index"})
|
||||
singer_result = self.spider.categoryContent(
|
||||
"singer",
|
||||
"1",
|
||||
False,
|
||||
{"sex": "girl", "area": "huayu", "char": "index"},
|
||||
)
|
||||
mv_result = self.spider.categoryContent(
|
||||
"mv",
|
||||
"1",
|
||||
False,
|
||||
{"area": "index", "type": "index", "sort": "new"},
|
||||
)
|
||||
self.assertTrue(rank_result["list"][0]["vod_id"].startswith("rank:"))
|
||||
self.assertEqual(playlist_result["list"][0]["vod_id"], "playlist:gedan001")
|
||||
self.assertEqual(singer_result["list"][0]["vod_id"], "singer:zhoujielun")
|
||||
self.assertEqual(mv_result["list"][0]["vod_id"], "mv:999")
|
||||
|
||||
@patch.object(Spider, "fetch")
|
||||
def test_search_content_maps_link_types_and_blank_keyword(self, mock_fetch):
|
||||
mock_fetch.return_value = SimpleNamespace(status_code=200, text=SEARCH_HTML)
|
||||
result = self.spider.searchContent("周杰伦", False, "1")
|
||||
self.assertEqual(
|
||||
[item["vod_id"] for item in result["list"]],
|
||||
["song:777", "mv:778", "playlist:top777", "singer:jay"],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider.searchContent("", False, "1"),
|
||||
{"page": 1, "limit": 0, "total": 0, "list": []},
|
||||
)
|
||||
|
||||
def test_category_content_returns_empty_for_unknown_tid(self):
|
||||
self.assertEqual(
|
||||
self.spider.categoryContent("unknown", "1", False, {}),
|
||||
{"page": 1, "limit": 0, "total": 0, "list": []},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+90
-2
@@ -27,6 +27,11 @@ class Spider(BaseSpider):
|
||||
{"type_id": "singer", "type_name": "歌手"},
|
||||
{"type_id": "mv", "type_name": "MV"},
|
||||
]
|
||||
self.rank_list = [
|
||||
("rise", "音乐飙升榜"),
|
||||
("new", "新歌排行榜"),
|
||||
("top", "Top热歌榜"),
|
||||
]
|
||||
|
||||
def init(self, extend=""):
|
||||
return None
|
||||
@@ -50,11 +55,22 @@ class Spider(BaseSpider):
|
||||
def _encode_vod_id(self, href):
|
||||
href = str(href or "")
|
||||
if "/mp3/" in href:
|
||||
return "song:" + href.rsplit("/", 1)[-1].replace(".html", "")
|
||||
return "song:" + self._extract_site_id(href, "mp3")
|
||||
if "/mp4/" in href:
|
||||
return "mv:" + href.rsplit("/", 1)[-1].replace(".html", "")
|
||||
return "mv:" + self._extract_site_id(href, "mp4")
|
||||
if "/playlist/" in href:
|
||||
return "playlist:" + self._extract_site_id(href, "playlist")
|
||||
if "/singer/" in href:
|
||||
return "singer:" + self._extract_site_id(href, "singer")
|
||||
return ""
|
||||
|
||||
def _extract_site_id(self, href, prefix):
|
||||
text = str(href or "")
|
||||
marker = f"/{prefix}/"
|
||||
if marker not in text:
|
||||
return ""
|
||||
return text.split(marker, 1)[1].split(".html", 1)[0].split("/", 1)[0]
|
||||
|
||||
def _build_filters(self):
|
||||
return {"singer": [], "mv": [], "playlist": []}
|
||||
|
||||
@@ -81,9 +97,81 @@ class Spider(BaseSpider):
|
||||
)
|
||||
return items
|
||||
|
||||
def _page_result(self, items, pg):
|
||||
page = int(pg)
|
||||
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
||||
|
||||
def _parse_list_cards(self, html, expected_prefixes):
|
||||
root = self._load_html(html)
|
||||
items = []
|
||||
seen = set()
|
||||
for node in root.xpath("//li"):
|
||||
href = "".join(node.xpath(".//a[1]/@href")).strip()
|
||||
vod_id = self._encode_vod_id(href)
|
||||
if not vod_id or vod_id in seen:
|
||||
continue
|
||||
if expected_prefixes and not any(vod_id.startswith(prefix) for prefix in expected_prefixes):
|
||||
continue
|
||||
seen.add(vod_id)
|
||||
items.append(
|
||||
{
|
||||
"vod_id": vod_id,
|
||||
"vod_name": self._clean_text(
|
||||
"".join(node.xpath(".//div[contains(@class,'name')]//text()"))
|
||||
or "".join(node.xpath(".//a[1]//text()"))
|
||||
),
|
||||
"vod_pic": self._build_url("".join(node.xpath(".//img[1]/@src"))),
|
||||
"vod_remarks": "",
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
def homeContent(self, filter):
|
||||
items = self._parse_home_items(self._fetch_html("/"))
|
||||
return {"class": list(self.classes), "filters": self._build_filters(), "list": items}
|
||||
|
||||
def homeVideoContent(self):
|
||||
return {"list": self.homeContent(False).get("list", [])}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
if tid == "home":
|
||||
return self._page_result(self.homeContent(False).get("list", []), 1)
|
||||
if tid == "rank_list":
|
||||
return self._page_result(
|
||||
[
|
||||
{
|
||||
"vod_id": f"rank:{rank_id}",
|
||||
"vod_name": title,
|
||||
"vod_pic": "",
|
||||
"vod_remarks": "排行榜",
|
||||
}
|
||||
for rank_id, title in self.rank_list
|
||||
],
|
||||
pg,
|
||||
)
|
||||
if tid == "playlist":
|
||||
return self._page_result(
|
||||
self._parse_list_cards(self._fetch_html("/playlists/index.html"), ["playlist:"]),
|
||||
pg,
|
||||
)
|
||||
if tid == "singer":
|
||||
return self._page_result(
|
||||
self._parse_list_cards(self._fetch_html("/singerlist/huayu/girl/index.html"), ["singer:"]),
|
||||
pg,
|
||||
)
|
||||
if tid == "mv":
|
||||
return self._page_result(
|
||||
self._parse_list_cards(self._fetch_html("/mvlist/index/index/new.html"), ["mv:"]),
|
||||
pg,
|
||||
)
|
||||
return {"page": 1, "limit": 0, "total": 0, "list": []}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
keyword = self._clean_text(key)
|
||||
if not keyword:
|
||||
return {"page": 1, "limit": 0, "total": 0, "list": []}
|
||||
items = self._parse_list_cards(
|
||||
self._fetch_html(f"/so.php?wd={keyword}&page={pg}"),
|
||||
["song:", "mv:", "playlist:", "singer:"],
|
||||
)
|
||||
return self._page_result(items, pg)
|
||||
|
||||
Reference in New Issue
Block a user