feat: add sjmusic detail parsing
This commit is contained in:
Harold
@@ -74,6 +74,54 @@ SEARCH_HTML = """
|
|||||||
</body></html>
|
</body></html>
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
RANK_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<ul class="play_list">
|
||||||
|
<li><div class="name"><a href="/mp3/321.html">搁浅</a></div></li>
|
||||||
|
<li><div class="name"><a href="/mp3/322.html">简单爱</a></div></li>
|
||||||
|
</ul>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
SONG_HTML = """
|
||||||
|
<html><head><title>夜曲_世纪音乐</title></head><body>
|
||||||
|
<h1>夜曲</h1>
|
||||||
|
<div class="play_singer"><div class="name"><a>周杰伦</a></div></div>
|
||||||
|
<div class="playhimg"><img src="/img/song_detail.jpg"></div>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
MV_DETAIL_HTML = """
|
||||||
|
<html><head><title>晴天MV_世纪音乐</title></head><body>
|
||||||
|
<h1>晴天MV</h1>
|
||||||
|
<div class="play_singer"><div class="name"><a>周杰伦</a></div></div>
|
||||||
|
<div class="playhimg"><img src="/img/mv_detail.jpg"></div>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
PLAYLIST_DETAIL_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<h1>周董歌单</h1>
|
||||||
|
<div class="pic"><img src="/img/playlist.jpg"></div>
|
||||||
|
<ul class="play_list">
|
||||||
|
<li><div class="name"><a href="/mp3/401.html">安静</a></div></li>
|
||||||
|
<li><div class="name"><a href="/mp3/402.html">晴天</a></div></li>
|
||||||
|
</ul>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
SINGER_DETAIL_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<h1>周杰伦</h1>
|
||||||
|
<div class="singer_info"><div class="info"><p>华语男歌手</p></div></div>
|
||||||
|
<div class="pic"><img src="/img/singer_detail.jpg"></div>
|
||||||
|
<ul class="play_list">
|
||||||
|
<li><div class="name"><a href="/mp3/501.html">青花瓷</a></div></li>
|
||||||
|
<li><div class="name"><a href="/mp3/502.html">稻香</a></div></li>
|
||||||
|
</ul>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class TestSJMusicSpider(unittest.TestCase):
|
class TestSJMusicSpider(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
@@ -145,6 +193,26 @@ class TestSJMusicSpider(unittest.TestCase):
|
|||||||
{"page": 1, "limit": 0, "total": 0, "list": []},
|
{"page": 1, "limit": 0, "total": 0, "list": []},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch.object(Spider, "fetch")
|
||||||
|
def test_detail_content_builds_rank_song_mv_playlist_and_singer(self, mock_fetch):
|
||||||
|
mock_fetch.side_effect = [
|
||||||
|
SimpleNamespace(status_code=200, text=RANK_HTML),
|
||||||
|
SimpleNamespace(status_code=200, text=SONG_HTML),
|
||||||
|
SimpleNamespace(status_code=200, text=MV_DETAIL_HTML),
|
||||||
|
SimpleNamespace(status_code=200, text=PLAYLIST_DETAIL_HTML),
|
||||||
|
SimpleNamespace(status_code=200, text=SINGER_DETAIL_HTML),
|
||||||
|
]
|
||||||
|
rank_vod = self.spider.detailContent(["rank:rise"])["list"][0]
|
||||||
|
song_vod = self.spider.detailContent(["song:123"])["list"][0]
|
||||||
|
mv_vod = self.spider.detailContent(["mv:456"])["list"][0]
|
||||||
|
playlist_vod = self.spider.detailContent(["playlist:top100"])["list"][0]
|
||||||
|
singer_vod = self.spider.detailContent(["singer:jay"])["list"][0]
|
||||||
|
self.assertEqual(rank_vod["vod_play_url"], "搁浅$music:321#简单爱$music:322")
|
||||||
|
self.assertEqual(song_vod["vod_play_url"], "周杰伦 - 夜曲$music:123")
|
||||||
|
self.assertEqual(mv_vod["vod_play_url"], "晴天MV$vplay:456:1080")
|
||||||
|
self.assertEqual(playlist_vod["vod_play_url"], "安静$music:401#晴天$music:402")
|
||||||
|
self.assertEqual(singer_vod["vod_play_url"], "青花瓷$music:501#稻香$music:502")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+110
@@ -101,6 +101,30 @@ class Spider(BaseSpider):
|
|||||||
page = int(pg)
|
page = int(pg)
|
||||||
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
||||||
|
|
||||||
|
def _decode_vod_id(self, vod_id):
|
||||||
|
raw = str(vod_id or "").strip()
|
||||||
|
if ":" not in raw:
|
||||||
|
return "", ""
|
||||||
|
prefix, value = raw.split(":", 1)
|
||||||
|
if prefix == "rank":
|
||||||
|
return prefix, f"/list/{value}.html"
|
||||||
|
if prefix == "song":
|
||||||
|
return prefix, f"/mp3/{value}.html"
|
||||||
|
if prefix == "mv":
|
||||||
|
return prefix, f"/mp4/{value}.html"
|
||||||
|
if prefix == "playlist":
|
||||||
|
return prefix, f"/playlist/{value}.html"
|
||||||
|
if prefix == "singer":
|
||||||
|
return prefix, f"/singer/{value}.html"
|
||||||
|
return "", ""
|
||||||
|
|
||||||
|
def _encode_play_id(self, kind, value):
|
||||||
|
if kind == "music":
|
||||||
|
return f"music:{value}"
|
||||||
|
if kind == "vplay":
|
||||||
|
return f"vplay:{value}:1080"
|
||||||
|
return ""
|
||||||
|
|
||||||
def _parse_list_cards(self, html, expected_prefixes):
|
def _parse_list_cards(self, html, expected_prefixes):
|
||||||
root = self._load_html(html)
|
root = self._load_html(html)
|
||||||
items = []
|
items = []
|
||||||
@@ -126,6 +150,21 @@ class Spider(BaseSpider):
|
|||||||
)
|
)
|
||||||
return items
|
return items
|
||||||
|
|
||||||
|
def _build_episode_rows(self, html):
|
||||||
|
root = self._load_html(html)
|
||||||
|
rows = []
|
||||||
|
for node in root.xpath("//*[contains(@class,'play_list')]//li"):
|
||||||
|
href = "".join(node.xpath(".//a[1]/@href")).strip()
|
||||||
|
song_id = self._extract_site_id(href, "mp3")
|
||||||
|
if not song_id:
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
self._clean_text("".join(node.xpath(".//a[1]//text()")))
|
||||||
|
+ "$"
|
||||||
|
+ self._encode_play_id("music", song_id)
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
def homeContent(self, filter):
|
def homeContent(self, filter):
|
||||||
items = self._parse_home_items(self._fetch_html("/"))
|
items = self._parse_home_items(self._fetch_html("/"))
|
||||||
return {"class": list(self.classes), "filters": self._build_filters(), "list": items}
|
return {"class": list(self.classes), "filters": self._build_filters(), "list": items}
|
||||||
@@ -175,3 +214,74 @@ class Spider(BaseSpider):
|
|||||||
["song:", "mv:", "playlist:", "singer:"],
|
["song:", "mv:", "playlist:", "singer:"],
|
||||||
)
|
)
|
||||||
return self._page_result(items, pg)
|
return self._page_result(items, pg)
|
||||||
|
|
||||||
|
def detailContent(self, ids):
|
||||||
|
vod_id = str((ids or [""])[0] or "").strip()
|
||||||
|
kind, path = self._decode_vod_id(vod_id)
|
||||||
|
if not path:
|
||||||
|
return {"list": []}
|
||||||
|
html = self._fetch_html(path)
|
||||||
|
root = self._load_html(html)
|
||||||
|
title = self._clean_text("".join(root.xpath("//h1[1]//text()")))
|
||||||
|
pic = self._build_url("".join(root.xpath("(//img[1]/@src)[1]")))
|
||||||
|
if kind == "rank":
|
||||||
|
return {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": title or "排行榜",
|
||||||
|
"vod_pic": pic,
|
||||||
|
"vod_remarks": "",
|
||||||
|
"vod_content": "",
|
||||||
|
"vod_play_from": self.name,
|
||||||
|
"vod_play_url": "#".join(self._build_episode_rows(html)),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
if kind == "song":
|
||||||
|
singer = self._clean_text("".join(root.xpath("//*[contains(@class,'play_singer')]//a[1]//text()")))
|
||||||
|
display = f"{singer} - {title}" if singer else title
|
||||||
|
return {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": title,
|
||||||
|
"vod_pic": pic,
|
||||||
|
"vod_remarks": "",
|
||||||
|
"vod_content": "",
|
||||||
|
"vod_actor": singer,
|
||||||
|
"vod_play_from": self.name,
|
||||||
|
"vod_play_url": display + "$" + self._encode_play_id("music", vod_id.split(":", 1)[1]),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
if kind == "mv":
|
||||||
|
return {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": title,
|
||||||
|
"vod_pic": pic,
|
||||||
|
"vod_remarks": "",
|
||||||
|
"vod_content": "",
|
||||||
|
"vod_play_from": self.name,
|
||||||
|
"vod_play_url": title + "$" + self._encode_play_id("vplay", vod_id.split(":", 1)[1]),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
if kind in ("playlist", "singer"):
|
||||||
|
content = self._clean_text("".join(root.xpath("//*[contains(@class,'info')]//p[1]//text()")))
|
||||||
|
return {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"vod_id": vod_id,
|
||||||
|
"vod_name": title,
|
||||||
|
"vod_pic": pic,
|
||||||
|
"vod_remarks": "",
|
||||||
|
"vod_content": content,
|
||||||
|
"vod_play_from": self.name,
|
||||||
|
"vod_play_url": "#".join(self._build_episode_rows(html)),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
return {"list": []}
|
||||||
|
|||||||
Reference in New Issue
Block a user