From 552243cccdcaa5be0cc86deaf219d3f8bea4da4c Mon Sep 17 00:00:00 2001 From: Harold <8866033@gmail.com> Date: Tue, 21 Apr 2026 19:51:42 +0800 Subject: [PATCH] feat: add muou and labi spiders --- py/tests/test_木偶.py | 225 ++++++++++++++++++++++++++++++++++ py/tests/test_蜡笔.py | 225 ++++++++++++++++++++++++++++++++++ py/木偶.py | 274 ++++++++++++++++++++++++++++++++++++++++++ py/蜡笔.py | 274 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 998 insertions(+) create mode 100644 py/tests/test_木偶.py create mode 100644 py/tests/test_蜡笔.py create mode 100644 py/木偶.py create mode 100644 py/蜡笔.py diff --git a/py/tests/test_木偶.py b/py/tests/test_木偶.py new file mode 100644 index 0000000..1e1fff3 --- /dev/null +++ b/py/tests/test_木偶.py @@ -0,0 +1,225 @@ +import unittest +from importlib.machinery import SourceFileLoader +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +MODULE = SourceFileLoader("muou_spider", str(ROOT / "木偶.py")).load_module() +Spider = MODULE.Spider + + +class TestMuOuSpider(unittest.TestCase): + def setUp(self): + Spider._instance = None + self.spider = Spider() + self.spider.init() + + def test_home_content_exposes_all_categories(self): + content = self.spider.homeContent(False) + self.assertEqual( + [(item["type_id"], item["type_name"]) for item in content["class"]], + [ + ("25", "木偶臻选"), + ("1", "木偶电影"), + ("2", "木偶电视剧"), + ("3", "木偶动漫"), + ("4", "木偶纪录片"), + ("29", "木偶综艺"), + ("30", "木偶原盘"), + ], + ) + + def test_home_video_content_returns_empty_list(self): + self.assertEqual(self.spider.homeVideoContent(), {"list": []}) + + def test_build_url_and_detect_pan_type(self): + self.assertEqual( + self.spider._build_url("/voddetail/1.html"), + "https://www.muou.site/voddetail/1.html", + ) + self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源")) + self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源")) + self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), ("", "")) + + def test_parse_cards_extracts_short_path_ids(self): + html = """ +
+
+
+ + 示例影片 +
+
HD
+
2025
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/voddetail/123.html", + "vod_name": "示例影片", + "vod_pic": "https://www.muou.site/poster.jpg", + "vod_remarks": "HD", + "vod_year": "2025", + } + ], + ) + + @patch.object(Spider, "_request_html") + def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): + mock_request_html.return_value = """ +
+
+
+ + 分类影片 +
+
更新至10集
+
2024
+
+
+ """ + result = self.spider.categoryContent("25", "3", False, {}) + self.assertEqual( + mock_request_html.call_args.args[0], + "https://www.muou.site/index.php/vod/type/id/25/page/3.html", + ) + self.assertEqual(result["page"], 3) + self.assertEqual(result["limit"], 1) + self.assertEqual(result["list"][0]["vod_name"], "分类影片") + self.assertNotIn("pagecount", result) + + @patch.object(Spider, "fetch") + def test_category_content_tries_next_domain_when_primary_fails(self, mock_fetch): + html = """ +
+
+
+ + 分类影片 +
+
更新至10集
+
2024
+
+
+ """ + + def fake_fetch(url, headers=None, timeout=10): + if url.startswith("https://www.muou.site"): + raise RuntimeError("boom") + return SimpleNamespace(status_code=200, text=html) + + mock_fetch.side_effect = fake_fetch + result = self.spider.categoryContent("25", "1", False, {}) + self.assertEqual(result["list"][0]["vod_name"], "分类影片") + self.assertEqual(self.spider.hosts[0], "https://www.muou.asia") + + @patch.object(Spider, "_request_html") + def test_search_content_builds_reference_search_url_and_parses_results(self, mock_request_html): + mock_request_html.return_value = """ +
+ 抢先版 +
+ 搜索影片 +
+
+ """ + result = self.spider.searchContent("繁花", False, "2") + self.assertEqual( + mock_request_html.call_args.args[0], + "https://www.muou.site/vodsearch/-------------.html?wd=%E7%B9%81%E8%8A%B1&page=2", + ) + self.assertEqual( + result["list"][0], + { + "vod_id": "/voddetail/789.html", + "vod_name": "搜索影片", + "vod_pic": "https://www.muou.site/search.jpg", + "vod_remarks": "抢先版", + }, + ) + + def test_search_content_returns_empty_list_for_blank_keyword(self): + self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []}) + + def test_build_pan_lines_deduplicates_and_sorts_supported_links(self): + detail = { + "pan_urls": [ + "https://pan.quark.cn/s/q1", + "https://pan.baidu.com/s/b1", + "https://pan.baidu.com/s/b1", + "https://example.com/ignored", + ] + } + self.assertEqual( + self.spider._build_pan_lines(detail), + [ + ("baidu#木偶", "百度资源$https://pan.baidu.com/s/b1"), + ("quark#木偶", "夸克资源$https://pan.quark.cn/s/q1"), + ], + ) + + def test_parse_detail_page_extracts_meta_content_and_pan_urls(self): + html = """ +
示例剧
+
+
年代
2024
+
导演
导演甲
+
主演
演员甲演员乙
+
剧情

一段剧情简介

+
+

https://pan.quark.cn/s/q1

+

https://pan.baidu.com/s/b1

+
+ """ + detail = self.spider._parse_detail_page("/voddetail/123.html", html) + self.assertEqual(detail["vod_name"], "示例剧") + self.assertEqual(detail["vod_pic"], "https://www.muou.site/poster.jpg") + self.assertEqual(detail["vod_year"], "2024") + self.assertEqual(detail["vod_director"], "导演甲") + self.assertEqual(detail["vod_actor"], "演员甲,演员乙") + self.assertEqual(detail["vod_content"], "一段剧情简介") + self.assertEqual(detail["pan_urls"], ["https://pan.quark.cn/s/q1", "https://pan.baidu.com/s/b1"]) + + @patch.object(Spider, "_request_html") + def test_detail_content_builds_pan_play_fields(self, mock_request_html): + mock_request_html.return_value = """ +
示例剧
+
+
年代
2024
+
导演
导演甲
+
主演
演员甲
+
剧情

一段剧情简介

+
+

https://pan.quark.cn/s/q1

+

https://pan.baidu.com/s/b1

+
+ """ + result = self.spider.detailContent(["/voddetail/123.html"]) + vod = result["list"][0] + self.assertEqual(vod["vod_name"], "示例剧") + self.assertEqual(vod["vod_play_from"], "baidu#木偶$$$quark#木偶") + self.assertEqual( + vod["vod_play_url"], + "百度资源$https://pan.baidu.com/s/b1$$$夸克资源$https://pan.quark.cn/s/q1", + ) + + def test_player_content_passthroughs_supported_pan_urls(self): + self.assertEqual( + self.spider.playerContent("baidu#木偶", "https://pan.baidu.com/s/demo", {}), + {"parse": 0, "playUrl": "", "url": "https://pan.baidu.com/s/demo"}, + ) + + def test_player_content_rejects_non_pan_url(self): + self.assertEqual( + self.spider.playerContent("site", "/vodplay/1-1-1.html", {}), + {"parse": 0, "playUrl": "", "url": ""}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/tests/test_蜡笔.py b/py/tests/test_蜡笔.py new file mode 100644 index 0000000..e15d5d1 --- /dev/null +++ b/py/tests/test_蜡笔.py @@ -0,0 +1,225 @@ +import unittest +from importlib.machinery import SourceFileLoader +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +MODULE = SourceFileLoader("labi_spider", str(ROOT / "蜡笔.py")).load_module() +Spider = MODULE.Spider + + +class TestLaBiSpider(unittest.TestCase): + def setUp(self): + Spider._instance = None + self.spider = Spider() + self.spider.init() + + def test_home_content_exposes_all_categories(self): + content = self.spider.homeContent(False) + self.assertEqual( + [(item["type_id"], item["type_name"]) for item in content["class"]], + [ + ("29", "蜡笔臻彩"), + ("1", "蜡笔电影"), + ("2", "蜡笔电视剧"), + ("3", "蜡笔动漫"), + ("4", "蜡笔综艺"), + ("5", "蜡笔短剧"), + ("24", "蜡笔4K"), + ], + ) + + def test_home_video_content_returns_empty_list(self): + self.assertEqual(self.spider.homeVideoContent(), {"list": []}) + + def test_build_url_and_detect_pan_type(self): + self.assertEqual( + self.spider._build_url("/voddetail/1.html"), + "http://xiaocgege.shop/voddetail/1.html", + ) + self.assertEqual(self.spider._detect_pan_type("https://pan.baidu.com/s/demo"), ("baidu", "百度资源")) + self.assertEqual(self.spider._detect_pan_type("https://pan.quark.cn/s/demo"), ("quark", "夸克资源")) + self.assertEqual(self.spider._detect_pan_type("https://example.com/video"), ("", "")) + + def test_parse_cards_extracts_short_path_ids(self): + html = """ +
+
+
+ + 示例影片 +
+
HD
+
2025
+
+
+ """ + self.assertEqual( + self.spider._parse_cards(html), + [ + { + "vod_id": "/voddetail/123.html", + "vod_name": "示例影片", + "vod_pic": "http://xiaocgege.shop/poster.jpg", + "vod_remarks": "HD", + "vod_year": "2025", + } + ], + ) + + @patch.object(Spider, "_request_html") + def test_category_content_builds_reference_url_and_returns_page_payload(self, mock_request_html): + mock_request_html.return_value = """ +
+
+
+ + 分类影片 +
+
更新至10集
+
2024
+
+
+ """ + result = self.spider.categoryContent("29", "3", False, {}) + self.assertEqual( + mock_request_html.call_args.args[0], + "http://xiaocgege.shop/index.php/vod/type/id/29/page/3.html", + ) + self.assertEqual(result["page"], 3) + self.assertEqual(result["limit"], 1) + self.assertEqual(result["list"][0]["vod_name"], "分类影片") + self.assertNotIn("pagecount", result) + + @patch.object(Spider, "fetch") + def test_category_content_tries_next_domain_when_primary_fails(self, mock_fetch): + html = """ +
+
+
+ + 分类影片 +
+
更新至10集
+
2024
+
+
+ """ + + def fake_fetch(url, headers=None, timeout=10): + if url.startswith("http://xiaocgege.shop") or url.startswith("http://xiaocge.fun"): + raise RuntimeError("boom") + return SimpleNamespace(status_code=200, text=html) + + mock_fetch.side_effect = fake_fetch + result = self.spider.categoryContent("29", "1", False, {}) + self.assertEqual(result["list"][0]["vod_name"], "分类影片") + self.assertEqual(self.spider.hosts[0], "http://fmao.shop") + + @patch.object(Spider, "_request_html") + def test_search_content_builds_reference_search_url_and_parses_results(self, mock_request_html): + mock_request_html.return_value = """ +
+ 抢先版 +
+ 搜索影片 +
+
+ """ + result = self.spider.searchContent("繁花", False, "2") + self.assertEqual( + mock_request_html.call_args.args[0], + "http://xiaocgege.shop/vodsearch/-------------.html?wd=%E7%B9%81%E8%8A%B1&page=2", + ) + self.assertEqual( + result["list"][0], + { + "vod_id": "/voddetail/789.html", + "vod_name": "搜索影片", + "vod_pic": "http://xiaocgege.shop/search.jpg", + "vod_remarks": "抢先版", + }, + ) + + def test_search_content_returns_empty_list_for_blank_keyword(self): + self.assertEqual(self.spider.searchContent("", False, "1"), {"page": 1, "total": 0, "list": []}) + + def test_build_pan_lines_deduplicates_and_sorts_supported_links(self): + detail = { + "pan_urls": [ + "https://pan.quark.cn/s/q1", + "https://pan.baidu.com/s/b1", + "https://pan.baidu.com/s/b1", + "https://example.com/ignored", + ] + } + self.assertEqual( + self.spider._build_pan_lines(detail), + [ + ("baidu#蜡笔", "百度资源$https://pan.baidu.com/s/b1"), + ("quark#蜡笔", "夸克资源$https://pan.quark.cn/s/q1"), + ], + ) + + def test_parse_detail_page_extracts_meta_content_and_pan_urls(self): + html = """ +
示例剧
+
+
年代
2024
+
导演
导演甲
+
主演
演员甲演员乙
+
剧情

一段剧情简介

+
+

https://pan.quark.cn/s/q1

+

https://pan.baidu.com/s/b1

+
+ """ + detail = self.spider._parse_detail_page("/voddetail/123.html", html) + self.assertEqual(detail["vod_name"], "示例剧") + self.assertEqual(detail["vod_pic"], "http://xiaocgege.shop/poster.jpg") + self.assertEqual(detail["vod_year"], "2024") + self.assertEqual(detail["vod_director"], "导演甲") + self.assertEqual(detail["vod_actor"], "演员甲,演员乙") + self.assertEqual(detail["vod_content"], "一段剧情简介") + self.assertEqual(detail["pan_urls"], ["https://pan.quark.cn/s/q1", "https://pan.baidu.com/s/b1"]) + + @patch.object(Spider, "_request_html") + def test_detail_content_builds_pan_play_fields(self, mock_request_html): + mock_request_html.return_value = """ +
示例剧
+
+
年代
2024
+
导演
导演甲
+
主演
演员甲
+
剧情

一段剧情简介

+
+

https://pan.quark.cn/s/q1

+

https://pan.baidu.com/s/b1

+
+ """ + result = self.spider.detailContent(["/voddetail/123.html"]) + vod = result["list"][0] + self.assertEqual(vod["vod_name"], "示例剧") + self.assertEqual(vod["vod_play_from"], "baidu#蜡笔$$$quark#蜡笔") + self.assertEqual( + vod["vod_play_url"], + "百度资源$https://pan.baidu.com/s/b1$$$夸克资源$https://pan.quark.cn/s/q1", + ) + + def test_player_content_passthroughs_supported_pan_urls(self): + self.assertEqual( + self.spider.playerContent("baidu#蜡笔", "https://pan.baidu.com/s/demo", {}), + {"parse": 0, "playUrl": "", "url": "https://pan.baidu.com/s/demo"}, + ) + + def test_player_content_rejects_non_pan_url(self): + self.assertEqual( + self.spider.playerContent("site", "/vodplay/1-1-1.html", {}), + {"parse": 0, "playUrl": "", "url": ""}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/木偶.py b/py/木偶.py new file mode 100644 index 0000000..96258b4 --- /dev/null +++ b/py/木偶.py @@ -0,0 +1,274 @@ +# coding=utf-8 +import re +import sys +from urllib.parse import quote, urljoin, urlsplit + +from base.spider import Spider as BaseSpider + +sys.path.append("..") + + +class Spider(BaseSpider): + def __init__(self): + self.name = "木偶" + self.hosts = ["https://www.muou.site", "https://www.muou.asia", "https://666.666291.xyz"] + self.headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/136.0.0.0 Safari/537.36" + ), + "Referer": self.hosts[0] + "/", + } + self.categories = [ + {"type_id": "25", "type_name": "木偶臻选"}, + {"type_id": "1", "type_name": "木偶电影"}, + {"type_id": "2", "type_name": "木偶电视剧"}, + {"type_id": "3", "type_name": "木偶动漫"}, + {"type_id": "4", "type_name": "木偶纪录片"}, + {"type_id": "29", "type_name": "木偶综艺"}, + {"type_id": "30", "type_name": "木偶原盘"}, + ] + self.pan_patterns = [ + ("baidu", "百度资源", r"pan\.baidu\.com|yun\.baidu\.com"), + ("a139", "139资源", r"yun\.139\.com"), + ("a189", "天翼资源", r"cloud\.189\.cn"), + ("a123", "123资源", r"123684\.com|123865\.com|123912\.com|123pan\.com"), + ("a115", "115资源", r"115\.com|115cdn\.com"), + ("quark", "夸克资源", r"pan\.quark\.cn|quark\.cn"), + ("xunlei", "迅雷资源", r"pan\.xunlei\.com|xunlei\.com"), + ("aliyun", "阿里资源", r"aliyundrive\.com|alipan\.com"), + ("uc", "UC资源", r"drive\.uc\.cn|uc\.cn"), + ] + self.pan_priority = { + "baidu": 1, + "a139": 2, + "a189": 3, + "a123": 4, + "a115": 5, + "quark": 6, + "xunlei": 7, + "aliyun": 8, + "uc": 9, + } + + def init(self, extend=""): + return None + + def getName(self): + return self.name + + def homeContent(self, filter): + return {"class": self.categories} + + def homeVideoContent(self): + return {"list": []} + + def _build_url(self, path): + return urljoin(self.hosts[0] + "/", str(path or "").strip()) + + def _clean_text(self, text): + return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip() + + def _detect_pan_type(self, url): + raw = str(url or "").strip() + for pan_type, title, pattern in self.pan_patterns: + if re.search(pattern, raw, re.I): + return pan_type, title + return "", "" + + def _request_html(self, path_or_url): + raw = str(path_or_url or "").strip() + relative = raw + if raw.startswith("http"): + parsed = urlsplit(raw) + relative = parsed.path or "/" + if parsed.query: + relative += "?" + parsed.query + + last_error = None + for index, host in enumerate(list(self.hosts)): + target = raw if raw.startswith("http") and raw.startswith(host) else urljoin(host + "/", relative) + try: + headers = dict(self.headers) + headers["Referer"] = host + "/" + response = self.fetch(target, headers=headers, timeout=10) + if response.status_code == 200 and response.text: + if index > 0: + self.hosts.insert(0, self.hosts.pop(index)) + self.headers["Referer"] = self.hosts[0] + "/" + return response.text or "" + except Exception as exc: + last_error = exc + if last_error: + raise last_error + return "" + + def _page_result(self, items, pg): + page = int(pg) + return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items} + + def _parse_cards(self, html): + root = self.html(html) + if root is None: + return [] + + items = [] + seen = set() + for node in root.xpath( + "//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]" + ): + href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip() + title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip() + pic = ( + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() + ) + remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()"))) + year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()"))) + if not href or not title or href in seen: + continue + seen.add(href) + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(pic), + "vod_remarks": remarks, + "vod_year": year, + } + ) + return items + + def categoryContent(self, tid, pg, filter, extend): + url = self._build_url(f"/index.php/vod/type/id/{tid}/page/{int(pg)}.html") + return self._page_result(self._parse_cards(self._request_html(url)), pg) + + def searchContent(self, key, quick, pg="1"): + keyword = self._clean_text(key) + page = int(pg) + if not keyword: + return {"page": page, "total": 0, "list": []} + + url = self._build_url(f"/vodsearch/-------------.html?wd={quote(keyword)}&page={page}") + root = self.html(self._request_html(url)) + if root is None: + return {"page": page, "total": 0, "list": []} + + items = [] + for node in root.xpath("//*[contains(@class,'module-search-item')]"): + href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() + title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() + pic = ( + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() + ) + remarks = self._clean_text( + "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) + or "".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")) + ) + if not href or not title: + continue + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(pic), + "vod_remarks": remarks, + } + ) + return {"page": page, "total": len(items), "list": items} + + def _parse_detail_page(self, vod_id, html): + root = self.html(html) + if root is None: + return { + "vod_id": vod_id, + "vod_name": "", + "vod_pic": "", + "vod_year": "", + "vod_director": "", + "vod_actor": "", + "vod_content": "", + "pan_urls": [], + } + + detail = { + "vod_id": vod_id, + "vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))), + "vod_pic": self._build_url( + "".join( + root.xpath( + "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | " + "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src" + ) + ).strip() + ), + "vod_year": "", + "vod_director": "", + "vod_actor": "", + "vod_content": "", + "pan_urls": [], + } + + for label_node in root.xpath("//*[contains(@class,'video-info-itemtitle')]"): + key = self._clean_text("".join(label_node.xpath(".//text()"))) + sibling = label_node.getnext() + if sibling is None: + continue + values = [self._clean_text(text) for text in sibling.xpath(".//a//text()")] + joined = ",".join([value for value in values if value]) + text_value = self._clean_text("".join(sibling.xpath(".//text()"))) + if "年代" in key: + detail["vod_year"] = joined or text_value + elif "导演" in key: + detail["vod_director"] = joined or text_value + elif "主演" in key: + detail["vod_actor"] = joined or text_value + elif "剧情" in key: + detail["vod_content"] = text_value + + for node in root.xpath("//*[contains(@class,'module-row-info')]//p"): + text = self._clean_text("".join(node.xpath(".//text()"))) + if text: + detail["pan_urls"].append(text) + return detail + + def _build_pan_lines(self, detail): + lines = [] + seen = set() + for url in detail.get("pan_urls", []): + pan_type, title = self._detect_pan_type(url) + if not pan_type or url in seen: + continue + seen.add(url) + lines.append((self.pan_priority.get(pan_type, 999), f"{pan_type}#木偶", f"{title}${url}")) + lines.sort(key=lambda item: item[0]) + return [(item[1], item[2]) for item in lines] + + def detailContent(self, ids): + result = {"list": []} + for raw_id in ids: + vod_id = str(raw_id or "").strip() + detail = self._parse_detail_page(vod_id, self._request_html(self._build_url(vod_id))) + lines = self._build_pan_lines(detail) + result["list"].append( + { + "vod_id": vod_id, + "vod_name": detail["vod_name"], + "vod_pic": detail["vod_pic"], + "vod_year": detail["vod_year"], + "vod_director": detail["vod_director"], + "vod_actor": detail["vod_actor"], + "vod_content": detail["vod_content"], + "vod_play_from": "$$$".join([item[0] for item in lines]), + "vod_play_url": "$$$".join([item[1] for item in lines]), + } + ) + return result + + def playerContent(self, flag, id, vipFlags): + pan_type, _ = self._detect_pan_type(id) + if pan_type: + return {"parse": 0, "playUrl": "", "url": id} + return {"parse": 0, "playUrl": "", "url": ""} diff --git a/py/蜡笔.py b/py/蜡笔.py new file mode 100644 index 0000000..5c47e2a --- /dev/null +++ b/py/蜡笔.py @@ -0,0 +1,274 @@ +# coding=utf-8 +import re +import sys +from urllib.parse import quote, urljoin, urlsplit + +from base.spider import Spider as BaseSpider + +sys.path.append("..") + + +class Spider(BaseSpider): + def __init__(self): + self.name = "蜡笔" + self.hosts = ["http://xiaocgege.shop", "http://fmao.shop"] + self.headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/136.0.0.0 Safari/537.36" + ), + "Referer": self.hosts[0] + "/", + } + self.categories = [ + {"type_id": "29", "type_name": "蜡笔臻彩"}, + {"type_id": "1", "type_name": "蜡笔电影"}, + {"type_id": "2", "type_name": "蜡笔电视剧"}, + {"type_id": "3", "type_name": "蜡笔动漫"}, + {"type_id": "4", "type_name": "蜡笔综艺"}, + {"type_id": "5", "type_name": "蜡笔短剧"}, + {"type_id": "24", "type_name": "蜡笔4K"}, + ] + self.pan_patterns = [ + ("baidu", "百度资源", r"pan\.baidu\.com|yun\.baidu\.com"), + ("a139", "139资源", r"yun\.139\.com"), + ("a189", "天翼资源", r"cloud\.189\.cn"), + ("a123", "123资源", r"123684\.com|123865\.com|123912\.com|123pan\.com"), + ("a115", "115资源", r"115\.com|115cdn\.com"), + ("quark", "夸克资源", r"pan\.quark\.cn|quark\.cn"), + ("xunlei", "迅雷资源", r"pan\.xunlei\.com|xunlei\.com"), + ("aliyun", "阿里资源", r"aliyundrive\.com|alipan\.com"), + ("uc", "UC资源", r"drive\.uc\.cn|uc\.cn"), + ] + self.pan_priority = { + "baidu": 1, + "a139": 2, + "a189": 3, + "a123": 4, + "a115": 5, + "quark": 6, + "xunlei": 7, + "aliyun": 8, + "uc": 9, + } + + def init(self, extend=""): + return None + + def getName(self): + return self.name + + def homeContent(self, filter): + return {"class": self.categories} + + def homeVideoContent(self): + return {"list": []} + + def _build_url(self, path): + return urljoin(self.hosts[0] + "/", str(path or "").strip()) + + def _clean_text(self, text): + return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip() + + def _detect_pan_type(self, url): + raw = str(url or "").strip() + for pan_type, title, pattern in self.pan_patterns: + if re.search(pattern, raw, re.I): + return pan_type, title + return "", "" + + def _request_html(self, path_or_url): + raw = str(path_or_url or "").strip() + relative = raw + if raw.startswith("http"): + parsed = urlsplit(raw) + relative = parsed.path or "/" + if parsed.query: + relative += "?" + parsed.query + + last_error = None + for index, host in enumerate(list(self.hosts)): + target = raw if raw.startswith("http") and raw.startswith(host) else urljoin(host + "/", relative) + try: + headers = dict(self.headers) + headers["Referer"] = host + "/" + response = self.fetch(target, headers=headers, timeout=10) + if response.status_code == 200 and response.text: + if index > 0: + self.hosts.insert(0, self.hosts.pop(index)) + self.headers["Referer"] = self.hosts[0] + "/" + return response.text or "" + except Exception as exc: + last_error = exc + if last_error: + raise last_error + return "" + + def _page_result(self, items, pg): + page = int(pg) + return {"page": page, "limit": len(items), "total": page * 20 + len(items), "list": items} + + def _parse_cards(self, html): + root = self.html(html) + if root is None: + return [] + + items = [] + seen = set() + for node in root.xpath( + "//*[@id='main']//*[contains(concat(' ', normalize-space(@class), ' '), ' module-item ')]" + ): + href = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//a[@href])[1]/@href") or [""])[0]).strip() + title = ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@alt])[1]/@alt") or [""])[0]).strip() + pic = ( + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() + ) + remarks = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-text')])[1]//text()"))) + year = self._clean_text("".join(node.xpath("(.//*[contains(@class,'module-item-caption')])[1]//span[1]//text()"))) + if not href or not title or href in seen: + continue + seen.add(href) + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(pic), + "vod_remarks": remarks, + "vod_year": year, + } + ) + return items + + def categoryContent(self, tid, pg, filter, extend): + url = self._build_url(f"/index.php/vod/type/id/{tid}/page/{int(pg)}.html") + return self._page_result(self._parse_cards(self._request_html(url)), pg) + + def searchContent(self, key, quick, pg="1"): + keyword = self._clean_text(key) + page = int(pg) + if not keyword: + return {"page": page, "total": 0, "list": []} + + url = self._build_url(f"/vodsearch/-------------.html?wd={quote(keyword)}&page={page}") + root = self.html(self._request_html(url)) + if root is None: + return {"page": page, "total": 0, "list": []} + + items = [] + for node in root.xpath("//*[contains(@class,'module-search-item')]"): + href = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@href")).strip() + title = "".join(node.xpath(".//*[contains(@class,'video-serial')][1]/@title")).strip() + pic = ( + ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@data-src])[1]/@data-src") or [""])[0]).strip() + or ((node.xpath("(.//*[contains(@class,'module-item-pic')]//img[@src])[1]/@src") or [""])[0]).strip() + ) + remarks = self._clean_text( + "".join(node.xpath(".//*[contains(@class,'video-serial')][1]//text()")) + or "".join(node.xpath(".//*[contains(@class,'module-item-text')][1]//text()")) + ) + if not href or not title: + continue + items.append( + { + "vod_id": href, + "vod_name": title, + "vod_pic": self._build_url(pic), + "vod_remarks": remarks, + } + ) + return {"page": page, "total": len(items), "list": items} + + def _parse_detail_page(self, vod_id, html): + root = self.html(html) + if root is None: + return { + "vod_id": vod_id, + "vod_name": "", + "vod_pic": "", + "vod_year": "", + "vod_director": "", + "vod_actor": "", + "vod_content": "", + "pan_urls": [], + } + + detail = { + "vod_id": vod_id, + "vod_name": self._clean_text("".join(root.xpath("//*[contains(@class,'page-title')][1]//text()"))), + "vod_pic": self._build_url( + "".join( + root.xpath( + "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@data-src | " + "//*[contains(@class,'mobile-play')]//*[contains(@class,'lazyload')][1]/@src" + ) + ).strip() + ), + "vod_year": "", + "vod_director": "", + "vod_actor": "", + "vod_content": "", + "pan_urls": [], + } + + for label_node in root.xpath("//*[contains(@class,'video-info-itemtitle')]"): + key = self._clean_text("".join(label_node.xpath(".//text()"))) + sibling = label_node.getnext() + if sibling is None: + continue + values = [self._clean_text(text) for text in sibling.xpath(".//a//text()")] + joined = ",".join([value for value in values if value]) + text_value = self._clean_text("".join(sibling.xpath(".//text()"))) + if "年代" in key: + detail["vod_year"] = joined or text_value + elif "导演" in key: + detail["vod_director"] = joined or text_value + elif "主演" in key: + detail["vod_actor"] = joined or text_value + elif "剧情" in key: + detail["vod_content"] = text_value + + for node in root.xpath("//*[contains(@class,'module-row-info')]//p"): + text = self._clean_text("".join(node.xpath(".//text()"))) + if text: + detail["pan_urls"].append(text) + return detail + + def _build_pan_lines(self, detail): + lines = [] + seen = set() + for url in detail.get("pan_urls", []): + pan_type, title = self._detect_pan_type(url) + if not pan_type or url in seen: + continue + seen.add(url) + lines.append((self.pan_priority.get(pan_type, 999), f"{pan_type}#蜡笔", f"{title}${url}")) + lines.sort(key=lambda item: item[0]) + return [(item[1], item[2]) for item in lines] + + def detailContent(self, ids): + result = {"list": []} + for raw_id in ids: + vod_id = str(raw_id or "").strip() + detail = self._parse_detail_page(vod_id, self._request_html(self._build_url(vod_id))) + lines = self._build_pan_lines(detail) + result["list"].append( + { + "vod_id": vod_id, + "vod_name": detail["vod_name"], + "vod_pic": detail["vod_pic"], + "vod_year": detail["vod_year"], + "vod_director": detail["vod_director"], + "vod_actor": detail["vod_actor"], + "vod_content": detail["vod_content"], + "vod_play_from": "$$$".join([item[0] for item in lines]), + "vod_play_url": "$$$".join([item[1] for item in lines]), + } + ) + return result + + def playerContent(self, flag, id, vipFlags): + pan_type, _ = self._detect_pan_type(id) + if pan_type: + return {"parse": 0, "playUrl": "", "url": id} + return {"parse": 0, "playUrl": "", "url": ""}