feat: scaffold rrdy spider
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
import unittest
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE = SourceFileLoader("rrdy_spider", str(ROOT / "人人电影.py")).load_module()
|
||||
Spider = MODULE.Spider
|
||||
|
||||
|
||||
class TestRenRenDianYingSpider(unittest.TestCase):
|
||||
def setUp(self):
|
||||
Spider._instance = None
|
||||
self.spider = Spider()
|
||||
self.spider.init()
|
||||
|
||||
def test_home_content_exposes_reference_categories(self):
|
||||
content = self.spider.homeContent(False)
|
||||
self.assertEqual(
|
||||
[(item["type_id"], item["type_name"]) for item in content["class"]],
|
||||
[
|
||||
("movie/list_2", "电影"),
|
||||
("dianshiju/list_6", "电视剧"),
|
||||
("dongman/list_13", "动漫"),
|
||||
("zongyi/list_10", "老电影"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_home_video_content_returns_empty_list(self):
|
||||
self.assertEqual(self.spider.homeVideoContent(), {"list": []})
|
||||
|
||||
def test_build_url_and_pan_detection_helpers(self):
|
||||
self.assertEqual(self.spider._build_url("/movie/1.html"), "https://www.rrdynb.com/movie/1.html")
|
||||
self.assertEqual(
|
||||
self.spider._build_url("https://pan.baidu.com/s/demo"),
|
||||
"https://pan.baidu.com/s/demo",
|
||||
)
|
||||
self.assertTrue(self.spider._is_supported_pan_url("https://pan.baidu.com/s/demo"))
|
||||
self.assertTrue(self.spider._is_supported_pan_url("https://pan.quark.cn/s/demo"))
|
||||
self.assertFalse(self.spider._is_supported_pan_url("https://pan.xunlei.com/s/demo"))
|
||||
|
||||
def test_clean_search_title_and_normalize_title(self):
|
||||
self.assertEqual(
|
||||
self.spider._clean_search_title("<font color='red'>剑来</font> 第二季"),
|
||||
"剑来 第二季",
|
||||
)
|
||||
self.assertEqual(self.spider._normalize_title("《繁花》"), "繁花")
|
||||
self.assertEqual(self.spider._normalize_title("「诛仙」特别篇"), "诛仙")
|
||||
self.assertEqual(self.spider._normalize_title("普通标题"), "普通标题")
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from base.spider import Spider as BaseSpider
|
||||
|
||||
sys.path.append("..")
|
||||
|
||||
|
||||
class Spider(BaseSpider):
|
||||
def __init__(self):
|
||||
self.name = "人人电影"
|
||||
self.host = "https://www.rrdynb.com"
|
||||
self.headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/136.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": self.host + "/",
|
||||
}
|
||||
self.categories = [
|
||||
{"type_id": "movie/list_2", "type_name": "电影"},
|
||||
{"type_id": "dianshiju/list_6", "type_name": "电视剧"},
|
||||
{"type_id": "dongman/list_13", "type_name": "动漫"},
|
||||
{"type_id": "zongyi/list_10", "type_name": "老电影"},
|
||||
]
|
||||
self.supported_pan_patterns = [
|
||||
r"pan\.baidu\.com|yun\.baidu\.com",
|
||||
r"pan\.quark\.cn",
|
||||
r"drive\.uc\.cn",
|
||||
r"115\.com",
|
||||
r"123pan\.com|123684\.com|123865\.com|123912\.com",
|
||||
r"cloud\.189\.cn",
|
||||
r"yun\.139\.com",
|
||||
]
|
||||
self.excluded_pan_patterns = [
|
||||
r"pan\.xunlei\.com",
|
||||
r"aliyundrive\.com",
|
||||
r"alipan\.com",
|
||||
]
|
||||
|
||||
def init(self, extend=""):
|
||||
return None
|
||||
|
||||
def getName(self):
|
||||
return self.name
|
||||
|
||||
def homeContent(self, filter):
|
||||
return {"class": self.categories}
|
||||
|
||||
def homeVideoContent(self):
|
||||
return {"list": []}
|
||||
|
||||
def _build_url(self, path):
|
||||
raw = str(path or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
if raw.startswith(("http://", "https://")):
|
||||
return raw
|
||||
return urljoin(self.host + "/", raw)
|
||||
|
||||
def _clean_search_title(self, text):
|
||||
return str(text or "").replace("<font color='red'>", "").replace("</font>", "").strip()
|
||||
|
||||
def _normalize_title(self, raw_title):
|
||||
cleaned = self._clean_search_title(raw_title)
|
||||
matched = re.search(r"《(.*?)》|「(.*?)」", cleaned)
|
||||
if matched:
|
||||
return (matched.group(1) or matched.group(2) or "").strip()
|
||||
return cleaned.strip()
|
||||
|
||||
def _is_supported_pan_url(self, url):
|
||||
raw = str(url or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
if any(re.search(pattern, raw, re.I) for pattern in self.excluded_pan_patterns):
|
||||
return False
|
||||
return any(re.search(pattern, raw, re.I) for pattern in self.supported_pan_patterns)
|
||||
Reference in New Issue
Block a user