feat: add 两个BT list parsing
This commit is contained in:
Harold
@@ -0,0 +1,139 @@
|
||||
import unittest
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE = SourceFileLoader("lianggebt_spider", str(ROOT / "两个BT.py")).load_module()
|
||||
Spider = MODULE.Spider
|
||||
|
||||
|
||||
class TestLiangGeBTSpider(unittest.TestCase):
|
||||
def setUp(self):
|
||||
Spider._instance = None
|
||||
self.spider = Spider()
|
||||
self.spider.init()
|
||||
|
||||
def test_home_content_exposes_expected_categories(self):
|
||||
content = self.spider.homeContent(False)
|
||||
self.assertEqual(
|
||||
[item["type_id"] for item in content["class"]],
|
||||
[
|
||||
"zgjun",
|
||||
"meiju",
|
||||
"jpsrtv",
|
||||
"movie_bt_tags/xiju",
|
||||
"movie_bt_tags/aiqing",
|
||||
"movie_bt_tags/adt",
|
||||
"movie_bt_tags/at",
|
||||
"movie_bt_tags/donghua",
|
||||
"movie_bt_tags/qihuan",
|
||||
"movie_bt_tags/xuanni",
|
||||
"movie_bt_tags/kehuan",
|
||||
"movie_bt_tags/juqing",
|
||||
"movie_bt_tags/kongbu",
|
||||
"gf",
|
||||
],
|
||||
)
|
||||
self.assertNotIn("filters", content)
|
||||
|
||||
def test_extract_cards_supports_short_ids_and_relative_images(self):
|
||||
html = """
|
||||
<ul class="item">
|
||||
<li>
|
||||
<a href="/movie/123.html" title="示例影片">
|
||||
<img data-original="/cover.jpg" />
|
||||
</a>
|
||||
<h3><a href="/movie/123.html">示例影片</a></h3>
|
||||
<span class="status">更新至10集</span>
|
||||
</li>
|
||||
</ul>
|
||||
"""
|
||||
cards = self.spider._extract_cards(html)
|
||||
self.assertEqual(
|
||||
cards,
|
||||
[
|
||||
{
|
||||
"vod_id": "123",
|
||||
"vod_name": "示例影片",
|
||||
"vod_pic": "https://www.bttwoo.com/cover.jpg",
|
||||
"vod_remarks": "更新至10集",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_home_video_content_requests_home_page(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<li>
|
||||
<a href="/movie/200.html" title="首页影片">
|
||||
<img src="/home.jpg" />
|
||||
</a>
|
||||
<h3>首页影片</h3>
|
||||
</li>
|
||||
"""
|
||||
result = self.spider.homeVideoContent()
|
||||
self.assertEqual(mock_request_html.call_args.args[0], "https://www.bttwoo.com")
|
||||
self.assertEqual(result["list"][0]["vod_id"], "200")
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_category_content_builds_expected_urls(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<li>
|
||||
<a href="/movie/301.html" title="分类影片">
|
||||
<img src="/cate.jpg" />
|
||||
</a>
|
||||
<h3>分类影片</h3>
|
||||
<span class="rating">HD</span>
|
||||
</li>
|
||||
"""
|
||||
page_one = self.spider.categoryContent("meiju", "1", False, {})
|
||||
page_three = self.spider.categoryContent("movie_bt_tags/xiju", "3", False, {})
|
||||
self.assertEqual(mock_request_html.call_args_list[0].args[0], "https://www.bttwoo.com/meiju")
|
||||
self.assertEqual(
|
||||
mock_request_html.call_args_list[1].args[0],
|
||||
"https://www.bttwoo.com/movie_bt_tags/xiju?paged=3",
|
||||
)
|
||||
self.assertEqual(page_one["page"], 1)
|
||||
self.assertEqual(page_one["list"][0]["vod_name"], "分类影片")
|
||||
self.assertNotIn("pagecount", page_one)
|
||||
self.assertEqual(page_three["page"], 3)
|
||||
|
||||
@patch.object(Spider, "_request_html")
|
||||
def test_search_content_builds_query_and_filters_irrelevant_results(self, mock_request_html):
|
||||
mock_request_html.return_value = """
|
||||
<li>
|
||||
<a href="/movie/401.html" title="繁花">
|
||||
<img src="/match.jpg" />
|
||||
</a>
|
||||
<h3>繁花</h3>
|
||||
</li>
|
||||
<li>
|
||||
<a href="/movie/402.html" title="无关结果">
|
||||
<img src="/other.jpg" />
|
||||
</a>
|
||||
<h3>无关结果</h3>
|
||||
</li>
|
||||
"""
|
||||
result = self.spider.searchContent("繁花", False, "2")
|
||||
self.assertEqual(
|
||||
mock_request_html.call_args.args[0],
|
||||
"https://www.bttwoo.com/xssssearch?q=%E7%B9%81%E8%8A%B1&p=2",
|
||||
)
|
||||
self.assertEqual(result["page"], 2)
|
||||
self.assertEqual(
|
||||
result["list"],
|
||||
[
|
||||
{
|
||||
"vod_id": "401",
|
||||
"vod_name": "繁花",
|
||||
"vod_pic": "https://www.bttwoo.com/match.jpg",
|
||||
"vod_remarks": "",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
from lxml import html as lxml_html
|
||||
|
||||
from base.spider import Spider as BaseSpider
|
||||
|
||||
sys.path.append("..")
|
||||
|
||||
|
||||
class Spider(BaseSpider):
|
||||
def __init__(self):
|
||||
self.name = "两个BT"
|
||||
self.host = "https://www.bttwoo.com"
|
||||
self.headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Referer": self.host + "/",
|
||||
}
|
||||
self.classes = [
|
||||
{"type_id": "zgjun", "type_name": "国产剧"},
|
||||
{"type_id": "meiju", "type_name": "美剧"},
|
||||
{"type_id": "jpsrtv", "type_name": "日韩剧"},
|
||||
{"type_id": "movie_bt_tags/xiju", "type_name": "喜剧"},
|
||||
{"type_id": "movie_bt_tags/aiqing", "type_name": "爱情"},
|
||||
{"type_id": "movie_bt_tags/adt", "type_name": "冒险"},
|
||||
{"type_id": "movie_bt_tags/at", "type_name": "动作"},
|
||||
{"type_id": "movie_bt_tags/donghua", "type_name": "动画"},
|
||||
{"type_id": "movie_bt_tags/qihuan", "type_name": "奇幻"},
|
||||
{"type_id": "movie_bt_tags/xuanni", "type_name": "悬疑"},
|
||||
{"type_id": "movie_bt_tags/kehuan", "type_name": "科幻"},
|
||||
{"type_id": "movie_bt_tags/juqing", "type_name": "剧情"},
|
||||
{"type_id": "movie_bt_tags/kongbu", "type_name": "恐怖"},
|
||||
{"type_id": "gf", "type_name": "高分电影"},
|
||||
]
|
||||
|
||||
def init(self, extend=""):
|
||||
return None
|
||||
|
||||
def getName(self):
|
||||
return self.name
|
||||
|
||||
def homeContent(self, filter):
|
||||
return {"class": self.classes}
|
||||
|
||||
def homeVideoContent(self):
|
||||
return {"list": self._extract_cards(self._request_html(self.host))}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
page = max(1, self._to_int(pg, 1))
|
||||
items = self._extract_cards(self._request_html(self._build_category_url(tid, page)))
|
||||
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
page = max(1, self._to_int(pg, 1))
|
||||
keyword = self._clean_text(key)
|
||||
if not keyword:
|
||||
return {"page": page, "limit": 0, "total": 0, "list": []}
|
||||
url = self.host + f"/xssssearch?q={quote(keyword)}"
|
||||
if page > 1:
|
||||
url += f"&p={page}"
|
||||
items = self._extract_cards(self._request_html(url), keyword=keyword)
|
||||
return {"page": page, "limit": len(items), "total": len(items), "list": items}
|
||||
|
||||
def detailContent(self, ids):
|
||||
return {"list": []}
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
return {"parse": 1, "jx": 1, "playUrl": "", "url": "", "header": {}}
|
||||
|
||||
def _request_html(self, url, referer=None):
|
||||
headers = dict(self.headers)
|
||||
headers["Referer"] = referer or self.headers["Referer"]
|
||||
try:
|
||||
response = self.fetch(url, headers=headers, timeout=10, verify=False)
|
||||
except Exception:
|
||||
return ""
|
||||
if response.status_code != 200:
|
||||
return ""
|
||||
return str(response.text or "")
|
||||
|
||||
def _build_category_url(self, tid, page):
|
||||
path = str(tid or "").strip().lstrip("/")
|
||||
url = self.host + "/" + path
|
||||
if page > 1:
|
||||
url += f"?paged={page}"
|
||||
return url
|
||||
|
||||
def _extract_cards(self, html, keyword=None):
|
||||
root = self._parse_html(html)
|
||||
if root is None:
|
||||
return []
|
||||
results = []
|
||||
seen = set()
|
||||
for node in root.xpath("//li[.//a[contains(@href,'/movie/')]]"):
|
||||
href = self._first_attr(node, ".//a[contains(@href,'/movie/')][1]", "href")
|
||||
vod_id = self._extract_vod_id(href)
|
||||
title = (
|
||||
self._first_text(node, ".//h3//a[1]")
|
||||
or self._first_text(node, ".//h3[1]")
|
||||
or self._clean_text(self._first_attr(node, ".//a[@title][1]", "title"))
|
||||
or self._first_text(node, ".//*[contains(@class,'title')][1]")
|
||||
or self._first_text(node, ".//*[contains(@class,'name')][1]")
|
||||
)
|
||||
if not vod_id or not title or vod_id in seen:
|
||||
continue
|
||||
if keyword and not self._is_relevant_search_result(title, keyword):
|
||||
continue
|
||||
pic = (
|
||||
self._first_attr(node, ".//img[@data-original][1]", "data-original")
|
||||
or self._first_attr(node, ".//img[@data-src][1]", "data-src")
|
||||
or self._first_attr(node, ".//img[@src][1]", "src")
|
||||
)
|
||||
remarks = (
|
||||
self._first_text(node, ".//*[contains(@class,'rating')][1]")
|
||||
or self._first_text(node, ".//*[contains(@class,'status')][1]")
|
||||
)
|
||||
seen.add(vod_id)
|
||||
results.append(
|
||||
{
|
||||
"vod_id": vod_id,
|
||||
"vod_name": title,
|
||||
"vod_pic": self._abs_url(pic),
|
||||
"vod_remarks": remarks,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
def _is_relevant_search_result(self, title, keyword):
|
||||
title_text = self._clean_text(title).lower()
|
||||
keyword_text = self._clean_text(keyword).lower()
|
||||
if not title_text or not keyword_text:
|
||||
return False
|
||||
if keyword_text in title_text:
|
||||
return True
|
||||
if len(keyword_text) <= 2:
|
||||
return False
|
||||
key_chars = set(keyword_text.replace(" ", ""))
|
||||
title_chars = set(title_text.replace(" ", ""))
|
||||
if not key_chars:
|
||||
return False
|
||||
return len(key_chars & title_chars) / float(len(key_chars)) >= 0.6
|
||||
|
||||
def _extract_vod_id(self, href):
|
||||
matched = re.search(r"/movie/(\d+)\.html", str(href or "").strip())
|
||||
return matched.group(1) if matched else ""
|
||||
|
||||
def _abs_url(self, value):
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
if raw.startswith(("http://", "https://")):
|
||||
return raw
|
||||
if raw.startswith("//"):
|
||||
return "https:" + raw
|
||||
return urljoin(self.host + "/", raw.lstrip("/"))
|
||||
|
||||
def _parse_html(self, text):
|
||||
body = str(text or "").strip()
|
||||
if not body:
|
||||
return None
|
||||
try:
|
||||
return lxml_html.fromstring(body)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _clean_text(self, text):
|
||||
return re.sub(r"\s+", " ", str(text or "").replace("\xa0", " ")).strip()
|
||||
|
||||
def _first_attr(self, node, xpath, attr):
|
||||
if node is None:
|
||||
return ""
|
||||
for item in node.xpath(xpath):
|
||||
value = str(item.get(attr) or "").strip()
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
def _first_text(self, node, xpath):
|
||||
if node is None:
|
||||
return ""
|
||||
for item in node.xpath(xpath):
|
||||
text = self._clean_text(item.text_content())
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
def _to_int(self, value, default=0):
|
||||
try:
|
||||
return int(str(value))
|
||||
except Exception:
|
||||
return default
|
||||
Reference in New Issue
Block a user