372 lines
13 KiB
Python
372 lines
13 KiB
Python
import sys
|
|
import json
|
|
import requests
|
|
import random
|
|
from datetime import datetime
|
|
from typing import List
|
|
from urllib.parse import urlparse, urlencode
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.retry import Retry
|
|
|
|
sys.path.append('..')
|
|
from base.spider import Spider
|
|
|
|
class CloudSearchSpider(Spider):
|
|
"""网盘资源搜索爬虫"""
|
|
|
|
DEFAULT_BASE_URL = "https://so.252035.xyz"
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
|
"Content-Type": "application/json"
|
|
}
|
|
SEARCH_PAGE_SIZE = 300
|
|
REQUEST_TIMEOUT = 30
|
|
MAX_RETRIES = 3
|
|
BACKOFF_FACTOR = 0.5
|
|
|
|
PAN_CONFIG = {
|
|
'ali': {
|
|
'api_type': 'aliyun',
|
|
'name': '阿里',
|
|
'keywords': ['alipan.com', 'aliyundrive.com'],
|
|
'icon': 'ali.png'
|
|
},
|
|
'quark': {
|
|
'api_type': 'quark',
|
|
'name': '夸克',
|
|
'keywords': ['pan.quark.cn'],
|
|
'icon': 'quark.png'
|
|
},
|
|
'uc': {
|
|
'api_type': 'uc',
|
|
'name': 'UC',
|
|
'keywords': ['drive.uc.cn'],
|
|
'icon': 'uc.png'
|
|
},
|
|
'xunlei': {
|
|
'api_type': 'xunlei',
|
|
'name': '迅雷',
|
|
'keywords': ['xunlei', 'thunder'],
|
|
'icon': 'xunlei.png'
|
|
},
|
|
'a123': {
|
|
'api_type': '123',
|
|
'name': '123',
|
|
'keywords': ['123684.com', '123685.com', '123912.com', '123pan.com', '123pan.cn', '123592.com'],
|
|
'icon': '123.png'
|
|
},
|
|
'a189': {
|
|
'api_type': 'tianyi',
|
|
'name': '天翼',
|
|
'keywords': ['cloud.189.cn'],
|
|
'icon': '189.png'
|
|
},
|
|
'a139': {
|
|
'api_type': 'mobile',
|
|
'name': '移动',
|
|
'keywords': ['caiyun.139.com'],
|
|
'icon': '139.png'
|
|
},
|
|
'a115': {
|
|
'api_type': '115',
|
|
'name': '115',
|
|
'keywords': ['115cdn.com', '115.com', 'anxia.com'],
|
|
'icon': '115.png'
|
|
},
|
|
'baidu': {
|
|
'api_type': 'baidu',
|
|
'name': '百度',
|
|
'keywords': ['baidu'],
|
|
'icon': 'baidu.png'
|
|
},
|
|
'pikpak': {
|
|
'api_type': 'pikpak',
|
|
'name': 'PikPak',
|
|
'keywords': ['pikpak'],
|
|
'icon': 'pikpak.png'
|
|
},
|
|
'magnet': {
|
|
'api_type': 'magnet',
|
|
'name': '磁力',
|
|
'keywords': ['magnet'],
|
|
'icon': 'cili.png'
|
|
},
|
|
'ed2k': {
|
|
'api_type': 'ed2k',
|
|
'name': '电驴',
|
|
'keywords': ['ed2k'],
|
|
'icon': ''
|
|
}
|
|
}
|
|
|
|
REVERSE_PAN_MAP = {v['api_type']: k for k, v in PAN_CONFIG.items()}
|
|
# 优化：从 PAN_CONFIG 自动派生网盘简称，消除数据冗余
|
|
PAN_SHORT_NAMES = {k: v['name'] for k, v in PAN_CONFIG.items()}
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.base_url = self.DEFAULT_BASE_URL
|
|
self.proxy = ''
|
|
self.pan_priority = ''
|
|
self.pan_order = ''
|
|
self.channels = ''
|
|
self.plugins = ''
|
|
self.cloud_types = ''
|
|
self.session = requests.Session()
|
|
self.session.headers.update(self.HEADERS)
|
|
|
|
# 增强重试策略
|
|
retries = Retry(
|
|
total=self.MAX_RETRIES,
|
|
backoff_factor=self.BACKOFF_FACTOR,
|
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
raise_on_status=False
|
|
)
|
|
self.session.mount('http://', HTTPAdapter(max_retries=retries))
|
|
self.session.mount('https://', HTTPAdapter(max_retries=retries))
|
|
|
|
def init(self, extend):
|
|
try:
|
|
extend_dict = json.loads(extend) if extend else {}
|
|
self.base_url = extend_dict.get('server', self.DEFAULT_BASE_URL).rstrip('/')
|
|
self.proxy = extend_dict.get('proxy', '')
|
|
self.pan_priority = extend_dict.get('pan_priority', '')
|
|
self.pan_order = extend_dict.get('pan_order', '')
|
|
# 新增三个搜索过滤参数
|
|
self.channels = extend_dict.get('channels', '')
|
|
self.plugins = extend_dict.get('plugins', '')
|
|
self.cloud_types = extend_dict.get('cloud_types', '')
|
|
except json.JSONDecodeError:
|
|
self._reset_to_defaults()
|
|
|
|
# 代理协议分离
|
|
if self.proxy:
|
|
self.session.proxies = {
|
|
"http": self.proxy,
|
|
"https": self.proxy
|
|
}
|
|
|
|
def _reset_to_defaults(self):
|
|
self.base_url = self.DEFAULT_BASE_URL
|
|
self.proxy = ''
|
|
self.pan_priority = ''
|
|
self.pan_order = ''
|
|
self.channels = ''
|
|
self.plugins = ''
|
|
self.cloud_types = ''
|
|
|
|
def getName(self):
|
|
return "盘搜"
|
|
|
|
def homeContent(self, filter):
|
|
return {'class': [{"type_id": "1", "type_name": "盘搜|聚合搜索"}], 'list': []}
|
|
|
|
def homeVideoContent(self):
|
|
return {}
|
|
|
|
def categoryContent(self, cid, page, filter, ext):
|
|
return {
|
|
'list': [{"vod_id": "1", "vod_name": "请在搜索框中输入关键词搜索", "vod_pic": "", "vod_remarks": "盘搜"}],
|
|
'page': 1, 'pagecount': 1, 'limit': 1, 'total': 1
|
|
}
|
|
|
|
def detailContent(self, did):
|
|
result = {'list': []}
|
|
if not did or not did[0]:
|
|
return result
|
|
|
|
resource_url = did[0]
|
|
pan_type = self._extract_pan_type_from_url(resource_url)
|
|
pan_config = self.PAN_CONFIG.get(pan_type, {})
|
|
pan_name = pan_config.get('name', '网盘资源')
|
|
pan_icon = self._get_icon_url(pan_config.get('icon', ''))
|
|
|
|
result['list'].append({
|
|
"vod_id": resource_url,
|
|
"vod_name": f"{pan_name}资源",
|
|
"vod_pic": pan_icon,
|
|
"vod_play_from": "盘搜",
|
|
"vod_play_url": f"盘搜${resource_url}",
|
|
"vod_content": f"网盘类型: {pan_name}\n资源链接: {resource_url}"
|
|
})
|
|
return result
|
|
|
|
def searchContent(self, key, quick, pg="1"):
|
|
return self._perform_search(key, pg)
|
|
|
|
def searchContentPage(self, key, quick, page):
|
|
return self._perform_search(key, page)
|
|
|
|
def _perform_search(self, keywords, page_str):
|
|
try:
|
|
page = int(page_str)
|
|
except (ValueError, TypeError):
|
|
page = 1
|
|
|
|
result = {'list': [], 'page': page, 'pagecount': 1, 'limit': self.SEARCH_PAGE_SIZE, 'total': 0}
|
|
if not keywords:
|
|
return result
|
|
|
|
try:
|
|
# 构建查询参数
|
|
params = {'kw': keywords}
|
|
if self.channels:
|
|
params['channels'] = self.channels
|
|
if self.plugins:
|
|
params['plugins'] = self.plugins
|
|
if self.cloud_types:
|
|
params['cloud_types'] = self.cloud_types
|
|
|
|
url = f"{self.base_url}/api/search"
|
|
search_response = self.session.get(
|
|
url,
|
|
params=params,
|
|
timeout=self.REQUEST_TIMEOUT
|
|
)
|
|
search_response.raise_for_status()
|
|
|
|
search_data = search_response.json()
|
|
if search_data.get('code') != 0:
|
|
return result
|
|
|
|
all_results = self._parse_and_sort_results(search_data, keywords)
|
|
total_count = len(all_results)
|
|
start_index = (page - 1) * self.SEARCH_PAGE_SIZE
|
|
paged_results = all_results[start_index:start_index + self.SEARCH_PAGE_SIZE]
|
|
|
|
result.update({
|
|
'list': paged_results,
|
|
'total': total_count,
|
|
'pagecount': max(1, (total_count + self.SEARCH_PAGE_SIZE - 1) // self.SEARCH_PAGE_SIZE)
|
|
})
|
|
|
|
except requests.RequestException:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
return result
|
|
|
|
def _parse_and_sort_results(self, data, keywords):
|
|
# 优化：移除此处的硬编码字典，它已在类级别定义
|
|
|
|
if self.pan_order:
|
|
enabled_pan_types = [pan.strip() for pan in self.pan_order.split(',') if pan.strip()]
|
|
use_pan_order = True
|
|
elif self.pan_priority:
|
|
enabled_pan_types = [pan.strip() for pan in self.pan_priority.split(',') if pan.strip()]
|
|
use_pan_order = False
|
|
else:
|
|
enabled_pan_types = []
|
|
use_pan_order = False
|
|
|
|
all_items = []
|
|
all_images = []
|
|
|
|
# 收集所有图片用于兜底随机
|
|
for items in data.get('data', {}).get('merged_by_type', {}).values():
|
|
for item in items:
|
|
if item.get('images'):
|
|
all_images.extend(item['images'])
|
|
|
|
# 设置随机种子，确保同关键词结果一致
|
|
random.seed(hash(keywords))
|
|
|
|
for cloud_type, items in data.get('data', {}).get('merged_by_type', {}).items():
|
|
pan_type = self.REVERSE_PAN_MAP.get(cloud_type, cloud_type)
|
|
if enabled_pan_types and pan_type not in enabled_pan_types:
|
|
continue
|
|
|
|
for item in items:
|
|
url = item.get('url')
|
|
if not url:
|
|
continue
|
|
|
|
pan_config = self.PAN_CONFIG.get(pan_type, {})
|
|
pan_icon = self._get_icon_url(pan_config.get('icon', ''))
|
|
# 优化：使用 self.PAN_SHORT_NAMES 替代本地字典
|
|
pan_short = self.PAN_SHORT_NAMES.get(pan_type, '网盘')
|
|
|
|
# 简化图片逻辑：优先自带 → 全局随机 → 图标
|
|
vod_pic = self._get_best_image(item, all_images, pan_icon)
|
|
|
|
dt_obj = self._to_datetime(item.get('datetime'))
|
|
time_str = dt_obj.strftime("%m-%d") if dt_obj else ""
|
|
|
|
source = item.get('source', '盘搜')
|
|
remarks = f"{pan_short}:{time_str}|{source}" if time_str else f"{pan_short}|{source}"
|
|
|
|
all_items.append({
|
|
"vod_id": url,
|
|
"vod_name": item.get('note', ''),
|
|
"vod_pic": vod_pic,
|
|
"vod_remarks": remarks,
|
|
"_timestamp": dt_obj.timestamp() if dt_obj else 0,
|
|
"_pan_type": pan_type
|
|
})
|
|
|
|
# 排序逻辑
|
|
if use_pan_order:
|
|
all_items.sort(key=lambda x: -x['_timestamp'])
|
|
elif enabled_pan_types:
|
|
pan_priority_order = {pan: idx for idx, pan in enumerate(enabled_pan_types)}
|
|
all_items.sort(key=lambda x: (pan_priority_order.get(x['_pan_type'], len(enabled_pan_types)), -x['_timestamp']))
|
|
else:
|
|
all_items.sort(key=lambda x: -x['_timestamp'])
|
|
|
|
# 清理临时字段
|
|
for item in all_items:
|
|
item.pop('_timestamp', None)
|
|
item.pop('_pan_type', None)
|
|
|
|
return all_items
|
|
|
|
def _get_best_image(self, item, all_images, pan_icon):
|
|
if item.get('images') and item['images']:
|
|
return item['images'][0]
|
|
if all_images:
|
|
return random.choice(all_images)
|
|
return pan_icon
|
|
|
|
def playerContent(self, flag, pid, vipFlags):
|
|
result = {"parse": 0, "header": self.HEADERS, "url": ""}
|
|
if not pid:
|
|
return result
|
|
|
|
url = pid.strip()
|
|
if not url.startswith('push:'):
|
|
if not url.startswith(('http://', 'https://')):
|
|
url = f'https://{url}'
|
|
result['url'] = f"push:{url}"
|
|
else:
|
|
result['url'] = url
|
|
|
|
return result
|
|
|
|
def _to_datetime(self, time_str):
|
|
if not time_str or time_str == "0001-01-01T00:00:00Z":
|
|
return None
|
|
try:
|
|
time_str_clean = time_str.replace('Z', '+00:00')
|
|
return datetime.fromisoformat(time_str_clean)
|
|
except (ValueError, TypeError):
|
|
try:
|
|
return datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S')
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
def _extract_pan_type_from_url(self, url):
|
|
if not url:
|
|
return "unknown"
|
|
url_lower = urlparse(url).netloc.lower() or url.lower()
|
|
for pan_type, config in self.PAN_CONFIG.items():
|
|
if any(keyword in url_lower for keyword in config['keywords']):
|
|
return pan_type
|
|
return "unknown"
|
|
|
|
def _get_icon_url(self, icon_name):
|
|
if not icon_name:
|
|
return ""
|
|
return f"http://127.0.0.1:9978/file/Download/lib/icon/{icon_name}"
|
|
|
|
Spider = CloudSearchSpider |