diff --git a/CNAME b/CNAME index 9152dc455..848110ac2 100755 --- a/CNAME +++ b/CNAME @@ -1 +1 @@ -clun.top \ No newline at end of file +clun.cc \ No newline at end of file diff --git a/LICENSE b/LICENSE index ad5b9ba95..ef9d4a5c1 100755 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 ClunTop +Copyright (c) 2024 cluntop Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/box.json b/box.json index 0bda85224..a51addc23 100755 --- a/box.json +++ b/box.json @@ -1278,29 +1278,8 @@ ] } ], - "proxy": [ - "raw.github*.com", - "github.com", - "youtube.com", - ".*boku.*", - ".*nivod.*", - ".*ulivetv.*", - "missav.ws", - "javmulu.net", - "www.netflav.com" - ], - "hosts": [ - "cache.ott.ystenlive.itv.cmvideo.cn=base-v4-free-mghy.e.cdn.chinamobile.com", - "cache.ott.bestlive.itv.cmvideo.cn=base-v4-free-mghy.e.cdn.chinamobile.com", - "cache.ott.wasulive.itv.cmvideo.cn=base-v4-free-mghy.e.cdn.chinamobile.com", - "cache.ott.fifalive.itv.cmvideo.cn=base-v4-free-mghy.e.cdn.chinamobile.com", - "cache.ott.hnbblive.itv.cmvideo.cn=base-v4-free-mghy.e.cdn.chinamobile.com", - "hlsztemgsplive.miguvideo.com=hlsztemgsplive.miguvideo.com.b.cdn.chinamobile.com", - "push-rtmp-hs-spe-f5.douyincdn.com=source-fcdn-spe-push.s.bytefcdn.com", - "cdn9.163189.xyz=gcore.jsdelivr.net", - "cache.ott.fifalive.itv.cmvideo.cn=cache.ott.fifalive.itv.cmvideo.cn.e.cdn.chinamobile.com", - "studentlive.migucloud.com=base-v4v6-miguvideo.e.cdn.chinamobile.com" - ], + "proxy": [], + "hosts": [], "rules": [ { "host": "*", @@ -1445,30 +1424,6 @@ "script": [ "document.getElementsByClassName('swal-button swal-button--confirm')[0].click()" ] - }, - { - "name": "磁力", - "hosts": [ - "magnet" - ], - "regex": [ - "更多", - "社 區", - "x u u", - "最 新", - "直 播", - "更 新", - "社 区", - "有 趣", - "英皇体育", - "全中文AV在线", - "澳门皇冠赌场", - "哥哥快来", - "美女荷官", - "裸聊", - "新片首发", - "UUE29" - ] } ], "headers": [ @@ -1719,7 +1674,7 @@ ], "disabled_wallpaper": "", "wallpaper": "./img/0.jpg", + "spider": "./jar/xb.jar", "logo": "./img/pg.gif", - "spider": "./pg.jar", "warningText": "1" } \ No newline at end of file diff --git a/jar/pg.jar b/jar/pg.jar deleted file mode 100755 index 7eeb8409e..000000000 Binary files a/jar/pg.jar and /dev/null differ diff --git a/jar/xb.jar b/jar/xb.jar new file mode 100755 index 000000000..8b9203d5d Binary files /dev/null and b/jar/xb.jar differ diff --git a/py/base/local.py b/py/base/local.py new file mode 100755 index 000000000..a839577db --- /dev/null +++ b/py/base/local.py @@ -0,0 +1,106 @@ +#coding=utf-8 +#!/usr/bin/python +from re import sub +from requests import get +from urllib.parse import unquote +from threading import Thread, Event +from socketserver import ThreadingMixIn +from urllib.parse import urlparse, parse_qs +from importlib.machinery import SourceFileLoader +from http.server import BaseHTTPRequestHandler, HTTPServer + +cache = {} +class ProxyServer(BaseHTTPRequestHandler): + def do_GET(self): + urlParts = urlparse(self.path) + queryQarams = parse_qs(urlParts.query) + do = queryQarams['do'][0] + try: + key = queryQarams['key'][0] + except: + key = '' + try: + value = queryQarams['value'][0] + except: + value = '' + if do == 'set': + cache[key] = value + self.send_response(200) + self.end_headers() + if do == 'get': + self.send_response(200) + self.end_headers() + if key in cache: + self.wfile.write(cache[key].encode()) + elif do == 'delete': + cache.pop(key, None) + self.send_response(200) + self.end_headers() + else: + self.send_response(200) + self.end_headers() + + def do_POST(self): + urlParts = urlparse(self.path) + queryQarams = parse_qs(urlParts.query) + key = queryQarams['key'][0] + try: + contentLength = int(self.headers.get('Content-Length', 0)) + value = self.rfile.read(contentLength).decode().replace('+', ' ') + value = sub(r'value=(.*?)', '', unquote(value)) + except: + value = '' + cache[key] = value + self.send_response(200) + self.end_headers() + +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + """Handle requests in a separate thread.""" + +def serveForever(event): + try: + while not event.is_set(): + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).handle_request() + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close() + except Exception as erro: + print(erro) + finally: + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close() + +def loadFromDisk(fileName): + name = fileName.split('/')[-1].split('.')[0] + sp = SourceFileLoader(name, fileName).load_module().Spider() + return sp + +def run(fileName, proxy=False): + event = Event() + if proxy: + thread = Thread(target=serveForever, args=(event,), name='localProxy') + thread.start() + sp = loadFromDisk(f'../plugin/{fileName}.py') #载入本地脚本 + sp.init('') # 初始化 + try: + # formatJo = sp.decode('') + # formatJo = sp.homeContent(True) # 主页 + # formatJo = sp.homeVideoContent() # 主页视频 + formatJo = sp.searchContentPage("繁花", False, '1') # 搜索 + # formatJo = sp.categoryContent('bilibili', 1, False, {}) # 分类 + # formatJo = sp.detailContent(['']) # 详情 + # formatJo = sp.playerContent("", '', {}) # 播放 + # formatJo = sp.localProxy({}) # 本地代理 + print(formatJo) + except Exception as erro: + print(erro) + finally: + event.set() + try: + get('http://127.0.0.1:9978/cache?do=none') + except: + pass + +if __name__ == '__main__': + """ + run(PY爬虫文件名, 是否启用本地代理) + 再去run函数中修改函数参数 + """ + run('py_bilibilivd', True) \ No newline at end of file diff --git a/py/base/localProxy.py b/py/base/localProxy.py new file mode 100755 index 000000000..fc7eb0221 --- /dev/null +++ b/py/base/localProxy.py @@ -0,0 +1,6 @@ +class Proxy: + def getUrl(self, local): + return 'http://127.0.0.1:9978' + + def getPort(self): + return 9978 \ No newline at end of file diff --git a/py/base/spider.py b/py/base/spider.py new file mode 100755 index 000000000..80ab9d5cf --- /dev/null +++ b/py/base/spider.py @@ -0,0 +1,151 @@ +import re +import os +import json +import time +import requests +from lxml import etree +from abc import abstractmethod, ABCMeta +from importlib.machinery import SourceFileLoader +from base.localProxy import Proxy + +class Spider(metaclass=ABCMeta): + _instance = None + + def __init__(self): + self.extend = '' + + def __new__(cls, *args, **kwargs): + if cls._instance: + return cls._instance + else: + cls._instance = super().__new__(cls) + return cls._instance + + @abstractmethod + def init(self, extend=""): + pass + + def homeContent(self, filter): + pass + + def homeVideoContent(self): + pass + + def categoryContent(self, tid, pg, filter, extend): + pass + + def detailContent(self, ids): + pass + + def searchContent(self, key, quick, pg="1"): + pass + + def playerContent(self, flag, id, vipFlags): + pass + + def liveContent(self, url): + pass + + def localProxy(self, param): + pass + + def isVideoFormat(self, url): + pass + + def manualVideoCheck(self): + pass + + def action(self, action): + pass + + def destroy(self): + pass + + def getName(self): + pass + + def getDependence(self): + return [] + + def loadSpider(self, name): + return self.loadModule(name).Spider() + + def loadModule(self, name): + path = os.path.join(os.path.join("../plugin"), f'{name}.py') + return SourceFileLoader(name, path).load_module() + + def regStr(self, reg, src, group=1): + m = re.search(reg, src) + src = '' + if m: + src = m.group(group) + return src + + def removeHtmlTags(self, src): + clean = re.compile('<.*?>') + return re.sub(clean, '', src) + + def cleanText(self, src): + clean = re.sub('[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', '', + src) + return clean + + def fetch(self, url, params=None, cookies=None, headers=None, timeout=5, verify=True, stream=False, + allow_redirects=True): + rsp = requests.get(url, params=params, cookies=cookies, headers=headers, timeout=timeout, verify=verify, + stream=stream, allow_redirects=allow_redirects) + rsp.encoding = 'utf-8' + return rsp + + def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True, + stream=False, allow_redirects=True): + rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout, + verify=verify, stream=stream, allow_redirects=allow_redirects) + rsp.encoding = 'utf-8' + return rsp + + def html(self, content): + return etree.HTML(content) + + def str2json(str): + return json.loads(str) + + def json2str(str): + return json.dumps(str, ensure_ascii=False) + + def getProxyUrl(self, local=True): + return f'{Proxy.getUrl(local)}?do=py' + + def log(self, msg): + if isinstance(msg, dict) or isinstance(msg, list): + print(json.dumps(msg, ensure_ascii=False)) + else: + print(f'{msg}') + + def getCache(self, key): + value = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=get&key={key}', timeout=5).text + if len(value) > 0: + if value.startswith('{') and value.endswith('}') or value.startswith('[') and value.endswith(']'): + value = json.loads(value) + if type(value) == dict: + if not 'expiresAt' in value or value['expiresAt'] >= int(time.time()): + return value + else: + self.delCache(key) + return None + return value + else: + return None + + def setCache(self, key, value): + if type(value) in [int, float]: + value = str(value) + if len(value) > 0: + if type(value) == dict or type(value) == list: + value = json.dumps(value, ensure_ascii=False) + r = self.post(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=set&key={key}', data={"value": value}, timeout=5) + return 'succeed' if r.status_code == 200 else 'failed' + + def delCache(self, key): + r = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=del&key={key}', timeout=5) + return 'succeed' if r.status_code == 200 else 'failed' \ No newline at end of file diff --git a/tcp.sh b/tcp.sh index 4fb9acfe2..69f48a281 100755 --- a/tcp.sh +++ b/tcp.sh @@ -4,11 +4,11 @@ if [ "$(id -u)" -ne 0 ]; then exec sudo "$0" "$@" fi -clun_download() { +clun() { cd ~ || return 1 curl -s https://raw.githubusercontent.com/cluntop/sh/refs/heads/main/tcp.sh -o clun_tcp.sh || return 1 chmod +x clun_tcp.sh || return 1 ./clun_tcp.sh "$1" } -clun_download "$1" +clun "$1"