purge history
This commit is contained in:
GitHub Actions
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#coding=utf-8
|
||||
#!/usr/bin/python
|
||||
from re import sub
|
||||
from requests import get
|
||||
from urllib.parse import unquote
|
||||
from threading import Thread, Event
|
||||
from socketserver import ThreadingMixIn
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
cache = {}
|
||||
class ProxyServer(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
urlParts = urlparse(self.path)
|
||||
queryQarams = parse_qs(urlParts.query)
|
||||
do = queryQarams['do'][0]
|
||||
try:
|
||||
key = queryQarams['key'][0]
|
||||
except:
|
||||
key = ''
|
||||
try:
|
||||
value = queryQarams['value'][0]
|
||||
except:
|
||||
value = ''
|
||||
if do == 'set':
|
||||
cache[key] = value
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
if do == 'get':
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
if key in cache:
|
||||
self.wfile.write(cache[key].encode())
|
||||
elif do == 'delete':
|
||||
cache.pop(key, None)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
else:
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
urlParts = urlparse(self.path)
|
||||
queryQarams = parse_qs(urlParts.query)
|
||||
key = queryQarams['key'][0]
|
||||
try:
|
||||
contentLength = int(self.headers.get('Content-Length', 0))
|
||||
value = self.rfile.read(contentLength).decode().replace('+', ' ')
|
||||
value = sub(r'value=(.*?)', '', unquote(value))
|
||||
except:
|
||||
value = ''
|
||||
cache[key] = value
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
|
||||
"""Handle requests in a separate thread."""
|
||||
|
||||
def serveForever(event):
|
||||
try:
|
||||
while not event.is_set():
|
||||
ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).handle_request()
|
||||
ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close()
|
||||
except Exception as erro:
|
||||
print(erro)
|
||||
finally:
|
||||
ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close()
|
||||
|
||||
def loadFromDisk(fileName):
|
||||
name = fileName.split('/')[-1].split('.')[0]
|
||||
sp = SourceFileLoader(name, fileName).load_module().Spider()
|
||||
return sp
|
||||
|
||||
def run(fileName, proxy=False):
|
||||
event = Event()
|
||||
if proxy:
|
||||
thread = Thread(target=serveForever, args=(event,), name='localProxy')
|
||||
thread.start()
|
||||
sp = loadFromDisk(f'../plugin/{fileName}.py') #载入本地脚本
|
||||
sp.init('') # 初始化
|
||||
try:
|
||||
# formatJo = sp.decode('')
|
||||
# formatJo = sp.homeContent(True) # 主页
|
||||
# formatJo = sp.homeVideoContent() # 主页视频
|
||||
formatJo = sp.searchContentPage("繁花", False, '1') # 搜索
|
||||
# formatJo = sp.categoryContent('bilibili', 1, False, {}) # 分类
|
||||
# formatJo = sp.detailContent(['']) # 详情
|
||||
# formatJo = sp.playerContent("", '', {}) # 播放
|
||||
# formatJo = sp.localProxy({}) # 本地代理
|
||||
print(formatJo)
|
||||
except Exception as erro:
|
||||
print(erro)
|
||||
finally:
|
||||
event.set()
|
||||
try:
|
||||
get('http://127.0.0.1:9978/cache?do=none')
|
||||
except:
|
||||
pass
|
||||
|
||||
if __name__ == '__main__':
|
||||
"""
|
||||
run(PY爬虫文件名, 是否启用本地代理)
|
||||
再去run函数中修改函数参数
|
||||
"""
|
||||
run('py_bilibilivd', True)
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
class Proxy:
|
||||
def getUrl(self, local):
|
||||
return 'http://127.0.0.1:9978'
|
||||
|
||||
def getPort(self):
|
||||
return 9978
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
import re
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
from lxml import etree
|
||||
from abc import abstractmethod, ABCMeta
|
||||
from importlib.machinery import SourceFileLoader
|
||||
from base.localProxy import Proxy
|
||||
|
||||
class Spider(metaclass=ABCMeta):
|
||||
_instance = None
|
||||
|
||||
def __init__(self):
|
||||
self.extend = ''
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if cls._instance:
|
||||
return cls._instance
|
||||
else:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
@abstractmethod
|
||||
def init(self, extend=""):
|
||||
pass
|
||||
|
||||
def homeContent(self, filter):
|
||||
pass
|
||||
|
||||
def homeVideoContent(self):
|
||||
pass
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
pass
|
||||
|
||||
def detailContent(self, ids):
|
||||
pass
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
pass
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
pass
|
||||
|
||||
def liveContent(self, url):
|
||||
pass
|
||||
|
||||
def localProxy(self, param):
|
||||
pass
|
||||
|
||||
def isVideoFormat(self, url):
|
||||
pass
|
||||
|
||||
def manualVideoCheck(self):
|
||||
pass
|
||||
|
||||
def action(self, action):
|
||||
pass
|
||||
|
||||
def destroy(self):
|
||||
pass
|
||||
|
||||
def getName(self):
|
||||
pass
|
||||
|
||||
def getDependence(self):
|
||||
return []
|
||||
|
||||
def loadSpider(self, name):
|
||||
return self.loadModule(name).Spider()
|
||||
|
||||
def loadModule(self, name):
|
||||
path = os.path.join(os.path.join("../plugin"), f'{name}.py')
|
||||
return SourceFileLoader(name, path).load_module()
|
||||
|
||||
def regStr(self, reg, src, group=1):
|
||||
m = re.search(reg, src)
|
||||
src = ''
|
||||
if m:
|
||||
src = m.group(group)
|
||||
return src
|
||||
|
||||
def removeHtmlTags(self, src):
|
||||
clean = re.compile('<.*?>')
|
||||
return re.sub(clean, '', src)
|
||||
|
||||
def cleanText(self, src):
|
||||
clean = re.sub('[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', '',
|
||||
src)
|
||||
return clean
|
||||
|
||||
def fetch(self, url, params=None, cookies=None, headers=None, timeout=5, verify=True, stream=False,
|
||||
allow_redirects=True):
|
||||
rsp = requests.get(url, params=params, cookies=cookies, headers=headers, timeout=timeout, verify=verify,
|
||||
stream=stream, allow_redirects=allow_redirects)
|
||||
rsp.encoding = 'utf-8'
|
||||
return rsp
|
||||
|
||||
def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True,
|
||||
stream=False, allow_redirects=True):
|
||||
rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout,
|
||||
verify=verify, stream=stream, allow_redirects=allow_redirects)
|
||||
rsp.encoding = 'utf-8'
|
||||
return rsp
|
||||
|
||||
def html(self, content):
|
||||
return etree.HTML(content)
|
||||
|
||||
def str2json(str):
|
||||
return json.loads(str)
|
||||
|
||||
def json2str(str):
|
||||
return json.dumps(str, ensure_ascii=False)
|
||||
|
||||
def getProxyUrl(self, local=True):
|
||||
return f'{Proxy.getUrl(local)}?do=py'
|
||||
|
||||
def log(self, msg):
|
||||
if isinstance(msg, dict) or isinstance(msg, list):
|
||||
print(json.dumps(msg, ensure_ascii=False))
|
||||
else:
|
||||
print(f'{msg}')
|
||||
|
||||
def getCache(self, key):
|
||||
value = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=get&key={key}', timeout=5).text
|
||||
if len(value) > 0:
|
||||
if value.startswith('{') and value.endswith('}') or value.startswith('[') and value.endswith(']'):
|
||||
value = json.loads(value)
|
||||
if type(value) == dict:
|
||||
if not 'expiresAt' in value or value['expiresAt'] >= int(time.time()):
|
||||
return value
|
||||
else:
|
||||
self.delCache(key)
|
||||
return None
|
||||
return value
|
||||
else:
|
||||
return None
|
||||
|
||||
def setCache(self, key, value):
|
||||
if type(value) in [int, float]:
|
||||
value = str(value)
|
||||
if len(value) > 0:
|
||||
if type(value) == dict or type(value) == list:
|
||||
value = json.dumps(value, ensure_ascii=False)
|
||||
r = self.post(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=set&key={key}', data={"value": value}, timeout=5)
|
||||
return 'succeed' if r.status_code == 200 else 'failed'
|
||||
|
||||
def delCache(self, key):
|
||||
r = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=del&key={key}', timeout=5)
|
||||
return 'succeed' if r.status_code == 200 else 'failed'
|
||||
Reference in New Issue
Block a user