diff --git a/py/base/local.py b/py/base/local.py new file mode 100755 index 0000000..a839577 --- /dev/null +++ b/py/base/local.py @@ -0,0 +1,106 @@ +#coding=utf-8 +#!/usr/bin/python +from re import sub +from requests import get +from urllib.parse import unquote +from threading import Thread, Event +from socketserver import ThreadingMixIn +from urllib.parse import urlparse, parse_qs +from importlib.machinery import SourceFileLoader +from http.server import BaseHTTPRequestHandler, HTTPServer + +cache = {} +class ProxyServer(BaseHTTPRequestHandler): + def do_GET(self): + urlParts = urlparse(self.path) + queryQarams = parse_qs(urlParts.query) + do = queryQarams['do'][0] + try: + key = queryQarams['key'][0] + except: + key = '' + try: + value = queryQarams['value'][0] + except: + value = '' + if do == 'set': + cache[key] = value + self.send_response(200) + self.end_headers() + if do == 'get': + self.send_response(200) + self.end_headers() + if key in cache: + self.wfile.write(cache[key].encode()) + elif do == 'delete': + cache.pop(key, None) + self.send_response(200) + self.end_headers() + else: + self.send_response(200) + self.end_headers() + + def do_POST(self): + urlParts = urlparse(self.path) + queryQarams = parse_qs(urlParts.query) + key = queryQarams['key'][0] + try: + contentLength = int(self.headers.get('Content-Length', 0)) + value = self.rfile.read(contentLength).decode().replace('+', ' ') + value = sub(r'value=(.*?)', '', unquote(value)) + except: + value = '' + cache[key] = value + self.send_response(200) + self.end_headers() + +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + """Handle requests in a separate thread.""" + +def serveForever(event): + try: + while not event.is_set(): + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).handle_request() + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close() + except Exception as erro: + print(erro) + finally: + ThreadedHTTPServer(('0.0.0.0', 9978), ProxyServer).server_close() + +def loadFromDisk(fileName): + name = fileName.split('/')[-1].split('.')[0] + sp = SourceFileLoader(name, fileName).load_module().Spider() + return sp + +def run(fileName, proxy=False): + event = Event() + if proxy: + thread = Thread(target=serveForever, args=(event,), name='localProxy') + thread.start() + sp = loadFromDisk(f'../plugin/{fileName}.py') #载入本地脚本 + sp.init('') # 初始化 + try: + # formatJo = sp.decode('') + # formatJo = sp.homeContent(True) # 主页 + # formatJo = sp.homeVideoContent() # 主页视频 + formatJo = sp.searchContentPage("繁花", False, '1') # 搜索 + # formatJo = sp.categoryContent('bilibili', 1, False, {}) # 分类 + # formatJo = sp.detailContent(['']) # 详情 + # formatJo = sp.playerContent("", '', {}) # 播放 + # formatJo = sp.localProxy({}) # 本地代理 + print(formatJo) + except Exception as erro: + print(erro) + finally: + event.set() + try: + get('http://127.0.0.1:9978/cache?do=none') + except: + pass + +if __name__ == '__main__': + """ + run(PY爬虫文件名, 是否启用本地代理) + 再去run函数中修改函数参数 + """ + run('py_bilibilivd', True) \ No newline at end of file diff --git a/py/base/localProxy.py b/py/base/localProxy.py new file mode 100755 index 0000000..fc7eb02 --- /dev/null +++ b/py/base/localProxy.py @@ -0,0 +1,6 @@ +class Proxy: + def getUrl(self, local): + return 'http://127.0.0.1:9978' + + def getPort(self): + return 9978 \ No newline at end of file diff --git a/py/base/spider.py b/py/base/spider.py new file mode 100755 index 0000000..80ab9d5 --- /dev/null +++ b/py/base/spider.py @@ -0,0 +1,151 @@ +import re +import os +import json +import time +import requests +from lxml import etree +from abc import abstractmethod, ABCMeta +from importlib.machinery import SourceFileLoader +from base.localProxy import Proxy + +class Spider(metaclass=ABCMeta): + _instance = None + + def __init__(self): + self.extend = '' + + def __new__(cls, *args, **kwargs): + if cls._instance: + return cls._instance + else: + cls._instance = super().__new__(cls) + return cls._instance + + @abstractmethod + def init(self, extend=""): + pass + + def homeContent(self, filter): + pass + + def homeVideoContent(self): + pass + + def categoryContent(self, tid, pg, filter, extend): + pass + + def detailContent(self, ids): + pass + + def searchContent(self, key, quick, pg="1"): + pass + + def playerContent(self, flag, id, vipFlags): + pass + + def liveContent(self, url): + pass + + def localProxy(self, param): + pass + + def isVideoFormat(self, url): + pass + + def manualVideoCheck(self): + pass + + def action(self, action): + pass + + def destroy(self): + pass + + def getName(self): + pass + + def getDependence(self): + return [] + + def loadSpider(self, name): + return self.loadModule(name).Spider() + + def loadModule(self, name): + path = os.path.join(os.path.join("../plugin"), f'{name}.py') + return SourceFileLoader(name, path).load_module() + + def regStr(self, reg, src, group=1): + m = re.search(reg, src) + src = '' + if m: + src = m.group(group) + return src + + def removeHtmlTags(self, src): + clean = re.compile('<.*?>') + return re.sub(clean, '', src) + + def cleanText(self, src): + clean = re.sub('[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', '', + src) + return clean + + def fetch(self, url, params=None, cookies=None, headers=None, timeout=5, verify=True, stream=False, + allow_redirects=True): + rsp = requests.get(url, params=params, cookies=cookies, headers=headers, timeout=timeout, verify=verify, + stream=stream, allow_redirects=allow_redirects) + rsp.encoding = 'utf-8' + return rsp + + def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True, + stream=False, allow_redirects=True): + rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout, + verify=verify, stream=stream, allow_redirects=allow_redirects) + rsp.encoding = 'utf-8' + return rsp + + def html(self, content): + return etree.HTML(content) + + def str2json(str): + return json.loads(str) + + def json2str(str): + return json.dumps(str, ensure_ascii=False) + + def getProxyUrl(self, local=True): + return f'{Proxy.getUrl(local)}?do=py' + + def log(self, msg): + if isinstance(msg, dict) or isinstance(msg, list): + print(json.dumps(msg, ensure_ascii=False)) + else: + print(f'{msg}') + + def getCache(self, key): + value = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=get&key={key}', timeout=5).text + if len(value) > 0: + if value.startswith('{') and value.endswith('}') or value.startswith('[') and value.endswith(']'): + value = json.loads(value) + if type(value) == dict: + if not 'expiresAt' in value or value['expiresAt'] >= int(time.time()): + return value + else: + self.delCache(key) + return None + return value + else: + return None + + def setCache(self, key, value): + if type(value) in [int, float]: + value = str(value) + if len(value) > 0: + if type(value) == dict or type(value) == list: + value = json.dumps(value, ensure_ascii=False) + r = self.post(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=set&key={key}', data={"value": value}, timeout=5) + return 'succeed' if r.status_code == 200 else 'failed' + + def delCache(self, key): + r = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=del&key={key}', timeout=5) + return 'succeed' if r.status_code == 200 else 'failed' \ No newline at end of file