This commit is contained in:
qist
2026-05-21 18:44:59 +08:00
parent bca37f2b1f
commit d38766eb4a
+15 -5
View File
@@ -16,6 +16,7 @@ class Spider(Spider):
def __init__(self): def __init__(self):
self.host = "https://czzyv.com" self.host = "https://czzyv.com"
self.timeout = 20 self.timeout = 20
self._proxy = ""
self._hosts = [ self._hosts = [
"https://czzyv.com", "https://czzyv.com",
"https://www.cz4k.com", "https://www.cz4k.com",
@@ -29,7 +30,7 @@ class Spider(Spider):
self.session = None self.session = None
self._text_cache = {} self._text_cache = {}
self._text_cache_ttl = 300 self._text_cache_ttl = 300
self._last = {"url": "", "status": 0, "len": 0, "host": "", "ua": "", "err": ""} self._last = {"url": "", "status": 0, "len": 0, "host": "", "ua": "", "proxy": "", "err": ""}
self._api_base = "" self._api_base = ""
self._use_api = False self._use_api = False
self._ua_fallback = "Dalvik/2.1.0 (Linux; U; Android 10)" self._ua_fallback = "Dalvik/2.1.0 (Linux; U; Android 10)"
@@ -56,10 +57,12 @@ class Spider(Spider):
return "厂长资源" return "厂长资源"
def init(self, extend=""): def init(self, extend=""):
proxy = ""
if isinstance(extend, dict): if isinstance(extend, dict):
host = (extend.get("host") or extend.get("site") or "").strip() host = (extend.get("host") or extend.get("site") or "").strip()
if host: if host:
self.host = host.rstrip("/") self.host = host.rstrip("/")
proxy = (extend.get("proxy") or extend.get("http_proxy") or extend.get("https_proxy") or "").strip()
elif isinstance(extend, str) and extend.strip(): elif isinstance(extend, str) and extend.strip():
ext_str = extend.strip() ext_str = extend.strip()
if (ext_str.startswith("{") and ext_str.endswith("}")) or (ext_str.startswith("[") and ext_str.endswith("]")): if (ext_str.startswith("{") and ext_str.endswith("}")) or (ext_str.startswith("[") and ext_str.endswith("]")):
@@ -69,6 +72,7 @@ class Spider(Spider):
host = (ext_obj.get("host") or ext_obj.get("site") or "").strip() host = (ext_obj.get("host") or ext_obj.get("site") or "").strip()
if host: if host:
self.host = host.rstrip("/") self.host = host.rstrip("/")
proxy = (ext_obj.get("proxy") or ext_obj.get("http_proxy") or ext_obj.get("https_proxy") or "").strip()
except Exception: except Exception:
pass pass
elif ext_str.startswith("http"): elif ext_str.startswith("http"):
@@ -78,6 +82,9 @@ class Spider(Spider):
self.headers["Origin"] = self.host self.headers["Origin"] = self.host
self.session = requests.Session() self.session = requests.Session()
self.session.headers.update(self.headers) self.session.headers.update(self.headers)
self._proxy = proxy
if proxy:
self.session.proxies.update({"http": proxy, "https": proxy})
self._choose_host() self._choose_host()
self._detect_api() self._detect_api()
self._warmup() self._warmup()
@@ -167,8 +174,8 @@ class Spider(Spider):
try: try:
r = self.session.get(url, timeout=self.timeout, allow_redirects=True, verify=False) r = self.session.get(url, timeout=self.timeout, allow_redirects=True, verify=False)
break break
except Exception: except Exception as e:
last_exc = "exception" last_exc = f"{type(e).__name__}: {e}"
time.sleep(1) time.sleep(1)
if not r or r.status_code != 200: if not r or r.status_code != 200:
if r and r.status_code in (403, 406, 412): if r and r.status_code in (403, 406, 412):
@@ -184,6 +191,7 @@ class Spider(Spider):
"len": 0, "len": 0,
"host": self.host, "host": self.host,
"ua": (self.session.headers.get("User-Agent") if self.session else ""), "ua": (self.session.headers.get("User-Agent") if self.session else ""),
"proxy": self._proxy,
"err": last_exc or "bad_status", "err": last_exc or "bad_status",
} }
return "" return ""
@@ -204,19 +212,21 @@ class Spider(Spider):
"len": len(text), "len": len(text),
"host": self.host, "host": self.host,
"ua": (self.session.headers.get("User-Agent") if self.session else ""), "ua": (self.session.headers.get("User-Agent") if self.session else ""),
"proxy": self._proxy,
"err": "", "err": "",
} }
if text: if text:
self._text_cache[url] = (now + self._text_cache_ttl, text) self._text_cache[url] = (now + self._text_cache_ttl, text)
return text return text
except Exception: except Exception as e:
self._last = { self._last = {
"url": url, "url": url,
"status": 0, "status": 0,
"len": 0, "len": 0,
"host": self.host, "host": self.host,
"ua": (self.session.headers.get("User-Agent") if self.session else ""), "ua": (self.session.headers.get("User-Agent") if self.session else ""),
"err": "exception", "proxy": self._proxy,
"err": f"{type(e).__name__}: {e}",
} }
return "" return ""