Sync all projects
This commit is contained in:
github-actions[bot]
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re, json, requests
|
||||
from urllib.parse import quote
|
||||
try:
|
||||
from lxml import etree
|
||||
except Exception:
|
||||
etree = None
|
||||
from base.spider import Spider
|
||||
|
||||
BADIMG = ("logo", "loading", "nopic", "no-pic", "static/")
|
||||
|
||||
|
||||
class Spider(Spider):
|
||||
def getName(self): return "低端影视"
|
||||
|
||||
def init(self, extend=""):
|
||||
self.host = "https://www.ddys.run"
|
||||
try: ext = json.loads(extend) if str(extend).strip().startswith("{") else {}
|
||||
except Exception: ext = {}
|
||||
if ext.get("host"): self.host = ext["host"].rstrip("/")
|
||||
self.headers = {"User-Agent": ext.get("ua", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"), "Referer": self.host + "/", "Accept-Language": "zh-CN,zh;q=0.9", "Cookie": ext.get("cookie", "pageLogin=ok")}
|
||||
self.listFmt = ext.get("listFmt", "")
|
||||
self.categories = [{"type_id": "dianying", "type_name": "电影"}, {"type_id": "juji", "type_name": "剧集"}, {"type_id": "dongman", "type_name": "动漫"}]
|
||||
|
||||
def _fix(self, u):
|
||||
if not u: return ""
|
||||
if u.startswith("//"): return "https:" + u
|
||||
if u.startswith("/"): return self.host + u
|
||||
return u
|
||||
|
||||
def _get(self, path):
|
||||
url = path if path.startswith("http") else self.host + path
|
||||
try:
|
||||
r = requests.get(url, headers=self.headers, timeout=15); r.encoding = "utf-8"
|
||||
if r.status_code >= 400: print("[WARN] status=%s url=%s" % (r.status_code, url)); return None
|
||||
return r.text
|
||||
except requests.exceptions.Timeout: print("[ERROR] 请求超时: %s" % url)
|
||||
except requests.exceptions.ConnectionError: print("[ERROR] 连接错误: %s" % url)
|
||||
except Exception as e: print("[ERROR] 请求失败: %s, %s" % (url, str(e)))
|
||||
return None
|
||||
|
||||
def _post(self, path, data):
|
||||
try:
|
||||
r = requests.post(self.host + path, data=data, headers=self.headers, timeout=15); r.encoding = "utf-8"; return r.text
|
||||
except Exception as e: print("[ERROR] POST失败: %s, %s" % (path, str(e))); return None
|
||||
|
||||
def _img(self, node):
|
||||
for at in ("data-original", "data-src", "data-echo", "src"):
|
||||
for v in node.xpath('.//@%s' % at):
|
||||
if v and re.search(r'\.(jpg|jpeg|png|webp|avif)', v) and not any(b in v for b in BADIMG): return v
|
||||
return ""
|
||||
|
||||
def _parse_list(self, html):
|
||||
if not html: return []
|
||||
if etree is None:
|
||||
print("[WARN] lxml 不可用，降级为正则解析")
|
||||
out, seen = [], set()
|
||||
for slug, title in re.findall(r'href="/video/([^"]+)\.html"[^>]*title="([^"]*)"', html):
|
||||
if slug in seen: continue
|
||||
seen.add(slug); out.append({"vod_id": slug, "vod_name": title, "vod_pic": ""})
|
||||
return out
|
||||
tree = etree.HTML(html); results, seen = [], set()
|
||||
items = tree.xpath('//a[contains(@class,"stui-vodlist__thumb") and contains(@href,"/video/")]') or tree.xpath('//div[contains(@class,"stui-vodlist__box")]//a[contains(@href,"/video/")]') or tree.xpath('//a[contains(@href,"/video/") and @title]')
|
||||
for it in items:
|
||||
try:
|
||||
m = re.search(r'/video/([^/.]+)\.html', it.get("href", ""))
|
||||
if not m or m.group(1) in seen: continue
|
||||
name = (it.get("title") or "".join(it.xpath('.//img/@alt')[:1])).strip()
|
||||
if not name: continue
|
||||
seen.add(m.group(1))
|
||||
note = "".join(it.xpath('.//span[contains(@class,"pic-text")]//text()')).strip()
|
||||
results.append({"vod_id": m.group(1), "vod_name": name, "vod_pic": self._fix(self._img(it)), "vod_remarks": note})
|
||||
except Exception: continue
|
||||
return results
|
||||
|
||||
def _dcount(self, sep):
|
||||
try: return len(re.findall(r'/list/[a-z0-9]+(%s*)\.html' % re.escape(sep), self._home)[0])
|
||||
except Exception: return 11
|
||||
|
||||
def homeContent(self, filter):
|
||||
self._home = self._get("/") or ""
|
||||
fl = {}
|
||||
return {"class": self.categories, "list": self._parse_list(self._home), "filters": fl}
|
||||
|
||||
def homeVideoContent(self): return {"list": self._parse_list(self._get("/"))}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
pg = str(pg or "1")
|
||||
cands = ["/list/%s-%s----------.html" % (tid, pg), "/list/%s----------%s-.html" % (tid, pg), "/list/%s-----------.html?page=%s" % (tid, pg)]
|
||||
if self.listFmt: cands = [self.listFmt % (tid, pg)]
|
||||
lst = []
|
||||
for c in cands:
|
||||
lst = self._parse_list(self._get(c))
|
||||
if lst:
|
||||
self.listFmt = c.replace(tid, "%s", 1).replace("-%s-" % pg, "-%s-", 1).replace("%s-" % pg + "-", "%s--", 1) if not self.listFmt else self.listFmt
|
||||
self.listFmt = c.split(tid, 1)[0] + "%s" + c.split(tid, 1)[1].replace(pg, "%s", 1)
|
||||
break
|
||||
return {"page": int(pg), "pagecount": int(pg) + 1 if lst else int(pg), "limit": 48, "total": 999999, "list": lst}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
pg = str(pg or "1")
|
||||
lst = self._parse_list(self._get("/search/-------------.html?wd=%s&page=%s" % (quote(key), pg)))
|
||||
if not lst:
|
||||
lst = self._parse_list(self._post("/search/-------------.html", {"wd": key}))
|
||||
if not lst:
|
||||
lst = self._parse_list(self._get("/vodsearch/-------------.html?wd=%s" % quote(key)))
|
||||
return {"list": lst, "page": int(pg)}
|
||||
|
||||
def detailContent(self, ids):
|
||||
slug = str(ids[0])
|
||||
html = self._get("/video/%s.html" % slug)
|
||||
if not html or etree is None: return {"list": []}
|
||||
tree = etree.HTML(html)
|
||||
text = "\n".join(x.strip() for x in tree.xpath('//text()') if x.strip())
|
||||
pic = self._img(tree) or "".join(tree.xpath('//img[contains(@class,"lazyload")]/@data-original')[:1])
|
||||
vod = {"vod_id": slug,
|
||||
"vod_name": ("".join(tree.xpath('//h1//text()')).strip() or "".join(tree.xpath('//div[contains(@class,"stui-content__detail")]//h1/text()')).strip()),
|
||||
"vod_pic": self._fix(pic),
|
||||
"vod_year": self._field(text, "年份"), "vod_area": self._field(text, "地区"),
|
||||
"type_name": self._field(text, "分类") or self._field(text, "类型"),
|
||||
"vod_lang": self._field(text, "语言"),
|
||||
"vod_actor": self._field(text, "主演"), "vod_director": self._field(text, "导演"),
|
||||
"vod_remarks": self._field(text, "备注") or self._field(text, "状态"),
|
||||
"vod_content": re.sub(r'\s+', ' ', "".join(tree.xpath('//span[contains(@class,"detail-content")]//text() | //div[contains(@class,"detail")]//span[contains(@class,"content")]//text()'))).strip()}
|
||||
froms, urls = [], []
|
||||
heads = tree.xpath('//div[contains(@class,"stui-pannel__head") or contains(@class,"playlist")]//h3/text() | //div[contains(@class,"stui-vodlist__head")]//h3/text()')
|
||||
lists = tree.xpath('//ul[contains(@class,"stui-content__playlist") or contains(@class,"stui-content__list") or contains(@class,"content__playlist")]')
|
||||
for i, ul in enumerate(lists):
|
||||
eps = []
|
||||
for a in ul.xpath('.//a'):
|
||||
nm = ("".join(a.xpath('.//text()')).strip() or a.get("title", "")).strip()
|
||||
lk = a.get("href", "")
|
||||
if not nm or not lk: continue
|
||||
eps.append(nm.replace("$", "").replace("#", "") + "$" + self._fix(lk))
|
||||
if eps:
|
||||
froms.append(heads[i].strip() if i < len(heads) else "线路%d" % (i + 1))
|
||||
urls.append("#".join(eps))
|
||||
vod["vod_play_from"] = "$$$".join(froms) if froms else "低端影视"
|
||||
vod["vod_play_url"] = "$$$".join(urls) if urls else ("正片$%s/video/%s.html" % (self.host, slug))
|
||||
return {"list": [vod]}
|
||||
|
||||
def _field(self, text, key):
|
||||
m = re.search(r'%s\s*[:：]\s*([^\n]{1,200})' % key, text)
|
||||
return m.group(1).strip(" \u3000|/") if m else ""
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
pid = id if id.startswith("http") else self._fix(id)
|
||||
html = self._get(pid) or ""
|
||||
url = ""
|
||||
for p in [r'var\s+player_\w*\s*=\s*(\{.*?\})\s*[<;]', r'"url"\s*:\s*"([^"]+)"', r'var\s+now\s*=\s*["\']([^"\']+)["\']', r'url:\s*["\']([^"\']+\.(?:m3u8|mp4)[^"\']*)["\']', r'(https?://[^\s"\'\\<>]+\.(?:m3u8|mp4)[^\s"\'\\<>]*)']:
|
||||
m = re.search(p, html.replace("\\/", "/"), re.S)
|
||||
if not m: continue
|
||||
val = m.group(1)
|
||||
if val.startswith("{"):
|
||||
try: val = json.loads(val).get("url", "")
|
||||
except Exception:
|
||||
m2 = re.search(r'"url"\s*:\s*"([^"]+)"', val); val = m2.group(1).replace("\\/", "/") if m2 else ""
|
||||
if val: url = self._fix(val); break
|
||||
if not url: return {"parse": 1, "url": pid, "header": self.headers}
|
||||
return {"parse": 0, "url": url, "header": {"User-Agent": self.headers["User-Agent"], "Referer": self.host + "/"}}
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re, json, requests
|
||||
from urllib.parse import quote
|
||||
try:
|
||||
from lxml import etree
|
||||
except Exception:
|
||||
etree = None
|
||||
from base.spider import Spider
|
||||
|
||||
|
||||
class Spider(Spider):
|
||||
def getName(self): return "袋鼠影视"
|
||||
|
||||
def init(self, extend=""):
|
||||
self.host = "https://dsystv.com"
|
||||
try: ext = json.loads(extend) if str(extend).strip().startswith("{") else {}
|
||||
except Exception: ext = {}
|
||||
if ext.get("host"): self.host = ext["host"].rstrip("/")
|
||||
self.headers = {"User-Agent": ext.get("ua", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"), "Referer": self.host + "/", "Accept-Language": "zh-CN,zh;q=0.9"}
|
||||
self.categories = [{"type_id": "1", "type_name": "电影"}, {"type_id": "2", "type_name": "电视剧"}, {"type_id": "3", "type_name": "综艺"}, {"type_id": "4", "type_name": "动漫"}, {"type_id": "44", "type_name": "短剧"}]
|
||||
self.subs = {"1": [["全部", "1"], ["动作片", "5"], ["喜剧片", "10"], ["科幻片", "7"], ["恐怖片", "8"], ["战争片", "9"], ["动画片", "41"], ["剧情片", "12"], ["爱情片", "6"], ["纪录片", "11"]],
|
||||
"2": [["全部", "2"], ["国产剧", "13"], ["港台剧", "14"], ["欧美剧", "15"], ["日韩剧", "16"], ["海外剧", "42"]]}
|
||||
self.orders = [["默认", ""], ["最近更新", "time"], ["总排行", "hit"], ["月排行", "monthhit"], ["周排行", "weekhit"], ["豆瓣评分", "douban"]]
|
||||
|
||||
def _fix(self, u):
|
||||
if not u: return ""
|
||||
if u.startswith("//"): return "https:" + u
|
||||
if u.startswith("/"): return self.host + u
|
||||
return u
|
||||
|
||||
def _get(self, path):
|
||||
url = path if path.startswith("http") else self.host + path
|
||||
try:
|
||||
r = requests.get(url, headers=self.headers, timeout=15); r.encoding = "utf-8"
|
||||
if r.status_code >= 400: print("[WARN] status=%s url=%s" % (r.status_code, url))
|
||||
return r.text
|
||||
except requests.exceptions.Timeout: print("[ERROR] 请求超时: %s" % url)
|
||||
except requests.exceptions.ConnectionError: print("[ERROR] 连接错误: %s" % url)
|
||||
except Exception as e: print("[ERROR] 请求失败: %s, %s" % (url, str(e)))
|
||||
return None
|
||||
|
||||
def _post(self, path, data):
|
||||
try:
|
||||
r = requests.post(self.host + path, data=data, headers=self.headers, timeout=15); r.encoding = "utf-8"; return r.text
|
||||
except Exception as e: print("[ERROR] POST失败: %s, %s" % (path, str(e))); return None
|
||||
|
||||
def _parse_list(self, html):
|
||||
if not html: return []
|
||||
if etree is None:
|
||||
print("[WARN] lxml 不可用，降级为正则解析")
|
||||
out, seen = [], set()
|
||||
for vid, title in re.findall(r'href="[^"]*?/movie/index(\d+)\.html"[^>]*?title="([^"]*)"', html):
|
||||
if vid in seen: continue
|
||||
seen.add(vid); out.append({"vod_id": vid, "vod_name": title, "vod_pic": ""})
|
||||
return out
|
||||
tree = etree.HTML(html); results, seen = [], set()
|
||||
items = tree.xpath('//a[contains(@class,"videopic") and contains(@href,"/movie/index")]') + tree.xpath('//div[contains(@class,"item")]//a[contains(@href,"/movie/index") and .//img]') + tree.xpath('//a[contains(@href,"/movie/index") and .//img]')
|
||||
for it in items:
|
||||
try:
|
||||
m = re.search(r'/movie/index(\d+)\.html', it.get("href", ""))
|
||||
if not m or m.group(1) in seen: continue
|
||||
name = (it.get("title") or "".join(it.xpath('.//img/@alt')[:1])).strip()
|
||||
if not name: continue
|
||||
seen.add(m.group(1))
|
||||
pic = ""
|
||||
for at in ("data-original", "data-src", "data-echo", "data-lazy", "src"):
|
||||
cand = it.xpath('.//img/@%s' % at)
|
||||
if cand and "load.gif" not in cand[0] and "loading" not in cand[0]: pic = cand[0]; break
|
||||
note = " ".join(x.strip() for x in it.xpath('.//span//text()') if x.strip())
|
||||
results.append({"vod_id": m.group(1), "vod_name": name, "vod_pic": self._fix(pic), "vod_remarks": note[:40]})
|
||||
except Exception: continue
|
||||
return results
|
||||
|
||||
def _parse_playlist(self, tree, vid):
|
||||
groups = {}
|
||||
for a in tree.xpath('//a[contains(@href,"/play/")]'):
|
||||
m = re.search(r'/play/%s-(\d+)-(\d+)\.html' % vid, a.get("href", ""))
|
||||
if not m: continue
|
||||
s, e = int(m.group(1)), int(m.group(2))
|
||||
nm = (a.get("title") or "".join(a.xpath('.//text()'))).strip()
|
||||
groups.setdefault(s, {}).setdefault(e, [])
|
||||
if nm: groups[s][e].append(nm)
|
||||
froms, urls = [], []
|
||||
for s in sorted(groups):
|
||||
tab = tree.xpath('//a[@href="#playlist%d"]' % (s + 1))
|
||||
name = ((tab[0].get("title") or "".join(tab[0].xpath('.//text()')).strip().split(" ")[0]) if tab else "").strip() or "线路%d" % (s + 1)
|
||||
eps = []
|
||||
for e in sorted(groups[s]):
|
||||
cand = [x for x in groups[s][e] if re.search(r'第.*[集期话]|^\d+$|HD|BD|TS|正片|预告|番外|国语|粤语|中字', x)]
|
||||
nm = (cand[0] if cand else "第%d集" % (e + 1)).replace("$", "").replace("#", "")
|
||||
eps.append("%s$/play/%s-%d-%d.html" % (nm, vid, s, e))
|
||||
froms.append(name); urls.append("#".join(eps))
|
||||
return froms, urls
|
||||
|
||||
def _meta(self, tree, prop):
|
||||
v = tree.xpath('//meta[@property="%s"]/@content' % prop) or tree.xpath('//meta[@name="%s"]/@content' % prop)
|
||||
return v[0].strip() if v else ""
|
||||
|
||||
def _people(self, tree, label):
|
||||
v = tree.xpath('//*[contains(text(),"%s")]//a[contains(@href,"searchword=")]/text()' % label)
|
||||
return " ".join(x.strip() for x in v[:30] if x.strip())
|
||||
|
||||
def _field(self, text, key):
|
||||
m = re.search(r'%s\s*[:：]\s*([^\n]{1,120})' % key, text)
|
||||
return m.group(1).strip(" \u3000|/") if m else ""
|
||||
|
||||
def homeContent(self, filter):
|
||||
fl = {}
|
||||
for c in self.categories:
|
||||
f = []
|
||||
if c["type_id"] in self.subs:
|
||||
f.append({"key": "tid", "name": "类型", "value": [{"n": s[0], "v": s[1]} for s in self.subs[c["type_id"]]]})
|
||||
f.append({"key": "order", "name": "排序", "value": [{"n": o[0], "v": o[1]} for o in self.orders]})
|
||||
fl[c["type_id"]] = f
|
||||
return {"class": self.categories, "list": self._parse_list(self._get("/index.html")), "filters": fl}
|
||||
|
||||
def homeVideoContent(self): return {"list": self._parse_list(self._get("/index.html"))}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
pg = str(pg or "1"); ex = extend or {}
|
||||
real = ex.get("tid") or tid
|
||||
url = "/search.php?searchtype=5&tid=%s&page=%s" % (real, pg)
|
||||
if ex.get("order"): url += "&order=" + ex["order"]
|
||||
lst = self._parse_list(self._get(url))
|
||||
return {"page": int(pg), "pagecount": int(pg) + 1 if lst else int(pg), "limit": 24, "total": 999999, "list": lst}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
pg = str(pg or "1")
|
||||
lst = self._parse_list(self._get("/search.php?searchword=%s&page=%s" % (quote(key), pg)))
|
||||
if not lst and pg == "1":
|
||||
lst = self._parse_list(self._post("/search.php", {"searchword": key, "searchtype": "1"}))
|
||||
return {"list": lst, "page": int(pg)}
|
||||
|
||||
def detailContent(self, ids):
|
||||
vid = re.sub(r'\D', '', str(ids[0]))
|
||||
html = self._get("/movie/index%s.html" % vid)
|
||||
if not html or etree is None: return {"list": []}
|
||||
tree = etree.HTML(html)
|
||||
text = re.sub(r'[ \t\u3000]+', ' ', "\n".join(x.strip() for x in tree.xpath('//text()') if x.strip()))
|
||||
froms, urls = self._parse_playlist(tree, vid)
|
||||
vod = {"vod_id": vid,
|
||||
"vod_name": (self._meta(tree, "og:title") or "".join(tree.xpath('//h1//text()'))).strip().split("《")[-1].split("》")[0] or "".join(tree.xpath('//h1//text()')).strip(),
|
||||
"vod_pic": self._fix(self._meta(tree, "og:image")),
|
||||
"vod_year": self._field(text, "年份"), "vod_area": self._field(text, "地区"),
|
||||
"type_name": self._field(text, "类型"), "vod_lang": self._field(text, "语言"),
|
||||
"vod_actor": self._people(tree, "主演") or self._field(text, "主演"),
|
||||
"vod_director": self._people(tree, "导演") or self._field(text, "导演"),
|
||||
"vod_remarks": self._field(text, "豆瓣"),
|
||||
"vod_content": self._meta(tree, "og:description") or self._meta(tree, "description"),
|
||||
"vod_play_from": "$$$".join(froms), "vod_play_url": "$$$".join(urls)}
|
||||
return {"list": [vod]}
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
pid = id if id.startswith("http") else self._fix(id)
|
||||
html = self._get(pid) or ""
|
||||
url = ""
|
||||
for p in [r'var\s+now\s*=\s*["\']([^"\']+)["\']', r'var\s+player_\w+\s*=\s*(\{.*?\})\s*[;<]', r'"url"\s*:\s*"([^"]+\.(?:m3u8|mp4)[^"]*)"', r'url:\s*["\']([^"\']+\.(?:m3u8|mp4)[^"\']*)["\']', r'(https?://[^\s"\'\\]+\.(?:m3u8|mp4)[^\s"\'\\]*)']:
|
||||
m = re.search(p, html.replace("\\/", "/"), re.S)
|
||||
if not m: continue
|
||||
val = m.group(1)
|
||||
if val.startswith("{"):
|
||||
try: val = json.loads(val).get("url", "")
|
||||
except Exception:
|
||||
m2 = re.search(r'"(https?://[^"]+\.(?:m3u8|mp4)[^"]*)"', val); val = m2.group(1) if m2 else ""
|
||||
if val and not val.startswith("#"): url = self._fix(val); break
|
||||
if not url: return {"parse": 1, "url": pid, "header": self.headers}
|
||||
return {"parse": 0, "url": url, "header": {"User-Agent": self.headers["User-Agent"], "Referer": self.host + "/"}}
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re, json, requests
|
||||
from urllib.parse import quote, urlencode
|
||||
try:
|
||||
from lxml import etree
|
||||
except Exception:
|
||||
etree = None
|
||||
from base.spider import Spider
|
||||
|
||||
|
||||
class Spider(Spider):
|
||||
def getName(self): return "PTT视频"
|
||||
|
||||
def init(self, extend=""):
|
||||
self.host = "https://ptt.red"
|
||||
try: ext = json.loads(extend) if str(extend).strip().startswith("{") else {}
|
||||
except Exception: ext = {}
|
||||
self.lang = ext.get("lang", "")
|
||||
self.relay = ext.get("relay", "").rstrip("/")
|
||||
self.searchPath = ""
|
||||
self.headers = {"User-Agent": ext.get("ua", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"), "Referer": self.host + "/", "Accept-Language": "zh-TW,zh;q=0.9"}
|
||||
if ext.get("cookie"): self.headers["Cookie"] = ext["cookie"]
|
||||
self.categories = [{"type_id": "3", "type_name": "电视剧"}, {"type_id": "1", "type_name": "电影"}, {"type_id": "4", "type_name": "动漫"}, {"type_id": "2", "type_name": "综艺"}, {"type_id": "66", "type_name": "短剧"}, {"type_id": "53", "type_name": "体育"}]
|
||||
self.areas = [["全部", ""], ["大陆", "19"], ["香港", "20"], ["台湾", "81"], ["日本", "83"], ["韩国", "82"], ["欧美", "22"], ["泰国", "92"], ["其他", "23"]]
|
||||
self.years = ["2026", "2025", "2024", "2023", "2022", "2021", "2020", "2019", "2018", "2017", "2016"]
|
||||
|
||||
def _fix(self, u):
|
||||
if not u: return ""
|
||||
if u.startswith("//"): return "https:" + u
|
||||
if u.startswith("/"): return self.host + u
|
||||
return u
|
||||
|
||||
def _blocked(self, t): return not t or "Just a moment" in t or "cf-chl" in t or "Enable JavaScript and cookies" in t or "__cdnlah_pow_config" in t or "_chg_waf_pow" in t
|
||||
|
||||
def _get(self, path):
|
||||
url = path if path.startswith("http") else self.host + self.lang + path
|
||||
txt = None
|
||||
try:
|
||||
r = requests.get(url, headers=self.headers, timeout=15); r.encoding = "utf-8"; txt = r.text
|
||||
if self._blocked(txt): print("[WARN] 被网关拦截 status=%s len=%s url=%s" % (r.status_code, len(txt), url))
|
||||
except requests.exceptions.Timeout: print("[ERROR] 请求超时: %s" % url)
|
||||
except requests.exceptions.ConnectionError: print("[ERROR] 连接错误: %s" % url)
|
||||
except Exception as e: print("[ERROR] 请求失败: %s, %s" % (url, str(e)))
|
||||
if self._blocked(txt) and self.relay:
|
||||
try:
|
||||
r = requests.get(self.relay + "/?url=" + quote(url, safe=""), headers={"User-Agent": self.headers["User-Agent"]}, timeout=45); r.encoding = "utf-8"; txt = r.text
|
||||
except Exception as e: print("[ERROR] relay失败: %s" % str(e))
|
||||
return None if self._blocked(txt) else txt
|
||||
|
||||
def _pic(self, vid):
|
||||
try: return "%s/images/node/%d/%s.avif" % (self.host, int(vid) // 10000, vid)
|
||||
except Exception: return ""
|
||||
|
||||
def _parse_list(self, html):
|
||||
if not html: return []
|
||||
if etree is None:
|
||||
print("[WARN] lxml 不可用，降级为正则解析")
|
||||
return [{"vod_id": v, "vod_name": n, "vod_pic": self._fix(p)} for v, p, n in dict((m[0], m) for m in re.findall(r'href="/(\d+)"><img[^>]*?src="([^"]+)"[^>]*?alt="([^"]*)"', html)).values()]
|
||||
tree = etree.HTML(html); results, seen = [], set()
|
||||
items = tree.xpath('//div[@id="videos"]//div[contains(@class,"item")]') or tree.xpath('//div[contains(@class,"card")][.//img[@alt]]') or tree.xpath('//a[.//img[@alt]]')
|
||||
for it in items:
|
||||
try:
|
||||
href = "".join(it.xpath('.//a/@href')[:1]) or it.get("href", "")
|
||||
vid = href.strip("/").split("/")[-1]
|
||||
if not vid.isdigit() or vid in seen: continue
|
||||
name = "".join(it.xpath('.//img/@alt')[:1]).strip() or "".join(it.xpath('.//div[contains(@class,"lines")]//a//text()')).strip()
|
||||
if not name: continue
|
||||
seen.add(vid)
|
||||
pic = "".join(it.xpath('.//img/@data-original | .//img/@data-src | .//img/@src')[:1])
|
||||
note = "".join(it.xpath('.//div[contains(@class,"imagelabel-bottom-right")]//text()')).strip()
|
||||
year = "".join(it.xpath('.//div[contains(@class,"imagelabel-bottom-left")]//text()')).strip()
|
||||
results.append({"vod_id": vid, "vod_name": name, "vod_pic": self._fix(pic) or self._pic(vid), "vod_year": year.replace("年", ""), "vod_remarks": note})
|
||||
except Exception: continue
|
||||
return results
|
||||
|
||||
def _get_pagination(self, html, pg):
|
||||
if not html: return int(pg)
|
||||
tree = etree.HTML(html)
|
||||
nums = [int(x) for x in tree.xpath('//ul[contains(@class,"pagination")]//a[@data-page]/text()') if x.strip().isdigit()]
|
||||
nxt = tree.xpath('//li[contains(@class,"page-item") and contains(@class,"next")]/@class')
|
||||
return max(nums + [int(pg)]) + (1 if nxt and "disabled" not in nxt[0] else 0)
|
||||
|
||||
def _extract_m3u8(self, html):
|
||||
if not html: return None
|
||||
for p in [r'var\s+now\s*=\s*["\']([^"\']+\.m3u8[^"\']*)["\']', r'var\s+player_data\s*=\s*(\{.*?\})', r'player_\w+\s*=\s*(\{.*?\})\s*[;<]', r'url:\s*["\']([^"\']+\.m3u8[^"\']*)["\']', r'var\s+playurl\s*=\s*["\']([^"\']+)["\']', r'(https?://[^\s"\'\\]+\.(?:m3u8|mp4)[^\s"\'\\]*)']:
|
||||
m = re.search(p, html.replace("\\/", "/"), re.S)
|
||||
if not m: continue
|
||||
val = m.group(1)
|
||||
if val.startswith("{"):
|
||||
try: val = json.loads(val).get("url", "")
|
||||
except Exception:
|
||||
m2 = re.search(r'"(https?://[^"]+\.(?:m3u8|mp4)[^"]*)"', val); val = m2.group(1) if m2 else ""
|
||||
if val: return self._fix(val)
|
||||
return None
|
||||
|
||||
def homeContent(self, filter):
|
||||
fl = {c["type_id"]: [{"key": "category_id", "name": "地区", "value": [{"n": a[0], "v": a[1]} for a in self.areas]}, {"key": "year", "name": "年份", "value": [{"n": "全部", "v": ""}] + [{"n": y, "v": y} for y in self.years]}] for c in self.categories}
|
||||
return {"class": self.categories, "list": self._parse_list(self._get("/")), "filters": fl}
|
||||
|
||||
def homeVideoContent(self): return {"list": self._parse_list(self._get("/"))}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
pg = str(pg or "1")
|
||||
qs = {"page": pg}
|
||||
for k in ("category_id", "year"):
|
||||
if (extend or {}).get(k): qs[k] = extend[k]
|
||||
html = self._get("/p/%s?%s" % (tid, urlencode(qs)))
|
||||
lst = self._parse_list(html)
|
||||
return {"page": int(pg), "pagecount": self._get_pagination(html, pg) if lst else int(pg), "limit": 48, "total": 999999, "list": lst}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
pg = str(pg or "1")
|
||||
for p in ([self.searchPath] if self.searchPath else ["/node/search?q={k}&page={p}", "/node/search?keyword={k}&page={p}", "/node/search?title={k}&page={p}", "/s/{k}?page={p}"]):
|
||||
lst = self._parse_list(self._get(p.format(k=quote(key), p=pg)))
|
||||
if lst:
|
||||
self.searchPath = p
|
||||
return {"list": lst, "page": int(pg)}
|
||||
return {"list": [], "page": int(pg)}
|
||||
|
||||
def detailContent(self, ids):
|
||||
vid = str(ids[0]).strip("/").split("/")[-1]
|
||||
html = self._get("/" + vid)
|
||||
if not html: return {"list": []}
|
||||
tree = etree.HTML(html)
|
||||
title = ("".join(tree.xpath('//h1//text()')).strip() or "".join(tree.xpath('//title/text()')).split(" - ")[0].strip())
|
||||
eps, seen = [], set()
|
||||
for a in tree.xpath('//a[@href]'):
|
||||
lk = a.get("href", "")
|
||||
if lk in seen or not re.match(r'^/%s[/\-_]|^/play/|^/v/%s' % (vid, vid), lk): continue
|
||||
nm = ("".join(a.xpath('.//text()')).strip() or a.get("title", "")).strip()
|
||||
if not nm: continue
|
||||
seen.add(lk)
|
||||
eps.append(nm.replace("$", "").replace("#", "") + "$" + self._fix(lk))
|
||||
return {"list": [{"vod_id": vid, "vod_name": title, "vod_pic": self._pic(vid), "vod_content": "".join(tree.xpath('//div[contains(@class,"content-wrapper")]//p[1]//text()')).strip(), "vod_play_from": "PTT", "vod_play_url": "#".join(eps or ["正片$" + self.host + "/" + vid])}]}
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
pid = id if id.startswith("http") else self.host + "/" + id.strip("/")
|
||||
url = self._extract_m3u8(self._get(pid))
|
||||
if not url: return {"parse": 1, "url": pid, "header": self.headers}
|
||||
return {"parse": 0, "url": url, "header": {"User-Agent": self.headers["User-Agent"], "Referer": self.host + "/"}}
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re, json, requests
|
||||
from urllib.parse import quote
|
||||
try:
|
||||
from lxml import etree
|
||||
except Exception:
|
||||
etree = None
|
||||
from base.spider import Spider
|
||||
|
||||
BADIMG = ("poster_loading", "logo", "thumb.png", "playing.gif", "favicon", "doubanio", "discord")
|
||||
PAGEFMT = ["%s/p/%s", "%s?page=%s", "%s/page/%s", "%s/%s"]
|
||||
SEARCHFMT = ["/search/%s", "/vod/search/%s", "/search/index/wd/%s", "/index.php/vod/search/wd/%s", "/search?wd=%s"]
|
||||
|
||||
|
||||
class Spider(Spider):
|
||||
def getName(self): return "影视天堂"
|
||||
|
||||
def init(self, extend=""):
|
||||
self.host = "https://ysttv.com"
|
||||
try: ext = json.loads(extend) if str(extend).strip().startswith("{") else {}
|
||||
except Exception: ext = {}
|
||||
if ext.get("host"): self.host = ext["host"].rstrip("/")
|
||||
self.pageFmt = ext.get("pageFmt", "")
|
||||
self.searchFmt = ext.get("searchFmt", "")
|
||||
self.headers = {"User-Agent": ext.get("ua", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"), "Referer": self.host + "/", "Accept-Language": "zh-CN,zh;q=0.9"}
|
||||
self.categories = [{"type_id": "movie", "type_name": "电影"}, {"type_id": "teleplay", "type_name": "剧集"}, {"type_id": "variety", "type_name": "综艺"}, {"type_id": "anime", "type_name": "动漫"}, {"type_id": "playlet", "type_name": "短剧"}]
|
||||
self.genres = [["全部", ""], ["动作", "action"], ["喜剧", "comedy"], ["科幻", "sci-fi"], ["悬疑", "mystery"], ["爱情", "romance"], ["犯罪", "crime"], ["恐怖", "horror"], ["剧情", "drama"], ["奇幻", "fantasy"], ["惊悚", "thriller"], ["冒险", "adventure"], ["动画", "animation"], ["历史", "history"], ["同性", "lgbt"], ["纪录片", "documentary"], ["古装", "costume"], ["武侠", "wuxia"], ["音乐", "music"], ["歌舞", "musical"], ["运动", "sports"], ["灾难", "disaster"], ["传记", "biography"], ["儿童", "kids"]]
|
||||
self.years = [["全部", ""]] + [[str(y), "year%d" % y] for y in range(2025, 2014, -1)]
|
||||
self.areas = [["全部", ""], ["大陆", "area-china"], ["台湾", "area-taiwan"], ["香港", "area-hong-kong"], ["美国", "area-usa"], ["韩国", "area-korea"], ["日本", "area-japan"], ["英国", "area-uk"], ["法国", "area-france"], ["泰国", "area-thailand"], ["印度", "area-india"], ["加拿大", "area-canada"]]
|
||||
self.sorts = [["最新", ""], ["人气", "hot"], ["评分", "rating"]]
|
||||
|
||||
def _fix(self, u):
|
||||
if not u: return ""
|
||||
if u.startswith("//"): return "https:" + u
|
||||
if u.startswith("/"): return self.host + u
|
||||
return u
|
||||
|
||||
def _get(self, path):
|
||||
url = path if path.startswith("http") else self.host + path
|
||||
try:
|
||||
r = requests.get(url, headers=self.headers, timeout=15); r.encoding = "utf-8"
|
||||
if r.status_code >= 400: print("[WARN] status=%s url=%s" % (r.status_code, url)); return None
|
||||
return r.text
|
||||
except requests.exceptions.Timeout: print("[ERROR] 请求超时: %s" % url)
|
||||
except requests.exceptions.ConnectionError: print("[ERROR] 连接错误: %s" % url)
|
||||
except Exception as e: print("[ERROR] 请求失败: %s, %s" % (url, str(e)))
|
||||
return None
|
||||
|
||||
def _img(self, node):
|
||||
for at in ("data-original", "data-src", "data-echo", "src"):
|
||||
for v in node.xpath('.//img/@%s' % at):
|
||||
if v and not any(b in v for b in BADIMG): return v
|
||||
return ""
|
||||
|
||||
def _parse_list(self, html):
|
||||
if not html: return []
|
||||
if etree is None:
|
||||
print("[WARN] lxml 不可用，降级为正则解析")
|
||||
out, seen = [], set()
|
||||
for vid, title in re.findall(r'/detail/(\d+)/?"[^>]*?title="([^"]*)"', html):
|
||||
if vid in seen: continue
|
||||
seen.add(vid); out.append({"vod_id": vid, "vod_name": title, "vod_pic": ""})
|
||||
return out
|
||||
tree = etree.HTML(html); results, seen = [], set()
|
||||
items = tree.xpath('//a[contains(@href,"/detail/") and .//img]') + tree.xpath('//a[contains(@href,"/detail/")]')
|
||||
for it in items:
|
||||
try:
|
||||
m = re.search(r'/detail/(\d+)', it.get("href", ""))
|
||||
if not m or m.group(1) in seen: continue
|
||||
name = (it.get("title") or "".join(it.xpath('.//img/@alt')[:1])).strip()
|
||||
if not name: continue
|
||||
seen.add(m.group(1))
|
||||
note = [x.strip() for x in it.xpath('.//span//text() | .//em//text() | .//i//text()') if x.strip() and x.strip() != name]
|
||||
results.append({"vod_id": m.group(1), "vod_name": name, "vod_pic": self._fix(self._img(it)), "vod_remarks": " ".join(note)[:30]})
|
||||
except Exception: continue
|
||||
return results
|
||||
|
||||
def _first(self, lst): return lst[0]["vod_id"] if lst else ""
|
||||
|
||||
def _paged(self, base, pg):
|
||||
if pg == "1": return self._get(base)
|
||||
if self.pageFmt: return self._get(self.pageFmt % (base, pg))
|
||||
first = self._first(self._parse_list(self._get(base)))
|
||||
for f in PAGEFMT:
|
||||
html = self._get(f % (base, pg))
|
||||
got = self._parse_list(html)
|
||||
if got and self._first(got) != first:
|
||||
self.pageFmt = f
|
||||
print("[INFO] 分页格式确定: %s" % (f % ("{base}", "{pg}")))
|
||||
return html
|
||||
print("[WARN] 未能确定分页格式，仅返回首页")
|
||||
return None
|
||||
|
||||
def homeContent(self, filter):
|
||||
fl = {}
|
||||
for c in self.categories:
|
||||
fl[c["type_id"]] = [{"key": "genre", "name": "类型", "value": [{"n": g[0], "v": g[1]} for g in self.genres]},
|
||||
{"key": "year", "name": "年份", "value": [{"n": y[0], "v": y[1]} for y in self.years]},
|
||||
{"key": "area", "name": "地区", "value": [{"n": a[0], "v": a[1]} for a in self.areas]},
|
||||
{"key": "sort", "name": "排序", "value": [{"n": s[0], "v": s[1]} for s in self.sorts]}]
|
||||
return {"class": self.categories, "list": self._parse_list(self._get("/")), "filters": fl}
|
||||
|
||||
def homeVideoContent(self): return {"list": self._parse_list(self._get("/"))}
|
||||
|
||||
def categoryContent(self, tid, pg, filter, extend):
|
||||
pg = str(pg or "1"); ex = extend or {}
|
||||
facet = ex.get("genre") or ex.get("year") or ex.get("area") or ex.get("sort") or ""
|
||||
base = "/vod/%s" % tid + ("/%s" % facet if facet else "")
|
||||
lst = self._parse_list(self._paged(base, pg))
|
||||
return {"page": int(pg), "pagecount": int(pg) + 1 if lst else int(pg), "limit": 32, "total": 999999, "list": lst}
|
||||
|
||||
def searchContent(self, key, quick, pg="1"):
|
||||
pg = str(pg or "1")
|
||||
for f in ([self.searchFmt] if self.searchFmt else SEARCHFMT):
|
||||
lst = self._parse_list(self._get(f % quote(key)))
|
||||
if lst:
|
||||
self.searchFmt = f
|
||||
print("[INFO] 搜索格式确定: %s" % f)
|
||||
return {"list": lst, "page": int(pg)}
|
||||
print("[WARN] 未能确定搜索接口，需抓包补 searchFmt")
|
||||
return {"list": [], "page": int(pg)}
|
||||
|
||||
def _field(self, text, key):
|
||||
m = re.search(r'%s\s*[:：]\s*([^\n]{1,200})' % key, text)
|
||||
return m.group(1).strip(" \u3000|/") if m else ""
|
||||
|
||||
def detailContent(self, ids):
|
||||
vid = re.sub(r'\D', '', str(ids[0]))
|
||||
html = self._get("/detail/%s/" % vid)
|
||||
if not html or etree is None: return {"list": []}
|
||||
tree = etree.HTML(html)
|
||||
text = "\n".join(x.strip() for x in tree.xpath('//text()') if x.strip())
|
||||
pic = ""
|
||||
for v in tree.xpath('//img/@src | //img/@data-original | //img/@data-src'):
|
||||
if v and not any(b in v for b in BADIMG) and re.search(r'/cover/|/upload/|jinyingimage', v): pic = v; break
|
||||
vod = {"vod_id": vid,
|
||||
"vod_name": ("".join(tree.xpath('//h1//text()')).strip() or self._field(text, "og:title")).strip("《》"),
|
||||
"vod_pic": self._fix(pic),
|
||||
"vod_year": "".join(tree.xpath('//a[contains(@href,"/year")]/text()')[:1]).strip(),
|
||||
"vod_area": "".join(tree.xpath('//a[contains(@href,"/area-")]/text()')[:1]).strip(),
|
||||
"type_name": " ".join(x.strip() for x in tree.xpath('//a[contains(@href,"/vod/") and not(contains(@href,"/year")) and not(contains(@href,"/area-"))]/text()')[:3] if x.strip()),
|
||||
"vod_director": self._field(text, "导演"), "vod_actor": self._field(text, "主演"),
|
||||
"vod_remarks": ("共%s集" % self._field(text, "集数")) if self._field(text, "集数").isdigit() and self._field(text, "集数") != "1" else (self._field(text, "评分") or self._field(text, "集数")),
|
||||
"vod_content": self._field(text, "剧情") or self._field(text, "简介")}
|
||||
ph = self._get("/player/%s/" % vid)
|
||||
eps = []
|
||||
if ph:
|
||||
pt = etree.HTML(ph)
|
||||
for a in pt.xpath('//a[contains(@href,"/player/%s/")]' % vid):
|
||||
lk = a.get("href", "")
|
||||
nm = ("".join(a.xpath('.//text()')).strip() or a.get("title", "")).strip()
|
||||
if not lk or not nm or lk.rstrip("/").endswith("/%s" % vid): continue
|
||||
item = nm.replace("$", "").replace("#", "") + "$" + self._fix(lk)
|
||||
if item not in eps: eps.append(item)
|
||||
vod["vod_play_from"] = "影视天堂"
|
||||
vod["vod_play_url"] = "#".join(eps or ["正片$%s/player/%s/" % (self.host, vid)])
|
||||
return {"list": [vod]}
|
||||
|
||||
def playerContent(self, flag, id, vipFlags):
|
||||
pid = id if id.startswith("http") else self._fix(id)
|
||||
html = self._get(pid) or ""
|
||||
url = ""
|
||||
for p in [r'var\s+now\s*=\s*["\']([^"\']+)["\']', r'var\s+player_\w+\s*=\s*(\{.*?\})\s*[;<]', r'"(?:url|video_url|playUrl)"\s*:\s*"([^"]+\.(?:m3u8|mp4)[^"]*)"', r'url:\s*["\']([^"\']+\.(?:m3u8|mp4)[^"\']*)["\']', r'(https?://[^\s"\'\\<>]+\.(?:m3u8|mp4)[^\s"\'\\<>]*)']:
|
||||
m = re.search(p, html.replace("\\/", "/"), re.S)
|
||||
if not m: continue
|
||||
val = m.group(1)
|
||||
if val.startswith("{"):
|
||||
try: val = json.loads(val).get("url", "")
|
||||
except Exception:
|
||||
m2 = re.search(r'"(https?://[^"]+\.(?:m3u8|mp4)[^"]*)"', val); val = m2.group(1) if m2 else ""
|
||||
if val: url = self._fix(val); break
|
||||
if not url: return {"parse": 1, "url": pid, "header": self.headers}
|
||||
return {"parse": 0, "url": url, "header": {"User-Agent": self.headers["User-Agent"], "Referer": self.host + "/"}}
|
||||
+891
-57
@@ -6,6 +6,8 @@ import sys
|
||||
import json
|
||||
import html
|
||||
import time
|
||||
import socket
|
||||
import hashlib
|
||||
from urllib.parse import quote, unquote, parse_qs, urlencode, urlparse, urlunparse, urljoin
|
||||
|
||||
import requests
|
||||
@@ -14,8 +16,15 @@ from base.spider import Spider
|
||||
sys.path.append('..')
|
||||
|
||||
# ---------- 日志路径 ----------
|
||||
# DEBUG_LOG = '/storage/emulated/0/源码/ytb_debug.log'
|
||||
# 置空 = 关闭日志(不再每次调用都抛 NameError)
|
||||
# 需要排障时改成:
|
||||
DEBUG_LOG = '/storage/emulated/0/源码/ytb_debug.log'
|
||||
#DEBUG_LOG = ''
|
||||
|
||||
|
||||
def _ensure_log_dir():
|
||||
if not DEBUG_LOG:
|
||||
return
|
||||
try:
|
||||
log_dir = os.path.dirname(DEBUG_LOG)
|
||||
if log_dir and not os.path.exists(log_dir):
|
||||
@@ -25,6 +34,8 @@ def _ensure_log_dir():
|
||||
_ensure_log_dir()
|
||||
|
||||
def debug_log(message, data=None):
|
||||
if not DEBUG_LOG:
|
||||
return
|
||||
try:
|
||||
log_dir = os.path.dirname(DEBUG_LOG)
|
||||
if log_dir and not os.path.exists(log_dir):
|
||||
@@ -1305,6 +1316,322 @@ try:
|
||||
except NameError:
|
||||
CATEGORY_FILTERS = {}
|
||||
|
||||
# ==================== 节点风险 / 健康度管理 ====================
|
||||
# 说明: 这里的"节点"指 Innertube 客户端节点(ANDROID_VR / ANDROID / IOS / WEB ...),
|
||||
# 与下方 Spider._auto_detect_proxy 里的"代理节点"是两层不同的东西。
|
||||
|
||||
# 风险等级: 0=可用, 1=可疑(降权), 2=高风险(冷却), 3=不可用(长冷却)
|
||||
RISK_OK, RISK_SUSPECT, RISK_HIGH, RISK_DEAD = 0, 1, 2, 3
|
||||
|
||||
# 需要人机校验 / 登录 / 年龄验证等"风险弹窗"类状态
|
||||
RISK_STATUS = {
|
||||
'LOGIN_REQUIRED': RISK_HIGH,
|
||||
'AGE_VERIFICATION_REQUIRED': RISK_HIGH,
|
||||
'AGE_CHECK_REQUIRED': RISK_HIGH,
|
||||
'CONTENT_CHECK_REQUIRED': RISK_SUSPECT,
|
||||
'UNPLAYABLE': RISK_SUSPECT,
|
||||
'ERROR': RISK_HIGH,
|
||||
'LIVE_STREAM_OFFLINE': RISK_OK,
|
||||
}
|
||||
|
||||
# reason 文案里出现这些词 = 触发了 Google 的风控/人机校验
|
||||
BOT_REASON_PATTERNS = (
|
||||
'not a bot',
|
||||
'confirm you',
|
||||
'sign in to confirm',
|
||||
'登录以确认',
|
||||
'確認您不是機器人',
|
||||
'确认您不是机器人',
|
||||
'inappropriate for some users',
|
||||
'this helps protect our community',
|
||||
)
|
||||
|
||||
# 地区/版权类, 换客户端节点没用, 直接判死
|
||||
FATAL_REASON_PATTERNS = (
|
||||
'not available in your country',
|
||||
'blocked it in your country',
|
||||
'在您所在的国家',
|
||||
'video is private',
|
||||
'video unavailable',
|
||||
'has been removed',
|
||||
'account associated with this video has been terminated',
|
||||
)
|
||||
|
||||
|
||||
class NodeHealth:
|
||||
"""客户端节点健康表: 记录成功/失败, 高风险节点进入冷却, 排序时降权。"""
|
||||
|
||||
def __init__(self):
|
||||
self.state = {}
|
||||
|
||||
def _slot(self, name):
|
||||
if len(self.state) > 64:
|
||||
self.state.clear()
|
||||
return self.state.setdefault(name, {
|
||||
'ok': 0, 'fail': 0, 'risk': RISK_OK, 'cooldown_until': 0.0, 'last_reason': '',
|
||||
})
|
||||
|
||||
def markOk(self, name):
|
||||
s = self._slot(name)
|
||||
s['ok'] += 1
|
||||
s['risk'] = RISK_OK
|
||||
s['cooldown_until'] = 0.0
|
||||
s['last_reason'] = ''
|
||||
|
||||
def markRisk(self, name, risk, reason=''):
|
||||
s = self._slot(name)
|
||||
s['fail'] += 1
|
||||
s['risk'] = max(s['risk'], risk)
|
||||
s['last_reason'] = str(reason)[:200]
|
||||
# 冷却时长随风险等级递增, 避免反复撞同一个被风控的节点
|
||||
cool = {RISK_OK: 0, RISK_SUSPECT: 60, RISK_HIGH: 600, RISK_DEAD: 1800}.get(risk, 60)
|
||||
if cool:
|
||||
s['cooldown_until'] = time.time() + cool
|
||||
|
||||
def isCooling(self, name):
|
||||
return self._slot(name)['cooldown_until'] > time.time()
|
||||
|
||||
def weight(self, name):
|
||||
"""排序权重, 越小越优先。"""
|
||||
s = self._slot(name)
|
||||
base = s['risk'] * 10
|
||||
if self.isCooling(name):
|
||||
base += 100
|
||||
base += min(s['fail'], 9)
|
||||
base -= min(s['ok'], 5)
|
||||
return base
|
||||
|
||||
def snapshot(self):
|
||||
now = time.time()
|
||||
return {k: {'risk': v['risk'], 'ok': v['ok'], 'fail': v['fail'],
|
||||
'cool': max(0, int(v['cooldown_until'] - now))}
|
||||
for k, v in self.state.items()}
|
||||
|
||||
|
||||
# 进程级共享, 跨 extract 调用保留节点健康度
|
||||
NODE_HEALTH = NodeHealth()
|
||||
|
||||
|
||||
# ==================== 全局 bot-check 熔断 ====================
|
||||
# 实测(ytb_debug.log): IP 被标记后, 8 个客户端节点会全部返回 LOGIN_REQUIRED。
|
||||
# 此时继续轮询没有任何意义, 只会
|
||||
# a) 让 UI 卡 3~5 秒
|
||||
# b) 短时间内打出十几个请求, 进一步加深 IP 标记
|
||||
# 因此一轮全灭后开启熔断, 期间直接短路到兜底。
|
||||
BOT_BREAKER = {'until': 0.0, 'hits': 0, 'reason': ''}
|
||||
|
||||
|
||||
def breakerOpen():
|
||||
return time.time() < BOT_BREAKER['until']
|
||||
|
||||
|
||||
def breakerTrip(seconds, reason=''):
|
||||
BOT_BREAKER['hits'] += 1
|
||||
# 连续命中则指数退避, 上限 30 分钟
|
||||
factor = min(2 ** max(0, BOT_BREAKER['hits'] - 1), 8)
|
||||
BOT_BREAKER['until'] = time.time() + min(seconds * factor, 1800)
|
||||
BOT_BREAKER['reason'] = str(reason)[:200]
|
||||
debug_log('bot breaker tripped', {'hits': BOT_BREAKER['hits'],
|
||||
'seconds': int(BOT_BREAKER['until'] - time.time()),
|
||||
'reason': BOT_BREAKER['reason'][:80]})
|
||||
|
||||
|
||||
def breakerReset():
|
||||
if BOT_BREAKER['hits'] or BOT_BREAKER['until']:
|
||||
debug_log('bot breaker reset')
|
||||
BOT_BREAKER['until'] = 0.0
|
||||
BOT_BREAKER['hits'] = 0
|
||||
BOT_BREAKER['reason'] = ''
|
||||
|
||||
|
||||
def breakerRemain():
|
||||
return max(0, int(BOT_BREAKER['until'] - time.time()))
|
||||
|
||||
|
||||
# ==================== 登录态 Cookie ====================
|
||||
def parseCookieInput(raw_value):
|
||||
"""接受三种写法, 统一返回 Cookie 请求头字符串:
|
||||
1) 直接就是 Cookie 头: "SID=...; HSID=...; SAPISID=..."
|
||||
2) dict: {"SID": "...", "SAPISID": "..."}
|
||||
3) Netscape cookies.txt 的多行文本
|
||||
"""
|
||||
if not raw_value:
|
||||
return ''
|
||||
if isinstance(raw_value, dict):
|
||||
return '; '.join('%s=%s' % (k, v) for k, v in raw_value.items() if k and v)
|
||||
value = str(raw_value).strip()
|
||||
if '\n' in value or '\t' in value:
|
||||
pairs = []
|
||||
for line in value.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
cols = line.split('\t')
|
||||
if len(cols) >= 7 and cols[5]:
|
||||
pairs.append('%s=%s' % (cols[5], cols[6]))
|
||||
if pairs:
|
||||
return '; '.join(pairs)
|
||||
return value
|
||||
|
||||
|
||||
def sapisidHash(cookie_header, origin='https://www.youtube.com'):
|
||||
"""Google 内部接口鉴权头。
|
||||
格式: SAPISIDHASH <ts>_<sha1(ts + ' ' + SAPISID + ' ' + origin)>
|
||||
只用本机 Cookie 计算, 不做任何网络请求。
|
||||
"""
|
||||
try:
|
||||
jar = {}
|
||||
for part in (cookie_header or '').split(';'):
|
||||
if '=' in part:
|
||||
k, v = part.split('=', 1)
|
||||
jar[k.strip()] = v.strip()
|
||||
sapisid = jar.get('SAPISID') or jar.get('__Secure-3PAPISID') or jar.get('__Secure-1PAPISID')
|
||||
if not sapisid:
|
||||
return None
|
||||
ts = str(int(time.time()))
|
||||
digest = hashlib.sha1(('%s %s %s' % (ts, sapisid, origin)).encode('utf-8')).hexdigest()
|
||||
return 'SAPISIDHASH %s_%s' % (ts, digest)
|
||||
except Exception as e:
|
||||
debug_log('sapisidHash error', repr(e))
|
||||
return None
|
||||
|
||||
|
||||
def authHeaders(cookie_header):
|
||||
"""有 Cookie 就补齐 Google 需要的一整套认证头。"""
|
||||
headers = {}
|
||||
if not cookie_header:
|
||||
return headers
|
||||
headers['Cookie'] = cookie_header
|
||||
auth = sapisidHash(cookie_header)
|
||||
if auth:
|
||||
headers['Authorization'] = auth
|
||||
headers['X-Origin'] = 'https://www.youtube.com'
|
||||
headers['X-Goog-AuthUser'] = '0'
|
||||
return headers
|
||||
|
||||
|
||||
|
||||
# ==================== po_token 远程 provider ====================
|
||||
class PoTokenProvider:
|
||||
"""对接 bgutil-ytdlp-pot-provider 的 HTTP server。
|
||||
|
||||
协议(见文末引用):
|
||||
GET {base}/ping -> 健康检查
|
||||
POST {base}/get_pot -> {"content_binding": "...", "proxy": "..."}
|
||||
-> {"po_token": "...", ...}
|
||||
注意: 新版 provider 已弃用 visitor_data / data_sync_id 字段, 必须用 content_binding,
|
||||
否则返回 400。
|
||||
|
||||
设计原则: 全程 fail-soft —— provider 挂了只会拿不到 token, 绝不能拖垮播放或抛异常。
|
||||
"""
|
||||
|
||||
def __init__(self, session, config=None):
|
||||
self.session = session
|
||||
self.config = config or {}
|
||||
base = self.config.get('po_token_url') or ''
|
||||
if base and '://' not in base:
|
||||
base = 'http://' + base
|
||||
self.base = base.rstrip('/')
|
||||
self.timeout = float(self.config.get('po_token_timeout') or 6)
|
||||
# provider 的 token 默认按小时算, 这里换成秒
|
||||
self.ttl = int(float(self.config.get('po_token_ttl') or 6) * 3600)
|
||||
self.cache = {}
|
||||
self.cooldown_until = 0.0
|
||||
self.enabled = bool(self.base)
|
||||
|
||||
def _cool(self, seconds=180):
|
||||
self.cooldown_until = time.time() + seconds
|
||||
|
||||
def available(self):
|
||||
return self.enabled and time.time() >= self.cooldown_until
|
||||
|
||||
def ping(self):
|
||||
if not self.available():
|
||||
return False
|
||||
try:
|
||||
r = self.session.get(self.base + '/ping', timeout=self.timeout)
|
||||
ok = r.status_code == 200
|
||||
if not ok:
|
||||
self._cool()
|
||||
return ok
|
||||
except Exception as e:
|
||||
debug_log('pot ping failed', repr(e))
|
||||
self._cool()
|
||||
return False
|
||||
|
||||
def get(self, content_binding, proxy=None):
|
||||
"""按 content_binding 取 token, 带本地 TTL 缓存。失败返回 None。"""
|
||||
if not self.available() or not content_binding:
|
||||
return None
|
||||
now = time.time()
|
||||
hit = self.cache.get(content_binding)
|
||||
if hit and hit.get('expires', 0) > now:
|
||||
return hit.get('token')
|
||||
payload = {'content_binding': str(content_binding)}
|
||||
if proxy:
|
||||
payload['proxy'] = proxy if '://' in str(proxy) else 'http://' + str(proxy)
|
||||
try:
|
||||
r = self.session.post(self.base + '/get_pot', json=payload, timeout=self.timeout)
|
||||
if r.status_code >= 400:
|
||||
debug_log('pot http error', {'status': r.status_code, 'body': r.text[:200]})
|
||||
# 400 = 用了被弃用字段; 5xx = 生成失败。都冷却一段时间
|
||||
self._cool(300 if r.status_code >= 500 else 600)
|
||||
return None
|
||||
data = r.json() or {}
|
||||
token = data.get('po_token') or data.get('poToken') or data.get('token')
|
||||
if not token:
|
||||
debug_log('pot empty token', {'keys': list(data.keys())[:8]})
|
||||
self._cool()
|
||||
return None
|
||||
# 缓存封顶, 防止长跑内存膨胀
|
||||
if len(self.cache) > 128:
|
||||
self.cache.clear()
|
||||
self.cache[content_binding] = {'token': token, 'expires': now + self.ttl}
|
||||
debug_log('pot ok', {'binding': str(content_binding)[:24], 'len': len(token)})
|
||||
return token
|
||||
except Exception as e:
|
||||
debug_log('pot request error', repr(e))
|
||||
self._cool()
|
||||
return None
|
||||
|
||||
|
||||
def capCache(store, limit):
|
||||
"""字典缓存封顶: 超过上限直接清空(比 LRU 简单, 且不会在弱设备上抖动)。"""
|
||||
try:
|
||||
if isinstance(store, dict) and len(store) > limit:
|
||||
store.clear()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def classifyPlayability(status, reason):
|
||||
"""把 playabilityStatus 归类成 (风险等级, 类别)。"""
|
||||
text_reason = (str(reason or '')).lower()
|
||||
for pat in FATAL_REASON_PATTERNS:
|
||||
if pat.lower() in text_reason:
|
||||
return RISK_DEAD, 'FATAL'
|
||||
for pat in BOT_REASON_PATTERNS:
|
||||
if pat.lower() in text_reason:
|
||||
return RISK_HIGH, 'BOT_CHECK'
|
||||
if status in RISK_STATUS:
|
||||
return RISK_STATUS[status], status
|
||||
if status and status != 'OK':
|
||||
return RISK_SUSPECT, status
|
||||
return RISK_OK, 'OK'
|
||||
|
||||
|
||||
class RiskBlocked(Exception):
|
||||
"""所有客户端节点都被风控拦截时抛出, 携带结构化信息供上层兜底。"""
|
||||
|
||||
def __init__(self, video_id, category, reason, nodes=None):
|
||||
self.video_id = video_id
|
||||
self.category = category
|
||||
self.reason = reason
|
||||
self.nodes = nodes or {}
|
||||
super().__init__('[%s] %s' % (category, reason))
|
||||
|
||||
|
||||
# ==================== 核心提取类（合并优化） ====================
|
||||
class YouTubeLite:
|
||||
"""普通视频提取，合并 0712 优化：快速 API、编码优先级、SDR/HDR 识别"""
|
||||
@@ -1316,14 +1643,38 @@ class YouTubeLite:
|
||||
self.extract_cache = {}
|
||||
self.sig_plan_cache = {}
|
||||
self.extract_cache_ttl = int(self.config.get('extract_cache_ttl') or 300)
|
||||
# ---- 节点风险 / 探测开关 (均可由 ext 覆盖) ----
|
||||
self.node_health = NODE_HEALTH
|
||||
# probe=0 关闭播放前探测(最快); 1=仅 best 探测; 2=全部探测
|
||||
self.probe_level = int(self.config.get('probe', 1) or 0)
|
||||
self.probe_timeout = float(self.config.get('probe_timeout') or 4)
|
||||
self.probe_max = int(self.config.get('probe_max') or 3)
|
||||
# 允许 av01 / 无 contentLength 等高风险码流(默认不允许)
|
||||
self.allow_risky = bool(self.config.get('allow_risky'))
|
||||
self.last_block = None
|
||||
self.visitor_data = None
|
||||
self.pot = PoTokenProvider(session, self.config)
|
||||
# 缓存上限(条), 防止长时间运行 OOM 导致闪退
|
||||
self.cache_limit = int(self.config.get('cache_limit') or 64)
|
||||
self._current_video_id = None
|
||||
self.cookie_header = parseCookieInput(self.config.get('cookie'))
|
||||
|
||||
def extract(self, url_or_id):
|
||||
def extract(self, url_or_id, force=False):
|
||||
video_id = self.extract_video_id(url_or_id)
|
||||
cached = self.extract_cache.get(video_id)
|
||||
now = time.time()
|
||||
if force:
|
||||
self.extract_cache.pop(video_id, None)
|
||||
cached = None
|
||||
if cached and cached.get('expires', 0) > now:
|
||||
debug_log('extract cache hit', {'video_id': video_id, 'ttl': int(cached.get('expires', 0) - now)})
|
||||
return cached.get('data')
|
||||
# 熔断期间直接短路, 一个网络请求都不发
|
||||
if breakerOpen() and not self.config.get('ignore_breaker'):
|
||||
debug_log('extract short-circuit by breaker', {'video_id': video_id, 'remain': breakerRemain()})
|
||||
raise RiskBlocked(video_id, 'BOT_CHECK',
|
||||
BOT_BREAKER['reason'] or '触发人机校验, 冷却中',
|
||||
self.node_health.snapshot())
|
||||
watch_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
debug_log('extract start', {'input': url_or_id, 'video_id': video_id})
|
||||
page_resp = self._get(watch_url)
|
||||
@@ -1334,7 +1685,13 @@ class YouTubeLite:
|
||||
player_url = self._extract_player_url(page)
|
||||
api_key = ytcfg.get('INNERTUBE_API_KEY') or self._search(r'"INNERTUBE_API_KEY":"([^"]+)"', page)
|
||||
visitor_data = self._extract_visitor_data(ytcfg, player_response)
|
||||
self.visitor_data = visitor_data
|
||||
self._current_video_id = video_id
|
||||
# 原代码这里恒为 None, 导致 _extract_signature_timestamp 是死代码,
|
||||
# WEB / MWEB 节点缺 signatureTimestamp 时签名会失效。
|
||||
sts = None
|
||||
if player_url and self.config.get('sts', True) is not False:
|
||||
sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
|
||||
debug_log('page parsed', {'has_ytcfg': bool(ytcfg), 'has_initial_pr': bool(player_response), 'initial_status': (player_response.get('playabilityStatus') or {}).get('status'), 'initial_has_streaming': bool(player_response.get('streamingData')), 'has_api_key': bool(api_key), 'has_visitor': bool(visitor_data), 'sts': sts, 'player_url': player_url})
|
||||
context = ytcfg.get('INNERTUBE_CONTEXT') or {
|
||||
'client': {'clientName': 'WEB', 'clientVersion': '2.20240310.01.00', 'hl': 'en', 'gl': 'US'}
|
||||
@@ -1351,7 +1708,11 @@ class YouTubeLite:
|
||||
streaming = player_response.get('streamingData') or {}
|
||||
if status and status not in ('OK', 'LIVE_STREAM_OFFLINE') and not streaming:
|
||||
reason = (player_response.get('playabilityStatus') or {}).get('reason') or status
|
||||
raise Exception(f'YouTube 不可播放: {reason}')
|
||||
risk, category = classifyPlayability(status, reason)
|
||||
debug_log('extract blocked', {'video_id': video_id, 'status': status,
|
||||
'category': category, 'reason': str(reason)[:160],
|
||||
'health': self.node_health.snapshot()})
|
||||
raise RiskBlocked(video_id, category, reason, self.node_health.snapshot())
|
||||
details = player_response.get('videoDetails') or {}
|
||||
raw_formats = []
|
||||
seen_raw = set()
|
||||
@@ -1379,13 +1740,20 @@ class YouTubeLite:
|
||||
formats.append(item)
|
||||
debug_log('normalized formats', {'count': len(formats), 'cipher_count': cipher_count, 'progressive': len([x for x in formats if x.get('vcodec') != 'none' and x.get('acodec') != 'none'])})
|
||||
if not formats:
|
||||
raise Exception('未获取到可用播放地址')
|
||||
raise RiskBlocked(video_id, 'NO_FORMAT', '所有客户端节点均未产出可用码流',
|
||||
self.node_health.snapshot())
|
||||
data = {
|
||||
'id': video_id,
|
||||
'title': details.get('title') or video_id,
|
||||
'duration': int(details.get('lengthSeconds') or 0),
|
||||
'formats': formats,
|
||||
# 点播也可能带 HLS, 作为最后的兜底播放地址
|
||||
'hls_url': streaming.get('hlsManifestUrl') or '',
|
||||
}
|
||||
breakerReset() # 拿到码流 = IP 目前没被拦, 复位熔断
|
||||
capCache(self.extract_cache, self.cache_limit)
|
||||
capCache(self.player_cache, 2)
|
||||
capCache(self.sig_plan_cache, 8)
|
||||
self.extract_cache[video_id] = {'data': data, 'expires': time.time() + self.extract_cache_ttl}
|
||||
return data
|
||||
|
||||
@@ -1408,6 +1776,7 @@ class YouTubeLite:
|
||||
'ANDROID': 3,
|
||||
'IOS': 5,
|
||||
'TVHTML5': 7,
|
||||
'TVHTML5_SIMPLY_EMBEDDED_PLAYER': 85,
|
||||
'ANDROID_VR': 28,
|
||||
'WEB_EMBEDDED_PLAYER': 56,
|
||||
'WEB_REMIX': 67,
|
||||
@@ -1431,12 +1800,31 @@ class YouTubeLite:
|
||||
return None
|
||||
|
||||
def _get_po_token(self, client_name, context='gvs'):
|
||||
"""先用 ext 里写死的静态 token, 没有再走远程 provider。"""
|
||||
tokens = self.config.get('po_token') or self.config.get('po_tokens') or {}
|
||||
if isinstance(tokens, str):
|
||||
if isinstance(tokens, str) and tokens:
|
||||
return tokens
|
||||
if isinstance(tokens, dict):
|
||||
return tokens.get(f'{client_name}.{context}') or tokens.get(client_name) or tokens.get(context)
|
||||
return None
|
||||
static = tokens.get(f'{client_name}.{context}') or tokens.get(client_name) or tokens.get(context)
|
||||
if static:
|
||||
return static
|
||||
# 远程 provider: gvs 上下文绑定 visitorData, player 上下文绑定 videoId
|
||||
try:
|
||||
if not self.pot.available():
|
||||
return None
|
||||
binding = self.visitor_data if context == 'gvs' else (self._current_video_id or self.visitor_data)
|
||||
if not binding:
|
||||
return None
|
||||
proxy = None
|
||||
try:
|
||||
proxy = (self.session.proxies or {}).get('https') or (self.session.proxies or {}).get('http')
|
||||
except Exception:
|
||||
proxy = None
|
||||
return self.pot.get(binding, proxy)
|
||||
except Exception as e:
|
||||
debug_log('po_token provider error', repr(e))
|
||||
return None
|
||||
|
||||
|
||||
def _video_codec_priority(self, item):
|
||||
mime = (item.get('mimeType') or '').lower()
|
||||
@@ -1457,9 +1845,51 @@ class YouTubeLite:
|
||||
color = item.get('colorInfo') or {}
|
||||
return 'vp9.2' in mime or 'vp09.02' in codecs or bool(color.get('hdrMetadataInfo'))
|
||||
|
||||
def _is_risky_best_video(self, item):
|
||||
@staticmethod
|
||||
def url_expired(media_url, slack=60):
|
||||
"""YouTube 直链带 expire=<unix秒>, 过期后必然 403 -> 表现为"无法播放"。"""
|
||||
try:
|
||||
q = parse_qs(urlparse(media_url or '').query)
|
||||
exp = int((q.get('expire') or ['0'])[0])
|
||||
return bool(exp) and (time.time() + slack) >= exp
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# ---------- 码流(节点产物)风险评分 ----------
|
||||
# 分值越高越容易 403 / 解码失败 / 卡顿, 排序时降权
|
||||
def _format_risk(self, item):
|
||||
score = 0
|
||||
codecs = (item.get('codecs') or '').lower()
|
||||
return 'av01' in codecs
|
||||
client = (item.get('client') or '').upper()
|
||||
url = item.get('url') or ''
|
||||
# 1) AV1: 大量电视盒子无硬解, 软解直接卡死
|
||||
if 'av01' in codecs:
|
||||
score += 4
|
||||
# 2) WEB / MWEB 节点直链普遍需要 po_token, 无 pot 极易 403
|
||||
if client in ('WEB', 'MWEB', 'WEB_EMBEDDED_PLAYER') and 'pot=' not in url:
|
||||
score += 5
|
||||
# 3) 缺 contentLength 的分片流在 DASH 组装时容易失败
|
||||
if not item.get('contentLength'):
|
||||
score += 2
|
||||
# 4) n 参数未解密成功(仍是密文)会被服务端限速到几十 KB/s
|
||||
if 'n=' in url and item.get('_nsig_failed'):
|
||||
score += 3
|
||||
# 5) HDR(VP9.2) 在 SDR 设备上偏色/黑屏
|
||||
if self._is_hdr_video(item):
|
||||
score += 1
|
||||
# 5.5) 直链已过期 -> 必 403
|
||||
if self.url_expired(url):
|
||||
score += 9
|
||||
# 6) 超高码率 4K 在弱网/弱盒子上必然卡
|
||||
if int(item.get('height') or 0) >= 2160 and int(item.get('bitrate') or 0) > 25_000_000:
|
||||
score += 2
|
||||
return score
|
||||
|
||||
def _is_risky_best_video(self, item):
|
||||
"""兼容旧调用点: 风险分 >= 4 视为高风险。"""
|
||||
if self.allow_risky:
|
||||
return False
|
||||
return self._format_risk(item) >= 4
|
||||
|
||||
def choose_playable(self, formats, quality=None):
|
||||
all_videos = [x for x in formats if x.get('vcodec') != 'none' and x.get('acodec') == 'none']
|
||||
@@ -1480,23 +1910,80 @@ class YouTubeLite:
|
||||
candidates = all_videos
|
||||
if not candidates:
|
||||
return None
|
||||
# 风险分升序优先, 其次编码优先级/高度/码率降序
|
||||
candidates.sort(key=lambda x: (
|
||||
self._video_codec_priority(x),
|
||||
int(x.get('height') or 0),
|
||||
int(x.get('bitrate') or 0)
|
||||
), reverse=True)
|
||||
self._format_risk(x),
|
||||
-self._video_codec_priority(x),
|
||||
-int(x.get('height') or 0),
|
||||
-int(x.get('bitrate') or 0),
|
||||
))
|
||||
selected = candidates[0]
|
||||
debug_log('video selected fast', {
|
||||
'quality': quality,
|
||||
'itag': selected.get('itag'),
|
||||
'height': selected.get('height'),
|
||||
'mime': selected.get('mimeType'),
|
||||
'risk': self._format_risk(selected),
|
||||
'codec_priority': self._video_codec_priority(selected),
|
||||
'candidates': len(candidates),
|
||||
'probe_skipped': True,
|
||||
})
|
||||
return selected
|
||||
|
||||
# ---------- 风险感知的候选队列 ----------
|
||||
def rank_candidates(self, formats, quality=None):
|
||||
videos = [x for x in formats if x.get('vcodec') != 'none' and x.get('acodec') == 'none']
|
||||
if quality == '4k':
|
||||
pool = [x for x in videos if int(x.get('height') or 0) >= 2160]
|
||||
elif quality == '2k':
|
||||
pool = [x for x in videos if 1440 <= int(x.get('height') or 0) < 2160]
|
||||
elif quality == '1080p':
|
||||
pool = [x for x in videos if 1000 <= int(x.get('height') or 0) < 1440]
|
||||
else:
|
||||
pool = videos[:]
|
||||
pool = pool or videos
|
||||
pool.sort(key=lambda x: (
|
||||
self._format_risk(x),
|
||||
-self._video_codec_priority(x),
|
||||
-int(x.get('height') or 0),
|
||||
-int(x.get('bitrate') or 0),
|
||||
))
|
||||
return pool
|
||||
|
||||
def progressive_fallback(self, formats):
|
||||
"""音视频合一的 progressive 流(itag 18/22), 兼容性最好, 用于最后兜底。"""
|
||||
prog = [x for x in formats if x.get('vcodec') != 'none' and x.get('acodec') != 'none']
|
||||
if not prog:
|
||||
return None
|
||||
prog.sort(key=lambda x: (self._format_risk(x), -int(x.get('height') or 0)))
|
||||
return prog[0]
|
||||
|
||||
def pick_playable_with_probe(self, formats, quality=None, force_probe=False):
|
||||
"""按风险从低到高逐个轻探测(Range: bytes=0-1), 403/404 就换下一个码流节点。
|
||||
|
||||
probe_level: 0=不探测 1=仅 best/4k 探测 2=全部探测
|
||||
"""
|
||||
candidates = self.rank_candidates(formats, quality)
|
||||
if not candidates:
|
||||
return None, []
|
||||
need_probe = force_probe or self.probe_level >= 2 or (self.probe_level == 1 and quality in ('best', '4k'))
|
||||
if not need_probe:
|
||||
return candidates[0], []
|
||||
tried = []
|
||||
for item in candidates[:max(1, self.probe_max)]:
|
||||
ok, code = self._probe_format(item)
|
||||
tried.append({'itag': item.get('itag'), 'client': item.get('client'),
|
||||
'risk': self._format_risk(item), 'ok': ok, 'code': code})
|
||||
if ok:
|
||||
debug_log('probe picked', {'itag': item.get('itag'), 'tried': tried})
|
||||
return item, tried
|
||||
# 该码流不可用 -> 给产出它的客户端节点记一次风险
|
||||
client = item.get('client')
|
||||
if client:
|
||||
self.node_health.markRisk(client, RISK_SUSPECT, 'probe %s' % code)
|
||||
debug_log('probe all failed', tried)
|
||||
return candidates[0], tried
|
||||
|
||||
def choose_video_tracks(self, formats, quality=None):
|
||||
videos = [x for x in formats if x.get('vcodec') != 'none' and x.get('acodec') == 'none']
|
||||
cap = 2160 if quality in ('best', '4k') else 1440 if quality == '2k' else 1080
|
||||
@@ -1549,7 +2036,8 @@ class YouTubeLite:
|
||||
headers = self.headers.copy()
|
||||
headers.update(item.get('headers') or {})
|
||||
headers['Range'] = 'bytes=0-1'
|
||||
r = self.session.get(item.get('url'), headers=headers, stream=True, timeout=10)
|
||||
r = self.session.get(item.get('url'), headers=headers, stream=True,
|
||||
timeout=getattr(self, 'probe_timeout', 4))
|
||||
if r.url and r.url != item.get('url'):
|
||||
item['url'] = r.url
|
||||
item['redirected'] = True
|
||||
@@ -1594,18 +2082,50 @@ class YouTubeLite:
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def _call_player_api(self, video_id, api_key, context, referer, visitor_data=None, sts=None):
|
||||
clients = [
|
||||
{'client': {'clientName': 'ANDROID_VR', 'clientVersion': '1.65.10', 'deviceMake': 'Oculus', 'deviceModel': 'Quest 3', 'androidSdkVersion': 32, 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip', 'osName': 'Android', 'osVersion': '12L', 'hl': 'en', 'gl': 'US'}},
|
||||
{'client': {'clientName': 'ANDROID', 'clientVersion': '21.02.35', 'androidSdkVersion': 30, 'userAgent': 'com.google.android.youtube/21.02.35 (Linux; U; Android 11) gzip', 'osName': 'Android', 'osVersion': '11', 'hl': 'en', 'gl': 'US'}},
|
||||
{'client': {'clientName': 'IOS', 'clientVersion': '21.02.3', 'deviceMake': 'Apple', 'deviceModel': 'iPhone16,2', 'userAgent': 'com.google.ios.youtube/21.02.3 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)', 'osName': 'iPhone', 'osVersion': '18.3.2.22D82', 'hl': 'en', 'gl': 'US'}},
|
||||
context,
|
||||
{'client': {'clientName': 'MWEB', 'clientVersion': '2.20260115.01.00', 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', 'hl': 'en', 'gl': 'US'}},
|
||||
# ---------- 客户端节点注册表 ----------
|
||||
def _client_nodes(self, context):
|
||||
"""返回节点列表; risky=该节点产出的直链更容易 403(需 po_token)。"""
|
||||
return [
|
||||
{'name': 'ANDROID_VR', 'risky': False, 'client': {'clientName': 'ANDROID_VR', 'clientVersion': '1.65.10', 'deviceMake': 'Oculus', 'deviceModel': 'Quest 3', 'androidSdkVersion': 32, 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip', 'osName': 'Android', 'osVersion': '12L', 'hl': 'en', 'gl': 'US'}},
|
||||
{'name': 'ANDROID', 'risky': False, 'client': {'clientName': 'ANDROID', 'clientVersion': '21.02.35', 'androidSdkVersion': 30, 'userAgent': 'com.google.android.youtube/21.02.35 (Linux; U; Android 11) gzip', 'osName': 'Android', 'osVersion': '11', 'hl': 'en', 'gl': 'US'}},
|
||||
{'name': 'IOS', 'risky': False, 'client': {'clientName': 'IOS', 'clientVersion': '21.02.3', 'deviceMake': 'Apple', 'deviceModel': 'iPhone16,2', 'userAgent': 'com.google.ios.youtube/21.02.3 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)', 'osName': 'iPhone', 'osVersion': '18.3.2.22D82', 'hl': 'en', 'gl': 'US'}},
|
||||
{'name': 'TVHTML5', 'risky': False, 'client': {'clientName': 'TVHTML5', 'clientVersion': '7.20250101.00.00', 'userAgent': 'Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15', 'hl': 'en', 'gl': 'US'}},
|
||||
{'name': (((context or {}).get('client') or {}).get('clientName') or 'WEB'), 'risky': True, 'raw': context},
|
||||
# TVHTML5_SIMPLY_EMBEDDED_PLAYER 已被 YouTube 下线, 实测固定返回
|
||||
# "YouTube is no longer supported in this application or device.", 故移除。
|
||||
# WEB_EMBEDDED_PLAYER 必须带 thirdParty.embedUrl, 否则返回 "This video is unavailable"。
|
||||
{'name': 'WEB_EMBEDDED_PLAYER', 'risky': True, 'embed': True, 'client': {'clientName': 'WEB_EMBEDDED_PLAYER', 'clientVersion': '1.20260115.01.00', 'userAgent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'hl': 'en', 'gl': 'US'}},
|
||||
{'name': 'MWEB', 'risky': True, 'client': {'clientName': 'MWEB', 'clientVersion': '2.20260115.01.00', 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', 'hl': 'en', 'gl': 'US'}},
|
||||
]
|
||||
|
||||
def _call_player_api(self, video_id, api_key, context, referer, visitor_data=None, sts=None):
|
||||
nodes = self._client_nodes(context)
|
||||
# ext.clients 可指定只用哪些节点(按给定顺序), 例如 ["ANDROID_VR","IOS"]
|
||||
wanted = self.config.get('clients')
|
||||
if isinstance(wanted, str):
|
||||
wanted = [x.strip() for x in wanted.split(',') if x.strip()]
|
||||
if isinstance(wanted, list) and wanted:
|
||||
table = {n['name']: n for n in nodes}
|
||||
nodes = [table[x] for x in wanted if x in table] or nodes
|
||||
# 按健康度排序: 冷却中/高风险的节点自动排到最后, 但不丢弃(全挂时仍可兜底)
|
||||
nodes.sort(key=lambda n: (self.node_health.weight(n['name']), 1 if n.get('risky') else 0))
|
||||
debug_log('node order', {'order': [n['name'] for n in nodes], 'health': self.node_health.snapshot()})
|
||||
|
||||
results = []
|
||||
fallback = None
|
||||
for ctx in clients:
|
||||
client_name = (ctx.get('client') or {}).get('clientName')
|
||||
blocked = [] # 被风险拦截的节点
|
||||
fatal = None # 地区/删除类, 换节点也没用
|
||||
|
||||
for node in nodes:
|
||||
client_name = node['name']
|
||||
ctx = node.get('raw') or {'client': node['client']}
|
||||
if node.get('embed'):
|
||||
ctx = dict(ctx)
|
||||
ctx['thirdParty'] = {'embedUrl': 'https://www.youtube.com/'}
|
||||
client = ctx.get('client') or {}
|
||||
if self.node_health.isCooling(client_name) and results:
|
||||
debug_log('node skipped (cooling)', {'client': client_name})
|
||||
continue
|
||||
try:
|
||||
url = f'https://www.youtube.com/youtubei/v1/player?key={api_key}&prettyPrint=false'
|
||||
payload = {
|
||||
@@ -1615,7 +2135,10 @@ class YouTubeLite:
|
||||
'contentCheckOk': True,
|
||||
'racyCheckOk': True,
|
||||
}
|
||||
client = ctx.get('client') or {}
|
||||
# 有 po_token 就带上, 显著降低 WEB/MWEB 节点被风控的概率
|
||||
po = self._get_po_token(client_name, 'player')
|
||||
if po:
|
||||
payload['serviceIntegrityDimensions'] = {'poToken': po}
|
||||
headers = {
|
||||
'Referer': referer,
|
||||
'X-YouTube-Client-Name': str(self._client_name_id(client.get('clientName'))),
|
||||
@@ -1623,32 +2146,73 @@ class YouTubeLite:
|
||||
}
|
||||
if visitor_data:
|
||||
headers['X-Goog-Visitor-Id'] = visitor_data
|
||||
# 登录态: Cookie + SAPISIDHASH。目前对付 bot check 最有效的手段
|
||||
headers.update(authHeaders(self.cookie_header))
|
||||
client_ua = client.get('userAgent')
|
||||
if client_ua:
|
||||
headers['User-Agent'] = client_ua
|
||||
|
||||
data = self._post_json(url, payload, headers=headers)
|
||||
status = (data.get('playabilityStatus') or {}).get('status')
|
||||
ps = data.get('playabilityStatus') or {}
|
||||
status = ps.get('status')
|
||||
reason = ps.get('reason') or (((ps.get('errorScreen') or {}).get('playerErrorMessageRenderer') or {}).get('subreason') or {}).get('simpleText') or ''
|
||||
risk, category = classifyPlayability(status, reason)
|
||||
|
||||
streaming = data.get('streamingData') or {}
|
||||
formats = streaming.get('formats') or []
|
||||
adaptive = streaming.get('adaptiveFormats') or []
|
||||
direct_video = [x for x in adaptive if (x.get('url') or x.get('signatureCipher') or x.get('cipher')) and str(x.get('mimeType') or '').startswith('video/')]
|
||||
direct_any = [x for x in formats + adaptive if x.get('url') or x.get('signatureCipher') or x.get('cipher')]
|
||||
has_streaming = bool(streaming)
|
||||
debug_log('player api client', {'client': client_name, 'status': status, 'has_streaming': has_streaming, 'formats': len(formats), 'adaptive': len(adaptive), 'direct_any': len(direct_any), 'direct_video': len(direct_video)})
|
||||
|
||||
debug_log('player api client', {'client': client_name, 'status': status, 'risk': risk, 'category': category,
|
||||
'has_streaming': has_streaming, 'formats': len(formats),
|
||||
'adaptive': len(adaptive), 'direct_any': len(direct_any),
|
||||
'direct_video': len(direct_video)})
|
||||
|
||||
if category == 'FATAL':
|
||||
fatal = fatal or (category, reason or status)
|
||||
self.node_health.markRisk(client_name, RISK_DEAD, reason)
|
||||
# 地区/下架类: 换节点无意义, 但仍继续跑完(某些节点 gl 不同可能可用)
|
||||
blocked.append({'client': client_name, 'status': status, 'category': category, 'reason': reason})
|
||||
continue
|
||||
|
||||
if not has_streaming and risk >= RISK_SUSPECT:
|
||||
# 风险拦截: 记账 + 冷却 + 换下一个节点, 不上抛
|
||||
self.node_health.markRisk(client_name, risk, reason)
|
||||
blocked.append({'client': client_name, 'status': status, 'category': category, 'reason': reason})
|
||||
debug_log('node blocked', {'client': client_name, 'category': category, 'reason': reason[:120]})
|
||||
continue
|
||||
|
||||
if has_streaming:
|
||||
self.node_health.markOk(client_name)
|
||||
data['_client_name'] = client_name
|
||||
data['_client_ua'] = client_ua
|
||||
data['_client_risky'] = bool(node.get('risky'))
|
||||
results.append(data)
|
||||
if client_name == 'ANDROID_VR' and direct_video:
|
||||
# 低风险节点直接拿到视频直链 -> 快速返回
|
||||
if not node.get('risky') and direct_video and status == 'OK':
|
||||
debug_log('player api fast return', {'client': client_name, 'direct_video': len(direct_video)})
|
||||
return results
|
||||
if has_streaming and fallback is None:
|
||||
fallback = data
|
||||
if fallback is None:
|
||||
fallback = data
|
||||
elif fallback is None:
|
||||
fallback = data
|
||||
except Exception as e:
|
||||
self.node_health.markRisk(client_name, RISK_SUSPECT, repr(e))
|
||||
debug_log('player api client error', {'client': client_name, 'error': repr(e)})
|
||||
continue
|
||||
|
||||
self.last_block = {'blocked': blocked, 'fatal': fatal, 'health': self.node_health.snapshot()}
|
||||
# 一轮下来全灭且主因是人机校验 -> 开熔断, 避免继续空轮询加深 IP 标记
|
||||
if not results and blocked:
|
||||
bot_hits = [b for b in blocked if b.get('category') in ('BOT_CHECK', 'LOGIN_REQUIRED')]
|
||||
if len(bot_hits) >= max(2, len(nodes) // 2):
|
||||
breakerTrip(int(self.config.get('bot_cooldown') or 300),
|
||||
bot_hits[0].get('reason') or 'BOT_CHECK')
|
||||
if not results and blocked:
|
||||
category, reason = fatal if fatal else (blocked[0].get('category'), blocked[0].get('reason') or blocked[0].get('status'))
|
||||
raise RiskBlocked(video_id, category, reason, self.node_health.snapshot())
|
||||
return results or ([fallback] if fallback else [])
|
||||
|
||||
def _normalize_format(self, fmt, player_url):
|
||||
@@ -1659,7 +2223,9 @@ class YouTubeLite:
|
||||
media_url = self._decrypt_signature_cipher(cipher, player_url)
|
||||
if not media_url:
|
||||
return None
|
||||
before_n = media_url
|
||||
media_url = self._decrypt_nsig(media_url, player_url)
|
||||
nsig_failed = ('n=' in media_url) and (media_url == before_n)
|
||||
client_name = fmt.get('_client_name')
|
||||
po_token = self._get_po_token(client_name, 'gvs') if client_name else None
|
||||
if po_token:
|
||||
@@ -1692,6 +2258,8 @@ class YouTubeLite:
|
||||
'vcodec': codecs if has_video else 'none',
|
||||
'acodec': codecs if has_audio else 'none',
|
||||
'headers': headers,
|
||||
'_nsig_failed': nsig_failed,
|
||||
'_client_risky': bool(fmt.get('_client_risky')),
|
||||
}
|
||||
|
||||
def _decrypt_signature_cipher(self, cipher, player_url):
|
||||
@@ -1985,6 +2553,10 @@ class YouTubeLiveLite:
|
||||
raise Exception('无法识别 YouTube 视频 ID')
|
||||
|
||||
def extract_live(self, url_or_id):
|
||||
if breakerOpen() and not (self.config or {}).get('ignore_breaker'):
|
||||
debug_log('live short-circuit by breaker', {'remain': breakerRemain()})
|
||||
raise RiskBlocked(self.extract_video_id(url_or_id), 'BOT_CHECK',
|
||||
BOT_BREAKER['reason'] or '触发人机校验, 冷却中')
|
||||
video_id = self.extract_video_id(url_or_id)
|
||||
now = time.time()
|
||||
cached = self.cache.get(video_id)
|
||||
@@ -2074,12 +2646,16 @@ class YouTubeLiveLite:
|
||||
return response.json()
|
||||
|
||||
def _call_player_api(self, video_id, api_key, ytcfg, referer, visitor_data=None):
|
||||
# 直播侧原本没有任何风险识别, 这里与点播侧对齐
|
||||
self.node_health = getattr(self, 'node_health', NODE_HEALTH)
|
||||
self.cookie_header = getattr(self, 'cookie_header', None) or parseCookieInput((self.config or {}).get('cookie'))
|
||||
context = ytcfg.get('INNERTUBE_CONTEXT') or {
|
||||
'client': {'clientName': 'WEB', 'clientVersion': '2.20240310.01.00', 'hl': 'en', 'gl': 'US'}
|
||||
}
|
||||
clients = [
|
||||
{'client': {'clientName': 'ANDROID', 'clientVersion': '21.02.35', 'androidSdkVersion': 30, 'userAgent': 'com.google.android.youtube/21.02.35 (Linux; U; Android 11) gzip', 'osName': 'Android', 'osVersion': '11', 'hl': 'en', 'gl': 'US'}},
|
||||
{'client': {'clientName': 'IOS', 'clientVersion': '21.02.3', 'deviceMake': 'Apple', 'deviceModel': 'iPhone16,2', 'userAgent': 'com.google.ios.youtube/21.02.3 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)', 'osName': 'iPhone', 'osVersion': '18.3.2.22D82', 'hl': 'en', 'gl': 'US'}},
|
||||
{'client': {'clientName': 'TVHTML5', 'clientVersion': '7.20250101.00.00', 'userAgent': 'Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15', 'hl': 'en', 'gl': 'US'}},
|
||||
{'client': {'clientName': 'MWEB', 'clientVersion': '2.20260115.01.00', 'userAgent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1', 'hl': 'en', 'gl': 'US'}},
|
||||
context,
|
||||
]
|
||||
@@ -2187,6 +2763,7 @@ class YouTubeLiveLite:
|
||||
'ANDROID': 3,
|
||||
'IOS': 5,
|
||||
'TVHTML5': 7,
|
||||
'TVHTML5_SIMPLY_EMBEDDED_PLAYER': 85,
|
||||
'ANDROID_VR': 28,
|
||||
'WEB_EMBEDDED_PLAYER': 56,
|
||||
'WEB_REMIX': 67,
|
||||
@@ -2199,6 +2776,38 @@ class Spider(Spider):
|
||||
return 'YouTube 视频+直播（优化版）'
|
||||
|
||||
def init(self, extend):
|
||||
# init 抛异常 = TVBox 认为源加载失败 -> 反复重载接口
|
||||
try:
|
||||
self._init(extend)
|
||||
except Exception as e:
|
||||
debug_log('init fatal', repr(e))
|
||||
self._init_minimal()
|
||||
|
||||
def _init_minimal(self):
|
||||
"""init 失败时的最小可用状态, 保证后续方法不会 AttributeError。"""
|
||||
self.extendDict = getattr(self, 'extendDict', {}) or {}
|
||||
if not hasattr(self, 'session'):
|
||||
self.session = requests.Session()
|
||||
self.proxy_str = getattr(self, 'proxy_str', '')
|
||||
self.yt_classes = getattr(self, 'yt_classes', None) or YOUTUBE_CLASSES
|
||||
self.yt_filters = getattr(self, 'yt_filters', None) or CATEGORY_FILTERS
|
||||
self.header = getattr(self, 'header', None) or {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'Referer': 'https://www.youtube.com/',
|
||||
}
|
||||
if not hasattr(self, 'yt_video'):
|
||||
self.yt_video = YouTubeLite(self.session, self.header, self.extendDict)
|
||||
if not hasattr(self, 'yt_live'):
|
||||
self.yt_live = YouTubeLiveLite(self.session, self.header, self.extendDict)
|
||||
for attr in ('search_page_cache', 'live_search_cache', 'hls_url_cache'):
|
||||
if not hasattr(self, attr):
|
||||
setattr(self, attr, {})
|
||||
self.hls_proxy_enabled = getattr(self, 'hls_proxy_enabled', True)
|
||||
self._hls_key_seq = getattr(self, '_hls_key_seq', 0)
|
||||
self.direct_segments = getattr(self, 'direct_segments', False)
|
||||
|
||||
def _init(self, extend):
|
||||
try:
|
||||
self.extendDict = json.loads(extend) if extend else {}
|
||||
except Exception:
|
||||
@@ -2269,6 +2878,21 @@ class Spider(Spider):
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'Referer': 'https://www.youtube.com/'
|
||||
}
|
||||
# 登录态 Cookie: ext.cookie 直接给字符串/字典, 或 ext.cookie_file 给 cookies.txt 路径
|
||||
cookie_raw = self.extendDict.get('cookie')
|
||||
cookie_file = self.extendDict.get('cookie_file')
|
||||
if not cookie_raw and cookie_file:
|
||||
try:
|
||||
with open(cookie_file, 'r', encoding='utf-8', errors='ignore') as fp:
|
||||
cookie_raw = fp.read()
|
||||
except Exception as e:
|
||||
debug_log('cookie_file 读取失败', repr(e))
|
||||
self.cookie_header = parseCookieInput(cookie_raw)
|
||||
if self.cookie_header:
|
||||
self.extendDict['cookie'] = self.cookie_header
|
||||
self.header.update(authHeaders(self.cookie_header))
|
||||
debug_log('已启用登录态 Cookie', {'len': len(self.cookie_header),
|
||||
'has_sapisid': 'SAPISID' in self.cookie_header})
|
||||
self.session.headers.update(self.header)
|
||||
|
||||
# 初始化两个提取器
|
||||
@@ -2280,10 +2904,109 @@ class Spider(Spider):
|
||||
self.hls_proxy_enabled = self.extendDict.get('hls_proxy', True) is not False
|
||||
self._hls_key_seq = 0
|
||||
self.direct_segments = str(self.extendDict.get('seg') or 'proxy').lower() == 'direct'
|
||||
# po_token provider 自检(可选, 失败只记日志不影响启动)
|
||||
if self.extendDict.get('po_token_url') and self.extendDict.get('po_token_ping', True) is not False:
|
||||
debug_log('po_token provider', {'base': self.yt_video.pot.base,
|
||||
'alive': self.yt_video.pot.ping()})
|
||||
|
||||
# ---------- 统一兜底: 拦截风险弹窗 / 无法播放 ----------
|
||||
def _risk_message(self, err):
|
||||
if isinstance(err, RiskBlocked):
|
||||
mapping = {
|
||||
'BOT_CHECK': ('触发 YouTube 人机校验(出口 IP 被标记)。'
|
||||
'按有效性排序: 1) 配 ext.cookie 登录态 2) 换出口 IP/节点 3) po_token'
|
||||
+ (' [冷却 %ds]' % breakerRemain() if breakerRemain() else '')),
|
||||
'LOGIN_REQUIRED': '该视频需要登录账号才能播放',
|
||||
'AGE_VERIFICATION_REQUIRED': '该视频有年龄限制, 需登录验证',
|
||||
'AGE_CHECK_REQUIRED': '该视频有年龄限制, 需登录验证',
|
||||
'FATAL': '该视频已下架/私享/在当前地区不可用',
|
||||
'NO_FORMAT': '未取到可用码流, 请稍后重试或更换代理节点',
|
||||
}
|
||||
return mapping.get(err.category, str(err.reason or err.category))
|
||||
return str(err)
|
||||
|
||||
def _fallback_play_result(self, video_id, err, formats=None, hls_url=''):
|
||||
"""降级链: progressive 直链 -> 最低风险码流 -> embed(可关) -> 静默失败。
|
||||
|
||||
目的是尽量不让 TVBox 弹出 WebView 风险窗口。
|
||||
"""
|
||||
message = self._risk_message(err)
|
||||
debug_log('fallback play', {'video_id': video_id, 'msg': message,
|
||||
'breaker_remain': breakerRemain(),
|
||||
'cookie': bool(getattr(self, 'cookie_header', ''))})
|
||||
|
||||
# 0) 点播 HLS(hlsManifestUrl): 单地址、自适应, 盒子兼容性最好
|
||||
try:
|
||||
if hls_url:
|
||||
play_url = self._cache_hls_url(hls_url, video_id, 'master') if self.hls_proxy_enabled else hls_url
|
||||
debug_log('fallback -> hls')
|
||||
return {'parse': 0, 'jx': 0, 'url': play_url, 'header': self.header,
|
||||
'format': 'application/x-mpegURL'}
|
||||
except Exception as e:
|
||||
debug_log('fallback hls error', repr(e))
|
||||
|
||||
# 1) progressive(音视频合一)直链, 兼容性最好
|
||||
try:
|
||||
if formats:
|
||||
prog = self.yt_video.progressive_fallback(formats)
|
||||
if prog and prog.get('url'):
|
||||
headers = self.header.copy()
|
||||
headers.update(prog.get('headers') or {})
|
||||
debug_log('fallback -> progressive', {'itag': prog.get('itag')})
|
||||
return {'parse': 0, 'jx': 0, 'url': prog['url'], 'header': headers}
|
||||
except Exception as e:
|
||||
debug_log('fallback progressive error', repr(e))
|
||||
|
||||
# 2) 风险最低的纯视频码流(无音轨也好过黑屏)
|
||||
try:
|
||||
if formats:
|
||||
ranked = self.yt_video.rank_candidates(formats, '1080p')
|
||||
if ranked and ranked[0].get('url'):
|
||||
headers = self.header.copy()
|
||||
headers.update(ranked[0].get('headers') or {})
|
||||
debug_log('fallback -> lowest risk video', {'itag': ranked[0].get('itag')})
|
||||
return {'parse': 0, 'jx': 0, 'url': ranked[0]['url'], 'header': headers}
|
||||
except Exception as e:
|
||||
debug_log('fallback lowrisk error', repr(e))
|
||||
|
||||
# 3) embed(会弹 WebView), 默认开启; ext 里 embed_fallback=false 可彻底关闭弹窗
|
||||
if self.extendDict.get('embed_fallback', True) is not False:
|
||||
fatal = isinstance(err, RiskBlocked) and err.category == 'FATAL'
|
||||
if not fatal:
|
||||
return {'parse': 1, 'jx': 1,
|
||||
'url': f'https://www.youtube.com/embed/{video_id}?autoplay=1',
|
||||
'header': json.dumps(self.header)}
|
||||
|
||||
# 4) 完全不弹窗: 返回空 url, 播放器直接提示失败
|
||||
return {'parse': 0, 'jx': 0, 'url': '', 'msg': message}
|
||||
|
||||
# 进程级缓存: 避免每次 init 都重新探测
|
||||
_PROXY_CACHE = {'proxy': None, 'expires': 0.0}
|
||||
_PROXY_BLACKLIST = {}
|
||||
|
||||
def _auto_detect_proxy(self):
|
||||
"""代理逻辑 2 & 3：探测内置列表，若都不行则清空，回退使用系统/全局代理"""
|
||||
proxy_list = [
|
||||
"""代理节点探测。
|
||||
|
||||
用裸 TCP connect 代替 HTTP 请求:
|
||||
- 本机端口没开时 connect 立刻 ECONNREFUSED, 12 个端口串行也在 50ms 内跑完
|
||||
- 不需要线程, 避免 Android Python 沙箱里线程残留导致的闪退
|
||||
"""
|
||||
now = time.time()
|
||||
cache = Spider._PROXY_CACHE
|
||||
ttl = int(self.extendDict.get('proxy_cache_ttl') or 300)
|
||||
|
||||
if cache.get('expires', 0) > now:
|
||||
cached = cache.get('proxy')
|
||||
if cached:
|
||||
self.session.proxies = {'http': cached, 'https': cached}
|
||||
self.proxy_str = cached.replace('http://', '').replace('https://', '')
|
||||
else:
|
||||
self.session.proxies = {}
|
||||
self.proxy_str = ''
|
||||
debug_log('代理节点缓存命中', {'proxy': cached})
|
||||
return
|
||||
|
||||
proxy_list = self.extendDict.get('proxy_list') or [
|
||||
"http://127.0.0.1:2080",
|
||||
"http://127.0.0.1:7890",
|
||||
"http://127.0.0.1:10809",
|
||||
@@ -2295,29 +3018,73 @@ class Spider(Spider):
|
||||
"http://127.0.0.1:3128",
|
||||
"http://127.0.0.1:1080",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://127.0.0.1:9090"
|
||||
"http://127.0.0.1:9090",
|
||||
]
|
||||
|
||||
# 第二级：探测内置列表
|
||||
for p in proxy_list:
|
||||
candidates = [p for p in proxy_list if Spider._PROXY_BLACKLIST.get(p, 0) < now] or proxy_list
|
||||
connect_timeout = float(self.extendDict.get('proxy_timeout') or 0.35)
|
||||
|
||||
picked = None
|
||||
for p in candidates:
|
||||
try:
|
||||
test_proxies = {'http': p, 'https': p}
|
||||
# 设置较短超时时间，快速失败
|
||||
r = requests.get('https://www.youtube.com', proxies=test_proxies, timeout=2)
|
||||
if r.status_code < 400:
|
||||
self.session.proxies = test_proxies
|
||||
self.proxy_str = p.replace('http://', '').replace('https://', '')
|
||||
debug_log('内置代理探测成功，使用内置', {'proxy': p})
|
||||
return
|
||||
parsed = urlparse(p if '://' in p else 'http://' + p)
|
||||
host = parsed.hostname or '127.0.0.1'
|
||||
port = parsed.port or 80
|
||||
sock = socket.create_connection((host, port), timeout=connect_timeout)
|
||||
sock.close()
|
||||
picked = p
|
||||
break
|
||||
except Exception:
|
||||
Spider._PROXY_BLACKLIST[p] = now + 300
|
||||
continue
|
||||
|
||||
# 第三级：如果所有预设都失败，清空 proxies 字典，回退到全局/系统代理
|
||||
|
||||
# 黑名单封顶, 防止长期运行后无限增长
|
||||
if len(Spider._PROXY_BLACKLIST) > 64:
|
||||
Spider._PROXY_BLACKLIST.clear()
|
||||
|
||||
if picked:
|
||||
self.session.proxies = {'http': picked, 'https': picked}
|
||||
self.proxy_str = picked.replace('http://', '').replace('https://', '')
|
||||
Spider._PROXY_CACHE = {'proxy': picked, 'expires': now + ttl}
|
||||
debug_log('代理节点探测成功', {'proxy': picked, 'tried': len(candidates)})
|
||||
return
|
||||
|
||||
self.session.proxies = {}
|
||||
self.proxy_str = ''
|
||||
debug_log('所有内置代理均不可用，清空设置，回退使用系统/全局代理')
|
||||
Spider._PROXY_CACHE = {'proxy': None, 'expires': now + min(ttl, 60)}
|
||||
debug_log('无可用代理节点, 回退系统/全局代理')
|
||||
|
||||
# ---------- 对外方法防崩护栏 ----------
|
||||
# catvod / TVBox 二开在这些方法抛异常时, 轻则重载接口, 重则闪退
|
||||
def homeContent(self, filter):
|
||||
try:
|
||||
return self._homeContent(filter)
|
||||
except Exception as e:
|
||||
debug_log('homeContent error', repr(e))
|
||||
return {'class': getattr(self, 'yt_classes', []) or []}
|
||||
|
||||
def categoryContent(self, cid, page, filter, ext):
|
||||
try:
|
||||
return self._categoryContent(cid, page, filter, ext)
|
||||
except Exception as e:
|
||||
debug_log('categoryContent error', repr(e))
|
||||
return {'list': [], 'page': int(page or 1), 'pagecount': int(page or 1),
|
||||
'limit': 30, 'total': 0}
|
||||
|
||||
def searchContent(self, key, quick, pg=1):
|
||||
try:
|
||||
return self._searchContent(key, quick, pg)
|
||||
except Exception as e:
|
||||
debug_log('searchContent error', repr(e))
|
||||
return {'list': [], 'page': int(pg or 1)}
|
||||
|
||||
def detailContent(self, did):
|
||||
try:
|
||||
return self._detailContent(did)
|
||||
except Exception as e:
|
||||
debug_log('detailContent error', repr(e))
|
||||
return {'list': []}
|
||||
|
||||
def _homeContent(self, filter):
|
||||
result = {'class': self.yt_classes}
|
||||
if filter:
|
||||
video_filters = {}
|
||||
@@ -2331,7 +3098,7 @@ class Spider(Spider):
|
||||
def homeVideoContent(self):
|
||||
return {'list': []}
|
||||
|
||||
def categoryContent(self, cid, page, filter, ext):
|
||||
def _categoryContent(self, cid, page, filter, ext):
|
||||
page = int(page or 1)
|
||||
filters = ext if isinstance(ext, dict) else {}
|
||||
if self._is_live_category(cid):
|
||||
@@ -2348,7 +3115,7 @@ class Spider(Spider):
|
||||
'total': len(videos)
|
||||
}
|
||||
|
||||
def searchContent(self, key, quick, pg=1):
|
||||
def _searchContent(self, key, quick, pg=1):
|
||||
page = int(pg or 1)
|
||||
keyword = str(key or '').strip()
|
||||
videos_v, _ = self._search_video_page(keyword, page)
|
||||
@@ -2422,7 +3189,7 @@ class Spider(Spider):
|
||||
return []
|
||||
|
||||
# ---------- detailContent（方案A：按高度分 SDR/HDR） ----------
|
||||
def detailContent(self, did):
|
||||
def _detailContent(self, did):
|
||||
video_id = did[0]
|
||||
# 检测是否为直播
|
||||
try:
|
||||
@@ -2529,7 +3296,19 @@ class Spider(Spider):
|
||||
|
||||
# ---------- playerContent ----------
|
||||
def playerContent(self, flag, pid, vipFlags):
|
||||
raw_pid = pid.split('$')[-1]
|
||||
# catvod / 影视仓在这里拿到异常会直接重载接口甚至闪退, 因此整体兜住
|
||||
try:
|
||||
return self._playerContent(flag, pid, vipFlags)
|
||||
except Exception as e:
|
||||
debug_log('playerContent fatal', repr(e))
|
||||
try:
|
||||
vid = str(pid).split('$')[-1].split('@')[0]
|
||||
except Exception:
|
||||
vid = ''
|
||||
return self._fallback_play_result(vid, e, None)
|
||||
|
||||
def _playerContent(self, flag, pid, vipFlags):
|
||||
raw_pid = str(pid or '').split('$')[-1]
|
||||
if '@' in raw_pid:
|
||||
video_id, quality_or_type = raw_pid.rsplit('@', 1)
|
||||
else:
|
||||
@@ -2561,6 +3340,7 @@ class Spider(Spider):
|
||||
|
||||
# ---------- 按高度和 SDR/HDR 类型播放点播 ----------
|
||||
def _play_video_by_height_and_type(self, video_id, target_height, hdr_type):
|
||||
formats = None
|
||||
try:
|
||||
data = self.yt_video.extract(video_id)
|
||||
formats = data.get('formats', [])
|
||||
@@ -2603,7 +3383,7 @@ class Spider(Spider):
|
||||
return {'parse': 0, 'jx': 0, 'url': selected_video['url'], 'header': headers}
|
||||
except Exception as e:
|
||||
debug_log('_play_video_by_height_and_type error', {'video_id': video_id, 'height': target_height, 'type': hdr_type, 'error': repr(e)})
|
||||
return {'parse': 1, 'url': f'https://www.youtube.com/embed/{video_id}?autoplay=1', 'header': json.dumps(self.header)}
|
||||
return self._fallback_play_result(video_id, e, formats)
|
||||
|
||||
|
||||
def _play_live_by_height(self, video_id, target_height):
|
||||
@@ -2669,10 +3449,17 @@ class Spider(Spider):
|
||||
return ' '.join([self._normalize_filter_term(item) for item in value.values() if item])
|
||||
return re.sub(r'\s+', ' ', str(value or '')).strip()[:180]
|
||||
|
||||
def _cap_spider_caches(self):
|
||||
limit = int(self.extendDict.get('cache_limit') or 64)
|
||||
capCache(self.search_page_cache, limit)
|
||||
capCache(self.live_search_cache, limit)
|
||||
capCache(self.hls_url_cache, limit * 4)
|
||||
|
||||
def _search_cache_key(self, key):
|
||||
return re.sub(r'\s+', ' ', str(key or '')).strip().lower()
|
||||
|
||||
def _search_video_page(self, key, page=1):
|
||||
self._cap_spider_caches()
|
||||
page = max(1, int(page or 1))
|
||||
cache_key = self._search_cache_key(key)
|
||||
session = self.search_page_cache.get(cache_key)
|
||||
@@ -2767,7 +3554,9 @@ class Spider(Spider):
|
||||
|
||||
def _extract_continuation_token(self, data):
|
||||
tokens = []
|
||||
def scan(obj):
|
||||
def scan(obj, _depth=0):
|
||||
if _depth > 40:
|
||||
return
|
||||
if isinstance(obj, dict):
|
||||
endpoint = obj.get('continuationEndpoint') or {}
|
||||
token = endpoint.get('continuationCommand', {}).get('token')
|
||||
@@ -2788,7 +3577,9 @@ class Spider(Spider):
|
||||
def _extract_videos_from_api(self, data, limit=30):
|
||||
videos = []
|
||||
seen = set()
|
||||
def scan(obj):
|
||||
def scan(obj, _depth=0):
|
||||
if _depth > 40:
|
||||
return
|
||||
if len(videos) >= limit:
|
||||
return
|
||||
if isinstance(obj, dict):
|
||||
@@ -2809,7 +3600,9 @@ class Spider(Spider):
|
||||
def _extract_live_videos_from_api(self, data, limit=30):
|
||||
videos = []
|
||||
seen = set()
|
||||
def scan(obj):
|
||||
def scan(obj, _depth=0):
|
||||
if _depth > 40:
|
||||
return
|
||||
if len(videos) >= limit:
|
||||
return
|
||||
if isinstance(obj, dict):
|
||||
@@ -2899,12 +3692,29 @@ class Spider(Spider):
|
||||
}
|
||||
except Exception as e:
|
||||
debug_log('_play_live error', {'video_id': video_id, 'error': repr(e)})
|
||||
return {'parse': 1, 'jx': 1, 'url': f'https://www.youtube.com/embed/{video_id}?autoplay=1'}
|
||||
return self._fallback_play_result(video_id, e, None)
|
||||
|
||||
def _play_video(self, video_id, quality):
|
||||
formats = None
|
||||
try:
|
||||
data = self.yt_video.extract(video_id)
|
||||
playable = self.yt_video.choose_playable(data['formats'], quality)
|
||||
formats = data.get('formats') or []
|
||||
# 命中缓存但直链已过期 -> 强制重取一次, 这是"点了没反应/无法播放"的高频原因
|
||||
if formats and self.yt_video.url_expired(formats[0].get('url')):
|
||||
debug_log('cached url expired, re-extract', {'video_id': video_id})
|
||||
data = self.yt_video.extract(video_id, force=True)
|
||||
formats = data.get('formats') or []
|
||||
playable, probe_log = self.yt_video.pick_playable_with_probe(formats, quality)
|
||||
if probe_log and not any(x.get('ok') for x in probe_log):
|
||||
# 全部探测失败 -> 换一批节点重取
|
||||
debug_log('all probe failed, re-extract', {'video_id': video_id})
|
||||
data = self.yt_video.extract(video_id, force=True)
|
||||
formats = data.get('formats') or []
|
||||
playable, probe_log = self.yt_video.pick_playable_with_probe(formats, quality)
|
||||
if probe_log:
|
||||
debug_log('probe log', probe_log)
|
||||
if not playable:
|
||||
playable = self.yt_video.choose_playable(formats, quality)
|
||||
if playable:
|
||||
audio = self.yt_video.choose_audio(data['formats'])
|
||||
debug_log('selected playable', {'itag': playable.get('itag'), 'client': playable.get('client'), 'mime': playable.get('mimeType'), 'height': playable.get('height'), 'has_n': 'n=' in playable.get('url', ''), 'redirected': bool(playable.get('redirected')), 'ua': (playable.get('headers') or {}).get('User-Agent', '')[:60], 'url_len': len(playable.get('url', ''))})
|
||||
@@ -2927,7 +3737,12 @@ class Spider(Spider):
|
||||
raise Exception(f'没有可直接播放的 {quality} 视频流格式')
|
||||
except Exception as e:
|
||||
debug_log('_play_video error', repr(e))
|
||||
return {'parse': 1, 'url': f'https://www.youtube.com/embed/{video_id}?autoplay=1', 'header': json.dumps(self.header)}
|
||||
hls = ''
|
||||
try:
|
||||
hls = ((self.yt_video.extract_cache.get(video_id) or {}).get('data') or {}).get('hls_url') or ''
|
||||
except Exception:
|
||||
hls = ''
|
||||
return self._fallback_play_result(video_id, e, formats, hls)
|
||||
|
||||
# ------------------ HLS 代理（原 youtubelive） ------------------
|
||||
def _probe_hls(self, video_id, hls_url):
|
||||
@@ -3222,6 +4037,25 @@ class Spider(Spider):
|
||||
return [500, 'text/plain', f'HLS 代理失败: {str(e)}']
|
||||
|
||||
def destroy(self):
|
||||
# 释放引用, 减少反复重载接口时的内存堆积
|
||||
try:
|
||||
for store in (getattr(self, 'search_page_cache', None),
|
||||
getattr(self, 'live_search_cache', None),
|
||||
getattr(self, 'hls_url_cache', None)):
|
||||
if isinstance(store, dict):
|
||||
store.clear()
|
||||
yv = getattr(self, 'yt_video', None)
|
||||
if yv:
|
||||
for name in ('extract_cache', 'player_cache', 'sig_plan_cache'):
|
||||
store = getattr(yv, name, None)
|
||||
if isinstance(store, dict):
|
||||
store.clear()
|
||||
sess = getattr(self, 'session', None)
|
||||
if sess:
|
||||
sess.close()
|
||||
except Exception as e:
|
||||
debug_log('destroy error', repr(e))
|
||||
|
||||
try:
|
||||
self.session.close()
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user