1
This commit is contained in:
qist
+74
-87
@@ -7,7 +7,6 @@ import time
|
|||||||
from urllib.parse import urljoin, quote, unquote, urlparse, parse_qs
|
from urllib.parse import urljoin, quote, unquote, urlparse, parse_qs
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
from base.spider import Spider
|
from base.spider import Spider
|
||||||
|
|
||||||
@@ -27,6 +26,7 @@ class Spider(Spider):
|
|||||||
self._text_cache_ttl = 300
|
self._text_cache_ttl = 300
|
||||||
self._api_base = ""
|
self._api_base = ""
|
||||||
self._use_api = False
|
self._use_api = False
|
||||||
|
self._ua_fallback = "Dalvik/2.1.0 (Linux; U; Android 10)"
|
||||||
|
|
||||||
self._class_map = [
|
self._class_map = [
|
||||||
("最新电影", "/zuixindianying"),
|
("最新电影", "/zuixindianying"),
|
||||||
@@ -44,9 +44,14 @@ class Spider(Spider):
|
|||||||
return "厂长资源"
|
return "厂长资源"
|
||||||
|
|
||||||
def init(self, extend=""):
|
def init(self, extend=""):
|
||||||
if extend:
|
if isinstance(extend, dict):
|
||||||
|
host = (extend.get("host") or extend.get("site") or "").strip()
|
||||||
|
if host:
|
||||||
|
self.host = host.rstrip("/")
|
||||||
|
elif isinstance(extend, str) and extend.strip():
|
||||||
self.host = extend.strip().rstrip("/")
|
self.host = extend.strip().rstrip("/")
|
||||||
self.headers["Referer"] = self.host + "/"
|
|
||||||
|
self.headers["Referer"] = self.host + "/"
|
||||||
self.headers["Origin"] = self.host
|
self.headers["Origin"] = self.host
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
self.session.headers.update(self.headers)
|
self.session.headers.update(self.headers)
|
||||||
@@ -103,9 +108,25 @@ class Spider(Spider):
|
|||||||
except Exception:
|
except Exception:
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
if not r or r.status_code != 200:
|
if not r or r.status_code != 200:
|
||||||
return ""
|
if r and r.status_code in (403, 406, 412):
|
||||||
|
self.session.headers["User-Agent"] = self._ua_fallback
|
||||||
|
try:
|
||||||
|
r = self.session.get(url, timeout=self.timeout, allow_redirects=True)
|
||||||
|
except Exception:
|
||||||
|
r = None
|
||||||
|
if not r or r.status_code != 200:
|
||||||
|
return ""
|
||||||
r.encoding = "utf-8"
|
r.encoding = "utf-8"
|
||||||
text = r.text or ""
|
text = r.text or ""
|
||||||
|
if "访问已被拦截" in text or "已被拦截" in text:
|
||||||
|
self.session.headers["User-Agent"] = self._ua_fallback
|
||||||
|
try:
|
||||||
|
r2 = self.session.get(url, timeout=self.timeout, allow_redirects=True)
|
||||||
|
if r2 and r2.status_code == 200:
|
||||||
|
r2.encoding = "utf-8"
|
||||||
|
text = r2.text or ""
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
if text:
|
if text:
|
||||||
self._text_cache[url] = (now + self._text_cache_ttl, text)
|
self._text_cache[url] = (now + self._text_cache_ttl, text)
|
||||||
return text
|
return text
|
||||||
@@ -123,60 +144,36 @@ class Spider(Spider):
|
|||||||
return 999
|
return 999
|
||||||
|
|
||||||
def _parse_vod_list(self, html):
|
def _parse_vod_list(self, html):
|
||||||
soup = BeautifulSoup(html or "", "html.parser")
|
html = html or ""
|
||||||
vods = []
|
vods = []
|
||||||
for li in soup.select("div.bt_img ul li"):
|
|
||||||
a = None
|
|
||||||
for x in li.select("a[href]"):
|
|
||||||
href = x.get("href") or ""
|
|
||||||
if "/movie/" in href and ".html" in href:
|
|
||||||
a = x
|
|
||||||
break
|
|
||||||
if not a:
|
|
||||||
continue
|
|
||||||
href = a.get("href") or ""
|
|
||||||
href = self._abs(href)
|
|
||||||
mid = ""
|
|
||||||
mm = re.search(r"/movie/(\d+)\.html", href)
|
|
||||||
if mm:
|
|
||||||
mid = mm.group(1)
|
|
||||||
if not mid:
|
|
||||||
continue
|
|
||||||
|
|
||||||
img = li.select_one("img")
|
blocks = re.findall(r"(?is)<li\b[^>]*>.*?</li>", html)
|
||||||
pic = ""
|
for b in blocks:
|
||||||
name = ""
|
if "/movie/" not in b:
|
||||||
if img:
|
continue
|
||||||
name = (img.get("alt") or "").strip()
|
m_id = re.search(r'(?is)href=["\'](?:https?://[^"\']+)?/movie/(\d+)\.html["\']', b)
|
||||||
pic = (
|
if not m_id:
|
||||||
(img.get("data-original") or "")
|
continue
|
||||||
or (img.get("data-src") or "")
|
mid = m_id.group(1)
|
||||||
or (img.get("data-lazy-src") or "")
|
|
||||||
or (img.get("src") or "")
|
|
||||||
).strip()
|
|
||||||
if not name:
|
|
||||||
h3a = li.select_one("h3 a") or li.select_one("a")
|
|
||||||
name = (h3a.get_text(strip=True) if h3a else "").strip()
|
|
||||||
if pic.endswith("/blank.gif") and img and img.get("data-original"):
|
|
||||||
pic = (img.get("data-original") or "").strip()
|
|
||||||
|
|
||||||
qb = li.select_one(".hdinfo .qb") or li.select_one(".hdinfo")
|
m_alt = re.search(r'(?is)<img\b[^>]*\balt=["\']([^"\']+)["\']', b)
|
||||||
rating = li.select_one(".rating")
|
name = (m_alt.group(1).strip() if m_alt else "") or mid
|
||||||
remark = (qb.get_text(" ", strip=True) if qb else "").strip()
|
|
||||||
score = (rating.get_text(" ", strip=True) if rating else "").strip()
|
m_pic = re.search(r'(?is)<img\b[^>]*(?:data-original|data-src|data-lazy-src|src)=["\']([^"\']+)["\']', b)
|
||||||
|
pic = (m_pic.group(1).strip() if m_pic else "")
|
||||||
|
|
||||||
|
remark = ""
|
||||||
|
m_qb = re.search(r'(?is)<div\b[^>]*class=["\'][^"\']*\bhdinfo\b[^"\']*["\'][^>]*>.*?<span\b[^>]*>(.*?)</span>', b)
|
||||||
|
if m_qb:
|
||||||
|
remark = re.sub(r"(?is)<[^>]+>", "", m_qb.group(1)).strip()
|
||||||
|
m_score = re.search(r'(?is)<div\b[^>]*class=["\'][^"\']*\brating\b[^"\']*["\'][^>]*>\s*([^<]+)\s*</div>', b)
|
||||||
|
score = (m_score.group(1).strip() if m_score else "")
|
||||||
if remark and score and score not in remark:
|
if remark and score and score not in remark:
|
||||||
remark = f"{remark} {score}"
|
remark = f"{remark} {score}"
|
||||||
elif not remark:
|
elif not remark:
|
||||||
remark = score
|
remark = score
|
||||||
|
|
||||||
vods.append(
|
vods.append({"vod_id": mid, "vod_name": name, "vod_pic": pic, "vod_remarks": remark})
|
||||||
{
|
|
||||||
"vod_id": mid,
|
|
||||||
"vod_name": name or mid,
|
|
||||||
"vod_pic": pic,
|
|
||||||
"vod_remarks": remark,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
seen = set()
|
seen = set()
|
||||||
unique = []
|
unique = []
|
||||||
@@ -238,45 +235,37 @@ class Spider(Spider):
|
|||||||
url = f"{self.host}/movie/{vid}.html"
|
url = f"{self.host}/movie/{vid}.html"
|
||||||
|
|
||||||
html = self._fetch_text(url)
|
html = self._fetch_text(url)
|
||||||
soup = BeautifulSoup(html or "", "html.parser")
|
name = ""
|
||||||
|
m_title = re.search(r"(?is)<h1[^>]*>\s*([^<]+)\s*</h1>", html or "")
|
||||||
|
if m_title:
|
||||||
|
name = m_title.group(1).strip()
|
||||||
|
|
||||||
h1 = soup.select_one("h1")
|
|
||||||
name = (h1.get_text(" ", strip=True) if h1 else "").strip()
|
|
||||||
|
|
||||||
img = soup.select_one(".dyimg img") or soup.select_one(".movimg img") or soup.select_one("img")
|
|
||||||
pic = ""
|
pic = ""
|
||||||
if img:
|
m_pic = re.search(r'(?is)<div\b[^>]*class=["\'][^"\']*\bdyimg\b[^"\']*["\'][^>]*>[\s\S]*?<img\b[^>]*(?:data-original|data-src|data-lazy-src|src)=["\']([^"\']+)["\']', html or "")
|
||||||
pic = (
|
if not m_pic:
|
||||||
(img.get("data-original") or "")
|
m_pic = re.search(r'(?is)<img\b[^>]*(?:data-original|data-src|data-lazy-src|src)=["\']([^"\']+)["\']', html or "")
|
||||||
or (img.get("data-src") or "")
|
if m_pic:
|
||||||
or (img.get("data-lazy-src") or "")
|
pic = m_pic.group(1).strip()
|
||||||
or (img.get("src") or "")
|
|
||||||
).strip()
|
|
||||||
|
|
||||||
desc = ""
|
desc = ""
|
||||||
desc_div = soup.select_one("div.yp_context")
|
m_desc = re.search(r'(?is)<div\b[^>]*class=["\'][^"\']*\byp_context\b[^"\']*["\'][^>]*>\s*([\s\S]*?)\s*</div>', html or "")
|
||||||
if desc_div:
|
if m_desc:
|
||||||
desc = desc_div.get_text("\n", strip=True)
|
desc = re.sub(r"(?is)<[^>]+>", "", m_desc.group(1)).strip()
|
||||||
|
|
||||||
text_lines = [x.strip() for x in soup.get_text("\n", strip=True).split("\n") if x.strip()]
|
actor = ""
|
||||||
|
director = ""
|
||||||
def pick(prefix):
|
m_actor = re.search(r"(?is)主演：\s*([^<\n\r]+)", html or "")
|
||||||
for line in text_lines:
|
if m_actor:
|
||||||
if line.startswith(prefix):
|
actor = m_actor.group(1).strip()
|
||||||
return line.split("：", 1)[-1].strip()
|
m_director = re.search(r"(?is)导演：\s*([^<\n\r]+)", html or "")
|
||||||
return ""
|
if m_director:
|
||||||
|
director = m_director.group(1).strip()
|
||||||
actor = pick("主演：")
|
|
||||||
director = pick("导演：")
|
|
||||||
|
|
||||||
play_items = []
|
play_items = []
|
||||||
for a in soup.select('a[href*="/v_play/"]'):
|
for m in re.finditer(r'(?is)<a\b[^>]*href=["\']([^"\']*/v_play/[^"\']+)["\'][^>]*>\s*([^<]+)\s*</a>', html or ""):
|
||||||
t = a.get_text(" ", strip=True).strip()
|
href = self._abs(m.group(1).strip())
|
||||||
href = a.get("href") or ""
|
t = m.group(2).strip()
|
||||||
if not href:
|
if t and href:
|
||||||
continue
|
|
||||||
href = self._abs(href)
|
|
||||||
if t:
|
|
||||||
play_items.append(f"{t}${href}")
|
play_items.append(f"{t}${href}")
|
||||||
|
|
||||||
if not play_items:
|
if not play_items:
|
||||||
@@ -326,11 +315,9 @@ class Spider(Spider):
|
|||||||
return {"parse": 1, "url": play_url, "header": h, "playUrl": ""}
|
return {"parse": 1, "url": play_url, "header": h, "playUrl": ""}
|
||||||
|
|
||||||
html = self._fetch_text(play_url)
|
html = self._fetch_text(play_url)
|
||||||
soup = BeautifulSoup(html or "", "html.parser")
|
m_iframe = re.search(r'(?is)<iframe\b[^>]*\bsrc=["\']([^"\']+)["\']', html or "")
|
||||||
|
if m_iframe:
|
||||||
iframe = soup.select_one("iframe.viframe") or soup.find("iframe")
|
src = m_iframe.group(1).strip()
|
||||||
if iframe:
|
|
||||||
src = (iframe.get("src") or "").strip()
|
|
||||||
if src:
|
if src:
|
||||||
qs = parse_qs(urlparse(src).query)
|
qs = parse_qs(urlparse(src).query)
|
||||||
raw = (qs.get("url") or [""])[0]
|
raw = (qs.get("url") or [""])[0]
|
||||||
|
|||||||
Reference in New Issue
Block a user