diff --git a/app/src/main/java/com/github/catvod/spider/Ddys.java b/app/src/main/java/com/github/catvod/spider/Ddys.java new file mode 100644 index 0000000..784637b --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Ddys.java @@ -0,0 +1,627 @@ +package com.github.catvod.spider; + +import android.text.TextUtils; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.AES; +import com.github.catvod.utils.CBC; +import com.github.catvod.utils.gZip; +import com.github.catvod.utils.okhttp.OKCallBack; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.StringReader; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Response; +import okhttp3.ResponseBody; + +/** + * Demo for self study + *

+ * Source from Author: CatVod + */ + +public class Ddys extends Spider { + + private static final String siteUrl = "https://ddys.tv"; + private static final String siteHost = "ddys.tv"; + private String cookie=""; + + protected JSONObject filterConfig; + + protected Pattern regexCategory = Pattern.compile("/category/(\\S+)/"); + protected Pattern regexVid = Pattern.compile("https://ddys.tv/(\\S+)/"); + + protected Pattern regexPage = Pattern.compile("\\S+/page/(\\S+)\\S+"); + protected Pattern m = Pattern.compile("\\S+(http\\S+g)"); + protected Pattern mark = Pattern.compile("\\S+(.*)"); + + // protected Pattern t = Pattern.compile("(\\S+)"); + + + /** + * 爬虫headers + * + * @param url + * @return + */ + protected HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36"); + headers.put("Referer", "https://ddys.tv/"); + if(cookie.length()>0 && !url.contains(("google"))){ + headers.put("Cookie", cookie); + } + return headers; + } + + protected static HashMap Headers() { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + headers.put("Referer", "https://ddys.tv/"); + return headers; + } + + /** + * 获取分类数据 + 首页最近更新视频列表数据 + * + * @param filter 是否开启筛选 关联的是 软件设置中 首页数据源里的筛选开关 + * @return + */ + @Override + public String homeContent(boolean filter) { + try { + String url = siteUrl + '/'; + cookie=""; + Map> cookies = new HashMap<>(); + Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url),cookies)); + for( Map.Entry> entry : cookies.entrySet() ){ + if(entry.getKey().equals("set-cookie")){ + cookie = TextUtils.join(";",entry.getValue()); + break; + } + } + Elements elements = doc.select("li.menu-item a"); + JSONArray classes = new JSONArray(); + ArrayList allClass = new ArrayList<>(); + for (Element ele : elements) { + String name = ele.attr("title"); + boolean show = !filter || (name.equals("热映中") + || name.equals("欧美剧") + || name.equals("日剧") + || name.equals("韩剧") + || name.equals("华语剧") + || name.equals("其他地区") + || name.equals("全部") + || name.equals("欧美电影") + || name.equals("日韩电影") + || name.equals("华语电影") + || name.equals("新番") + || name.equals("动画") + || name.equals("纪录片") + || name.equals("综艺")); + if (allClass.contains(name)) + show = false; + if (show) { + allClass.add(name); + Matcher mather = regexCategory.matcher(ele.attr("href")); + if (!mather.find()) + continue; + // 把分类的id和名称取出来加到列表里 + String id = mather.group(1).trim(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type_id", id); + jsonObject.put("type_name", name); + classes.put(jsonObject); + } + } + JSONObject result = new JSONObject(); + if (filter) { + result.put("filters", filterConfig); + } + result.put("class", classes); + try { + // 取首页推荐视频列表 + Elements list = doc.select("div.post-box-container"); + JSONArray videos = new JSONArray(); + for (int i = 0; i < list.size(); i++) { + Element vod = list.get(i); + String a = vod.selectFirst(".post-box-title a").text(); + if (a.contains("(")) { + String[] item = a.split("\\("); + String title = item[0]; + String remark = item[1].replace(")", ""); + String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style")); + Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href")); + if (!matcher.find()) + continue; + String id = matcher.group(1); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } else { + String title = a; + String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style")); + String remark = doReplaceRegex(mark, vod.selectFirst(".post-box-title a").text()); + Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href")); + if (!matcher.find()) + continue; + String id = matcher.group(1); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } + } + + result.put("list", videos); + } catch (Exception e) { + SpiderDebug.log(e); + } + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + + /** + * 获取分类信息数据 + * + * @param tid 分类id + * @param pg 页数 + * @param filter 同homeContent方法中的filter + * @param extend 筛选参数{k:v, k1:v1} + * @return + */ + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + String url = ""; + try { + if (extend != null && extend.size() > 0) { + for (Iterator it = extend.keySet().iterator(); it.hasNext(); ) { + String key = it.next(); + String value = extend.get(key); + if (value != null && value.length() != 0 && value != " ") { + url = siteUrl + "/category/" + tid + "/" + value; + } else { + url = siteUrl + "/category/" + tid; + } + ; + } + } else { + url = siteUrl + "/category/" + tid; + } + ; + if (pg.equals("1")) { + url = url + "/"; + } else { + url = url + "/page/" + pg + "/"; + } + //System.out.println(url); + String html = OkHttpUtil.string(url, getHeaders(url)); + Document doc = Jsoup.parse(html); + JSONObject result = new JSONObject(); + int pageCount = 0; + int page = -1; + + // 取页码相关信息 + Elements pageInfo = doc.select("div.nav-links"); + if (pageInfo.size() == 0) { + page = Integer.parseInt(pg); + pageCount = page; + } else { + for (int i = 0; i < pageInfo.size(); i++) { + Element li = pageInfo.get(i); + Element a = li.selectFirst("a"); + if (a == null) + continue; + String wy = doc.select("div.nav-links a").last().attr("href"); + String span = doc.select("span.current").text().trim(); + if (page == -1) { + page = Integer.parseInt(span); + } else { + + page = 0; + } + Matcher matcher = regexPage.matcher(wy); + if (matcher.find()) { + //System.out.println("尾页" + matcher.group(1)); + pageCount = Integer.parseInt(matcher.group(1)); + } else { + pageCount = 0; + } + break; + } + } + + JSONArray videos = new JSONArray(); + if (!html.contains("没有找到您想要的结果哦")) { + // 取当前分类页的视频列表 + Elements list = doc.select("div.post-box-container"); + for (int i = 0; i < list.size(); i++) { + Element vod = list.get(i); + String a = vod.selectFirst(".post-box-title a").text(); + if (a.contains("(")) { + String[] item = a.split("\\("); + String title = item[0]; + String remark = item[1].replace(")", ""); + String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style")); + Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href")); + if (!matcher.find()) + continue; + String id = matcher.group(1); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } else { + String title = a; + String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style")); + String remark = doReplaceRegex(mark, vod.selectFirst(".post-box-title a").text()); + Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href")); + if (!matcher.find()) + continue; + String id = matcher.group(1); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } + } + } + result.put("page", page); + result.put("pagecount", pageCount); + result.put("limit", 24); + result.put("total", pageCount <= 1 ? videos.length() : pageCount * 24); + + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + /** + * 视频详情信息 + * + * @param ids 视频id + * @return + */ + @Override + public String detailContent(List ids) { + try { + // 视频详情url + String url = siteUrl + "/" + ids.get(0) + "/"; + Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url))); + JSONObject result = new JSONObject(); + JSONObject vodList = new JSONObject(); + + // 取基本数据 + String cover = doc.select("div.post img").attr("src"); + String ab = doc.select("h1.post-title").text(); + if (ab.contains("(")) { + String[] b = ab.split("\\("); + String title = b[0]; + String remark = b[1].replace("(", ""); + vodList.put("vod_name", title); + vodList.put("vod_remarks", remark); + } else { + vodList.put("vod_name", ab); + String remark = doc.select("time").text().trim(); + vodList.put("vod_remarks", "全"); + } + String str2 = doc.select("div.abstract").text().replace(" ", ""); + String replace = str2.replace("
", ""); + String text = replace.replace("

", ""); + Pattern categorys = Pattern.compile("类型:(.*)制"); + String category = doReplaceRegex(categorys, text); + Pattern a = Pattern.compile("年份:(.*)简"); + String year = doReplaceRegex(a, text); + Pattern b = Pattern.compile("地区:(.*)年份"); + String area = doReplaceRegex(b, text); + Pattern c = Pattern.compile("演员:(.*)类"); + String actor = doReplaceRegex(c, text); + Pattern d = Pattern.compile("导演:(.*)演"); + String director = doReplaceRegex(d, text); + Pattern e = Pattern.compile("简介:(.*)"); + String desc = doReplaceRegex(e, text); + + + vodList.put("vod_id", ids.get(0)); + vodList.put("vod_pic", cover); + vodList.put("type_name", category); + vodList.put("vod_year", year); + vodList.put("vod_area", area); + vodList.put("vod_actor", actor); + vodList.put("vod_director", director); + vodList.put("vod_content", desc); + + List vodItems = new ArrayList<>(); + List vodItems2 = new ArrayList<>(); + List vodItemsos = new ArrayList<>(); + List vodItems2os = new ArrayList<>(); + Map vod_play = new LinkedHashMap<>(); + + Elements allScript = doc.select(".wp-playlist-script"); + String sourceName = "第1季"; + for (int j = 0; j < allScript.size(); j++) { + String scContent = allScript.get(j).html().trim(); + int start = scContent.indexOf('{'); + int end = scContent.lastIndexOf('}') + 1; + String json = scContent.substring(start, end); + JSONObject UJson = new JSONObject(json); + JSONArray Track = UJson.getJSONArray("tracks"); + for (int k = 0; k < Track.length(); k++) { + JSONObject src = Track.getJSONObject(k); + String adk = src.getString("src1"); + String vodName = src.getString("caption"); + String playURL = "https://ddys.tv/getvddr/video?id=" + adk +"&dim=1080P+&type=mix"; + String zm = "https://ddys.tv/subddr/" + src.getString("subsrc"); + String pzm = playURL + "|" + zm; + vodItems.add(vodName + "$" + pzm); + } + for (int k = 0; k < Track.length(); k++) { + JSONObject src = Track.getJSONObject(k); + String adksrc0 = src.getString("src0"); + String adksrc2 = src.getString("src2"); + String vodName = src.getString("caption"); + String playURL = "https://w.ddys.tv" + adksrc0 +"?ddrkey=" + adksrc2; + String zm = "https://ddys.tv/subddr/" + src.getString("subsrc"); + String pzm = playURL + "|" + zm; + vodItemsos.add(vodName + "$" + pzm); + } + vod_play.put(sourceName, TextUtils.join("#", vodItems)); + vod_play.put(sourceName+":海外", TextUtils.join("#", vodItemsos)); + } + + + Elements sources = doc.select(".post-page-numbers"); + if (!sources.isEmpty()) + for (int i = 0; i < sources.size(); i++) { + Element source = sources.get(i); + sourceName = "第" + source.text() + "季"; + String Purl = siteUrl + "/" + ids.get(0) + "/" + source.text() + "/"; + Document docs = Jsoup.parse(OkHttpUtil.string(Purl, getHeaders(Purl))); + Elements allScripts = docs.select(".wp-playlist-script"); + for (int j = 0; j < allScripts.size(); j++) { + String scContent = allScripts.get(j).html().trim(); + int start = scContent.indexOf('{'); + int end = scContent.lastIndexOf('}') + 1; + String json = scContent.substring(start, end); + JSONObject UJson = new JSONObject(json); + JSONArray Track = UJson.getJSONArray("tracks"); + for (int k = 0; k < Track.length(); k++) { + JSONObject src = Track.getJSONObject(k); + String adk = src.getString("src1"); + String vodName = src.getString("caption"); + String playURL = "https://ddys.tv/getvddr/video?id=" + adk +"&dim=1080P+&type=mix"; + String zm = "https://ddys.tv/subddr/" + src.getString("subsrc"); + String pzm = playURL + "|" + zm; + vodItems2.add(vodName + "$" + pzm); + } + for (int k = 0; k < Track.length(); k++) { + JSONObject src = Track.getJSONObject(k); + String adksrc0 = src.getString("src0"); + String adksrc2 = src.getString("src2"); + String vodName = src.getString("caption"); + String playURL = "https://w.ddys.tv" + adksrc0 +"?ddrkey=" + adksrc2; + String zm = "https://ddys.tv/subddr/" + src.getString("subsrc"); + String pzm = playURL + "|" + zm; + vodItems2os.add(vodName + "$" + pzm); + } + vod_play.put(sourceName, TextUtils.join("#", vodItems2)); + vod_play.put(sourceName+":海外", TextUtils.join("#", vodItems2os)); + } + vodItems2.removeAll(vodItems2); + vodItems2os.removeAll(vodItems2os); + } + + String vod_play_from = TextUtils.join("$$$", vod_play.keySet()); + String vod_play_url = TextUtils.join("$$$", vod_play.values()); + vodList.put("vod_play_from", vod_play_from); + vodList.put("vod_play_url", vod_play_url); + + JSONArray list = new JSONArray(); + list.put(vodList); + result.put("list", list); + return result.toString(); + + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + public static Object[] loadsub(String sub) { + try { + OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() { + @Override + protected void onFailure(Call call, Exception e) { + + } + + @Override + protected void onResponse(Response response) { + + } + }; + OkHttpClient YM = OkHttpUtil.defaultClient(); + OkHttpUtil.get(YM, sub, null, Headers(), callBack); + Response result = callBack.getResult(); + int code = result.code(); + if (code == 404) { + return new Object[]{200, "application/octet-stream", new ByteArrayInputStream("WEBVTT".getBytes())}; + } + ResponseBody body = result.body(); + byte[] bytes = body.bytes(); + byte[] tokenkey = Arrays.copyOfRange(bytes, 0, 16); + byte[] data = Arrays.copyOfRange(bytes, 16, bytes.length); + byte[] KS = CBC.CBC(data, tokenkey, tokenkey); + String vtt = gZip.KS(KS); + + + vtt = vtt.replaceAll("(\\d{2}:\\d{2}:\\d{2}.\\d{3}.+\\d{2}:\\d{2}:\\d{2}.\\d{3}).*", "$1"); + vtt = vtt.replaceAll("(\\d{2}:\\d{2}.\\d{3}).*?( --> )(\\d{2}:\\d{2}.\\d{3}).*", "00:$1$200:$3"); + BufferedReader br = new BufferedReader(new StringReader(vtt)); + ArrayList lines = new ArrayList<>(); + int captionNumber = 1; + String line = br.readLine(); + while (line != null) { + if (line.matches("\\d{2}:\\d{2}:\\d{2}.\\d{3}.+\\d{2}:\\d{2}:\\d{2}.\\d{3}")) { + if (lines.get(lines.size() - 1).trim().isEmpty()) { + lines.add(captionNumber + ""); + captionNumber++; + } + } + lines.add(line); + line = br.readLine(); + } + String join = TextUtils.join("\n", lines); + + return new Object[]{200, "application/octet-stream", new ByteArrayInputStream(join.getBytes())}; + } catch (Exception e) { + e.printStackTrace(); + SpiderDebug.log(e); + return null; + } + } + + + /** + * 获取视频播放信息 + * + * @param flag 播放源 + * @param id 视频id + * @param vipFlags 所有可能需要vip解析的源 + * @return + */ + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + JSONObject result = new JSONObject(); + String[] item = id.split("\\|"); + String playUrl = item[0]; + String ZiMu = item[1]; + String RealUrl=""; + if(playUrl.contains("ddrkey")){ + Map> respHeaders = new HashMap<>(); + OkHttpUtil.stringNoRedirect(playUrl, getHeaders(playUrl),respHeaders); + RealUrl = OkHttpUtil.getRedirectLocation(respHeaders); + } else { + String content = OkHttpUtil.string(playUrl, getHeaders(playUrl)); + JSONObject obj = new JSONObject(content); + RealUrl = obj.optString("url"); + } + + result.put("parse", 0); + result.put("playUrl", ""); + result.put("url", RealUrl); + result.put("header", ""); + // if (!TextUtils.isEmpty(str4)) { + result.put("subf", "/vtt/utf-8"); + result.put("subt", Proxy.localProxyUrl() + "?do=ddys&url=" + URLEncoder.encode(ZiMu)); + // result.put("subt", ZiMu); + // } + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + + @Override + public String searchContent(String key, boolean quick) { + try { + String url = "https://www.google.com/search?q=site%3Addys.tv+" + URLEncoder.encode(key); + Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url))); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + String sourceName = doc.select("div.yuRUbf a h3").text(); + if (sourceName.contains(key)) { + String list1 = doc.select("div.yuRUbf a").attr("href"); + Document ddrklink = Jsoup.parse(OkHttpUtil.string(list1, getHeaders(list1))); + String DDlink = ddrklink.select("link[rel=canonical]").attr("href"); + Matcher matcher = regexVid.matcher(DDlink); + if (matcher.find()) { + JSONObject v = new JSONObject(); + String group = matcher.group(1); + String ab = ddrklink.select("h1.post-title").text(); + String cover = ddrklink.select("div.post img").attr("src"); + if (ab.contains("(")) { + String[] b = ab.split("\\("); + String title = b[0]; + String remark = b[1].replace("(", ""); + v.put("vod_name", title); + v.put("vod_remarks", remark); + } else { + v.put("vod_name", ab); + String remark = "全"; + v.put("vod_remarks", remark); + } + v.put("vod_id", group); + v.put("vod_pic", cover); + videos.put(v); + } + } + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + private static String doReplaceRegex(Pattern pattern, String src) { + if (pattern == null) + return src; + try { + Matcher matcher = pattern.matcher(src); + if (matcher.find()) { + return matcher.group(1).trim(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return src; + } + + +} diff --git a/app/src/main/java/com/github/catvod/spider/Proxy.java b/app/src/main/java/com/github/catvod/spider/Proxy.java new file mode 100644 index 0000000..575fe10 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Proxy.java @@ -0,0 +1,69 @@ +package com.github.catvod.spider; + +import android.util.Base64; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.live.TxtSubscribe; +import com.github.catvod.utils.okhttp.OkHttpUtil; +import com.github.catvod.parser.MixDemo; +import com.github.catvod.parser.MixWeb; + +import java.io.ByteArrayInputStream; +import java.util.Map; + +public class Proxy extends Spider { + + public static int localPort = -1; + + static void adjustLocalPort() { + if (localPort > 0) + return; + int port = 9978; + while (port < 10000) { + String resp = OkHttpUtil.string("http://127.0.0.1:" + port + "/proxy?do=ck", null); + if (resp.equals("ok")) { + SpiderDebug.log("Found local server port " + port); + localPort = port; + break; + } + port++; + } + } + + public static String localProxyUrl() { + adjustLocalPort(); + return "http://127.0.0.1:" + Proxy.localPort + "/proxy"; + } + + public static Object[] proxy(Map params) { + try { + String what = params.get("do"); + if (what.equals("live")) { + String type = params.get("type"); + if (type.equals("txt")) { + String ext = params.get("ext"); + if (!ext.startsWith("http")) { + ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), "UTF-8"); + } + return TxtSubscribe.load(ext); + } + } else if (what.equals("ck")) { + Object[] result = new Object[3]; + result[0] = 200; + result[1] = "text/plain; charset=utf-8"; + ByteArrayInputStream baos = new ByteArrayInputStream("ok".getBytes("UTF-8")); + result[2] = baos; + return result; + } + else if (what.equals("MixWeb")) { + return MixWeb.loadHtml(params.get("flag"), params.get("url")); + } + + } catch (Throwable th) { + + } + return null; + } + +}