package com.github.catvod.spider;
import android.text.TextUtils;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.AES;
import com.github.catvod.utils.CBC;
import com.github.catvod.utils.gZip;
import com.github.catvod.utils.okhttp.OKCallBack;
import com.github.catvod.utils.okhttp.OkHttpUtil;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.StringReader;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Response;
import okhttp3.ResponseBody;
/**
* Demo for self study
*
* Source from Author: CatVod
*/
public class Ddys extends Spider {
private static final String siteUrl = "https://ddys.tv";
private static final String siteHost = "ddys.tv";
private String cookie="";
protected JSONObject filterConfig;
protected Pattern regexCategory = Pattern.compile("/category/(\\S+)/");
protected Pattern regexVid = Pattern.compile("https://ddys.tv/(\\S+)/");
protected Pattern regexPage = Pattern.compile("\\S+/page/(\\S+)\\S+");
protected Pattern m = Pattern.compile("\\S+(http\\S+g)");
protected Pattern mark = Pattern.compile("\\S+(.*)");
// protected Pattern t = Pattern.compile("(\\S+)");
/**
* 爬虫headers
*
* @param url
* @return
*/
protected HashMap getHeaders(String url) {
HashMap headers = new HashMap<>();
headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36");
headers.put("Referer", "https://ddys.tv/");
if(cookie.length()>0 && !url.contains(("google"))){
headers.put("Cookie", cookie);
}
return headers;
}
protected static HashMap Headers() {
HashMap headers = new HashMap<>();
headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36");
headers.put("Referer", "https://ddys.tv/");
return headers;
}
/**
* 获取分类数据 + 首页最近更新视频列表数据
*
* @param filter 是否开启筛选 关联的是 软件设置中 首页数据源里的筛选开关
* @return
*/
@Override
public String homeContent(boolean filter) {
try {
String url = siteUrl + '/';
cookie="";
Map> cookies = new HashMap<>();
Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url),cookies));
for( Map.Entry> entry : cookies.entrySet() ){
if(entry.getKey().equals("set-cookie")){
cookie = TextUtils.join(";",entry.getValue());
break;
}
}
Elements elements = doc.select("li.menu-item a");
JSONArray classes = new JSONArray();
ArrayList allClass = new ArrayList<>();
for (Element ele : elements) {
String name = ele.attr("title");
boolean show = !filter || (name.equals("热映中")
|| name.equals("欧美剧")
|| name.equals("日剧")
|| name.equals("韩剧")
|| name.equals("华语剧")
|| name.equals("其他地区")
|| name.equals("全部")
|| name.equals("欧美电影")
|| name.equals("日韩电影")
|| name.equals("华语电影")
|| name.equals("新番")
|| name.equals("动画")
|| name.equals("纪录片")
|| name.equals("综艺"));
if (allClass.contains(name))
show = false;
if (show) {
allClass.add(name);
Matcher mather = regexCategory.matcher(ele.attr("href"));
if (!mather.find())
continue;
// 把分类的id和名称取出来加到列表里
String id = mather.group(1).trim();
JSONObject jsonObject = new JSONObject();
jsonObject.put("type_id", id);
jsonObject.put("type_name", name);
classes.put(jsonObject);
}
}
JSONObject result = new JSONObject();
if (filter) {
result.put("filters", filterConfig);
}
result.put("class", classes);
try {
// 取首页推荐视频列表
Elements list = doc.select("div.post-box-container");
JSONArray videos = new JSONArray();
for (int i = 0; i < list.size(); i++) {
Element vod = list.get(i);
String a = vod.selectFirst(".post-box-title a").text();
if (a.contains("(")) {
String[] item = a.split("\\(");
String title = item[0];
String remark = item[1].replace(")", "");
String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style"));
Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href"));
if (!matcher.find())
continue;
String id = matcher.group(1);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", title);
v.put("vod_pic", cover);
v.put("vod_remarks", remark);
videos.put(v);
} else {
String title = a;
String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style"));
String remark = doReplaceRegex(mark, vod.selectFirst(".post-box-title a").text());
Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href"));
if (!matcher.find())
continue;
String id = matcher.group(1);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", title);
v.put("vod_pic", cover);
v.put("vod_remarks", remark);
videos.put(v);
}
}
result.put("list", videos);
} catch (Exception e) {
SpiderDebug.log(e);
}
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
/**
* 获取分类信息数据
*
* @param tid 分类id
* @param pg 页数
* @param filter 同homeContent方法中的filter
* @param extend 筛选参数{k:v, k1:v1}
* @return
*/
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap extend) {
String url = "";
try {
if (extend != null && extend.size() > 0) {
for (Iterator it = extend.keySet().iterator(); it.hasNext(); ) {
String key = it.next();
String value = extend.get(key);
if (value != null && value.length() != 0 && value != " ") {
url = siteUrl + "/category/" + tid + "/" + value;
} else {
url = siteUrl + "/category/" + tid;
}
;
}
} else {
url = siteUrl + "/category/" + tid;
}
;
if (pg.equals("1")) {
url = url + "/";
} else {
url = url + "/page/" + pg + "/";
}
//System.out.println(url);
String html = OkHttpUtil.string(url, getHeaders(url));
Document doc = Jsoup.parse(html);
JSONObject result = new JSONObject();
int pageCount = 0;
int page = -1;
// 取页码相关信息
Elements pageInfo = doc.select("div.nav-links");
if (pageInfo.size() == 0) {
page = Integer.parseInt(pg);
pageCount = page;
} else {
for (int i = 0; i < pageInfo.size(); i++) {
Element li = pageInfo.get(i);
Element a = li.selectFirst("a");
if (a == null)
continue;
String wy = doc.select("div.nav-links a").last().attr("href");
String span = doc.select("span.current").text().trim();
if (page == -1) {
page = Integer.parseInt(span);
} else {
page = 0;
}
Matcher matcher = regexPage.matcher(wy);
if (matcher.find()) {
//System.out.println("尾页" + matcher.group(1));
pageCount = Integer.parseInt(matcher.group(1));
} else {
pageCount = 0;
}
break;
}
}
JSONArray videos = new JSONArray();
if (!html.contains("没有找到您想要的结果哦")) {
// 取当前分类页的视频列表
Elements list = doc.select("div.post-box-container");
for (int i = 0; i < list.size(); i++) {
Element vod = list.get(i);
String a = vod.selectFirst(".post-box-title a").text();
if (a.contains("(")) {
String[] item = a.split("\\(");
String title = item[0];
String remark = item[1].replace(")", "");
String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style"));
Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href"));
if (!matcher.find())
continue;
String id = matcher.group(1);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", title);
v.put("vod_pic", cover);
v.put("vod_remarks", remark);
videos.put(v);
} else {
String title = a;
String cover = doReplaceRegex(m, vod.selectFirst(".post-box-image").attr("style"));
String remark = doReplaceRegex(mark, vod.selectFirst(".post-box-title a").text());
Matcher matcher = regexVid.matcher(vod.selectFirst(".post-box-title a").attr("href"));
if (!matcher.find())
continue;
String id = matcher.group(1);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", title);
v.put("vod_pic", cover);
v.put("vod_remarks", remark);
videos.put(v);
}
}
}
result.put("page", page);
result.put("pagecount", pageCount);
result.put("limit", 24);
result.put("total", pageCount <= 1 ? videos.length() : pageCount * 24);
result.put("list", videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
/**
* 视频详情信息
*
* @param ids 视频id
* @return
*/
@Override
public String detailContent(List ids) {
try {
// 视频详情url
String url = siteUrl + "/" + ids.get(0) + "/";
Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url)));
JSONObject result = new JSONObject();
JSONObject vodList = new JSONObject();
// 取基本数据
String cover = doc.select("div.post img").attr("src");
String ab = doc.select("h1.post-title").text();
if (ab.contains("(")) {
String[] b = ab.split("\\(");
String title = b[0];
String remark = b[1].replace("(", "");
vodList.put("vod_name", title);
vodList.put("vod_remarks", remark);
} else {
vodList.put("vod_name", ab);
String remark = doc.select("time").text().trim();
vodList.put("vod_remarks", "全");
}
String str2 = doc.select("div.abstract").text().replace(" ", "");
String replace = str2.replace("
", "");
String text = replace.replace("", "");
Pattern categorys = Pattern.compile("类型:(.*)制");
String category = doReplaceRegex(categorys, text);
Pattern a = Pattern.compile("年份:(.*)简");
String year = doReplaceRegex(a, text);
Pattern b = Pattern.compile("地区:(.*)年份");
String area = doReplaceRegex(b, text);
Pattern c = Pattern.compile("演员:(.*)类");
String actor = doReplaceRegex(c, text);
Pattern d = Pattern.compile("导演:(.*)演");
String director = doReplaceRegex(d, text);
Pattern e = Pattern.compile("简介:(.*)");
String desc = doReplaceRegex(e, text);
vodList.put("vod_id", ids.get(0));
vodList.put("vod_pic", cover);
vodList.put("type_name", category);
vodList.put("vod_year", year);
vodList.put("vod_area", area);
vodList.put("vod_actor", actor);
vodList.put("vod_director", director);
vodList.put("vod_content", desc);
List vodItems = new ArrayList<>();
List vodItems2 = new ArrayList<>();
List vodItemsos = new ArrayList<>();
List vodItems2os = new ArrayList<>();
Map vod_play = new LinkedHashMap<>();
Elements allScript = doc.select(".wp-playlist-script");
String sourceName = "第1季";
for (int j = 0; j < allScript.size(); j++) {
String scContent = allScript.get(j).html().trim();
int start = scContent.indexOf('{');
int end = scContent.lastIndexOf('}') + 1;
String json = scContent.substring(start, end);
JSONObject UJson = new JSONObject(json);
JSONArray Track = UJson.getJSONArray("tracks");
for (int k = 0; k < Track.length(); k++) {
JSONObject src = Track.getJSONObject(k);
String adk = src.getString("src1");
String vodName = src.getString("caption");
String playURL = "https://ddys.tv/getvddr/video?id=" + adk +"&dim=1080P+&type=mix";
String zm = "https://ddys.tv/subddr/" + src.getString("subsrc");
String pzm = playURL + "|" + zm;
vodItems.add(vodName + "$" + pzm);
}
for (int k = 0; k < Track.length(); k++) {
JSONObject src = Track.getJSONObject(k);
String adksrc0 = src.getString("src0");
String adksrc2 = src.getString("src2");
String vodName = src.getString("caption");
String playURL = "https://w.ddys.tv" + adksrc0 +"?ddrkey=" + adksrc2;
String zm = "https://ddys.tv/subddr/" + src.getString("subsrc");
String pzm = playURL + "|" + zm;
vodItemsos.add(vodName + "$" + pzm);
}
vod_play.put(sourceName, TextUtils.join("#", vodItems));
vod_play.put(sourceName+":海外", TextUtils.join("#", vodItemsos));
}
Elements sources = doc.select(".post-page-numbers");
if (!sources.isEmpty())
for (int i = 0; i < sources.size(); i++) {
Element source = sources.get(i);
sourceName = "第" + source.text() + "季";
String Purl = siteUrl + "/" + ids.get(0) + "/" + source.text() + "/";
Document docs = Jsoup.parse(OkHttpUtil.string(Purl, getHeaders(Purl)));
Elements allScripts = docs.select(".wp-playlist-script");
for (int j = 0; j < allScripts.size(); j++) {
String scContent = allScripts.get(j).html().trim();
int start = scContent.indexOf('{');
int end = scContent.lastIndexOf('}') + 1;
String json = scContent.substring(start, end);
JSONObject UJson = new JSONObject(json);
JSONArray Track = UJson.getJSONArray("tracks");
for (int k = 0; k < Track.length(); k++) {
JSONObject src = Track.getJSONObject(k);
String adk = src.getString("src1");
String vodName = src.getString("caption");
String playURL = "https://ddys.tv/getvddr/video?id=" + adk +"&dim=1080P+&type=mix";
String zm = "https://ddys.tv/subddr/" + src.getString("subsrc");
String pzm = playURL + "|" + zm;
vodItems2.add(vodName + "$" + pzm);
}
for (int k = 0; k < Track.length(); k++) {
JSONObject src = Track.getJSONObject(k);
String adksrc0 = src.getString("src0");
String adksrc2 = src.getString("src2");
String vodName = src.getString("caption");
String playURL = "https://w.ddys.tv" + adksrc0 +"?ddrkey=" + adksrc2;
String zm = "https://ddys.tv/subddr/" + src.getString("subsrc");
String pzm = playURL + "|" + zm;
vodItems2os.add(vodName + "$" + pzm);
}
vod_play.put(sourceName, TextUtils.join("#", vodItems2));
vod_play.put(sourceName+":海外", TextUtils.join("#", vodItems2os));
}
vodItems2.removeAll(vodItems2);
vodItems2os.removeAll(vodItems2os);
}
String vod_play_from = TextUtils.join("$$$", vod_play.keySet());
String vod_play_url = TextUtils.join("$$$", vod_play.values());
vodList.put("vod_play_from", vod_play_from);
vodList.put("vod_play_url", vod_play_url);
JSONArray list = new JSONArray();
list.put(vodList);
result.put("list", list);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
public static Object[] loadsub(String sub) {
try {
OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() {
@Override
protected void onFailure(Call call, Exception e) {
}
@Override
protected void onResponse(Response response) {
}
};
OkHttpClient YM = OkHttpUtil.defaultClient();
OkHttpUtil.get(YM, sub, null, Headers(), callBack);
Response result = callBack.getResult();
int code = result.code();
if (code == 404) {
return new Object[]{200, "application/octet-stream", new ByteArrayInputStream("WEBVTT".getBytes())};
}
ResponseBody body = result.body();
byte[] bytes = body.bytes();
byte[] tokenkey = Arrays.copyOfRange(bytes, 0, 16);
byte[] data = Arrays.copyOfRange(bytes, 16, bytes.length);
byte[] KS = CBC.CBC(data, tokenkey, tokenkey);
String vtt = gZip.KS(KS);
vtt = vtt.replaceAll("(\\d{2}:\\d{2}:\\d{2}.\\d{3}.+\\d{2}:\\d{2}:\\d{2}.\\d{3}).*", "$1");
vtt = vtt.replaceAll("(\\d{2}:\\d{2}.\\d{3}).*?( --> )(\\d{2}:\\d{2}.\\d{3}).*", "00:$1$200:$3");
BufferedReader br = new BufferedReader(new StringReader(vtt));
ArrayList lines = new ArrayList<>();
int captionNumber = 1;
String line = br.readLine();
while (line != null) {
if (line.matches("\\d{2}:\\d{2}:\\d{2}.\\d{3}.+\\d{2}:\\d{2}:\\d{2}.\\d{3}")) {
if (lines.get(lines.size() - 1).trim().isEmpty()) {
lines.add(captionNumber + "");
captionNumber++;
}
}
lines.add(line);
line = br.readLine();
}
String join = TextUtils.join("\n", lines);
return new Object[]{200, "application/octet-stream", new ByteArrayInputStream(join.getBytes())};
} catch (Exception e) {
e.printStackTrace();
SpiderDebug.log(e);
return null;
}
}
/**
* 获取视频播放信息
*
* @param flag 播放源
* @param id 视频id
* @param vipFlags 所有可能需要vip解析的源
* @return
*/
@Override
public String playerContent(String flag, String id, List vipFlags) {
try {
JSONObject result = new JSONObject();
String[] item = id.split("\\|");
String playUrl = item[0];
String ZiMu = item[1];
String RealUrl="";
if(playUrl.contains("ddrkey")){
Map> respHeaders = new HashMap<>();
OkHttpUtil.stringNoRedirect(playUrl, getHeaders(playUrl),respHeaders);
RealUrl = OkHttpUtil.getRedirectLocation(respHeaders);
} else {
String content = OkHttpUtil.string(playUrl, getHeaders(playUrl));
JSONObject obj = new JSONObject(content);
RealUrl = obj.optString("url");
}
result.put("parse", 0);
result.put("playUrl", "");
result.put("url", RealUrl);
result.put("header", "");
// if (!TextUtils.isEmpty(str4)) {
result.put("subf", "/vtt/utf-8");
result.put("subt", Proxy.localProxyUrl() + "?do=ddys&url=" + URLEncoder.encode(ZiMu));
// result.put("subt", ZiMu);
// }
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public String searchContent(String key, boolean quick) {
try {
String url = "https://www.google.com/search?q=site%3Addys.tv+" + URLEncoder.encode(key);
Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url)));
JSONObject result = new JSONObject();
JSONArray videos = new JSONArray();
String sourceName = doc.select("div.yuRUbf a h3").text();
if (sourceName.contains(key)) {
String list1 = doc.select("div.yuRUbf a").attr("href");
Document ddrklink = Jsoup.parse(OkHttpUtil.string(list1, getHeaders(list1)));
String DDlink = ddrklink.select("link[rel=canonical]").attr("href");
Matcher matcher = regexVid.matcher(DDlink);
if (matcher.find()) {
JSONObject v = new JSONObject();
String group = matcher.group(1);
String ab = ddrklink.select("h1.post-title").text();
String cover = ddrklink.select("div.post img").attr("src");
if (ab.contains("(")) {
String[] b = ab.split("\\(");
String title = b[0];
String remark = b[1].replace("(", "");
v.put("vod_name", title);
v.put("vod_remarks", remark);
} else {
v.put("vod_name", ab);
String remark = "全";
v.put("vod_remarks", remark);
}
v.put("vod_id", group);
v.put("vod_pic", cover);
videos.put(v);
}
}
result.put("list", videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
private static String doReplaceRegex(Pattern pattern, String src) {
if (pattern == null)
return src;
try {
Matcher matcher = pattern.matcher(src);
if (matcher.find()) {
return matcher.group(1).trim();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return src;
}
}