diff --git a/app/src/main/java/com/github/catvod/spider/Bili.java b/app/src/main/java/com/github/catvod/spider/Bili.java index 0accb46..f8d33de 100644 --- a/app/src/main/java/com/github/catvod/spider/Bili.java +++ b/app/src/main/java/com/github/catvod/spider/Bili.java @@ -2,10 +2,10 @@ package com.github.catvod.spider; import android.content.Context; import android.text.TextUtils; -import android.util.Base64; import com.github.catvod.crawler.Spider; import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.okhttp.OKCallBack; import com.github.catvod.utils.okhttp.OkHttpUtil; import org.json.JSONArray; @@ -17,12 +17,24 @@ import org.jsoup.nodes.Document; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.Iterator; import java.util.List; +import okhttp3.Call; +import okhttp3.Cookie; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Response; + public class Bili extends Spider { protected JSONObject ext = null; + protected static String siteUrl = "https://www.bilibili.com"; + protected static String sitHost = "www.bilibili.com"; @Override public void init(Context context, String extend) { @@ -49,13 +61,47 @@ public class Bili extends Spider { return results.toString(); } + public static String cookie = ""; + public static String getCookie(){ + if(cookie.isEmpty()){ + OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() { + @Override + protected void onFailure(Call call, Exception e) { + + } + + @Override + protected void onResponse(Response response) { + + } + }; + OkHttpUtil.get(OkHttpUtil.defaultClient(),siteUrl,callBack); + List cookies = OkHttpUtil.cookieStore.get(sitHost); + cookie = TextUtils.join(";",cookies); + } + return cookie; + } + + protected static HashMap CookieHeaders() { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + headers.put("Cookie", getCookie()); + return headers; + } + + @Override public String homeVideoContent() { try { JSONArray videos = new JSONArray(); try { String url = "https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword=窗 白噪音"; - String content = OkHttpUtil.string(url, null); + String content = OkHttpUtil.string(url, CookieHeaders()); + int code = new JSONObject(content).getInt("code"); + if(code != 0){ + cookie = ""; + content = OkHttpUtil.string(url, CookieHeaders()); + } JSONObject data = new JSONObject(content).getJSONObject("data"); JSONArray RArray = data.getJSONArray("result"); for (int i = 0; i < RArray.length(); i++) { @@ -108,7 +154,12 @@ public class Bili extends Spider { } } url += "&page=" + pg; - String content = OkHttpUtil.string(url, null); + String content = OkHttpUtil.string(url, CookieHeaders()); + int code = new JSONObject(content).getInt("code"); + if(code != 0){ + cookie = ""; + content = OkHttpUtil.string(url, CookieHeaders()); + } JSONObject data = new JSONObject(content).getJSONObject("data"); JSONArray list = new JSONArray(); JSONArray RSArray = data.getJSONArray("result"); diff --git a/app/src/main/java/com/github/catvod/spider/Kunyu77.java b/app/src/main/java/com/github/catvod/spider/Kunyu77.java index ef6d1ff..f56b9a0 100644 --- a/app/src/main/java/com/github/catvod/spider/Kunyu77.java +++ b/app/src/main/java/com/github/catvod/spider/Kunyu77.java @@ -31,6 +31,12 @@ public class Kunyu77 extends Spider { private HashMap getHeaders(String url) { HashMap headers = new HashMap<>(); headers.put("user-agent", uAgent); + String t = String.valueOf(System.currentTimeMillis()/ 1000) ; + + String zuhe=url+t+"XSpeUFjJ"; + String TK=Misc.MD5(zuhe, Misc.CharsetUTF8); + headers.put("TK",TK); + headers.put("t",t); return headers; } @@ -224,8 +230,9 @@ public class Kunyu77 extends Spider { @Override public String detailContent(List ids) { try { - String url = siteUrl + "/api.php/provide/videoDetail?ids=" + ids.get(0); - String content = OkHttpUtil.string(url, getHeaders(url)); + String url = siteUrl + "/api.php/provide/videoDetail?ids=" + ids.get(0) + "&version=2.0.4&sj=1661957675"; + String urlx="/api.php/provide/videoDetail" + ids.get(0); + String content = OkHttpUtil.string(url, getHeaders(urlx)); JSONObject dataObject = new JSONObject(decryptResponse(content)); JSONObject vObj = dataObject.getJSONObject("data"); JSONObject result = new JSONObject(); @@ -243,7 +250,7 @@ public class Kunyu77 extends Spider { vodAtom.put("vod_director", vObj.getString("director")); vodAtom.put("vod_content", vObj.getString("brief").trim()); - url = siteUrl + "/api.php/provide/videoPlaylist?ids=" + ids.get(0); + url = siteUrl + "/api.php/provide/videoPlaylist?ids=" + ids.get(0) + "&version=2.0.4&sj=1661957675"; content = OkHttpUtil.string(url, getHeaders(url)); JSONArray episodes = new JSONObject(content).getJSONObject("data").getJSONArray("episodes"); LinkedHashMap> playlist = new LinkedHashMap<>(); diff --git a/app/src/main/java/com/github/catvod/spider/PushAgent.java b/app/src/main/java/com/github/catvod/spider/PushAgent.java index 5354a0e..c411122 100644 --- a/app/src/main/java/com/github/catvod/spider/PushAgent.java +++ b/app/src/main/java/com/github/catvod/spider/PushAgent.java @@ -201,7 +201,6 @@ public class PushAgent extends Spider { } - private static HashMap Headers() { HashMap headers = new HashMap<>(); headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"); @@ -242,7 +241,6 @@ public class PushAgent extends Spider { } } - private static String VideoDetail(String shareid, String token, String fileid) { int i; try { @@ -357,8 +355,9 @@ public class PushAgent extends Spider { if (replace.length() > 20) { replace = replace.substring(0, 10) + "..." + replace.substring(replace.length() - 10); } + String category = jSONObject3.getString("category"); String fileIds = jSONObject3.getString("file_id"); - map.put(replace, str + "+" + str2 + "+" + fileIds); + map.put(replace, str + "+" + str2 + "+" + fileIds+ "+" + category); } } i2++; @@ -379,7 +378,7 @@ public class PushAgent extends Spider { } } - public String getAliContent(List list,String pic,JSONObject jSONObject6) { + public String getAliContent(List list,JSONObject jSONObject6) { String str; try { String url = list.get(0).trim(); @@ -432,7 +431,7 @@ public class PushAgent extends Spider { jSONObject6.put("vod_id", url); String string3 = jSONObject3.getString("share_name"); jSONObject6.put("vod_name", string3); - jSONObject6.put("vod_play_from", "AliYun"); + jSONObject6.put("vod_play_from", "AliYun$$$原画"); ArrayList arrayList = new ArrayList(); String string4 = jSONObject4.getString("type"); if (!string4.equals("folder")) { @@ -463,7 +462,9 @@ public class PushAgent extends Spider { ArrayList arrayList3 = new ArrayList(); for (int i2 = 0; i2 < 4; i2++) { String join = TextUtils.join("#", arrayList); + String join4 = TextUtils.join("#", arrayList); arrayList3.add(join); + arrayList3.add(join4); } String join2 = TextUtils.join("$$$", arrayList3); jSONObject6.put("vod_play_url", join2); @@ -628,7 +629,7 @@ public class PushAgent extends Spider { result.put("list", lists); return result.toString(); } else if (url.startsWith("http") && (matcher2.find())) { - return getAliContent(list,pic,vodAtom); + return getAliContent(list,vodAtom); } else if (url.startsWith("http") && (!matcher.find()) && (!matcher2.find())) { Document doc = null; String baseUrl = url.replaceAll("(^https?://.*?)(:\\d+)?/.*$", "$1");//https://www.dyk9.com @@ -833,6 +834,23 @@ public class PushAgent extends Spider { result.put("url", url); result.put("header", ""); return result.toString(); + case "原画": { + refreshTk(); + split = id.split("\\+"); + String DownLoadUrl = downLoadUrl(split[0], split[1], split[2], split[3]); + HashMap hashMap = new HashMap(); + OkHttpUtil.stringNoRedirect(DownLoadUrl,Headers(),hashMap); + url= OkHttpUtil.getRedirectLocation(hashMap); + JSONObject jSONObject4 = new JSONObject(); + jSONObject4.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"); + jSONObject4.put("Referer", "https://www.aliyundrive.com/"); + JSONObject jSONObject = new JSONObject(); + jSONObject.put("parse", "0"); + jSONObject.put("playUrl", ""); + jSONObject.put("url", url); + jSONObject.put("header",jSONObject4.toString()); + return jSONObject.toString(); + } } if(flag.startsWith("嗅探")){ result.put("parse", 1); @@ -848,4 +866,58 @@ public class PushAgent extends Spider { return ""; } + private static String downLoadUrl(String shareid, String token, String fileid, String type) { + try { + HashMap z = Headers(); + z.put("x-share-token", token); + z.put("authorization", j); + if (type.equals("video")) { + JSONObject jSONObject = new JSONObject(); + jSONObject.put("share_id", shareid); + jSONObject.put("category", "live_transcoding"); + jSONObject.put("file_id", fileid); + jSONObject.put("template_id", ""); + JSONObject jSONObject2 = new JSONObject(Post("https://api.aliyundrive.com/v2/file/get_share_link_video_preview_play_info", jSONObject.toString(), z)); + shareid = jSONObject2.getString("share_id"); + fileid = jSONObject2.getString("file_id"); + } + JSONObject jSONObject3 = new JSONObject(); + if (type.equals("video")) { + jSONObject3.put("expire_sec", 600); + jSONObject3.put("file_id", fileid); + jSONObject3.put("share_id", shareid); + } + if (type.equals("audio")) { + jSONObject3.put("share_id", shareid); + jSONObject3.put("get_audio_play_info", true); + jSONObject3.put("file_id", fileid); + } + return new JSONObject(Post("https://api.aliyundrive.com/v2/file/get_share_link_download_url", jSONObject3.toString(), z)).getString("download_url"); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + //修复软件不支持的格式无法嗅探的问题 + @Override + public boolean manualVideoCheck() { + return true; + } + + private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv", ".m4a",".mp3",".wma",".wmv"}; + + @Override + public boolean isVideoFormat(String url) { + url = url.toLowerCase(); + if (url.contains("=http") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; + } + for (String format : videoFormatList) { + if (url.contains(format)) { + return true; + } + } + return false; + } } \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/spider/XBiubiu.java b/app/src/main/java/com/github/catvod/spider/XBiubiu.java index ab071a9..77744d0 100644 --- a/app/src/main/java/com/github/catvod/spider/XBiubiu.java +++ b/app/src/main/java/com/github/catvod/spider/XBiubiu.java @@ -20,6 +20,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import okhttp3.Call; +import org.jsoup.Jsoup; public class XBiubiu extends Spider { private PushAgent pushAgent= null; @@ -101,11 +102,20 @@ public class XBiubiu extends Spider { private JSONObject category(String tid, String pg, boolean filter, HashMap extend) { try { fetchRule(); + if (tid.equals("空")) + tid = ""; + String qishiye = rule.optString("qishiye", "nil"); + if (qishiye.equals("空")) + pg = ""; + else if (!qishiye.equals("nil")) { + pg = String.valueOf(Integer.parseInt(pg) - 1 + Integer.parseInt(qishiye)); + } String baseUrl = getRuleVal("url"); String webUrl = baseUrl + tid + pg + getRuleVal("houzhui"); String html = fetch(webUrl); + html = removeUnicode(html); String parseContent = html; - boolean shifouercijiequ = getRuleVal("shifouercijiequ","1").equals("1"); + boolean shifouercijiequ = getRuleVal("shifouercijiequ").equals("1"); if (shifouercijiequ) { String jiequqian = getRuleVal("jiequqian"); String jiequhou = getRuleVal("jiequhou"); @@ -116,32 +126,30 @@ public class XBiubiu extends Spider { JSONArray videos = new JSONArray(); ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); for (int i = 0; i < jiequContents.size(); i++) { - //try { - String jiequContent = jiequContents.get(i); - String title = subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0); - String pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0); - pic = Misc.fixUrl(webUrl, pic); - String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0); - //String remarks = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0); - //String remarks = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); - String mark = ""; - if (!getRuleVal("gengxinqian").isEmpty() && !getRuleVal("gengxinhou").isEmpty()) { - try { - mark = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); - } catch (Exception e) { - SpiderDebug.log(e); + try { + String jiequContent = jiequContents.get(i); + String title = removeHtml(subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0)); + String pic = ""; + String tupianqian = getRuleVal("tupianqian").toLowerCase(); + if (tupianqian.startsWith("http://") || tupianqian.startsWith("https://")) { + pic = getRuleVal("tupianqian"); + } else { + pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0); } + pic = Misc.fixUrl(webUrl, pic); + String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0); + link = getRuleVal("ljqianzhui").isEmpty() ? (link + getRuleVal("ljhouzhui")) : ("x:" + getRuleVal("ljqianzhui")) + link + getRuleVal("ljhouzhui"); + String remark = !getRuleVal("fubiaotiqian").isEmpty() && !getRuleVal("fubiaotihou").isEmpty() ? + removeHtml(subContent(jiequContent, getRuleVal("fubiaotiqian"), getRuleVal("fubiaotihou")).get(0)) : ""; + JSONObject v = new JSONObject(); + v.put("vod_id", baseUrl+link + "$$$" + pic + "$$$" + title); + v.put("vod_name", title); + v.put("vod_pic", pic); + v.put("vod_remarks", remark); + videos.put(v); + } catch (Throwable th) { + th.printStackTrace(); } - JSONObject v = new JSONObject(); - v.put("vod_id", baseUrl+link + "$$$" + pic + "$$$" + title);// v.put("vod_id", title + "$$$" + pic + "$$$" + link); - v.put("vod_name", title); - v.put("vod_pic", pic); - v.put("vod_remarks", mark); - videos.put(v); - //} catch (Throwable th) { - //th.printStackTrace(); - //break; - //} } JSONObject result = new JSONObject(); result.put("page", pg); @@ -155,7 +163,21 @@ public class XBiubiu extends Spider { } return null; } + private static String removeUnicode(String str) { + Pattern pattern = Pattern.compile("(\\\\u(\\w{4}))"); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + String full = matcher.group(1); + String ucode = matcher.group(2); + char c = (char) Integer.parseInt(ucode, 16); + str = str.replace(full, c + ""); + } + return str; + } + String removeHtml(String text) { + return Jsoup.parse(text).text(); + } @Override public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { JSONObject obj = category(tid, pg, filter, extend); @@ -429,16 +451,28 @@ public class XBiubiu extends Spider { } private ArrayList subContent(String content, String startFlag, String endFlag) { - ArrayList result = new ArrayList<>(); - try { - Pattern pattern = Pattern.compile(startFlag + "(.*?)" + endFlag); - Matcher matcher = pattern.matcher(content); - while (matcher.find()) { - result.add(matcher.group(1)); - } - } catch (Throwable th) { - th.printStackTrace(); + return Misc.subContent(content,startFlag,endFlag); + } + + //修复软件不支持的格式无法嗅探的问题 + @Override + public boolean manualVideoCheck() { + return true; + } + + private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv", ".m4a",".mp3",".wma",".wmv"}; + + @Override + public boolean isVideoFormat(String url) { + url = url.toLowerCase(); + if (url.contains("=http") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; } - return result; + for (String format : videoFormatList) { + if (url.contains(format)) { + return true; + } + } + return false; } } \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/utils/Misc.java b/app/src/main/java/com/github/catvod/utils/Misc.java index e808be4..a2e6683 100644 --- a/app/src/main/java/com/github/catvod/utils/Misc.java +++ b/app/src/main/java/com/github/catvod/utils/Misc.java @@ -257,4 +257,34 @@ public class Misc { } return result; } + + public static ArrayList subContent(String content, String startFlag, String endFlag) { + ArrayList result = new ArrayList<>(); + if (startFlag.isEmpty() && endFlag.isEmpty()) { + result.add(content); + return result; + } + try { + Pattern pattern = Pattern.compile(escapeExprSpecialWord(startFlag) + "(.*?)" + escapeExprSpecialWord(endFlag)); + Matcher matcher = pattern.matcher(content); + while (matcher.find()) { + result.add(matcher.group(1)); + } + } catch (Throwable th) { + th.printStackTrace(); + } + return result; + } + + public static String escapeExprSpecialWord(String regexStr) { + if (!regexStr.isEmpty()) { + String[] fbsArr = {"\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"}; + for (String key : fbsArr) { + if (regexStr.contains(key)) { + regexStr = regexStr.replace(key, "\\" + key); + } + } + } + return regexStr; + } } \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java b/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java index 214d974..346cff5 100644 --- a/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java +++ b/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java @@ -3,11 +3,16 @@ package com.github.catvod.utils.okhttp; import com.github.catvod.crawler.SpiderDebug; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import okhttp3.Call; +import okhttp3.Cookie; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Response; @@ -16,7 +21,7 @@ public class OkHttpUtil { public static final String METHOD_GET = "GET"; public static final String METHOD_POST = "POST"; - public static final int DEFAULT_TIMEOUT = 20; + private static final int DEFAULT_TIMEOUT = 15; private static final Object lockO = new Object(); @@ -27,10 +32,24 @@ public class OkHttpUtil { */ private static OkHttpClient noRedirectClient = null; + public static HashMap> cookieStore = new HashMap<>(); + public static OkHttpClient defaultClient() { synchronized (lockO) { if (defaultClient == null) { OkHttpClient.Builder builder = new OkHttpClient.Builder() + .cookieJar(new CookieJar() { + @Override + public void saveFromResponse(HttpUrl httpUrl, List list) { + cookieStore.put(httpUrl.host(), list); + } + + @Override + public List loadForRequest(HttpUrl httpUrl) { + List cookies = cookieStore.get(httpUrl.host()); + return cookies != null ? cookies : new ArrayList<>(); + } + }) .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) @@ -164,7 +183,7 @@ public class OkHttpUtil { public static void cancel(Object tag) { cancel(defaultClient(), tag); } - + public static void cancelAll() { cancelAll(defaultClient()); } @@ -197,4 +216,4 @@ public class OkHttpUtil { return headers.get("Location").get(0); return null; } -} +} \ No newline at end of file