add - PushAgent XBiubiu

app/src/main/java/com/github/catvod/spider/Bili.java

@@ -2,10 +2,10 @@ package com.github.catvod.spider;
 
 import android.content.Context;
 import android.text.TextUtils;
-import android.util.Base64;
 
 import com.github.catvod.crawler.Spider;
 import com.github.catvod.crawler.SpiderDebug;
+import com.github.catvod.utils.okhttp.OKCallBack;
 import com.github.catvod.utils.okhttp.OkHttpUtil;
 
 import org.json.JSONArray;
@@ -17,12 +17,24 @@ import org.jsoup.nodes.Document;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.Iterator;
 import java.util.List;
 
+import okhttp3.Call;
+import okhttp3.Cookie;
+import okhttp3.CookieJar;
+import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
+import okhttp3.Response;
+
 public class Bili extends Spider {
 
     protected JSONObject ext = null;
+    protected static String siteUrl = "https://www.bilibili.com";
+    protected static String sitHost = "www.bilibili.com";
 
     @Override
     public void init(Context context, String extend) {
@@ -49,13 +61,47 @@ public class Bili extends Spider {
         return results.toString();
     }
 
+    public static String cookie = "";
+    public static String getCookie(){
+        if(cookie.isEmpty()){
+            OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() {
+                @Override
+                protected void onFailure(Call call, Exception e) {
+
+                }
+
+                @Override
+                protected void onResponse(Response response) {
+
+                }
+            };
+            OkHttpUtil.get(OkHttpUtil.defaultClient(),siteUrl,callBack);
+            List<Cookie> cookies = OkHttpUtil.cookieStore.get(sitHost);
+            cookie = TextUtils.join(";",cookies);
+        }
+        return cookie;
+    }
+
+    protected static HashMap<String, String> CookieHeaders() {
+        HashMap<String, String> headers = new HashMap<>();
+        headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36");
+        headers.put("Cookie", getCookie());
+        return headers;
+    }
+
+
     @Override
     public String homeVideoContent() {
         try {
             JSONArray videos = new JSONArray();
             try {
                 String url = "https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword=窗 白噪音";
-                String content = OkHttpUtil.string(url, null);
+                String content = OkHttpUtil.string(url, CookieHeaders());
+                int code = new JSONObject(content).getInt("code");
+                if(code != 0){
+                    cookie = "";
+                    content = OkHttpUtil.string(url, CookieHeaders());
+                }
                 JSONObject data = new JSONObject(content).getJSONObject("data");
                 JSONArray RArray = data.getJSONArray("result");
                 for (int i = 0; i < RArray.length(); i++) {
@@ -108,7 +154,12 @@ public class Bili extends Spider {
                 }
             }
             url += "&page=" + pg;
-            String content = OkHttpUtil.string(url, null);
+            String content = OkHttpUtil.string(url, CookieHeaders());
+            int code = new JSONObject(content).getInt("code");
+            if(code != 0){
+                cookie = "";
+                content = OkHttpUtil.string(url, CookieHeaders());
+            }
             JSONObject data = new JSONObject(content).getJSONObject("data");
             JSONArray list = new JSONArray();
             JSONArray RSArray = data.getJSONArray("result");

app/src/main/java/com/github/catvod/spider/Kunyu77.java

@@ -31,6 +31,12 @@ public class Kunyu77 extends Spider {
     private HashMap<String, String> getHeaders(String url) {
         HashMap<String, String> headers = new HashMap<>();
         headers.put("user-agent", uAgent);
+        String t = String.valueOf(System.currentTimeMillis()/ 1000) ;
+
+        String zuhe=url+t+"XSpeUFjJ";
+        String TK=Misc.MD5(zuhe, Misc.CharsetUTF8);
+        headers.put("TK",TK);
+        headers.put("t",t);
         return headers;
     }
 
@@ -224,8 +230,9 @@ public class Kunyu77 extends Spider {
     @Override
     public String detailContent(List<String> ids) {
         try {
-            String url = siteUrl + "/api.php/provide/videoDetail?ids=" + ids.get(0);
+            String url = siteUrl + "/api.php/provide/videoDetail?ids=" + ids.get(0) + "&version=2.0.4&sj=1661957675";
-            String content = OkHttpUtil.string(url, getHeaders(url));
+            String urlx="/api.php/provide/videoDetail" + ids.get(0);
+            String content = OkHttpUtil.string(url, getHeaders(urlx));
             JSONObject dataObject = new JSONObject(decryptResponse(content));
             JSONObject vObj = dataObject.getJSONObject("data");
             JSONObject result = new JSONObject();
@@ -243,7 +250,7 @@ public class Kunyu77 extends Spider {
             vodAtom.put("vod_director", vObj.getString("director"));
             vodAtom.put("vod_content", vObj.getString("brief").trim());
 
-            url = siteUrl + "/api.php/provide/videoPlaylist?ids=" + ids.get(0);
+            url = siteUrl + "/api.php/provide/videoPlaylist?ids=" + ids.get(0) + "&version=2.0.4&sj=1661957675";
             content = OkHttpUtil.string(url, getHeaders(url));
             JSONArray episodes = new JSONObject(content).getJSONObject("data").getJSONArray("episodes");
             LinkedHashMap<String, ArrayList<String>> playlist = new LinkedHashMap<>();

app/src/main/java/com/github/catvod/spider/PushAgent.java

@@ -201,7 +201,6 @@ public class PushAgent extends Spider {
 
     }
 
-
     private static HashMap<String, String> Headers() {
         HashMap<String, String> headers = new HashMap<>();
         headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36");
@@ -242,7 +241,6 @@ public class PushAgent extends Spider {
         }
     }
 
-
     private static String VideoDetail(String shareid, String token, String fileid) {
         int i;
         try {
@@ -357,8 +355,9 @@ public class PushAgent extends Spider {
                             if (replace.length() > 20) {
                                 replace = replace.substring(0, 10) + "..." + replace.substring(replace.length() - 10);
                             }
+                            String category = jSONObject3.getString("category");
                             String fileIds = jSONObject3.getString("file_id");
-                            map.put(replace, str + "+" + str2 + "+" + fileIds);
+                            map.put(replace, str + "+" + str2 + "+" + fileIds+ "+" + category);
                         }
                     }
                     i2++;
@@ -379,7 +378,7 @@ public class PushAgent extends Spider {
         }
     }
 
-    public String getAliContent(List<String> list,String pic,JSONObject jSONObject6) {
+    public String getAliContent(List<String> list,JSONObject jSONObject6) {
         String str;
         try {
             String url = list.get(0).trim();
@@ -432,7 +431,7 @@ public class PushAgent extends Spider {
             jSONObject6.put("vod_id", url);
             String string3 = jSONObject3.getString("share_name");
             jSONObject6.put("vod_name", string3);
-            jSONObject6.put("vod_play_from", "AliYun");
+            jSONObject6.put("vod_play_from", "AliYun$$$原画");
             ArrayList arrayList = new ArrayList();
             String string4 = jSONObject4.getString("type");
             if (!string4.equals("folder")) {
@@ -463,7 +462,9 @@ public class PushAgent extends Spider {
             ArrayList arrayList3 = new ArrayList();
             for (int i2 = 0; i2 < 4; i2++) {
                 String join = TextUtils.join("#", arrayList);
+                String join4 = TextUtils.join("#", arrayList);
                 arrayList3.add(join);
+                arrayList3.add(join4);
             }
             String join2 = TextUtils.join("$$$", arrayList3);
             jSONObject6.put("vod_play_url", join2);
@@ -628,7 +629,7 @@ public class PushAgent extends Spider {
                 result.put("list", lists);
                 return result.toString();
             } else if (url.startsWith("http") && (matcher2.find())) {
-                return getAliContent(list,pic,vodAtom);
+                return getAliContent(list,vodAtom);
             } else if (url.startsWith("http") && (!matcher.find()) && (!matcher2.find())) {
                 Document doc = null;
                 String baseUrl = url.replaceAll("(^https?://.*?)(:\\d+)?/.*$", "$1");//https://www.dyk9.com
@@ -833,6 +834,23 @@ public class PushAgent extends Spider {
                     result.put("url", url);
                     result.put("header", "");
                     return result.toString();
+                case "原画": {
+                    refreshTk();
+                    split = id.split("\\+");
+                    String DownLoadUrl = downLoadUrl(split[0], split[1], split[2], split[3]);
+                    HashMap hashMap = new HashMap();
+                    OkHttpUtil.stringNoRedirect(DownLoadUrl,Headers(),hashMap);
+                    url= OkHttpUtil.getRedirectLocation(hashMap);
+                    JSONObject jSONObject4 = new JSONObject();
+                    jSONObject4.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36");
+                    jSONObject4.put("Referer", "https://www.aliyundrive.com/");
+                    JSONObject jSONObject = new JSONObject();
+                    jSONObject.put("parse", "0");
+                    jSONObject.put("playUrl", "");
+                    jSONObject.put("url", url);
+                    jSONObject.put("header",jSONObject4.toString());
+                    return jSONObject.toString();
+                }
             }
             if(flag.startsWith("嗅探")){
                 result.put("parse", 1);
@@ -848,4 +866,58 @@ public class PushAgent extends Spider {
         return "";
     }
 
+    private static String downLoadUrl(String shareid, String token, String fileid, String type) {
+        try {
+            HashMap<String, String> z = Headers();
+            z.put("x-share-token", token);
+            z.put("authorization", j);
+            if (type.equals("video")) {
+                JSONObject jSONObject = new JSONObject();
+                jSONObject.put("share_id", shareid);
+                jSONObject.put("category", "live_transcoding");
+                jSONObject.put("file_id", fileid);
+                jSONObject.put("template_id", "");
+                JSONObject jSONObject2 = new JSONObject(Post("https://api.aliyundrive.com/v2/file/get_share_link_video_preview_play_info", jSONObject.toString(), z));
+                shareid = jSONObject2.getString("share_id");
+                fileid = jSONObject2.getString("file_id");
+            }
+            JSONObject jSONObject3 = new JSONObject();
+            if (type.equals("video")) {
+                jSONObject3.put("expire_sec", 600);
+                jSONObject3.put("file_id", fileid);
+                jSONObject3.put("share_id", shareid);
+            }
+            if (type.equals("audio")) {
+                jSONObject3.put("share_id", shareid);
+                jSONObject3.put("get_audio_play_info", true);
+                jSONObject3.put("file_id", fileid);
+            }
+            return new JSONObject(Post("https://api.aliyundrive.com/v2/file/get_share_link_download_url", jSONObject3.toString(), z)).getString("download_url");
+        } catch (Exception e) {
+            SpiderDebug.log(e);
+            return "";
+        }
+    }
+
+    //修复软件不支持的格式无法嗅探的问题
+    @Override
+    public boolean manualVideoCheck() {
+        return true;
+    }
+
+    private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv", ".m4a",".mp3",".wma",".wmv"};
+
+    @Override
+    public boolean isVideoFormat(String url) {
+        url = url.toLowerCase();
+        if (url.contains("=http") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) {
+            return false;
+        }
+        for (String format : videoFormatList) {
+            if (url.contains(format)) {
+                return true;
+            }
+        }
+        return false;
+    }
 }

app/src/main/java/com/github/catvod/spider/XBiubiu.java

@@ -20,6 +20,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import okhttp3.Call;
+import org.jsoup.Jsoup;
 
 public class XBiubiu extends Spider {
     private PushAgent pushAgent= null;
@@ -101,11 +102,20 @@ public class XBiubiu extends Spider {
     private JSONObject category(String tid, String pg, boolean filter, HashMap<String, String> extend) {
         try {
             fetchRule();
+            if (tid.equals("空"))
+                tid = "";
+            String qishiye = rule.optString("qishiye", "nil");
+            if (qishiye.equals("空"))
+                pg = "";
+            else if (!qishiye.equals("nil")) {
+                pg = String.valueOf(Integer.parseInt(pg) - 1 + Integer.parseInt(qishiye));
+            }
             String baseUrl = getRuleVal("url");
             String webUrl = baseUrl + tid + pg + getRuleVal("houzhui");
             String html = fetch(webUrl);
+            html = removeUnicode(html);
             String parseContent = html;
-            boolean shifouercijiequ = getRuleVal("shifouercijiequ","1").equals("1");
+            boolean shifouercijiequ = getRuleVal("shifouercijiequ").equals("1");
             if (shifouercijiequ) {
                 String jiequqian = getRuleVal("jiequqian");
                 String jiequhou = getRuleVal("jiequhou");
@@ -116,32 +126,30 @@ public class XBiubiu extends Spider {
             JSONArray videos = new JSONArray();
             ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
             for (int i = 0; i < jiequContents.size(); i++) {
-                //try {
+                try {
-                String jiequContent = jiequContents.get(i);
+                    String jiequContent = jiequContents.get(i);
-                String title = subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0);
+                    String title = removeHtml(subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0));
-                String pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0);
+                    String pic = "";
-                pic = Misc.fixUrl(webUrl, pic);
+                    String tupianqian = getRuleVal("tupianqian").toLowerCase();
-                String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0);
+                    if (tupianqian.startsWith("http://") || tupianqian.startsWith("https://")) {
-                //String remarks = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0);
+                        pic = getRuleVal("tupianqian");
-                //String remarks = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", "");
+                    } else {
-                String mark = "";
+                        pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0);
-                if (!getRuleVal("gengxinqian").isEmpty() && !getRuleVal("gengxinhou").isEmpty()) {
-                    try {
-                        mark = subContent(jiequContent, getRuleVal("gengxinqian"), getRuleVal("gengxinhou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", "");
-                    } catch (Exception e) {
-                        SpiderDebug.log(e);
                     }
+                    pic = Misc.fixUrl(webUrl, pic);
+                    String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0);
+                    link = getRuleVal("ljqianzhui").isEmpty() ? (link + getRuleVal("ljhouzhui")) : ("x:" + getRuleVal("ljqianzhui")) + link + getRuleVal("ljhouzhui");
+                    String remark = !getRuleVal("fubiaotiqian").isEmpty() && !getRuleVal("fubiaotihou").isEmpty() ?
+                            removeHtml(subContent(jiequContent, getRuleVal("fubiaotiqian"), getRuleVal("fubiaotihou")).get(0)) : "";
+                    JSONObject v = new JSONObject();
+                    v.put("vod_id", baseUrl+link + "$$$" + pic + "$$$" + title);
+                    v.put("vod_name", title);
+                    v.put("vod_pic", pic);
+                    v.put("vod_remarks", remark);
+                    videos.put(v);
+                } catch (Throwable th) {
+                    th.printStackTrace();
                 }
-                JSONObject v = new JSONObject();
-                v.put("vod_id", baseUrl+link + "$$$" + pic + "$$$" + title);// v.put("vod_id", title + "$$$" + pic + "$$$" + link);
-                v.put("vod_name", title);
-                v.put("vod_pic", pic);
-                v.put("vod_remarks", mark);
-                videos.put(v);
-                //} catch (Throwable th) {
-                //th.printStackTrace();
-                //break;
-                //}
             }
             JSONObject result = new JSONObject();
             result.put("page", pg);
@@ -155,7 +163,21 @@ public class XBiubiu extends Spider {
         }
         return null;
     }
+    private static String removeUnicode(String str) {
+        Pattern pattern = Pattern.compile("(\\\\u(\\w{4}))");
+        Matcher matcher = pattern.matcher(str);
+        while (matcher.find()) {
+            String full = matcher.group(1);
+            String ucode = matcher.group(2);
+            char c = (char) Integer.parseInt(ucode, 16);
+            str = str.replace(full, c + "");
+        }
+        return str;
+    }
 
+    String removeHtml(String text) {
+        return Jsoup.parse(text).text();
+    }
     @Override
     public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
         JSONObject obj = category(tid, pg, filter, extend);
@@ -429,16 +451,28 @@ public class XBiubiu extends Spider {
     }
 
     private ArrayList<String> subContent(String content, String startFlag, String endFlag) {
-        ArrayList<String> result = new ArrayList<>();
+        return Misc.subContent(content,startFlag,endFlag);
-        try {
+    }
-            Pattern pattern = Pattern.compile(startFlag + "(.*?)" + endFlag);
+
-            Matcher matcher = pattern.matcher(content);
+    //修复软件不支持的格式无法嗅探的问题
-            while (matcher.find()) {
+    @Override
-                result.add(matcher.group(1));
+    public boolean manualVideoCheck() {
-            }
+        return true;
-        } catch (Throwable th) {
+    }
-            th.printStackTrace();
+
+    private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv", ".m4a",".mp3",".wma",".wmv"};
+
+    @Override
+    public boolean isVideoFormat(String url) {
+        url = url.toLowerCase();
+        if (url.contains("=http") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) {
+            return false;
         }
-        return result;
+        for (String format : videoFormatList) {
+            if (url.contains(format)) {
+                return true;
+            }
+        }
+        return false;
     }
 }

app/src/main/java/com/github/catvod/utils/Misc.java

@@ -257,4 +257,34 @@ public class Misc {
         }
         return result;
     }
+
+    public static ArrayList<String> subContent(String content, String startFlag, String endFlag) {
+        ArrayList<String> result = new ArrayList<>();
+        if (startFlag.isEmpty() && endFlag.isEmpty()) {
+            result.add(content);
+            return result;
+        }
+        try {
+            Pattern pattern = Pattern.compile(escapeExprSpecialWord(startFlag) + "(.*?)" + escapeExprSpecialWord(endFlag));
+            Matcher matcher = pattern.matcher(content);
+            while (matcher.find()) {
+                result.add(matcher.group(1));
+            }
+        } catch (Throwable th) {
+            th.printStackTrace();
+        }
+        return result;
+    }
+
+    public static String escapeExprSpecialWord(String regexStr) {
+        if (!regexStr.isEmpty()) {
+            String[] fbsArr = {"\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"};
+            for (String key : fbsArr) {
+                if (regexStr.contains(key)) {
+                    regexStr = regexStr.replace(key, "\\" + key);
+                }
+            }
+        }
+        return regexStr;
+    }
 }

app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java

@@ -3,11 +3,16 @@ package com.github.catvod.utils.okhttp;
 import com.github.catvod.crawler.SpiderDebug;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import okhttp3.Call;
+import okhttp3.Cookie;
+import okhttp3.CookieJar;
+import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Response;
 
@@ -16,7 +21,7 @@ public class OkHttpUtil {
     public static final String METHOD_GET = "GET";
     public static final String METHOD_POST = "POST";
 
-    public static final int DEFAULT_TIMEOUT = 20;
+    private static final int DEFAULT_TIMEOUT = 15;
 
     private static final Object lockO = new Object();
 
@@ -27,10 +32,24 @@ public class OkHttpUtil {
      */
     private static OkHttpClient noRedirectClient = null;
 
+    public static HashMap<String, List<Cookie>> cookieStore = new HashMap<>();
+
     public static OkHttpClient defaultClient() {
         synchronized (lockO) {
             if (defaultClient == null) {
                 OkHttpClient.Builder builder = new OkHttpClient.Builder()
+                        .cookieJar(new CookieJar() {
+                            @Override
+                            public void saveFromResponse(HttpUrl httpUrl, List<Cookie> list) {
+                                cookieStore.put(httpUrl.host(), list);
+                            }
+
+                            @Override
+                            public List<Cookie> loadForRequest(HttpUrl httpUrl) {
+                                List<Cookie> cookies = cookieStore.get(httpUrl.host());
+                                return cookies != null ? cookies : new ArrayList<>();
+                            }
+                        })
                         .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS)
                         .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS)
                         .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS)