From 891a91b416369fa84081da7636e6bc49ba0bfcfc Mon Sep 17 00:00:00 2001 From: gzlinz2 <107169210+gzlinz2@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:19:19 +0800 Subject: [PATCH] Delete XPath.java --- .../java/com/github/catvod/spider/XPath.java | 483 ------------------ 1 file changed, 483 deletions(-) delete mode 100644 app/src/main/java/com/github/catvod/spider/XPath.java diff --git a/app/src/main/java/com/github/catvod/spider/XPath.java b/app/src/main/java/com/github/catvod/spider/XPath.java deleted file mode 100644 index 797ca25..0000000 --- a/app/src/main/java/com/github/catvod/spider/XPath.java +++ /dev/null @@ -1,483 +0,0 @@ -package com.github.catvod.spider; - -import android.content.Context; -import android.text.TextUtils; - -import com.github.catvod.crawler.Spider; -import com.github.catvod.crawler.SpiderDebug; -import com.github.catvod.utils.Misc; -import com.github.catvod.utils.okhttp.OkHttpUtil; -import com.github.catvod.xpath.XPathRule; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.seimicrawler.xpath.JXDocument; -import org.seimicrawler.xpath.JXNode; - -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; - -public class XPath extends Spider { - - @Override - public void init(Context context) { - super.init(context); - } - - public void init(Context context, String extend) { - super.init(context, extend); - this.ext = extend; - } - - @Override - public String homeContent(boolean filter) { - try { - fetchRule(); - JSONObject result = new JSONObject(); - JSONArray classes = new JSONArray(); - if (rule.getCateManual().size() > 0) { - Set keys = rule.getCateManual().keySet(); - for (String k : keys) { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("type_name", k); - jsonObject.put("type_id", rule.getCateManual().get(k)); - classes.put(jsonObject); - } - } - try { - String webUrl = rule.getHomeUrl(); - JXDocument doc = JXDocument.create(fetch(webUrl)); - if (rule.getCateManual().size() == 0) { - List navNodes = doc.selN(rule.getCateNode()); - for (int i = 0; i < navNodes.size(); i++) { - String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim(); - name = rule.getCateNameR(name); - String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim(); - id = rule.getCateIdR(id); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("type_id", id); - jsonObject.put("type_name", name); - classes.put(jsonObject); - } - } - if (!rule.getHomeVodNode().isEmpty()) { - try { - JSONArray videos = new JSONArray(); - List vodNodes = doc.selN(rule.getHomeVodNode()); - for (int i = 0; i < vodNodes.size(); i++) { - String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim(); - name = rule.getHomeVodNameR(name); - String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim(); - id = rule.getHomeVodIdR(id); - String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim(); - pic = rule.getHomeVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = ""; - if (!rule.getHomeVodMark().isEmpty()) { - try { - mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim(); - mark = rule.getHomeVodMarkR(mark); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - JSONObject v = new JSONObject(); - v.put("vod_id", id); - v.put("vod_name", name); - v.put("vod_pic", pic); - v.put("vod_remarks", mark); - videos.put(v); - } - result.put("list", videos); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - } catch (Exception e) { - SpiderDebug.log(e); - } - result.put("class", classes); - if (filter && rule.getFilter() != null) { - result.put("filters", rule.getFilter()); - } - return result.toString(); - } catch ( - Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - protected HashMap getHeaders(String url) { - HashMap headers = new HashMap<>(); - headers.put("User-Agent", rule.getUa().isEmpty() - ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36" - : rule.getUa()); - return headers; - } - - @Override - public String homeVideoContent() { - try { - fetchRule(); - } catch (Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - protected String categoryUrl(String tid, String pg, boolean filter, HashMap extend) { - return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg); - } - - protected String categoryUrl2(String tid, String pg, boolean filter, HashMap extend) { - return rule.getCateUrl2().replace("{cateId}", tid).replace("{catePg}", pg); - } - - @Override - public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { - try { - fetchRule(); - String webUrl = ""; - if (!tid.startsWith("@")) { - webUrl = categoryUrl(tid, pg, filter, extend); - } else { - webUrl = categoryUrl2(tid.replace("@",""), pg, filter, extend); - } - JSONArray videos = new JSONArray(); - JXDocument doc = JXDocument.create(fetch(webUrl)); - List vodNodes = doc.selN(rule.getCateVodNode()); - for (int i = 0; i < vodNodes.size(); i++) { - String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim(); - name = rule.getCateVodNameR(name); - String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim(); - id = rule.getCateVodIdR(id); - String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim(); - pic = rule.getCateVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = ""; - if (!rule.getCateVodMark().isEmpty()) { - try { - mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim(); - mark = rule.getCateVodMarkR(mark); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - JSONObject v = new JSONObject(); - v.put("vod_id", id); - v.put("vod_name", name); - v.put("vod_pic", pic); - v.put("vod_remarks", mark); - videos.put(v); - } - JSONObject result = new JSONObject(); - result.put("page", pg); - result.put("pagecount", Integer.MAX_VALUE); - result.put("limit", 90); - result.put("total", Integer.MAX_VALUE); - result.put("list", videos); - return result.toString(); - } catch (Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - protected void detailContentExt(String content, JSONObject vod) { - - } - - @Override - public String detailContent(List ids) { - try { - fetchRule(); - String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); - String webContent = fetch(webUrl); - JXDocument doc = JXDocument.create(webContent); - JXNode vodNode = doc.selNOne(rule.getDetailNode()); - - String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; - - title = vodNode.selOne(rule.getDetailName()).asString().trim(); - title = rule.getDetailNameR(title); - - cover = vodNode.selOne(rule.getDetailImg()).asString().trim(); - cover = rule.getDetailImgR(cover); - cover = Misc.fixUrl(webUrl, cover); - - if (!rule.getDetailCate().isEmpty()) { - try { - category = vodNode.selOne(rule.getDetailCate()).asString().trim(); - category = rule.getDetailCateR(category); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailYear().isEmpty()) { - try { - year = vodNode.selOne(rule.getDetailYear()).asString().trim(); - year = rule.getDetailYearR(year); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailArea().isEmpty()) { - try { - area = vodNode.selOne(rule.getDetailArea()).asString().trim(); - area = rule.getDetailAreaR(area); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailMark().isEmpty()) { - try { - remark = vodNode.selOne(rule.getDetailMark()).asString().trim(); - remark = rule.getDetailMarkR(remark); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailActor().isEmpty()) { - try { - actor = vodNode.selOne(rule.getDetailActor()).asString().trim(); - actor = rule.getDetailActorR(actor); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailDirector().isEmpty()) { - try { - director = vodNode.selOne(rule.getDetailDirector()).asString().trim(); - director = rule.getDetailDirectorR(director); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - if (!rule.getDetailDesc().isEmpty()) { - try { - desc = vodNode.selOne(rule.getDetailDesc()).asString().trim(); - desc = rule.getDetailDescR(desc); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - - JSONObject vod = new JSONObject(); - vod.put("vod_id", ids.get(0)); - vod.put("vod_name", title); - vod.put("vod_pic", cover); - vod.put("type_name", category); - vod.put("vod_year", year); - vod.put("vod_area", area); - vod.put("vod_remarks", remark); - vod.put("vod_actor", actor); - vod.put("vod_director", director); - vod.put("vod_content", desc); - - ArrayList playFrom = new ArrayList<>(); - - List fromNodes = doc.selN(rule.getDetailFromNode()); - for (int i = 0; i < fromNodes.size(); i++) { - String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim(); - name = rule.getDetailFromNameR(name); - playFrom.add(name); - } - - ArrayList playList = new ArrayList<>(); - List urlListNodes = doc.selN(rule.getDetailUrlNode()); - for (int i = 0; i < urlListNodes.size(); i++) { - List urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode()); - List vodItems = new ArrayList<>(); - for (int j = 0; j < urlNodes.size(); j++) { - String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim(); - name = rule.getDetailUrlNameR(name); - String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim(); - id = rule.getDetailUrlIdR(id); - vodItems.add(name + "$" + id); - } - // 排除播放列表为空的播放源 - if (vodItems.size() == 0 && playFrom.size() > i) { - playFrom.set(i, ""); - } - playList.add(TextUtils.join("#", vodItems)); - } - // 排除播放列表为空的播放源 - for (int i = playFrom.size() - 1; i >= 0; i--) { - if (playFrom.get(i).isEmpty()) - playFrom.remove(i); - } - for (int i = playList.size() - 1; i >= 0; i--) { - if (playList.get(i).isEmpty()) - playList.remove(i); - } - for (int i = playList.size() - 1; i >= 0; i--) { - if (i >= playFrom.size()) - playList.remove(i); - } - String vod_play_from = TextUtils.join("$$$", playFrom); - String vod_play_url = TextUtils.join("$$$", playList); - vod.put("vod_play_from", vod_play_from); - vod.put("vod_play_url", vod_play_url); - - detailContentExt(webContent, vod); - - JSONObject result = new JSONObject(); - JSONArray list = new JSONArray(); - list.put(vod); - result.put("list", list); - return result.toString(); - } catch (Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - @Override - public String playerContent(String flag, String id, List vipFlags) { - try { - fetchRule(); - String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); - SpiderDebug.log(webUrl); - JSONObject result = new JSONObject(); - result.put("parse", 1); - result.put("playUrl", ""); - if (!rule.getPlayUa().isEmpty()) { - result.put("header", rule.getPlayUa()); - } - result.put("url", webUrl); - return result.toString(); - } catch (Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - @Override - public String searchContent(String key, boolean quick) { - try { - fetchRule(); - if (rule.getSearchUrl().isEmpty()) { - return ""; - } - String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); - String webContent = fetch(webUrl); - JSONObject result = new JSONObject(); - JSONArray videos = new JSONArray(); - // add maccms suggest search api support - if (rule.getSearchVodNode().startsWith("json:")) { - String[] node = rule.getSearchVodNode().substring(5).split(">"); - JSONObject data = new JSONObject(webContent); - for (int i = 0; i < node.length; i++) { - if (i == node.length - 1) { - JSONArray vodArray = data.getJSONArray(node[i]); - for (int j = 0; j < vodArray.length(); j++) { - JSONObject vod = vodArray.getJSONObject(j); - String name = vod.optString(rule.getSearchVodName()).trim(); - name = rule.getSearchVodNameR(name); - String id = vod.optString(rule.getSearchVodId()).trim(); - id = rule.getSearchVodIdR(id); - String pic = vod.optString(rule.getSearchVodImg()).trim(); - pic = rule.getSearchVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = vod.optString(rule.getSearchVodMark()).trim(); - mark = rule.getSearchVodMarkR(mark); - JSONObject v = new JSONObject(); - v.put("vod_id", id); - v.put("vod_name", name); - v.put("vod_pic", pic); - v.put("vod_remarks", mark); - videos.put(v); - } - } else { - data = data.getJSONObject(node[i]); - } - } - } else { - JXDocument doc = JXDocument.create(webContent); - List vodNodes = doc.selN(rule.getSearchVodNode()); - for (int i = 0; i < vodNodes.size(); i++) { - String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); - name = rule.getSearchVodNameR(name); - String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); - id = rule.getSearchVodIdR(id); - String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim(); - pic = rule.getSearchVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = ""; - if (!rule.getCateVodMark().isEmpty()) { - try { - mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); - mark = rule.getSearchVodMarkR(mark); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - JSONObject v = new JSONObject(); - v.put("vod_id", id); - v.put("vod_name", name); - v.put("vod_pic", pic); - v.put("vod_remarks", mark); - videos.put(v); - } - } - result.put("list", videos); - return result.toString(); - } catch ( - Exception e) { - SpiderDebug.log(e); - } - return ""; - } - - @Override - public boolean manualVideoCheck() { - return false; - } - - private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv"}; - - @Override - public boolean isVideoFormat(String url) { - url = url.toLowerCase(); - if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { - return false; - } - for (String format : videoFormatList) { - if (url.contains(format)) { - return true; - } - } - return false; - } - - protected String ext = null; - protected XPathRule rule = null; - - protected void fetchRule() { - if (rule == null) { - if (ext != null) { - if (ext.startsWith("http")) { - String json = OkHttpUtil.string(ext, null); - rule = XPathRule.fromJson(json); - loadRuleExt(json); - } else { - rule = XPathRule.fromJson(ext); - loadRuleExt(ext); - } - } - } - } - - protected void loadRuleExt(String json) { - - } - - protected String fetch(String webUrl) { - SpiderDebug.log(webUrl); - return OkHttpUtil.string(webUrl, getHeaders(webUrl)); - } -} \ No newline at end of file