From a105f53f2f4b9399c788fbaf563a9a600c138237 Mon Sep 17 00:00:00 2001 From: gzlinz2 <107169210+gzlinz2@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:20:05 +0800 Subject: [PATCH] Add files via upload --- .../java/com/github/catvod/spider/XPath.java | 483 ++++++++++++++++++ 1 file changed, 483 insertions(+) create mode 100644 app/src/main/java/com/github/catvod/spider/XPath.java diff --git a/app/src/main/java/com/github/catvod/spider/XPath.java b/app/src/main/java/com/github/catvod/spider/XPath.java new file mode 100644 index 0000000..797ca25 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XPath.java @@ -0,0 +1,483 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.okhttp.OkHttpUtil; +import com.github.catvod.xpath.XPathRule; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.seimicrawler.xpath.JXDocument; +import org.seimicrawler.xpath.JXNode; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +public class XPath extends Spider { + + @Override + public void init(Context context) { + super.init(context); + } + + public void init(Context context, String extend) { + super.init(context, extend); + this.ext = extend; + } + + @Override + public String homeContent(boolean filter) { + try { + fetchRule(); + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + if (rule.getCateManual().size() > 0) { + Set keys = rule.getCateManual().keySet(); + for (String k : keys) { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type_name", k); + jsonObject.put("type_id", rule.getCateManual().get(k)); + classes.put(jsonObject); + } + } + try { + String webUrl = rule.getHomeUrl(); + JXDocument doc = JXDocument.create(fetch(webUrl)); + if (rule.getCateManual().size() == 0) { + List navNodes = doc.selN(rule.getCateNode()); + for (int i = 0; i < navNodes.size(); i++) { + String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim(); + name = rule.getCateNameR(name); + String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim(); + id = rule.getCateIdR(id); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type_id", id); + jsonObject.put("type_name", name); + classes.put(jsonObject); + } + } + if (!rule.getHomeVodNode().isEmpty()) { + try { + JSONArray videos = new JSONArray(); + List vodNodes = doc.selN(rule.getHomeVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim(); + name = rule.getHomeVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim(); + id = rule.getHomeVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim(); + pic = rule.getHomeVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getHomeVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim(); + mark = rule.getHomeVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } + result.put("list", videos); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } catch (Exception e) { + SpiderDebug.log(e); + } + result.put("class", classes); + if (filter && rule.getFilter() != null) { + result.put("filters", rule.getFilter()); + } + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", rule.getUa().isEmpty() + ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36" + : rule.getUa()); + return headers; + } + + @Override + public String homeVideoContent() { + try { + fetchRule(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected String categoryUrl(String tid, String pg, boolean filter, HashMap extend) { + return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg); + } + + protected String categoryUrl2(String tid, String pg, boolean filter, HashMap extend) { + return rule.getCateUrl2().replace("{cateId}", tid).replace("{catePg}", pg); + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + fetchRule(); + String webUrl = ""; + if (!tid.startsWith("@")) { + webUrl = categoryUrl(tid, pg, filter, extend); + } else { + webUrl = categoryUrl2(tid.replace("@",""), pg, filter, extend); + } + JSONArray videos = new JSONArray(); + JXDocument doc = JXDocument.create(fetch(webUrl)); + List vodNodes = doc.selN(rule.getCateVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim(); + name = rule.getCateVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim(); + id = rule.getCateVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim(); + pic = rule.getCateVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getCateVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim(); + mark = rule.getCateVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } + JSONObject result = new JSONObject(); + result.put("page", pg); + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit", 90); + result.put("total", Integer.MAX_VALUE); + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected void detailContentExt(String content, JSONObject vod) { + + } + + @Override + public String detailContent(List ids) { + try { + fetchRule(); + String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); + String webContent = fetch(webUrl); + JXDocument doc = JXDocument.create(webContent); + JXNode vodNode = doc.selNOne(rule.getDetailNode()); + + String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; + + title = vodNode.selOne(rule.getDetailName()).asString().trim(); + title = rule.getDetailNameR(title); + + cover = vodNode.selOne(rule.getDetailImg()).asString().trim(); + cover = rule.getDetailImgR(cover); + cover = Misc.fixUrl(webUrl, cover); + + if (!rule.getDetailCate().isEmpty()) { + try { + category = vodNode.selOne(rule.getDetailCate()).asString().trim(); + category = rule.getDetailCateR(category); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailYear().isEmpty()) { + try { + year = vodNode.selOne(rule.getDetailYear()).asString().trim(); + year = rule.getDetailYearR(year); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailArea().isEmpty()) { + try { + area = vodNode.selOne(rule.getDetailArea()).asString().trim(); + area = rule.getDetailAreaR(area); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailMark().isEmpty()) { + try { + remark = vodNode.selOne(rule.getDetailMark()).asString().trim(); + remark = rule.getDetailMarkR(remark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailActor().isEmpty()) { + try { + actor = vodNode.selOne(rule.getDetailActor()).asString().trim(); + actor = rule.getDetailActorR(actor); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailDirector().isEmpty()) { + try { + director = vodNode.selOne(rule.getDetailDirector()).asString().trim(); + director = rule.getDetailDirectorR(director); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailDesc().isEmpty()) { + try { + desc = vodNode.selOne(rule.getDetailDesc()).asString().trim(); + desc = rule.getDetailDescR(desc); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + + JSONObject vod = new JSONObject(); + vod.put("vod_id", ids.get(0)); + vod.put("vod_name", title); + vod.put("vod_pic", cover); + vod.put("type_name", category); + vod.put("vod_year", year); + vod.put("vod_area", area); + vod.put("vod_remarks", remark); + vod.put("vod_actor", actor); + vod.put("vod_director", director); + vod.put("vod_content", desc); + + ArrayList playFrom = new ArrayList<>(); + + List fromNodes = doc.selN(rule.getDetailFromNode()); + for (int i = 0; i < fromNodes.size(); i++) { + String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim(); + name = rule.getDetailFromNameR(name); + playFrom.add(name); + } + + ArrayList playList = new ArrayList<>(); + List urlListNodes = doc.selN(rule.getDetailUrlNode()); + for (int i = 0; i < urlListNodes.size(); i++) { + List urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode()); + List vodItems = new ArrayList<>(); + for (int j = 0; j < urlNodes.size(); j++) { + String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim(); + name = rule.getDetailUrlNameR(name); + String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim(); + id = rule.getDetailUrlIdR(id); + vodItems.add(name + "$" + id); + } + // 排除播放列表为空的播放源 + if (vodItems.size() == 0 && playFrom.size() > i) { + playFrom.set(i, ""); + } + playList.add(TextUtils.join("#", vodItems)); + } + // 排除播放列表为空的播放源 + for (int i = playFrom.size() - 1; i >= 0; i--) { + if (playFrom.get(i).isEmpty()) + playFrom.remove(i); + } + for (int i = playList.size() - 1; i >= 0; i--) { + if (playList.get(i).isEmpty()) + playList.remove(i); + } + for (int i = playList.size() - 1; i >= 0; i--) { + if (i >= playFrom.size()) + playList.remove(i); + } + String vod_play_from = TextUtils.join("$$$", playFrom); + String vod_play_url = TextUtils.join("$$$", playList); + vod.put("vod_play_from", vod_play_from); + vod.put("vod_play_url", vod_play_url); + + detailContentExt(webContent, vod); + + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + list.put(vod); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + fetchRule(); + String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); + SpiderDebug.log(webUrl); + JSONObject result = new JSONObject(); + result.put("parse", 1); + result.put("playUrl", ""); + if (!rule.getPlayUa().isEmpty()) { + result.put("header", rule.getPlayUa()); + } + result.put("url", webUrl); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String searchContent(String key, boolean quick) { + try { + fetchRule(); + if (rule.getSearchUrl().isEmpty()) { + return ""; + } + String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); + String webContent = fetch(webUrl); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + // add maccms suggest search api support + if (rule.getSearchVodNode().startsWith("json:")) { + String[] node = rule.getSearchVodNode().substring(5).split(">"); + JSONObject data = new JSONObject(webContent); + for (int i = 0; i < node.length; i++) { + if (i == node.length - 1) { + JSONArray vodArray = data.getJSONArray(node[i]); + for (int j = 0; j < vodArray.length(); j++) { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(rule.getSearchVodName()).trim(); + name = rule.getSearchVodNameR(name); + String id = vod.optString(rule.getSearchVodId()).trim(); + id = rule.getSearchVodIdR(id); + String pic = vod.optString(rule.getSearchVodImg()).trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = vod.optString(rule.getSearchVodMark()).trim(); + mark = rule.getSearchVodMarkR(mark); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } + } else { + data = data.getJSONObject(node[i]); + } + } + } else { + JXDocument doc = JXDocument.create(webContent); + List vodNodes = doc.selN(rule.getSearchVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); + name = rule.getSearchVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); + id = rule.getSearchVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getCateVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); + mark = rule.getSearchVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } + } + result.put("list", videos); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public boolean manualVideoCheck() { + return false; + } + + private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv"}; + + @Override + public boolean isVideoFormat(String url) { + url = url.toLowerCase(); + if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; + } + for (String format : videoFormatList) { + if (url.contains(format)) { + return true; + } + } + return false; + } + + protected String ext = null; + protected XPathRule rule = null; + + protected void fetchRule() { + if (rule == null) { + if (ext != null) { + if (ext.startsWith("http")) { + String json = OkHttpUtil.string(ext, null); + rule = XPathRule.fromJson(json); + loadRuleExt(json); + } else { + rule = XPathRule.fromJson(ext); + loadRuleExt(ext); + } + } + } + } + + protected void loadRuleExt(String json) { + + } + + protected String fetch(String webUrl) { + SpiderDebug.log(webUrl); + return OkHttpUtil.string(webUrl, getHeaders(webUrl)); + } +} \ No newline at end of file