From 57fc61d6b712619890ce31e79c4c2e8a3b53969c Mon Sep 17 00:00:00 2001 From: gzlinz2 <107169210+gzlinz2@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:31:01 +0800 Subject: [PATCH] Add files via upload --- .../com/github/catvod/spider/XPathRule.java | 696 ++++++++++++++++++ 1 file changed, 696 insertions(+) create mode 100644 app/src/main/java/com/github/catvod/spider/XPathRule.java diff --git a/app/src/main/java/com/github/catvod/spider/XPathRule.java b/app/src/main/java/com/github/catvod/spider/XPathRule.java new file mode 100644 index 0000000..b2d218d --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XPathRule.java @@ -0,0 +1,696 @@ +package com.github.catvod.xpath; + +import com.github.catvod.crawler.SpiderDebug; + +import org.json.JSONObject; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class XPathRule { + /** + * user-agent + */ + private String ua; + /** + * 获取分类和首页推荐的Url + */ + private String homeUrl; + /** + * 分类节点 xpath + */ + private String cateNode; + /** + * 分类节点名 xpath + */ + private String cateName; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateNameR; + /** + * 分类节点id xpath + */ + private String cateId; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateIdR; + /** + * 手动指定 分类 如果有则不从homeUrl中获取分类 + */ + private LinkedHashMap cateManual = new LinkedHashMap<>(); + + /** + * 筛选 + */ + private JSONObject filter; + + /** + * 更新推荐视频节点 xpath + */ + private String homeVodNode; + /** + * 更新推荐视频名称 xpath + */ + private String homeVodName; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern homeVodNameR; + /** + * 更新推荐视频id xpath + */ + private String homeVodId; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern homeVodIdR; + /** + * 更新推荐视频图片 xpath + */ + private String homeVodImg; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern homeVodImgR; + /** + * 更新推荐视频简介 xpath + */ + private String homeVodMark; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern homeVodMarkR; + /** + * 分类页地址 + */ + private String cateUrl; + /** + * 分类页地址2 + */ + private String cateUrl2; + /** + * 分类叶视频节点 xpath + */ + private String cateVodNode; + /** + * 分类叶视频名称 xpath + */ + private String cateVodName; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateVodNameR; + /** + * 分类叶视频视频id xpath + */ + private String cateVodId; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateVodIdR; + /** + * 分类叶视频视频图片 xpath + */ + private String cateVodImg; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateVodImgR; + /** + * 分类叶视频视频简介 xpath + */ + private String cateVodMark; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern cateVodMarkR; + + /** + * 详情页面 + */ + private String dtUrl; + /** + * 详情节点 xpath + */ + private String dtNode; + /** + * 详情 视频名 xpath + */ + private String dtName; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtNameR; + /** + * 详情视频图片 xpath + */ + private String dtImg; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtImgR; + /** + * 详情视频分类 xpath + */ + private String dtCate; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtCateR; + /** + * 详情视频年份 xpath + */ + private String dtYear; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtYearR; + /** + * 详情视频地区 xpath + */ + private String dtArea; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtAreaR; + /** + * 详情视频简介 xpath + */ + private String dtMark; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtMarkR; + /** + * 详情演员 xpath + */ + private String dtActor; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtActorR; + /** + * 详情导演 xpath + */ + private String dtDirector; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtDirectorR; + /** + * 详情 说明 长 xpath + */ + private String dtDesc; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern dtDescR; + + /** + * 详情播放来源节点 + */ + private String dtFromNode; + /** + * 详情播放来源名称 xpath + */ + private String dtFromName; + /** + * 详情 + */ + private Pattern dtFromNameR; + /** + * 详情播放地址列表节点 xpath + */ + private String dtUrlNode; + /** + * 详情播放地址节点 xpath + */ + private String dtUrlSubNode; + /** + * 详情播放地址id xpath + */ + private String dtUrlId; + /** + * 详情 + */ + private Pattern dtUrlIdR; + /** + * 详情播放地址名称 xpath + */ + private String dtUrlName; + /** + * 详情 + */ + private Pattern dtUrlNameR; + /** + * 播放页面url + */ + private String playUrl; + /** + * 播放解析调用ua + */ + private String playUa; + + /** + * 搜索页地址 + */ + private String searchUrl; + + /** + * 搜索页视频节点 xpath + */ + private String scVodNode; + /** + * 搜索页视频名称 xpath + */ + private String scVodName; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern scVodNameR; + /** + * 搜索页视频id xpath + */ + private String scVodId; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern scVodIdR; + /** + * 搜索页视频图片 xpath + */ + private String scVodImg; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern scVodImgR; + /** + * 搜索页视频简介 xpath + */ + private String scVodMark; + /** + * 正则对取到的数据进行二次处理 + */ + private Pattern scVodMarkR; + + private static Pattern getPattern(JSONObject json, String key) { + String v = json.optString(key).trim(); + if (v.isEmpty()) + return null; + else { + try { + return Pattern.compile(v); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + return null; + } + + private static String doReplaceRegex(Pattern pattern, String src) { + if (pattern == null) + return src; + try { + Matcher matcher = pattern.matcher(src); + if (matcher.find()) { + return matcher.group(1).trim(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return src; + } + + public static XPathRule fromJson(String json) { + try { + JSONObject jsonObj = new JSONObject(json); + XPathRule rule = new XPathRule(); + rule.ua = jsonObj.optString("ua"); + rule.homeUrl = jsonObj.optString("homeUrl").trim(); + rule.cateNode = jsonObj.optString("cateNode").trim(); + rule.cateName = jsonObj.optString("cateName").trim(); + rule.cateNameR = getPattern(jsonObj, "cateNameR"); + rule.cateId = jsonObj.optString("cateId").trim(); + rule.cateIdR = getPattern(jsonObj, "cateIdR"); + JSONObject navs = jsonObj.optJSONObject("cateManual"); + if (navs != null) { + Iterator keys = navs.keys(); + while (keys.hasNext()) { + String name = keys.next(); + rule.cateManual.put(name.trim(), navs.getString(name).trim()); + } + } + rule.filter = jsonObj.optJSONObject("filter"); + rule.homeVodNode = jsonObj.optString("homeVodNode").trim(); + rule.homeVodName = jsonObj.optString("homeVodName").trim(); + rule.homeVodNameR = getPattern(jsonObj, "homeVodNameR"); + rule.homeVodId = jsonObj.optString("homeVodId").trim(); + rule.homeVodIdR = getPattern(jsonObj, "homeVodIdR"); + rule.homeVodImg = jsonObj.optString("homeVodImg").trim(); + rule.homeVodImgR = getPattern(jsonObj, "homeVodImgR"); + rule.homeVodMark = jsonObj.optString("homeVodMark").trim(); + rule.homeVodMarkR = getPattern(jsonObj, "homeVodMarkR"); + rule.cateUrl = jsonObj.optString("cateUrl").trim(); + rule.cateUrl2 = jsonObj.optString("cateUrl2").trim(); + rule.cateVodNode = jsonObj.optString("cateVodNode").trim(); + rule.cateVodName = jsonObj.optString("cateVodName").trim(); + rule.cateVodNameR = getPattern(jsonObj, "cateVodNameR"); + rule.cateVodId = jsonObj.optString("cateVodId").trim(); + rule.cateVodIdR = getPattern(jsonObj, "cateVodIdR"); + rule.cateVodImg = jsonObj.optString("cateVodImg").trim(); + rule.cateVodImgR = getPattern(jsonObj, "cateVodImgR"); + rule.cateVodMark = jsonObj.optString("cateVodMark").trim(); + rule.cateVodMarkR = getPattern(jsonObj, "cateVodMarkR"); + rule.dtUrl = jsonObj.optString("dtUrl"); + rule.dtNode = jsonObj.optString("dtNode"); + rule.dtName = jsonObj.optString("dtName"); + rule.dtNameR = getPattern(jsonObj, "dtNameR"); + rule.dtImg = jsonObj.optString("dtImg"); + rule.dtImgR = getPattern(jsonObj, "dtImgR"); + rule.dtCate = jsonObj.optString("dtCate"); + rule.dtCateR = getPattern(jsonObj, "dtCateR"); + rule.dtYear = jsonObj.optString("dtYear"); + rule.dtYearR = getPattern(jsonObj, "dtYearR"); + rule.dtArea = jsonObj.optString("dtArea"); + rule.dtAreaR = getPattern(jsonObj, "dtAreaR"); + rule.dtMark = jsonObj.optString("dtMark"); + rule.dtMarkR = getPattern(jsonObj, "dtMarkR"); + rule.dtActor = jsonObj.optString("dtActor"); + rule.dtActorR = getPattern(jsonObj, "dtActorR"); + rule.dtDirector = jsonObj.optString("dtDirector"); + rule.dtDirectorR = getPattern(jsonObj, "dtDirectorR"); + rule.dtDesc = jsonObj.optString("dtDesc"); + rule.dtDescR = getPattern(jsonObj, "dtDescR"); + rule.dtFromNode = jsonObj.optString("dtFromNode"); + rule.dtFromName = jsonObj.optString("dtFromName"); + rule.dtFromNameR = getPattern(jsonObj, "dtFromNameR"); + rule.dtUrlNode = jsonObj.optString("dtUrlNode"); + rule.dtUrlSubNode = jsonObj.optString("dtUrlSubNode"); + rule.dtUrlId = jsonObj.optString("dtUrlId"); + rule.dtUrlIdR = getPattern(jsonObj, "dtUrlIdR"); + rule.dtUrlName = jsonObj.optString("dtUrlName"); + rule.dtUrlNameR = getPattern(jsonObj, "dtUrlNameR"); + rule.playUrl = jsonObj.optString("playUrl"); + rule.playUa = jsonObj.optString("playUa"); + rule.searchUrl = jsonObj.optString("searchUrl"); + rule.scVodNode = jsonObj.optString("scVodNode").trim(); + rule.scVodName = jsonObj.optString("scVodName").trim(); + rule.scVodNameR = getPattern(jsonObj, "scVodNameR"); + rule.scVodId = jsonObj.optString("scVodId").trim(); + rule.scVodIdR = getPattern(jsonObj, "scVodIdR"); + rule.scVodImg = jsonObj.optString("scVodImg").trim(); + rule.scVodImgR = getPattern(jsonObj, "scVodImgR"); + rule.scVodMark = jsonObj.optString("scVodMark").trim(); + rule.scVodMarkR = getPattern(jsonObj, "scVodMarkR"); + return rule; + } catch (Exception e) { + SpiderDebug.log(e); + } + return null; + } + + public String getUa() { + return ua; + } + + public String getHomeUrl() { + return homeUrl; + } + + public String getCateNode() { + return cateNode; + } + + public String getCateName() { + return cateName; + } + + public String getCateNameR(String src) { + return doReplaceRegex(cateNameR, src); + } + + public String getCateId() { + return cateId; + } + + public String getCateIdR(String src) { + return doReplaceRegex(cateIdR, src); + } + + public LinkedHashMap getCateManual() { + return cateManual; + } + + public JSONObject getFilter() { + return filter; + } + + public String getHomeVodNode() { + return homeVodNode; + } + + public String getHomeVodName() { + return homeVodName; + } + + public String getHomeVodNameR(String src) { + return doReplaceRegex(homeVodNameR, src); + } + + public String getHomeVodId() { + return homeVodId; + } + + public String getHomeVodIdR(String src) { + return doReplaceRegex(homeVodIdR, src); + } + + public String getHomeVodImg() { + return homeVodImg; + } + + public String getHomeVodImgR(String src) { + return doReplaceRegex(homeVodImgR, src); + } + + public String getHomeVodMark() { + return homeVodMark; + } + + public String getHomeVodMarkR(String src) { + return doReplaceRegex(homeVodMarkR, src); + } + + public String getCateUrl() { + return cateUrl; + } + + public String getCateUrl2() { + return cateUrl2; + } + + public String getCateVodNode() { + return cateVodNode; + } + + public String getCateVodName() { + return cateVodName; + } + + public String getCateVodNameR(String src) { + return doReplaceRegex(cateVodNameR, src); + } + + public String getCateVodId() { + return cateVodId; + } + + public String getCateVodIdR(String src) { + return doReplaceRegex(cateVodIdR, src); + } + + public String getCateVodImg() { + return cateVodImg; + } + + public String getCateVodImgR(String src) { + return doReplaceRegex(cateVodImgR, src); + } + + public String getCateVodMark() { + return cateVodMark; + } + + public String getCateVodMarkR(String src) { + return doReplaceRegex(cateVodNameR, src); + } + + public String getDetailUrl() { + return dtUrl; + } + + public String getDetailNode() { + return dtNode; + } + + public String getDetailName() { + return dtName; + } + + public String getDetailNameR(String src) { + return doReplaceRegex(dtNameR, src); + } + + public String getDetailImg() { + return dtImg; + } + + public String getDetailImgR(String src) { + return doReplaceRegex(dtImgR, src); + } + + public String getDetailCate() { + return dtCate; + } + + public String getDetailCateR(String src) { + return doReplaceRegex(dtCateR, src); + } + + public String getDetailYear() { + return dtYear; + } + + public String getDetailYearR(String src) { + return doReplaceRegex(dtYearR, src); + } + + public String getDetailArea() { + return dtArea; + } + + public String getDetailAreaR(String src) { + return doReplaceRegex(dtAreaR, src); + } + + public String getDetailMark() { + return dtMark; + } + + public String getDetailMarkR(String src) { + return doReplaceRegex(dtMarkR, src); + } + + public String getDetailActor() { + return dtActor; + } + + public String getDetailActorR(String src) { + return doReplaceRegex(dtActorR, src); + } + + public String getDetailDirector() { + return dtDirector; + } + + public String getDetailDirectorR(String src) { + return doReplaceRegex(dtDirectorR, src); + } + + public String getDetailDesc() { + return dtDesc; + } + + public String getDetailDescR(String src) { + return doReplaceRegex(dtDescR, src); + } + + public String getDetailFromNode() { + return dtFromNode; + } + + public String getDetailFromName() { + return dtFromName; + } + + public String getDetailFromNameR(String src) { + return doReplaceRegex(dtFromNameR, src); + } + + public String getDetailUrlNode() { + return dtUrlNode; + } + + public String getDetailUrlSubNode() { + return dtUrlSubNode; + } + + public String getDetailUrlId() { + return dtUrlId; + } + + public String getDetailUrlIdR(String src) { + return doReplaceRegex(dtUrlIdR, src); + } + + public String getDetailUrlName() { + return dtUrlName; + } + + public String getDetailUrlNameR(String src) { + return doReplaceRegex(dtUrlNameR, src); + } + + public String getPlayUrl() { + return playUrl; + } + + public String getPlayUa() { + return playUa; + } + + public String getSearchUrl() { + return searchUrl; + } + + public String getSearchVodNode() { + return scVodNode; + } + + public String getSearchVodName() { + return scVodName; + } + + public String getSearchVodNameR(String src) { + return doReplaceRegex(scVodNameR, src); + } + + public String getSearchVodId() { + return scVodId; + } + + public String getSearchVodIdR(String src) { + return doReplaceRegex(scVodIdR, src); + } + + public String getSearchVodImg() { + return scVodImg; + } + + public String getSearchVodImgR(String src) { + return doReplaceRegex(scVodImgR, src); + } + + public String getSearchVodMark() { + return scVodMark; + } + + public String getSearchVodMarkR(String src) { + return doReplaceRegex(scVodMarkR, src); + } +}