package com.github.catvod.spider;

import android.content.Context;
import android.text.TextUtils;
import android.util.Base64;
import android.util.Pair;

import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.Misc;
import com.github.catvod.utils.okhttp.OKCallBack;
import com.github.catvod.utils.okhttp.OkHttpUtil;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import okhttp3.Call;

public class XBiu extends Spider {

    protected String ext = null;
    public JSONObject rule = null;
    // 默认的视频类型
    private ArrayList<String> videoFormatList = new ArrayList<>(Arrays.asList(".m3u8", ".mp4", ".mpeg", ".flv", ".mkv"));


    protected final int base64Flag = Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP;
    // 一定是正确的分类名称，用来帮助定位分类列表，猜cateManual
    protected final ArrayList<String> cateManuals = new ArrayList<>(Arrays.asList("电影", "剧集", "电视剧", "连续剧", "综艺", "动漫"));
    // 无效的分类名，彡来过滤cateManual
    protected final ArrayList<String> invalidCateNames = new ArrayList<>(Arrays.asList("更多","下载", "首页", "资讯", "留言", "导航", "专题", "短视频", "热榜", "排行", "追剧","更新","APP", "直播", "label", "Netflix"));
    // 详情页 影片信息相关字段，猜详情页信息时用
    protected final ArrayList<String> detailItemNames = new ArrayList<>(Arrays.asList("导演", "主演", "演员", "地区", "类型", "年份", "年代"));
    protected final ArrayList<String> detailItemKeys = new ArrayList<>(Arrays.asList("vod_director", "vod_actor", "vod_actor", "vod_area", "type_name", "vod_year", "vod_year"));
    protected String splitFlag = ""; // 分段标志,猜cateManual时用

    // html标签查找时用到的辅助类
    protected class HtmlMatchInfo {
        public String group0; // 正则表达式匹配到的字符串
        /**
         * 一般用来放href中的内容
         */
        public String group1; //
        public String group2; //
        public String diff;   // 两个匹配结果比较group1得到的不同部分
        public int startPos;    // 正则匹配到的起始位置
        public int endPos;      // 正则匹配到的结束位置
        public ArrayList<Integer> uploads;  // 祖先结节的索引
        public int matchedUpNodePos = -1;   // 与其他HtmlMatchInfo最匹配的祖先节点位置
        public int diffStartIndex;   // 不同那部分数据的开始位置
        public int diffEndIndex; // 不同那部分数据的结束位置

        public void init(Matcher m) {
            this.group0 = m.group(0);
            if (m.groupCount() > 0)
                this.group1 = m.group(1);
            if (m.groupCount() > 1)
                this.group2 = m.group(2);
            this.startPos = m.start(0);
            this.endPos = m.end(0);
        }

        // 通过比较两个group1的不同部分，不同部分的内容以splitFlag中的字符为开始或结束位置
        public boolean findDiffStr(HtmlMatchInfo rhs, String splitFlag) {
            int len = Math.min(group1.length(), rhs.group1.length());
            // 找不同字符的开始位置
            for (int i =0; i < len; ++i){
                char a = group1.charAt(i);
                char b = rhs.group1.charAt(i);
                if(a== b &&  splitFlag.indexOf (a) != -1) {
                    diffStartIndex = i+1;
                    rhs.diffStartIndex = i+1;
                }
                if(a != b) break;
            }

            // 找不同字符的结束位置
            diffEndIndex = group1.length();
            rhs.diffEndIndex = rhs.group1.length();
            for (int i =1; i < len; ++i){
                char a = group1.charAt(group1.length()-i);
                char b = rhs.group1.charAt(rhs.group1.length()-i);
                if(a== b && splitFlag.indexOf (a) != -1) {
                    diffEndIndex = group1.length()-i;
                    rhs.diffEndIndex = rhs.group1.length()-i;
                }
                if(a != b) break;
            }
            if(this.diff == null || this.diff.isEmpty() && diffStartIndex < diffEndIndex) {
                diff = group1.substring(diffStartIndex, diffEndIndex);
            }else{
                if( diffEndIndex < diffStartIndex || !diff.equals(group1.substring(diffStartIndex, diffEndIndex))){
                    return false;
                }
            }
            if(rhs.diffStartIndex < rhs.diffEndIndex) {
                rhs.diff = rhs.group1.substring(rhs.diffStartIndex, rhs.diffEndIndex);
            }
            return true;
        }

        // 判断 rhs 与当前对象是有相同的祖先节点
        boolean hasSameUpNode(HtmlMatchInfo rhs) {
            if (rhs.uploads.size() != this.uploads.size()) return false;
            for (int i = 0; i < uploads.size(); ++i) {
                if (uploads.get(i).intValue() != rhs.uploads.get(i).intValue()) continue;
                if (matchedUpNodePos == -1 || uploads.get(i).intValue() == matchedUpNodePos) {
                    matchedUpNodePos = uploads.get(i).intValue();
                    rhs.matchedUpNodePos = uploads.get(i).intValue();
                    return true;
                }
                return false;
            }
            return false;
        }
    }

    public void init(Context context, String extend) {
        super.init(context, extend);
        this.ext = extend;
    }

    // 初始化抓虫规则
    protected void fetchRule() {
        if (rule == null) {
            if (ext != null) {
                try {

                    if (ext.startsWith("http")) {
                        if(ext.indexOf("{cateId}") != -1 || ext.indexOf("{catePg}") !=-1){
                            rule = new JSONObject();
                            rule.put("homeUrl", ext);
                        }else{
                            String json = OkHttpUtil.string(ext, null);
                            rule = new JSONObject(json);
                        }

                    } else {
                        rule = new JSONObject(ext);
                    }

                    if (!rule.has("list")) {
                        rule.put("list", new JSONObject());
                    }
                    JSONObject list = rule.getJSONObject("list");
                    // 初始化homeUrl,list.url
                    String homeUrl= rule.getString("homeUrl");
                    if(homeUrl.indexOf("{cateId}") != -1){
                        URL r = new URL(homeUrl);
                        String path =  r.getPath();
                        // 更新解析出来的homeUrl
                        rule.put("homeUrl", homeUrl.substring(0, homeUrl.indexOf(path)));
                        if(!list.has("url")){
                            list.put("url", homeUrl);
                        }
                    }
                    // 初始化截断标志
                    String listUrl = list.getString("url");
                    if(listUrl.indexOf("/") !=-1) splitFlag+='/';
                    if(listUrl.indexOf(".") !=-1) splitFlag+='.';
                    if(listUrl.indexOf("-") !=-1) splitFlag+='-';

                    if (!rule.has("detail")) {
                        rule.put("detail", new JSONObject());
                    }

                    if (!rule.has("playlist")) {
                        rule.put("playlist", new JSONObject());
                    }

                    // 如果没有search，则生成一个默认的search规则，大部分网站的search规则都一样
                    // 省掉一个search json
                    if (!rule.has("search")) {
                       String url = addHttpPrefix("index.php/ajax/suggest?mid=1&wd=阿凡达");
                        try {
                            // 尝试访问这个json接口，如果返回了正确的json格式，就认为支持json搜索
                            JSONObject result = new JSONObject( OkHttpUtil.string(url, getHeaders(url)));
                            JSONObject search = new JSONObject();
                            search.put("vod_id", "id");
                            search.put("vod_name", "name");
                            search.put("vod_pic", "pic");
                            search.put("url", addHttpPrefix("index.php/ajax/suggest?mid=1&wd={wd}"));
                            rule.put("search", search);
                        }
                        catch (Exception e){

                        }
                    }

                    // 部分网站的播放页上直接就有 播放地址，基本上就是一样的格式，可以尝试在playerContent中直接拿直链
                    if (!rule.has("play")) {
                        JSONObject play = new JSONObject();
                        JSONArray region = new JSONArray();
                        region.put("var player_aaaa=");
                        region.put(0);

                        JSONArray vod_url = new JSONArray();
                        vod_url.put("\"url\":\"");
                        vod_url.put("\"");
                        play.put("region", region);
                        play.put("vod_url", vod_url);
                        rule.put("play", play);
                    }

                    // play字段中可以填写播放连接的关键字用来帮助识别嗅探结果，
                    // 一般奇葩的网站会用到
                    if (rule.has("play")) { // 自定义嗅探关键字
                        JSONObject play = rule.getJSONObject("play");
                        JSONArray keywords = play.optJSONArray("keywords");
                        if (keywords != null) {
                            videoFormatList.clear();
                            for (int i = 0; i < keywords.length(); ++i) {
                                videoFormatList.add(keywords.getString(i));
                            }
                        }
                    }

                    // 猜cateManaul
                    JSONObject cateManual = rule.optJSONObject("cateManual");
                    String body = "";
                    if (cateManual == null) {
                        // 重建 cateManaul规则
                        body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header"));
                        if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); }
                        cateManual = this.guess_rule_cateManual(body);
                        if(cateManual != null){
                            rule.put("cateManual", cateManual);
                        }
                    }

                    // 猜list.vod_id
                    if (!list.has("vod_id")) {
                        if(body.isEmpty()){
                            body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header"));
                            if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); }
                        }
                        JSONArray listvodid = this.guess_rule_vod_id(body);
                        list.put("vod_id", listvodid);
                    }

                    // 如果没有json搜索接口，那么尝试在主页上找search 的接口 url
                    if (!rule.has("search")) {
                        if(body.isEmpty()){
                            body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header"));
                            if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); }
                        }
                        String url =  this.guess_rule_search_url(body);
                        if(!url.isEmpty()){
                            JSONObject search = new JSONObject();
                            search.put("url", url);
                            rule.put("search", search);
                        }
                    }

                    SpiderDebug.log(String.format("默认rule: %s", rule.toString()));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public String addHttpPrefix(String url) {
        try {
            if (url.isEmpty()) return "";
            if (url.startsWith("http")) return url;
            String result = rule.getString("homeUrl");
            if (result.endsWith("/")) {
                result = result.substring(0, result.length() - 1);
            }
            if (url.startsWith("/")) {
                result += url;
            } else {
                result += "/" + url;
            }
            return result;
        } catch (JSONException e) {
            e.printStackTrace();
        }
        return url;
    }

    protected HashMap<String, String> getHeaders(String url) {
        HashMap<String, String> headers = new HashMap<>();
        try {
            if (rule.has("header")) {
                JSONObject header = rule.getJSONObject("header");
                Iterator<String> iter = header.keys();
                while (iter.hasNext()) {
                    String key = iter.next();
                    headers.put(key, header.getString(key));
                }
            }
        } catch (JSONException e) {
            e.printStackTrace();
        }
        if (!headers.containsKey("User-Agent")) {
            headers.put("User-Agent", Misc.UaWinChrome);
        }
        return headers;
    }

    public static class HtmlNodeHlper{
        // 非正常配对的html标签，进行html层级查找时要用到
        protected static ArrayList<String> notPairedTag = new ArrayList<>(Arrays.asList("img", "br", "meta", "!--")); // <!---->为注释
        // 判断当前html标签是否为正常的标签
        public static boolean isPairedHtmlTag(String str, int startPos) {
            String tmp = str.substring(startPos, Math.min(str.length(), startPos + 10));
            for (String p : notPairedTag) {
                if (tmp.indexOf(p) != -1) {  // 找到了
                    // 找 > 如果匹配了 />  则认为是配对的
                    for (int i = startPos + 1; i < str.length(); ++i) {
                        String sm = str.substring(i);
                        if (str.charAt(i) == '>') {
                            if (str.charAt(i - 1) == '/') {
                                return true;
                            } else {
                                return false;
                            }
                        }
                    }
                    return false;
                }
            }
            return true;
        }

        // 查找当前标签的html代码 pos必须是标签的开始位置 <
        public static String nodeString(String str, int pos) {
            if (pos < 0 || pos >= str.length() || str.charAt(pos) != '<') return str;
            int isRightNode = 0;
            for (int i = pos; i < str.length() - 1; ++i) {
//            String sm = str.substring(i, i + 400);
                switch (str.charAt(i)) {
                    // 遇到 / 那么这个位置有可能是xml的结束标识,这种情况下再遇到<则不是当前节点的上级节点
                    case '/': {
                        if (str.charAt(i + 1) == '>') { // "/>" 认为是标签的结束位置
                            isRightNode--;
                        } else if (str.charAt(i - 1) == '<') { // "</" 认为是标签的结束位置
                            isRightNode--;
                        }
                        break;
                    }
                    case '>': {
                        if (isRightNode == 0) {
                            return str.substring(pos, i + 1);
                        }
                        break;
                    }
                    case '<': {
                        if (str.charAt(i + 1) != '/' && isPairedHtmlTag(str, i)) { // 不是 "</" 则认为是html标签的开始位置
                            ++isRightNode;
                        }
                        break;
                    }
                    default:
                        break;
                }

            }
            return str.substring(pos);
        }

        // lookback 回溯层级， 一般能找到共同的祖先结节就可以了
        public static ArrayList<Integer> findUpNodes(String str, int pos, int lookback) {
            ArrayList<Integer> nodes = new ArrayList<>();
            ArrayList<String> urls = new ArrayList<>();
            if (pos == -1) return nodes;
            int isUpNode = 0;
            for (int i = pos; i >= 0; --i) {
                switch (str.charAt(i)) {
                    // 遇到 / 那么这个位置有可能是xml的结束标识,这种情况下再遇到<则不是当前节点的上级节点
                    case '/': {
                        if (str.charAt(i + 1) == '>') {
                            isUpNode++;
//                        SpiderDebug.log(String.format("not xml %s", str.substring(i, i + 20)));
                        } else if (str.charAt(i - 1) == '<') {
                            isUpNode++;
                            --i;
//                        SpiderDebug.log(String.format("not xml %s", str.substring(i, i + 20)));
                        }
                        break;
                    }
                    case '<': {
                        if (isUpNode == 0) {
//                        SpiderDebug.log(String.format("find up node %d %s", i, str.substring(i, i + 30)));
                            urls.add(String.format("%5d", i));
                            nodes.add(i);
                        } else if (isPairedHtmlTag(str, i)) {
                            isUpNode--;
                            if (isUpNode < 0) isUpNode = 0;
//                        SpiderDebug.log(String.format("%s", str.substring(i, i + 30)));
                        }

                        break;
                    }
                    default:
                        break;
                }
                if (nodes.size() >= lookback) {
                    break;
                }
            }
            return nodes;
        }
        // 获取当前节点的所有子节点
        public static ArrayList<String> getChildNodes(String str) {
            ArrayList<String> arr = new ArrayList<>();
            int pos = 0;
            if (pos < 0 || pos >= str.length() || str.charAt(pos) != '<') return arr;
            ++pos;
            while (pos > -1 && pos < str.length()) {
                pos = str.indexOf('<', pos);
                String p = nodeString(str, pos);
                if (p.isEmpty()) {
                    break;
                }
                arr.add(p);
                pos += p.length();
            }
            return arr;
        }

        // 移除字符串的html标签
        public static String trimHtmlString(String str, String r) {
            String ret = str.replace("\r\n", "")
                    .replace("\n", "")
                    .replaceAll("<.+?>", r)
                    // .replace(" ", "")
                    .replaceAll("\\s+", " ")
                    .replace("&nbsp;", "")
                    .replace("&emsp;", "")
                    .trim();
            return ret;
        }

        public static String trimHtmlString(String str) {
            return trimHtmlString(str, "");
        }

    }

    public static class Utils{
        // 查找列表块的起始位置，取最靠近共同祖先节点的位置
        public static int findBlockPos(ArrayList<Integer> a, ArrayList<Integer> b) {
            int len = a.size() > b.size() ? b.size() : a.size();
            if(len ==1 ) return b.get(0);
            for (int i = 0; i < len; ++i) {
                if (a.get(i).intValue() == b.get(i).intValue()) {
                    return b.get(i - 1);
                }
            }
            return b.get(len - 1);
        }

        // 查找两个字符串之间的子串
        // keys 字段说明
        // 0 prefix 1 suffix 2 找到子串后左边index的偏移量 3 找到子串后右边index的偏移量
        public static String findSubString(String str, int startPos, JSONArray keys, String defaultVal) {
            try {
                if (keys == null) return defaultVal;
                String prefix = keys.getString(0);
                String suffix = keys.getString(1);
                int offsetl = 0;    // 左边的偏移量
                int offsetr = 0;    // 右边的偏移量
                if (keys.length() > 2) {
                    offsetl = keys.getInt(2);
                }
                if (keys.length() > 3) {
                    offsetr = keys.getInt(3);
                }
                int a = str.indexOf(prefix, startPos) + prefix.length();
                if (a < prefix.length()) return defaultVal;
                int b = str.indexOf(suffix, a);
                if (b < a) return defaultVal;
                return HtmlNodeHlper.trimHtmlString(str.substring(a + offsetl, b + offsetr));
            } catch (JSONException e) {
                e.printStackTrace();
            }
            return defaultVal;
        }

        public static String findSubString(String str, int startPos, JSONArray keys) {
            return findSubString(str, startPos, keys, "");
        }

        // 获取回看层数
        public static int getLookbackCount(JSONArray keys) {
            try {
                if (keys != null && keys.length() > 4) return keys.getInt(4);
            } catch (Exception e) {
                //e.printStackTrace();
            }
            return 0;
        }

        // 遍历JSONObect中的JSONArray查找回看的层数可用的规则
        public static JSONArray getLookbackArray(JSONObject obj) {
            try {
                Iterator iter = obj.keys();
                while (iter.hasNext()) {
                    String key = (String) iter.next();
                    Object val = obj.get(key);
                    if (val.getClass().getSimpleName().equals("JSONArray")) {
                        int c = getLookbackCount((JSONArray) val);
                        if (c > 0) return (JSONArray) val;
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }

        // 统计子串个数
        public static int getSubStringCount(String str, String sub){
            int pos =0;
            int count =0;
            while (pos < str.length()){
                pos = str.indexOf(sub, pos);
                if(pos == -1) break;
                pos += sub.length();
                ++count;
            }
            return  count;
        }

        // 获取指定区间的字符串
        public static String getRegion(String str, JSONObject obj) {
            try {
                if (obj == null) return str;
                JSONArray region = obj.optJSONArray("region");
                if (region == null) return str;
                String prefix = region.getString(0);
                int a = str.indexOf(prefix);
                if (a == -1) return str;
                int b = str.length();
                if (region.length() > 1) {
                    b = str.indexOf(region.getString(1), a + prefix.length());
                    if (b == -1) b = str.length();
                }
                return str.substring(a, b);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            return str;
        }

    }

    // 猜测分类列表的html区间代码
    protected String guessCateManualHtmlString(String body) {
        String regx = String.format("<a.+?href=\"(.+?)\".*?<", TextUtils.join("|", cateManuals));
        Pattern pattern = Pattern.compile(regx);
        Matcher m = pattern.matcher(body);
        ArrayList<HtmlMatchInfo> list = new ArrayList<>();
        int mcount = 0;
        while (m.find()   ) {
            ++mcount;
            if(mcount >30 && !list.isEmpty()){
                break;
            }
            HtmlMatchInfo cate = new HtmlMatchInfo();
            cate.init(m);
            cate.group2 = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(body, cate.startPos));
            if (cate.group2.isEmpty()) continue;
            boolean bOk = false;
            for (String v : cateManuals){
                if(cate.group2.indexOf(v) !=-1) {
                    bOk = true;
                    break;
                }
            }
            if(!bOk) continue;
            cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 3);
            if (!list.isEmpty()) {
                boolean b = list.get(0).hasSameUpNode(cate);
                if (!b) { // 当前找到的info和list中的匹配
                    if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表
                        return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos);
                    }
                    list.clear();
                }
            }
            list.add(cate);
        }
        if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表
            return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos);
        } else {
            return "";
        }
    }

    // 从html代码中猜测分类名和分类ID cateManual规则
    protected JSONObject guess_rule_cateManual(String body) {
        try {
            String str = this.guessCateManualHtmlString(body);
            if (str.isEmpty()) return new JSONObject();

            String regx = String.format("<a.+?href=\"(.+?)\".*?[\"|>](\\s*?\\S+?\\s*?)(\"|<)", TextUtils.join("|", cateManuals));
            Pattern pattern = Pattern.compile(regx, Pattern.CASE_INSENSITIVE);
            Matcher m = pattern.matcher(str);
            ArrayList<HtmlMatchInfo> list = new ArrayList<>();
            while (m.find()) {
                // HtmlMatchInfo 字段映射
                // HtmlMatchInfo.group1 -> href
                // HtmlMatchInfo.group2 -> name
                // HtmlMatchInfo.diff  -> id  分类ID
                HtmlMatchInfo cate = new HtmlMatchInfo();
                cate.init(m);
                if (cate.group1.length() < 5) continue;
                cate.group2 = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(str, cate.startPos));
                if(cate.group2.isEmpty()) continue;
                // 判断是否为正常的分类名
                boolean validCateName = true;
                for (int j = 0; j < invalidCateNames.size(); ++j) {
                    if (cate.group2.indexOf(invalidCateNames.get(j)) != -1) {
                        SpiderDebug.log(String.format("排除无效分类：%s --> %s", cate.group1, cate.group2));
                        validCateName = false;
                        break;
                    }
                }
                if(!validCateName) continue;

                if (!list.isEmpty()) {
                    if (!list.get(0).findDiffStr(cate, splitFlag)) {
                        SpiderDebug.log(String.format("排除可能无效的分类 %s <--> %s", cate.group1, cate.group2));
                        continue;
                    }
                }
                list.add(cate);
            }

            ArrayList<Integer> baseInfoIndexs = new ArrayList<>();
            // 找到最可能是正确的导航item
            for (int i =0; i < list.size(); ++i){
                list.get(i).diff = null;
                for (String v : cateManuals){
                    if(list.get(i).group2.indexOf(v) !=-1) {
                        baseInfoIndexs.add(i);
                        break;
                    }
                }
            }

            // 以找到的导航item为基准重建分类ID
            int baseInfoIndex=0;
            for (int i =1; i < baseInfoIndexs.size(); ++i){
                baseInfoIndex=baseInfoIndexs.get(0).intValue();
                list.get(baseInfoIndex).findDiffStr(list.get(baseInfoIndexs.get(i).intValue()), splitFlag);
            }

            JSONObject cateManual = new JSONObject();
            for (int i = 0; i < list.size(); ++i) {
                if(list.get(i).diff == null || list.get(i).diff.isEmpty()) {
                    if(!list.get(baseInfoIndex).findDiffStr(list.get(i), splitFlag)){
                        SpiderDebug.log(String.format("排除可能无效的分类 : %s", list.get(i).group0));
                        continue;
                    }
                }

                boolean validCateName = true;
                String name = list.get(i).group2;
                String id = list.get(i).diff;
                if (id == null || id.isEmpty()) continue;
                if (name == null || name.isEmpty()) continue;
                for (int k =0; k <id.length(); ++k){
                    if(splitFlag.indexOf(id.charAt(k)) != -1){
                        SpiderDebug.log(String.format("跳过无效的分类ID :%s", id));
                        continue;
                    }
                }
                for (int j = 0; j < invalidCateNames.size(); ++j) {
                    if (name.indexOf(invalidCateNames.get(j)) != -1) {
                        validCateName = false;
                        break;
                    }
                }
                if (validCateName && !cateManual.has(name)) {
                    cateManual.put(name, id);
                }
            }
            rule.put("cateManual", cateManual);
            return cateManual;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new JSONObject();
    }

    // 猜测搜索页的url规则
    protected String guess_rule_search_url(String body){
        // 找搜索页url的逻辑，不处理其他情情况
        // 1. 找包含 name input标签
        // 2. 找到后往上找三层，找action="" 如果有的话就找到了
        String regex = "<input.+?name=\"(.+?)\"";
        Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
        Matcher m = pattern.matcher(body);
        if(m.find()){
            String str = m.group(0);
            String wd = m.group(1);
            for (int i =1; i < 4; ++i){
                ArrayList<Integer> arr =  HtmlNodeHlper.findUpNodes(body,m.start(0), i);
                String r = HtmlNodeHlper.nodeString(body, arr.get(arr.size()-1));
                String regex2 = "action=\"(.+?)\"";
                Pattern pattern2 = Pattern.compile(regex2, Pattern.CASE_INSENSITIVE);
                Matcher m2 = pattern2.matcher(r);
                if(m2.find()){
                    String url = m2.group(1);
                    char ch = url.indexOf('?') ==-1 ? '?' : '&';
                    url = addHttpPrefix(url +   ch + wd + "={wd}");
                    return url;
                }

            }
        }
        return "";
    }
    // 猜测列表数据的 vod_id 规则
    public JSONArray guess_rule_vod_id(String body) {
        try {
            String regx = "<a.+?href=\"(.+?)\"";
            Pattern pattern = Pattern.compile(regx);
            Matcher m = pattern.matcher(body);
            HashMap<String, JSONArray> founds = new HashMap<>();
            ArrayList<HtmlMatchInfo> list = new ArrayList<>();
            while (m.find()) {
                HtmlMatchInfo cate = new HtmlMatchInfo();
                cate.init(m);
                cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 4);
//                String ms = this.findNodeString(body, cate.uploads.get(cate.uploads.size()-1));
                if (!list.isEmpty()) {

                    if(cate.group1.equals( list.get(list.size()-1).group1)) continue;
                    boolean b = list.get(list.size()-1).hasSameUpNode(cate);
                    if (!b) { // 当前找到的info和list中的匹配
                        if (list.size() > 1) {
                            HtmlMatchInfo info = list.get(0);
                            info.findDiffStr(list.get(1), splitFlag);
                            int id = 0;
                            boolean isNumberID = false;
                            try { id = Integer.valueOf(info.diff).intValue(); isNumberID = true; }catch (Exception e){}

                            if(id > 100 ){ // cateID一般都是小于100的
                                String url = (info.group1.replace(list.get(0).diff, "{vid}"));
                                JSONArray arr = new JSONArray();
                                String prefix = url.substring(0, url.indexOf("{vid}"));
                                String suffix = url.substring(prefix.length() + "{vid}".length());
                                int lookback = info.uploads.indexOf(info.matchedUpNodePos) - 1;
                                if (lookback < 1) lookback = 1;
                                arr.put(prefix);
                                arr.put(suffix);
                                arr.put(0);
                                arr.put(0);
                                arr.put(lookback);
                                arr.put(list.size());

                                if (!founds.containsKey(url)) {
                                    founds.put(url, arr);
                                } else {
                                    int nlen = founds.get(url).getInt(5) + list.size();
                                    arr.put(5, nlen);
                                    founds.put(url, arr);
                                    if(nlen >= 30){
                                        list.clear();
                                        break;
                                    }
                                }
                            }

                        }
                        list.clear();
                    }
                }
                list.add(cate);
                if(list.size()>30){
                    break;
                }
            }


            if (list.size() > 5 || (list.size()>1 && founds.isEmpty())) { // 如果list中的数据大于1 则认为找到了类型列表
                HtmlMatchInfo info = list.get(0);
                info.findDiffStr(list.get(1), splitFlag);
                int id = 0;
                boolean isNumberID = false;
                try { id = Integer.valueOf(info.diff).intValue(); isNumberID = true; }catch (Exception e){}

                if(id > 100 ){ // cateID一般都是小于100的

                    String url = (info.group1.replace(list.get(0).diff, "{vid}"));
                    JSONArray arr = new JSONArray();
                    String prefix = url.substring(0, url.indexOf("{vid}"));
                    String suffix = url.substring(prefix.length() + "{vid}".length());
                    int lookback = info.uploads.indexOf(info.matchedUpNodePos) - 1;
                    if (lookback < 1) lookback = 1;
                    arr.put(prefix);
                    arr.put(suffix);
                    arr.put(0);
                    arr.put(0);
                    arr.put(lookback);
                    arr.put(list.size());

                    if (!founds.containsKey(url)) {
                        founds.put(url, arr);
                    } else {
                        int nlen = founds.get(url).getInt(5) + list.size();
                        arr.put(5, nlen);
                        founds.put(url, arr);
                    }
                }
            }


            JSONArray c = null;
            for (String key : founds.keySet()) {
                JSONArray v = founds.get(key);
                if(c == null || c.getInt(5) < v.getInt(5)) c = v;
            }
            return  c;

        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    // 猜播放列表
    public JSONArray guess_rule_vod_play_url(String str, String vid) {
        String regex = "href=\"(/.+?)\"";
        Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
        Matcher m = pattern.matcher(str);
        HtmlMatchInfo info = new HtmlMatchInfo();
        ArrayList<String> vec = new ArrayList<>();
        boolean p0__ = false;
        while (m.find()){
            String sb = m.group(1);
            // 太长的url认为是错误的播放地址
            if(sb.length() > 100) continue;
            if(sb.indexOf(vid) == -1) continue;
            // 如果当前url的长度比上一个url的长度短也认为是无效的播放地址（带上了vod_id一般短一点的可能是详情页的地址）
            // 一般来讲 999-1-1.html 这种格式是播放页的地址，不排除这种地址
            boolean is__html = (sb.indexOf(vid+"-") != -1);
            if(!is__html  && vec.size() > 0 && vec.get(vec.size()-1).length() > sb.length()) continue;
            if(is__html && !p0__ ){ // 找到了准确度最高的播放连接格式，如果检查到列表头不是这种格式的话，清空列表从头开始
                vec.clear();
            }
            // 如果列表里面装了标准的连接格式，不标准的就不要了
            if(p0__ && !is__html) continue;

            info.init(m);
            if(vec.isEmpty()) p0__ = is__html;
            vec.add(m.group(1));
            if(vec.size() > 10  && vec.get(vec.size()-2).length() == sb.length()) {
                break;
            }
        }
        if(info.group0 != null){
            for (int i =1;i < 4; ++i){
                ArrayList<Integer> nodes = HtmlNodeHlper.findUpNodes(str, info.startPos, i);
                int startPos = nodes.get(nodes.size()-1).intValue();
                //String smd =  str.substring(startPos, startPos+10);
                String smd =  HtmlNodeHlper.nodeString(str, startPos);

                if(smd.indexOf("<ul") ==0 || smd.indexOf("<div") ==0  || i == 3){ // 最多退三层，找不到就算了
                    // found 播放列表的 根节节点
                    String prefix = info.group1.substring(0, info.group1.indexOf(vid));
                    String suffix = "\"";
                    JSONArray arr = new JSONArray();
                    arr.put(prefix);
                    arr.put(suffix);
                    arr.put(0 - prefix.length());
                    arr.put(0);
                    arr.put(i);
                    return arr;
                }
            }
        }
        return null;
    }

    //
    public String guess_value_vod_name(String nd, int startPos) {
        try {
            JSONArray vec = new JSONArray();
            vec.put("alt=\"");
            vec.put("\"");
            String val = Utils.findSubString(nd, startPos, vec);

            if (val.isEmpty()) {
                vec.put(0, "\" title=\"");
                val = Utils.findSubString(nd, startPos, vec);
            }
            if (val.isEmpty()) { // 如果没有通过title找到视频名，则取整个node的文本内容,取出现次数最多的项

                String all = HtmlNodeHlper.trimHtmlString(nd, "!!!!");
                String[] words = all.split("!!!!");
                HashMap<String, Integer> map = new HashMap<String, Integer>();

                for (int i = 0; i < words.length; ++i) {
                    words[i] =words[i].trim();
                    if (!words[i].isEmpty() && words[i].indexOf("更新") ==-1) {

                        int c = 1;
                        if (map.containsKey(words[i])) {
                            c = 1 + map.get(words[i]).intValue();
                            ;//[words[i]]
                        }
                        map.put(words[i], Integer.valueOf(c));
                    }
                }
                String s = "";
                int c = 0;
                for (String key : map.keySet()) {
                    int v = map.get(key).intValue();
                    if (v > c) {
                        c = v;
                        s = key;
                    }
                }
                val = s;
            }
            return val.replace("在线", "")
                    .replace("立即", "")
                    .replace("观看", "")
                    .replace("点播", "")
                    .replace("影片", "")
                    .replace("信息", "")
                    .replace("播放", "")
                    .trim();
            //return  val;
        } catch (Exception e) {

        }
        return "";
    }

    public String guess_value_vod_remarks(String nd, int startPos, String vod_name) {
        try {
            String all = HtmlNodeHlper.trimHtmlString(nd, "!!!!");
            String[] words = all.split("!!!!");
            String val = "";
            for (int i = 0; i < words.length; ++i) {
                String wd = words[i].trim();
                if (!wd.isEmpty() && wd.indexOf(vod_name) == -1) {
                    String dot = (!val.isEmpty()) ? "," : "";// val += ",";
                    String tmp = val + dot + wd;
                    if (tmp.length() > 20) {
                        break;
                    }
                    val = tmp;
                }
            }
            return val;
        } catch (Exception e) {

        }
        return "";

    }

    public String guess_value_vod_pic(String nd, int startPos) {
        try {
            JSONArray vec = new JSONArray();
            vec.put("data-original=\"");
            vec.put("\"");
            String val = Utils.findSubString(nd, startPos, vec);
            if (val.isEmpty()) {
                vec.put(0, "data-src=\"");
                val = Utils.findSubString(nd, startPos, vec);
            }
            if (val.isEmpty()) {
                vec.put(0, "src=\"");
                val = Utils.findSubString(nd, startPos, vec);
            }
            if (val.isEmpty()) {
                vec.put(0, "data-bg=\"");
                val = Utils.findSubString(nd, startPos, vec);
            }
            if (val.isEmpty()) {
                //TODO: 直接在nd中找个.jpg .png 之类的当图片
            }
            return addHttpPrefix(val);
        } catch (Exception e) {

        }
        return "";
    }

    @Override
    public boolean isVideoFormat(String url) {
        url = url.toLowerCase();
        if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) {
            return false;
        }
        for (String format : videoFormatList) {
            if (url.contains(format)) {
                return true;
            }
        }
        return false;
    }

    // 让当前爬虫自己判断是否为可播放的地址
    @Override
    public boolean manualVideoCheck() {
        return true;
    }


    @Override
    public String homeContent(boolean z) {
        try {
            fetchRule();

            JSONObject result = new JSONObject();
            JSONArray classes = new JSONArray();
            JSONObject cateManual = rule.optJSONObject("cateManual");

            Iterator<String> keys = cateManual.keys();
            while (keys.hasNext()) {
                String key = keys.next();
                JSONObject jsonObject = new JSONObject();
                jsonObject.put("type_name", key);
                jsonObject.put("type_id", cateManual.getString(key));
                classes.put(jsonObject);
            }

            result.put("class", classes);
            if (z && rule.has("filter")) {
                result.put("filters", rule.getJSONObject("filter"));
            }
            return result.toString();
        } catch (Exception e) {
            SpiderDebug.log(e);
        }
        return "";
    }

    // from xpath 加入过滤条件
    protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
        try {
            JSONObject list = this.rule.getJSONObject("list");
            String cateUrl = list.optString(pg, "");
            if(cateUrl.isEmpty())
                cateUrl = list.getString("url");
            if (filter && extend != null && extend.size() > 0) {
                for (Iterator<String> it = extend.keySet().iterator(); it.hasNext(); ) {
                    String key = it.next();
                    String value = extend.get(key);
                    if (value.length() > 0) {
                        cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
                    }
                }
            }
            cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
            Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
            while (m.find()) {
                String n = m.group(0).replace("{", "").replace("}", "");
                cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", "");
            }
            return cateUrl;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return "";
    }

    @Override
    public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
        try {
            JSONObject list = this.rule.getJSONObject("list");
            String url = categoryUrl(tid, pg, filter, extend);
            String body = fetchUrl(url, list.optJSONObject("header"));
            String str = Utils.getRegion(body, list);
            JSONArray videos = new JSONArray();
            JSONArray lookback = Utils.getLookbackArray(list);
            Set<String> set = new HashSet<String>();
            int pos = 0;
            ArrayList<Integer> urlnodes = null;
            int lookup = -1;
            while (lookback != null) {
                pos = str.indexOf(lookback.getString(0), pos);
                if (pos == -1) break;

                ArrayList<Integer> arr = null;
                int blockPos = 0;
                String nd ="";
                do {
                    arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4));
                    if (urlnodes == null) {
                        urlnodes = arr;
                        blockPos = arr.get(arr.size() - 1);
                    } else {
                        blockPos = Utils.findBlockPos(urlnodes, arr);
                    }
                    nd = HtmlNodeHlper.nodeString(str, blockPos);

                    // 检查是否回看层数过多，如果回看导数过多会导致加载不出来数据或一页只加载一条数据，需要进行修正
                    if(lookup < 0){
                        int count = Utils.getSubStringCount(nd, lookback.getString(0));
                        if(count > 3 && lookback.getInt(4)>1){
                            lookback.put(4, lookback.getInt(4)-1);
                            urlnodes = null;
                            blockPos=0;
                            nd="";
                            SpiderDebug.log(String.format("找到过多的url匹配项(%d)，降低匹配层级为%d", count, lookback.getInt(4)));
                        }else if(lookup == -1){
                            String pic = guess_value_vod_pic(nd,0); //尝试找一下图片，如果没找到的话增加一级
                            String vName = guess_value_vod_name(nd,0);
                            if(pic.isEmpty()||vName.isEmpty()){
                                lookback.put(4, lookback.getInt(4)+1);
                                urlnodes = null;
                                blockPos=0;
                                nd="";
                                lookup = -2; // 只退一次
                                SpiderDebug.log(String.format("当前层级未找到(%s)，增加匹配层级为%d",  pic.isEmpty()? "图片": "标题",  lookback.getInt(4)));
                            }else{
                                lookup = lookback.getInt(4);
                            }
                        }else{
                            lookup = lookback.getInt(4);
                        }
                    }
                }while (lookup < 0 );


                pos += nd.length();
                blockPos = 0;
                String vod_id = Utils.findSubString(nd, blockPos, list.getJSONArray("vod_id"));
                if (!set.contains(vod_id)) { // 排除重复数据
                    set.add(vod_id);
                    JSONObject v = new JSONObject();
                    v.put("vod_id", vod_id);
                    v.put("vod_name", Utils.findSubString(nd, blockPos, list.optJSONArray("vod_name")));
                    v.put("vod_pic", addHttpPrefix(Utils.findSubString(nd, blockPos, list.optJSONArray("vod_pic"))));
                    v.put("vod_remarks", Utils.findSubString(nd, blockPos, list.optJSONArray("vod_remarks")));

                    if (v.getString("vod_name").isEmpty()) {
                        v.put("vod_name", guess_value_vod_name(nd, 0));
                    }

                    if (v.getString("vod_pic").isEmpty()) {
                        v.put("vod_pic", guess_value_vod_pic(nd, 0));
                    }

                    // 随便整点remark
                    if (v.getString("vod_remarks").isEmpty()) {
                        String vod_name = v.getString("vod_name");
                        v.put("vod_remarks", guess_value_vod_remarks(nd, 0, vod_name));
                    }
                    v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag));
                    videos.put(v);
                }
//                pos += vod_id.length();
            }

            JSONObject result = new JSONObject();
            result.put("page", pg);
            result.put("pagecount", Integer.MAX_VALUE);
            result.put("limit", 90);
            result.put("total", Integer.MAX_VALUE);
            result.put("list", videos);
            return result.toString();
        } catch (Exception e) {
            e.printStackTrace();
            return "";
        }
    }

    // 生成播放的名称
    public ArrayList<String> makeVodPlayFrom(int sz) {
        ArrayList<String> vec = new ArrayList<String>();
        for (int i = 1; i <= sz; ++i) {
            vec.add("播放列表" + i);
        }
        return vec;
    }

    // 查找播放列表名
    public ArrayList<String> findVodPlayFrom(String str, int sz) {
        try {
            ArrayList<Integer> urlnodes = null;
            JSONObject playlist = this.rule.getJSONObject("playlist");
            if (!playlist.has("vod_play_from")) {
                return makeVodPlayFrom(sz);
            }
            ArrayList<Pair<Integer, String>> vod_play_from = new ArrayList<Pair<Integer, String>>();
            JSONArray rule_vod_play_from = playlist.getJSONArray("vod_play_from");
            for (int i = 0; i < rule_vod_play_from.length(); ++i) {
                String s = rule_vod_play_from.get(i).getClass().getSimpleName();
                String key = "";
                String alias = "";
                if (s.equals("String")) {
                    key = alias = rule_vod_play_from.getString(i);
                } else if (s.equals("JSONArray")) {
                    JSONArray item = rule_vod_play_from.getJSONArray(i);
                    key = alias = item.getString(0);
                    if (item.length() > 1) {
                        alias = item.getString(1);
                    }
                } else {
                    return makeVodPlayFrom(sz);
                }

                int pos = str.indexOf(key);
                if (pos == -1) continue;
                vod_play_from.add(new Pair<>(pos, alias));
            }
            // 找到的名称与实际需要的数量不匹配，返回默认的名称
            if (vod_play_from.size() != sz) {
                return makeVodPlayFrom(sz);
            }
            // 排序
            Collections.sort(vod_play_from, new Comparator<Pair<Integer, String>>() {
                @Override
                public int compare(Pair<Integer, String> a, Pair<Integer, String> b) {
                    return a.first.intValue() - b.first.intValue();
                }
            });

            ArrayList<String> vec = new ArrayList<String>();
            for (int i = 0; i < vod_play_from.size(); ++i) {
                vec.add(vod_play_from.get(i).second);
            }
            return vec;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return makeVodPlayFrom(sz);
    }

    // 查找播放列表
    public ArrayList<String> findVodPlayUrl(String str) {
        ArrayList<String> tmp_vod_play_url = new ArrayList<String>();
        ArrayList<String> vod_play_url = new ArrayList<String>();
        try {
            int pos = 0;
            ArrayList<Integer> urlnodes = null;
            JSONObject playlist = this.rule.getJSONObject("playlist");
            int sort = playlist.optInt("sort", 0);  // 如果这个值是0。表示要倒序播放列表
            HashMap<String, Integer> map = new HashMap<String, Integer>();
            Set<Integer> rmset = new HashSet<Integer>();
//            String tmp = "";
            ArrayList<String> tmp = new ArrayList<String>();
            JSONArray lookback = Utils.getLookbackArray(playlist);
            while (lookback != null) {
                JSONArray rule_vod_play_url = playlist.getJSONArray("vod_play_url");
                pos = str.indexOf(rule_vod_play_url.getString(0), pos);
                if (pos == -1) break;
                ArrayList<Integer> arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4));

                int blockPos = 0;
                if (urlnodes == null || arr.size() != urlnodes.size() || arr.get(arr.size() - 1).intValue() != urlnodes.get(urlnodes.size() - 1).intValue()) {
                    urlnodes = arr;
                    blockPos = arr.get(Math.max(0, arr.size() - 2));
                    // 如果上级节点不同，说明当前播放列表已经结束，可以
                    if (!tmp.isEmpty()) {
                        SpiderDebug.log("change play list ");
                        if (sort != 0) {
                            Collections.reverse(tmp);
                        }
                        tmp_vod_play_url.add(TextUtils.join("#", tmp));
                        tmp = new ArrayList<String>();
                    }
                } else {
                    blockPos = Utils.findBlockPos(urlnodes, arr);
                }
//                String bs = str.substring(blockPos, blockPos + 1000);
//                String b = this.findNodeString(str, blockPos);
                // 处理play_url 为空的情况
                String play_url = addHttpPrefix(Utils.findSubString(str, blockPos, playlist.getJSONArray("vod_play_url")));

                if (map.containsKey(play_url)) { // 如果以经找过当前播放的url,那么认为之前找到的都是垃圾数据，清空之间的成果
                    SpiderDebug.log("发现重复播放连接，清空已解析到的播放列表");
                    rmset.add(map.get(play_url)); // 添加移除标志
                }
                map.put(play_url, tmp_vod_play_url.size());
                String play_url_title = Utils.findSubString(str, blockPos, playlist.optJSONArray("vod_play_url_title"));
                if (play_url_title.isEmpty()) {
                    play_url_title = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(str, blockPos));
                    int dd = 3;
                }
                tmp.add(play_url_title + "$" + play_url);
//                SpiderDebug.log(String.format("%s$%s", play_url_title, play_url));
                pos += play_url.length();
            }

            if (!tmp.isEmpty()) {
                if (sort != 0) {
                    Collections.reverse(tmp);
                }
                tmp_vod_play_url.add(TextUtils.join("#", tmp));
            }

            for (int i = 0; i < tmp_vod_play_url.size(); ++i) {
                if (!rmset.contains(i)) {
                    vod_play_url.add(tmp_vod_play_url.get(i));
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return vod_play_url;
    }

    // 猜测详情数据的html区间
    protected String guessDetailContentRegion(String body) {
        String regx = String.format(">\\s*?(%s)|(%s)", TextUtils.join("|", detailItemNames), TextUtils.join("：|", detailItemNames));
        Pattern pattern = Pattern.compile(regx, Pattern.CASE_INSENSITIVE);
        Matcher m = pattern.matcher(body);
        ArrayList<HtmlMatchInfo> list = new ArrayList<>();
        while (m.find()) {
            HtmlMatchInfo cate = new HtmlMatchInfo();
            cate.init(m);
            cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 5);
            if (!list.isEmpty()) {
                boolean b = list.get(0).hasSameUpNode(cate);
                if (!b) { // 当前找到的info和list中的匹配
                    if (list.size() > 1) {
                        boolean found = false;
                        for (int i = 0; i < list.size(); ++i) {
                            if (list.get(i).group0.indexOf("导演") != -1) {
                                found = true;
                            }
                        }
                        if (found) {
                            return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos);
                        } else {
                            list.clear();
                        }
                    }
                    list.clear();
                }
            }
            list.add(cate);
        }
        if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表
            return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos);
        } else {
            return "";
        }
    }

    @Override
    public String detailContent(List<String> ids) {
        try {
            fetchRule();
            JSONObject vinfo = new JSONObject(new String(Base64.decode(ids.get(0), base64Flag), "UTF-8"));

            JSONObject detail = rule.optJSONObject("detail");
            if (detail == null) return "";
            // 如果不存在url,则使用list中的vod_id来生成url
            if (!detail.has("url")) {
                JSONObject list = rule.getJSONObject("list");
                JSONArray tmp = list.getJSONArray("vod_id");
                String u = addHttpPrefix(tmp.getString(0) + "{vid}" + tmp.getString(1));
                detail.put("url", u);
            }
            String url = detail.getString("url").replace("{vid}", vinfo.getString("vod_id"));
            String body = fetchUrl(url, detail.optJSONObject("header"));
            String str = Utils.getRegion(body, detail);
            int startPos = 0;

            String nodeString = "";
            // 圈定 详情数据的范围
            JSONArray lookback = Utils.getLookbackArray(detail);
            if (lookback != null) {
                int pos = str.indexOf(lookback.getString(0), 0);
                if (pos != -1) {
                    ArrayList<Integer> arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4));
                    if (arr.size() > 0) {
                        startPos = arr.get(arr.size() - 1);
                        nodeString = HtmlNodeHlper.nodeString(str, startPos); // 精确详情数据的范围
                    }
                }
            }
            // 没有指定详情数据范围则猜一个出来
            if (nodeString.isEmpty()) {
                nodeString = this.guessDetailContentRegion(body);
            }

            if (nodeString.length() != str.length()) {
                str = nodeString;
                startPos = 0;
            }
            ///////////////////////////////////////////////////////////////////////////////////////

            JSONObject vod = new JSONObject();
            vod.put("vod_id", ids.get(0));
            vod.put("vod_name", Utils.findSubString(str, startPos, detail.optJSONArray("vod_name")));
            vod.put("vod_pic", addHttpPrefix(Utils.findSubString(str, startPos, detail.optJSONArray("vod_pic"))));
            vod.put("type_name", Utils.findSubString(str, startPos, detail.optJSONArray("type_name")));
            vod.put("vod_year", Utils.findSubString(str, startPos, detail.optJSONArray("vod_year")));
            vod.put("vod_area", Utils.findSubString(str, startPos, detail.optJSONArray("vod_area")));
            vod.put("vod_remarks", Utils.findSubString(str, startPos, detail.optJSONArray("vod_remarks")));
            vod.put("vod_actor", Utils.findSubString(str, startPos, detail.optJSONArray("vod_actor")));
            vod.put("vod_director", Utils.findSubString(str, startPos, detail.optJSONArray("vod_director")));
            vod.put("vod_content", Utils.findSubString(str, startPos, detail.optJSONArray("vod_content")));


            ////////////////////////////////////////////////////////////////////////////////////////
            if (vod.getString("vod_name").isEmpty()) {
                vod.put("vod_name", vinfo.optString("vod_name", ""));
            }
            // 从页面中猜个视频名称出来
            if (vod.getString("vod_name").isEmpty()) {
                vod.put("vod_name", guess_value_vod_name(str, startPos));
            }

            ////////////////////////////////////////////////////////////////////////////////////////
            if (vod.getString("vod_pic").isEmpty()) {
                vod.put("vod_pic", vinfo.optString("vod_pic", ""));
            }
            if (vod.getString("vod_pic").isEmpty()) {
                vod.put("vod_pic", guess_value_vod_pic(str, startPos));
            }

            ////////////////////////////////////////////////////////////////////////////////////////
            if (lookback != null && lookback.length() > 1) {
                JSONArray key = new JSONArray();
                String name = lookback.getString(0);
                String skey = lookback.getString(0);

                ArrayList<String> detailItems = new ArrayList<>(Arrays.asList("导演", "演员", "类型", "年份"));
                for (String p : detailItems) {
                    if (name.indexOf(p) != -1) {
                        skey = p;
                        break;
                    }
                }
                key.put(name);
                key.put(lookback.getString(1));
                if (vod.getString("vod_director").isEmpty()) {
                    key.put(0, name.replace(skey, "导演"));
                    vod.put("vod_director", Utils.findSubString(str, startPos, key));
                }
                if (vod.getString("vod_actor").isEmpty()) {
                    key.put(0, name.replace(skey, "主演"));
                    vod.put("vod_actor", Utils.findSubString(str, startPos, key));
                }

                if (vod.getString("vod_content").isEmpty()) {
                    String all = HtmlNodeHlper.trimHtmlString(str, "!!!!");
                    String[] words = all.split("!!!!");
                    String v = "";
                    for (int i = 0; i < words.length; ++i) {
                        if (words[i].length() > v.length()) {
                            v = words[i];
                        }
                    }
                    vod.put("vod_content", HtmlNodeHlper.trimHtmlString(v));
                }

            } else { // 猜一下详情数据
                if (vod.getString("vod_director").isEmpty()) {
                    ArrayList<String> arr = HtmlNodeHlper.getChildNodes(nodeString);
                    String content = "";
                    String f = TextUtils.join("|", detailItemNames);
                    String regex = String.format("%s",f);
                    Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                    for (int i = 0; i < arr.size(); ++i) {
                        String p = HtmlNodeHlper.trimHtmlString(arr.get(i), " ").replace("：", "");
                        if (p.length() > content.length()) { content = p; }
                        String[] all = p.split(regex);
                        // split出来的可能存在空字符串，去除掉
                        ArrayList<String> items = new ArrayList<>();
                        for(String c: all){
                            if(c.isEmpty()) continue;
                            items.add(c);
                        }
                        Matcher m = pattern.matcher(p);
                        int index = 0;
                        while (m.find() && index < items.size()){
                            String s = m.group(0);
                            for (int j = 0; j < detailItemNames.size(); ++j) {
                                String name = detailItemNames.get(j);
                                String key = detailItemKeys.get(j);
                                if (s.indexOf(name) != -1) {
                                    if (vod.getString(key).isEmpty()) {
                                        vod.put(key, items.get(index).trim());
                                    }
                                    break;
                                }
                            }
                            ++index;
                        }
                    }

                    if (vod.getString("vod_content").isEmpty()) {
                        vod.put("vod_content", content);
                    }
                }
            }


            playlistContent(ids, vod, body);// 获取播放列表


            JSONObject result = new JSONObject();
            JSONArray list = new JSONArray();
            list.put(vod);
            result.put("list", list);
            return result.toString();
        } catch (Exception e) {
            e.printStackTrace();
            return "";
        }

    }

    // 播放页 str 为 detailContent 函数中http返回值
    protected void playlistContent(List<String> ids, JSONObject vod, String body) {
        try {
            fetchRule();
            JSONObject vinfo = new JSONObject(new String(Base64.decode(ids.get(0), base64Flag), "UTF-8"));

            JSONObject playlist = rule.optJSONObject("playlist");
            if (playlist == null) return;
            if (playlist.has("url")) {
                String detailUrl = rule.getJSONObject("detail").optString("url");
                String playListUrl = playlist.getString("url");
                if(!detailUrl.equals(playListUrl)){
                    String url = playlist.getString("url").replace("{vid}", vinfo.getString("vod_id"));
                    body = fetchUrl(url, playlist.optJSONObject("header"));
                }
            }
            String str = Utils.getRegion(body, playlist);

            ArrayList<String> vod_play_url = null;
            if (!playlist.has("vod_play_url")) {
                // 猜vod_play_url的查找规则
                JSONArray vod_play_url_rule = this.guess_rule_vod_play_url(str, vinfo.getString("vod_id"));
                if(vod_play_url_rule != null){
                    playlist.put("vod_play_url", vod_play_url_rule);
                }
            }
            vod_play_url = this.findVodPlayUrl(str);
            ArrayList<String> vod_play_from = this.findVodPlayFrom(str, vod_play_url.size());

            // 如果有说明播放源的名称，那么对播放源进行排序
            String f1 = TextUtils.join("$$$", vod_play_from);
            String f2 = TextUtils.join("$$$", makeVodPlayFrom(vod_play_url.size()));

            if (!f1.equals(f2)) {
                ArrayList<String> urls = new ArrayList<>();
                ArrayList<String> froms = new ArrayList<>();

                JSONArray rule_vod_play_from = playlist.getJSONArray("vod_play_from");
                for (int i = 0; i < rule_vod_play_from.length(); ++i) {
                    String s = rule_vod_play_from.get(i).getClass().getSimpleName();
                    String alias = "";
                    if (s.equals("String")) {
                        alias = rule_vod_play_from.getString(i);
                    } else if (s.equals("JSONArray")) {
                        JSONArray item = rule_vod_play_from.getJSONArray(i);
                        alias = item.getString(0);
                        if (item.length() > 1) {
                            alias = item.getString(1);
                        }
                    }

                    for (int j = 0; j < vod_play_from.size(); ++j) {
                        if (vod_play_from.get(j).equals(alias)) {
                            urls.add(vod_play_url.get(j));
                            froms.add(vod_play_from.get(j));
                        }
                    }

                }
                vod_play_url = urls;
                vod_play_from = froms;
            }
            vod.put("vod_play_url", TextUtils.join("$$$", vod_play_url));
            vod.put("vod_play_from", TextUtils.join("$$$", vod_play_from));
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    // 尝试从播放页中找播放url
    protected String parsePlayUrl(String str, String str2, List<String> list) {
        try {
            JSONObject play = rule.optJSONObject("play");
            if (play == null) {
                return "";
            }
            String tmp = fetchUrl(str2, play.optJSONObject("header"));
            String body = Utils.getRegion(tmp, play);
            int startPos = 0;
            JSONArray lookback = Utils.getLookbackArray(play);
            if (lookback != null) {
                int pos = body.indexOf(lookback.getString(0), 0);
                if (pos != -1) {
                    ArrayList<Integer> arr = HtmlNodeHlper.findUpNodes(body, pos - 1, lookback.getInt(4));
                    if (arr.size() > 0) {
                        startPos = arr.get(arr.size() - 1);
                    } else {
                        startPos = pos;
                    }
                }
            }

            String vod_url = Utils.findSubString(body, startPos, play.optJSONArray("vod_url"));
            vod_url = vod_url.replace("\\/", "/");
            if (vod_url.isEmpty() || !isVideoFormat(vod_url)) return "";
            JSONObject result = new JSONObject();
            result.put("parse", 0);
            result.put("playUrl", "");
            result.put("url", vod_url);
            return result.toString();

        } catch (Exception e) {
            e.printStackTrace();
        }

        return "";
    }

    @Override
    public String playerContent(String str, String str2, List<String> list) {
        try {
            fetchRule();
            // 先判断是否可以拿到直链
            String ret = parsePlayUrl(str, str2, list);
            if (!ret.isEmpty()) return ret;
            // 直接将网页地址返回回去进行嗅探
            JSONObject result = new JSONObject();
            result.put("parse", 1);
            result.put("playUrl", "");
            result.put("url", str2);
            return result.toString();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return "";
    }

    protected Object parseJsonSearchResult(Object obj) {
        try {
            if (obj == null) return null;
            JSONObject search = rule.optJSONObject("search");
            if (search == null) return null;
            String key_vod_id = search.getString("vod_id");
            String key_vod_name = search.getString("vod_name");
            String type = obj.getClass().getSimpleName();
            if (type.equals("JSONObject")) {
                JSONObject object = (JSONObject) obj;
                if (object.has(key_vod_id) && object.has(key_vod_name)) return object;
                for (Iterator<String> iter = object.keys(); iter.hasNext(); ) {
                    String k = iter.next();
                    Object r = parseJsonSearchResult(object.get(k));
                    if (r != null) {
                        return r;
                    }
                }
            } else if (type.equals("JSONArray")) {
                JSONArray array = (JSONArray) obj;
                for (int i = 0; i < array.length(); ++i) {
                    if (parseJsonSearchResult(array.get(i)) != null) {
                        return array;
                    }
                }
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    protected String parseSearchResult(String body) {
        try {
            JSONObject obj = new JSONObject(body);
            Object info = parseJsonSearchResult(obj);
            if (info == null) return "";
            JSONArray arr = new JSONArray();
            if (info.getClass().getSimpleName().equals("JSONObject")) {
                arr.put(info);
            } else {
                arr = (JSONArray) info;
            }
            JSONObject search = rule.optJSONObject("search");
            JSONArray videos = new JSONArray();
            for (int i = 0; i < arr.length(); ++i) {
                JSONObject v = new JSONObject();
                JSONObject o = arr.getJSONObject(i);
                if (search.has("vod_id") && o.has(search.getString("vod_id"))) {
                    v.put("vod_id", o.get(search.getString("vod_id")).toString());
                } else {
                    continue;
                }
                if (search.has("vod_name") && o.has(search.getString("vod_name"))) {
                    v.put("vod_name", o.get(search.getString("vod_name")).toString());
                } else {
                    v.put("vod_name", "未知");
                }

                if (search.has("vod_pic") && o.has(search.getString("vod_pic"))) {
                    v.put("vod_pic", o.get(search.getString("vod_pic")).toString());
                } else {
                    v.put("vod_pic", "");
                }

                if (search.has("vod_remarks") && o.has(search.getString("vod_remarks"))) {
                    v.put("vod_remarks", o.get(search.getString("vod_remarks")).toString());
                } else {
                    v.put("vod_remarks", "");
                }
                v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag));
                videos.put(v);
            }
            JSONObject result = new JSONObject();
            result.put("list", videos);
            return result.toString();

        } catch (Exception e) {
//            e.printStackTrace();
        }
        return "";
    }

    // post 搜索，只做了简单的支持
    protected String postSearch(String wd, boolean z) {
        try {
            JSONObject search = rule.optJSONObject("search");
            if (search == null) return "";
            String url = search.getString("url");
            JSONObject params = search.getJSONObject("post"); // 查询参数
            HashMap<String, String> reqpayload = new HashMap<>();
            Iterator<String> iter = params.keys();
            while (iter.hasNext()) {
                String key = iter.next();
                String value = params.getString(key).replace("{wd}", wd);
                reqpayload.put(key, value);
            }
            final String[] retval = {""};
            HashMap<String, String> header = getHeaders(url);
            header.put("content-type", "application/x-www-form-urlencoded");
            OkHttpUtil.post(OkHttpUtil.defaultClient(), url, reqpayload, header, new OKCallBack.OKCallBackString() {
                @Override
                protected void onFailure(Call call, Exception e) {

                }

                @Override
                protected void onResponse(String response) {
                    retval[0] = response;
                }
            });
            return retval[0];
        } catch (Exception e) {

        }
        return "";
    }

    @Override
    public String searchContent(String wd, boolean z) {
        try {
            fetchRule();
            JSONObject search = rule.optJSONObject("search");
            if (search == null) return "";
            String str = "";
            if (search.has("post")) {
                str = postSearch(wd, z);
            } else {
                String url = search.getString("url").replace("{wd}", wd);
                str = fetchUrl(url, search.optJSONObject("header"));
            }
            str = Utils.getRegion(str, search);
            // 先当JSON解析试试
            String r = parseSearchResult(str);
            if (r != null && !r.isEmpty()) {
                return r;
            }

            if (!search.has("vod_id")) {
                JSONObject list = rule.getJSONObject("list");
                search.put("vod_id", list.getJSONArray("vod_id"));
            }

            JSONArray videos = new JSONArray();
            Set<String> set = new HashSet<String>();
            int pos = 0;
            ArrayList<Integer> urlnodes = null;

            JSONArray lookback = Utils.getLookbackArray(search);

            int lookup = -1;
            while (lookback != null) {
                pos = str.indexOf(lookback.getString(0), pos);
                if (pos == -1) break;

                ArrayList<Integer> arr = null;
                int blockPos = 0;
                String nd ="";
                do {
                    arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4));
                    if (urlnodes == null) {
                        urlnodes = arr;
                        blockPos = arr.get(arr.size() - 1);
                    } else {
                        blockPos = Utils.findBlockPos(urlnodes, arr);
                    }
                    nd = HtmlNodeHlper.nodeString(str, blockPos);
                    // 检查是否回看层数过多，如果回看导数过多会导致加载不出来数据或一页只加载一条数据，需要进行修正
                    if(lookup < 0){
                        int count = Utils.getSubStringCount(nd, lookback.getString(0));
                        if(count > 3 && lookback.getInt(4)>1){
                            lookback.put(4, lookback.getInt(4)-1);
                            urlnodes = null;
                            blockPos=0;
                            nd="";
                            SpiderDebug.log(String.format("找到过多的url匹配项(%d)，降低匹配层级为%d", count, lookback.get(4)));
                        }
                        else if(lookup == -1){
                            String pic = guess_value_vod_pic(nd,0); //尝试找一下图片，如果没找到的话增加一级
                            String vName = guess_value_vod_name(nd,0);
                            if(pic.isEmpty()||vName.isEmpty()){
                                lookback.put(4, lookback.getInt(4)+1);
                                urlnodes = null;
                                blockPos=0;
                                nd="";
                                lookup = -2; // 只退一次
                                SpiderDebug.log(String.format("当前层级未找到(%s)，增加匹配层级为%d",  pic.isEmpty()? "图片": "标题",  lookback.getInt(4)));
                            }else{
                                lookup = lookback.getInt(4);
                            }
                        }else{
                            lookup = lookback.getInt(4);
                        }
                    }
                }while (lookup < 0);

                pos += nd.length();
                blockPos = 0;
                String vod_id = Utils.findSubString(nd, blockPos, search.getJSONArray("vod_id"));
                if (!set.contains(vod_id)) {
                    set.add(vod_id);
                    JSONObject v = new JSONObject();
                    v.put("vod_id", vod_id);
                    v.put("vod_name", Utils.findSubString(nd, blockPos, search.optJSONArray("vod_name")));
                    v.put("vod_pic", addHttpPrefix(Utils.findSubString(nd, blockPos, search.optJSONArray("vod_pic"))));
                    v.put("vod_remarks", Utils.findSubString(nd, blockPos, search.optJSONArray("vod_remarks")));

                    if (v.getString("vod_name").isEmpty()) {
                        v.put("vod_name", guess_value_vod_name(nd, 0));
                    }

                    if (v.getString("vod_pic").isEmpty()) {
                        v.put("vod_pic", guess_value_vod_pic(nd, 0));
                    }
                    // 随便整点remark
                    if (v.getString("vod_remarks").isEmpty()) {
                        String vod_name = v.getString("vod_name");
                        v.put("vod_remarks", guess_value_vod_remarks(nd, 0, vod_name));
                    }
                    v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag));
                    videos.put(v);
                }
            }

            JSONObject result = new JSONObject();
            result.put("list", videos);
            return result.toString();
        } catch (Exception e) {
            e.printStackTrace();
            return "";
        }
    }

    protected String fetchUrl(String url, JSONObject h) {
        String html = OkHttpUtil.string(url, getHeaders(url));
        html = this.jumpbtwaf(url, html, h);
        return html.replaceAll("<!--.+?-->", "").replace("\r\n","").replace("\n","");  // 移除注释
    }

    protected String jumpbtwaf(String webUrl, String html, JSONObject h) {

        try {
            // 没有配置btwaf不执行下面的代码
            if (!rule.optBoolean("btwaf", false)) {
                return html;
            }

            if (html.contains("检测中") && html.contains("跳转中") && html.contains("btwaf")) {
                JSONArray keys = new JSONArray();
                keys.put("btwaf=");
                keys.put("\"");
                String btwaf = Utils.findSubString(html, 0, keys);
                String bturl = webUrl + "?btwaf=" + btwaf;

                Map<String, List<String>> cookies = new HashMap<>();
                OkHttpUtil.string(bturl, getHeaders(webUrl), cookies);
                for (Map.Entry<String, List<String>> entry : cookies.entrySet()) {
                    if (entry.getKey().equals("set-cookie") || entry.getKey().equals("Set-Cookie")) {
                        String btcookie = TextUtils.join(";", entry.getValue());
                        if (!rule.has("header")) {
                            rule.put("header", new JSONObject());
                        }
                        rule.getJSONObject("header").put("cookie", btcookie);
                        break;
                    }
                }
                html = fetchUrl(webUrl, h);
            }
            if (!html.contains("检测中") && !html.contains("btwaf")) {
                return html;
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
        return html;
    }

}