From 6a214d9cb399d6f01ba5de2a504db66028571f70 Mon Sep 17 00:00:00 2001 From: leosam1024 <127861447+leosam1024@users.noreply.github.com> Date: Sun, 25 Jun 2023 08:12:29 +0800 Subject: [PATCH] =?UTF-8?q?MOD:=E6=94=AF=E6=8C=81=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?=E5=85=B6=E4=BB=96=E7=9B=AE=E5=BD=95=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 40 +++- data/dome.txt | 24 +++ .../com/leosam/tvbox/mv/MainVerticle.java | 27 ++- .../leosam/tvbox/mv/lucene/MvSearcher.java | 33 +++ .../leosam/tvbox/mv/service/MvService.java | 201 +++++++++++------- .../tvbox/mv/utils/ClassPathReaderUtils.java | 92 +++++++- .../tvbox/mv/utils/CollectionUtils.java | 4 + .../leosam/tvbox/mv/utils/StringUtils.java | 31 +++ .../resources/{tvbox => config}/home.json | 0 .../{tvbox => config}/singerPic.json | 0 src/main/resources/logback.xml | 4 +- src/main/resources/static/tvbox.json | 8 + 12 files changed, 372 insertions(+), 92 deletions(-) create mode 100644 data/dome.txt rename src/main/resources/{tvbox => config}/home.json (100%) rename src/main/resources/{tvbox => config}/singerPic.json (100%) diff --git a/README.md b/README.md index fba7fc8..d79f03a 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,11 @@ mv视频搜索服务 # 请求接口 -## Vod搜索接口 +## MV视频Vod搜索接口 http://localhost:7777/mv/vod?maxCount=100&wd=五月天后来的我们 +该接口只请求MV视频 + 请求参数定义 ~~~ wd ：搜索值，mv名称或者歌手名 都可以 @@ -31,8 +33,34 @@ maxCount：最大返回值，返回结果里面 list 最大数量。 最大值10 } ~~~ +## 自定义数据源-Vod搜索接口 +http://localhost:7777/vod/:index?maxCount=100&wd=五月天后来的我们 -## 搜素接口 +### 使用方法 +1. 在`jar`所在文件目录下创建`data`文件夹 +2. 将自定义数据源文件放入`data`文件夹下 +3. 自定义数据源名称为数据源文件名，如`dome.json`，则数据源名称为`dome` +4. 自定义数据源文件格式如下： `视频名称,视频链接` +4. 请求接口`http://localhost:7777/vod/dome?maxCount=100&wd=五月天后来的我们` + +请求路径参数定义 +~~~ +:index ：数据源名称，如：dome，不带文件后缀 +~~~ + +请求参数定义 +~~~ +wd ：搜索值，mv名称或者歌手名 都可以 +ids ：vodId,同wd +maxCount：最大返回值，返回结果里面 list 最大数量。 最大值1000 +~~~ +返回参数结果 +~~~ +同上 +~~~ + + +## MV视频搜素接口 http://localhost:7777/mv/search?maxCount=100&query=五月天后来的我们 请求参数定义 @@ -81,6 +109,14 @@ maxCount：最大返回值，返回结果里面 list 最大数量。 最大值10 "searchable": 1, "quickSearch": 1, "filterable": 1 + }, { + "key": "MV_vod_DOME", + "name": "👀┃DOME┃视频", + "type": 1, + "api": "http://你自己域名:7777/dome", + "searchable": 1, + "quickSearch": 1, + "filterable": 1 } ] ~~~ diff --git a/data/dome.txt b/data/dome.txt new file mode 100644 index 0000000..4b61caa --- /dev/null +++ b/data/dome.txt @@ -0,0 +1,24 @@ +水陆古法养鱼池塘缸，保姆级教程,https://www.bilibili.com/video/BV1Bg4y1K7yC/ +伤心情歌,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgmvmc4gUo8NSjoQI.mp4 +等你等到我心痛,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg5fTW4gUoypKv_AQ.mp4 +高安杭娇_一生无悔,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgtL6q4gUojIS-mQE.mp4 +刚好遇见你,http://15799848.s21v.faiusr.com/58/ABUIABA6GAAg3q6M5gUohPidmQc.mp4 +甘心情愿爱着你.安东阳张怡诺,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgq_2a4gUo686M8wc.mp4 +风中花雨楼.任妙音,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgypLV4gUo-ZmUhwc.mp4 +分手.冷漠,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgwsfE4gUokp27lAQ.mp4 +放狠爱.慕容晓晓肖玄,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgpoLG4gUo0JOBggI.mp4 +渡红尘.张碧晨,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgkN-64wUorOL7mQI.mp4 +都说.龙梅子老猫,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAghPbQ7AUohPLwrAM.mp4 +丁当_野兽,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg0MOm4gUo-LnKpQE.mp4 +邓紫棋_喜欢你,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg6uyj4QUosO5G.mp4 +等你一万年.白雪,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgyKyM5gUooJurKA.mp4 +等你一万年,http://15799848.s21v.faiusr.com/58/ABUIABA6GAAgyKyM5gUooJurKA.mp4 +等到山花开_钟小冰,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAggcOv5wUo4pShggc.mp4 +当爱离别时,http://15799848.s21v.faiusr.com/58/ABUIABA6GAAgstTE4gUo856frgY.mp4 +当,http://15799848.s21v.faiusr.com/58/ABUIABA6GAAgpPTW4gUoz_DgdQ.mp4 +单身情歌_黄晓凤,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgjfPW4gUo4umD3QE.mp4 +错过了缘分错过你_候俊辉,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAgqfLW4gUogNTT1wY.mp4 +崔子格老猫_老婆最大,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg29is4gUopvmF8wI.mp4 +传奇_黄晓凤,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg9b62-AUomeie4gY.mp4 +成龙金喜善_美丽的神话,https://15799848.s21v.faiusr.com/58/ABUIABA6GAAg_tma4gUowLvk9wM.mp4 +陈慧娴-逝去的诺言,http://bizcommon.alicdn.com/l2nDqpMmn6DGHnWzZQA/i78cWdjCSglPJHDaedL%40%40ld.m3u8 \ No newline at end of file diff --git a/src/main/java/com/leosam/tvbox/mv/MainVerticle.java b/src/main/java/com/leosam/tvbox/mv/MainVerticle.java index 156ae1f..2478bc0 100644 --- a/src/main/java/com/leosam/tvbox/mv/MainVerticle.java +++ b/src/main/java/com/leosam/tvbox/mv/MainVerticle.java @@ -53,6 +53,8 @@ public class MainVerticle extends AbstractVerticle { Router router = Router.router(vertx); router.route("/mv/search").handler(this::searchMv); router.route("/mv/vod").handler(this::searchMvVod); + router.route("/vod").handler(this::searchVod); + router.route("/vod/:index").handler(this::searchVod); router.route("/*").handler(StaticHandler.create("static")); router.route().handler(this::home); httpServer.requestHandler(router); @@ -63,39 +65,52 @@ public class MainVerticle extends AbstractVerticle { logger.info("HTTP server started on port 7777"); } else { startPromise.fail(http.cause()); + logger.info("HTTP server fail, on port 7777, 启动失败，更换端口重试", http.cause()); } }); port = httpServer.actualPort(); } + private void searchMvVod(RoutingContext req) { + searchVod(req, MvService.MV_FILE_NAME); + } + + private void searchVod(RoutingContext req) { + searchVod(req, StringUtils.EMPTY); + } + /** * 参考 ... * @param req */ - private void searchMvVod(RoutingContext req) { + private void searchVod(RoutingContext req, String index) { // 获取参数 String wd = VertxUtils.queryParam(req, "wd"); String ids = VertxUtils.queryParam(req, "ids"); - String query = StringUtils.isNotEmpty(wd) ? wd : ids; + String query = StringUtils.trimToEmpty(StringUtils.defaultIfEmpty(wd, ids)); String ac = VertxUtils.queryParam(req, "ac"); String type = VertxUtils.queryParam(req, "t"); - //query = StringUtils.isNotEmpty(query) ? query : "我"; String maxCount = VertxUtils.queryParam(req, "maxCount"); int max = Math.min(Math.max(NumberUtils.toInt(maxCount, 200), 10), 1000); String pg = VertxUtils.queryParam(req, "pg"); int page = NumberUtils.toInt(pg, 0); + if(StringUtils.isEmpty(index)){ + String pathIndex = req.pathParam("index"); + String queryIndex = VertxUtils.queryParam(req, "index"); + index = StringUtils.trimToEmpty(StringUtils.defaultIfEmpty(pathIndex, queryIndex)); + } // 处理结果 VodResult vodResult = null; try { // 搜索 if (mvService != null && StringUtils.isNotEmpty(query) && page <= 0) { - vodResult = mvService.searchVod(query, max); + vodResult = mvService.searchVod(index, query, max); } // 首页 if (mvService != null && StringUtils.isEmpty(query)) { - vodResult = mvService.searchVodHome(type, page); + vodResult = mvService.searchVodHome(index, type, page); } } catch (Exception e) { @@ -121,7 +136,7 @@ public class MainVerticle extends AbstractVerticle { MvResult search = null; try { if (mvService != null) { - search = mvService.search(query, max); + search = mvService.search(null, query, max); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/main/java/com/leosam/tvbox/mv/lucene/MvSearcher.java b/src/main/java/com/leosam/tvbox/mv/lucene/MvSearcher.java index 189a644..902eec3 100644 --- a/src/main/java/com/leosam/tvbox/mv/lucene/MvSearcher.java +++ b/src/main/java/com/leosam/tvbox/mv/lucene/MvSearcher.java @@ -1,5 +1,6 @@ package com.leosam.tvbox.mv.lucene; +import com.leosam.tvbox.mv.utils.StringUtils; import io.vertx.ext.web.impl.LRUCache; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer; @@ -7,10 +8,14 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -55,6 +60,34 @@ public class MvSearcher { return docs; } + public TopDocs searchTopDocs(String field, String queryStr, String termField, String termQueryStr, int maxHit) throws Exception { + String cacheKey = field + "_" + queryStr + "_" + termField + "_" + termQueryStr + "_" + maxHit; + TopDocs topDocs = topDocsLruCache.get(cacheKey); + if (topDocs != null) { + return topDocs; + } + + BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); + // 全文检索 + if(StringUtils.isNotEmpty(queryStr)){ + Analyzer analyzer = new SmartChineseAnalyzer(); + // Analyzer analyzer = new StandardAnalyzer(); + QueryParser parser = new QueryParser(field, analyzer); + Query query = parser.parse(queryStr); + queryBuilder.add(query, BooleanClause.Occur.MUST); + } + // 关键字检索 + if (StringUtils.isNotEmpty(termQueryStr)) { + Query keywordQuery = new TermQuery(new Term(termField, termQueryStr)); + queryBuilder.add(keywordQuery, BooleanClause.Occur.MUST); + } + BooleanQuery combinedQuery = queryBuilder.build(); + + TopDocs docs = searcher.search(combinedQuery, maxHit); + topDocsLruCache.put(cacheKey, docs); + return docs; + } + public void search(String field, String queryStr) throws Exception { Analyzer analyzer = new SmartChineseAnalyzer(); // Analyzer analyzer = new StandardAnalyzer(); diff --git a/src/main/java/com/leosam/tvbox/mv/service/MvService.java b/src/main/java/com/leosam/tvbox/mv/service/MvService.java index 8f00ec7..482d8bb 100644 --- a/src/main/java/com/leosam/tvbox/mv/service/MvService.java +++ b/src/main/java/com/leosam/tvbox/mv/service/MvService.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -41,11 +40,16 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class MvService { private static final Logger logger = LoggerFactory.getLogger(MvService.class); + public static final String MV_FILE_NAME = "16wMV"; private static final String indexDirectoryPath = "index"; - private static final String MV_FILE = "tvbox/16wMV.txt"; private static final Map> homeConfigMap = new LinkedHashMap<>(); private static final Map singerMap = new LinkedHashMap<>(); private static final String HOME_KEY = "HOME"; + private static final List FILE_PATH_LIST = List.of( + "classpath:tvbox/16wMV.txt", + "./data/", + "../data/" + ); /** * 最低相似分数 */ @@ -53,15 +57,18 @@ public class MvService { private MvSearcher mvSearcher; - public MvResult search(String query, int max) throws Exception { - if (StringUtils.isEmpty(query) || mvSearcher == null) { + public MvResult search(String index, String query, int max) throws Exception { + if (mvSearcher == null || max > 10000) { + return new MvResult(); + } + if (StringUtils.isEmpty(index) && StringUtils.isEmpty(query)) { return new MvResult(); } StopWatch stopWatch = new StopWatch(); stopWatch.start(); - TopDocs topDocs = mvSearcher.searchTopDocs("name", query, max); + TopDocs topDocs = mvSearcher.searchTopDocs("name", query, "index", index, max); MvResult result = new MvResult(); result.setQuery(query); @@ -69,7 +76,7 @@ public class MvService { result.setList(new ArrayList<>(topDocs.scoreDocs.length)); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { // 分数太低 忽略 - if (scoreDoc.score < MIN_QUERY_SCORE) { + if (scoreDoc.score < MIN_QUERY_SCORE && result.getList().size() > 50) { break; } Document document = mvSearcher.getDocument(scoreDoc.doc); @@ -103,16 +110,16 @@ public class MvService { return result; } - public VodResult searchVod(String wd, int max) throws Exception { - MvResult search = search(wd, max); + public VodResult searchVod(String index, String wd, int max) throws Exception { + MvResult search = this.search(index, wd, max); if (CollectionUtils.isEmpty(search.getList())) { return new VodResult(); } Vod vod = new Vod(); vod.setVodId(wd); vod.setVodName(wd); - vod.setVodPlayFrom("mv"); - vod.setVodPic(getVodPic(wd)); + vod.setVodPlayFrom(StringUtils.defaultIfEmpty(index, "mv")); + vod.setVodPic(getVodPic(index, wd)); List playUrlList = new LinkedList<>(); Set vodActorList = new LinkedHashSet<>(); for (MvContent content : search.getList()) { @@ -130,7 +137,7 @@ public class MvService { return vodResult; } - public VodResult searchVodHome(String type, int page) throws Exception { + public VodResult searchVodHome(String index, String type, int page) throws Exception { // 防止分页请求 if (page > 1) { return new VodResult(); @@ -141,50 +148,68 @@ public class MvService { VodResult vodResult = new VodResult().init(); - // 分类 - if (StringUtils.isEmpty(type)) { - vodResult.setVodClassList(new ArrayList<>(homeConfigMap.size())); - Set keySet = homeConfigMap.keySet(); - for (String key : keySet) { - VodClass vodClass = new VodClass().setTypeId(key).setTypeName(key); - if (HOME_KEY.equalsIgnoreCase(key)) { - vodClass.setTypeName("热门"); + if (MV_FILE_NAME.equals(index)) { + // 分类 + if (StringUtils.isEmpty(type)) { + vodResult.setVodClassList(new ArrayList<>(homeConfigMap.size())); + Set keySet = homeConfigMap.keySet(); + for (String key : keySet) { + VodClass vodClass = new VodClass().setTypeId(key).setTypeName(key); + if (HOME_KEY.equalsIgnoreCase(key)) { + vodClass.setTypeName("热门"); + } + vodResult.getVodClassList().add(vodClass); } - vodResult.getVodClassList().add(vodClass); } - } - // 首页mv推荐 - String thisType = StringUtils.isNotEmpty(type) ? type : HOME_KEY; - List singerList = homeConfigMap.getOrDefault(thisType, new ArrayList<>()); - vodResult.setList(new ArrayList<>(singerList.size())); - for (String singer : singerList) { - Vod vod = new Vod(); - vod.setVodId(singer); - vod.setVodName(singer); - vod.setVodPlayFrom("mv"); - vod.setVodPic(getVodPic(singer)); - vodResult.getList().add(vod); + // 首页mv推荐 + String thisType = StringUtils.isNotEmpty(type) ? type : HOME_KEY; + List singerList = homeConfigMap.getOrDefault(thisType, new ArrayList<>()); + vodResult.setList(new ArrayList<>(singerList.size())); + for (String singer : singerList) { + Vod vod = new Vod(); + vod.setVodId(singer); + vod.setVodName(singer); + vod.setVodPlayFrom(StringUtils.defaultIfEmpty(index, "mv")); + vod.setVodPic(getVodPic(index, singer)); + vodResult.getList().add(vod); + } + stopWatch.stop(); + logger.info("加载首页成功，类型={}, 返回={}条, 耗时{}毫秒", thisType, vodResult.getList().size(), stopWatch.getTotalTimeMillis()); + } else { + vodResult.setList(new ArrayList<>()); + MvResult search = search(index, "", 100); + if (CollectionUtils.isNotEmpty(search.getList())) { + for (MvContent content : search.getList()) { + Vod vod = new Vod(); + vod.setVodId(content.getName()); + vod.setVodName(content.getName()); + vod.setVodPlayFrom(StringUtils.defaultIfEmpty(index, "mv")); + vod.setVodPic(getVodPic(index, content.getName())); + vodResult.getList().add(vod); + } + } + logger.info("加载首页成功，文件={}, 返回={}条, 耗时{}毫秒", index, vodResult.getList().size(), stopWatch.getTotalTimeMillis()); } - - stopWatch.stop(); - logger.info("加载首页成功，类型={}, 返回={}条, 耗时{}毫秒", thisType, vodResult.getList().size(), stopWatch.getTotalTimeMillis()); return vodResult; } - private String getVodPic(String vodName) { + private String getVodPic(String index, String vodName) { String vodPic = null; - if (StringUtils.isEmpty(vodName)) { - vodPic = "http://yanxuan.nosdn.127.net/b6fb987ce79f308949e44f5129a4b51c.jpeg"; - } else if (singerMap.containsKey(vodName)) { - vodPic = singerMap.get(vodName); - } else if (vodName.contains(",")) { - String[] split = vodName.split(",", 2); - if (singerMap.containsKey(split[0])) { - vodPic = singerMap.get(split[0]); + if (StringUtils.isNotEmpty(vodName)) { + if (singerMap.containsKey(vodName)) { + vodPic = singerMap.get(vodName); + } else if (vodName.contains(",")) { + String[] split = vodName.split(",", 2); + if (singerMap.containsKey(split[0])) { + vodPic = singerMap.get(split[0]); + } } } - vodPic = StringUtils.isNotEmpty(vodPic) ? vodPic : "http://yanxuan.nosdn.127.net/b6fb987ce79f308949e44f5129a4b51c.jpeg"; + if (StringUtils.isEmpty(vodPic) && StringUtils.isNotEmpty(index)) { + vodPic = singerMap.get(index); + } + vodPic = StringUtils.defaultIfEmpty(vodPic, "http://yanxuan.nosdn.127.net/b6fb987ce79f308949e44f5129a4b51c.jpeg"); return vodPic; } @@ -200,13 +225,13 @@ public class MvService { mvSearcher = new MvSearcher(indexAbsolutePath); // 搜索一下，看看是否加载成功 - search("五月天", 10); + search(MV_FILE_NAME, "五月天", 10); } private void buildHomeConfig() { // 首页配置 try { - String content = ClassPathReaderUtils.getContent("tvbox/home.json"); + String content = ClassPathReaderUtils.getContent("config/home.json"); Map> map = new ObjectMapper().readValue(content, new TypeReference>>() { }); homeConfigMap.putAll(map); @@ -216,7 +241,7 @@ public class MvService { } // 歌手详情配置 try { - String content = ClassPathReaderUtils.getContent("tvbox/singerPic.json"); + String content = ClassPathReaderUtils.getContent("config/singerPic.json"); Map map = new ObjectMapper().readValue(content, new TypeReference>() { }); singerMap.putAll(map); @@ -228,14 +253,17 @@ public class MvService { } private static void reBuildIndex(String indexDirectoryPath) throws IOException { + Set dataFilePath = getDataFilePath(); + if (dataFilePath.isEmpty()) { + return; + } File file = new File(indexDirectoryPath).getAbsoluteFile(); - int mvFileSize = ClassPathReaderUtils.getSize(MV_FILE); - File mvFileSizeTxt = Path.of(file.getAbsolutePath(), "" + mvFileSize + ".txt").toFile(); + long dataFileSize = ClassPathReaderUtils.getAllFileSize(dataFilePath); + File mvFileSizeTxt = Path.of(file.getAbsolutePath(), "" + dataFileSize + ".txt").toFile(); if (mvFileSizeTxt.exists()) { - logger.info("{}已经索引, 跳过索引", MV_FILE); + logger.info("所有数据都已经索引, 跳过索引"); return; } - StopWatch stopWatch = new StopWatch(); logger.info("重建索引中...."); @@ -243,32 +271,57 @@ public class MvService { // 清空以前的索引 stopWatch.start("清空以前索引"); logger.info("清空历史索引...."); - - if (file.isDirectory()) { - File[] files = file.listFiles(); - for (File file1 : files) { - if (file1.isFile()) { - file1.delete(); - } - } - } + ClassPathReaderUtils.deleteAllFile(file); logger.info("清空历史索引....完成"); stopWatch.stop(); + // 创建索引 stopWatch.start("创建索引"); logger.info("创建索引中...."); - AtomicInteger line = new AtomicInteger(); MvIndex mvIndex = new MvIndex(indexDirectoryPath); - ClassPathReaderUtils.getBufferedReader(MV_FILE).lines() - .filter(l -> l.contains(",h")) - .filter(l -> l.contains("-")) + for (String path : dataFilePath) { + buildIndex(mvIndex, path); + } + mvIndex.close(); + mvFileSizeTxt.createNewFile(); + stopWatch.stop(); + logger.info("重建索引中....完成,耗时 {} 毫秒, 索引位置：{}", stopWatch.getTotalTimeMillis(), file.getPath()); + } + + private static Set getDataFilePath() throws IOException { + Map pathMap = new LinkedHashMap<>(); + for (String filePath : FILE_PATH_LIST) { + if (filePath.startsWith(ClassPathReaderUtils.CLASS_PATH_PREFIX)) { + pathMap.put(filePath, new File(filePath).getName()); + continue; + } + Map directoryFiles = ClassPathReaderUtils.getDirectoryFiles(filePath); + if (directoryFiles.isEmpty()) { + continue; + } + pathMap.putAll(directoryFiles); + } + + logger.info("↓↓↓↓↓ 搜集数据文件目录 ↓↓↓↓↓↓"); + for (Map.Entry entry : pathMap.entrySet()) { + logger.info("*** 数据文件：{}", entry.getValue()); + } + logger.info("↑↑↑↑↑ 搜集数据文件目录 ↑↑↑↑↑↑"); + return new LinkedHashSet<>(pathMap.keySet()); + } + + private static void buildIndex(MvIndex mvIndex, String path) { + String fileName = ClassPathReaderUtils.extractFileNameWithoutExtension(path); + AtomicInteger line = new AtomicInteger(); + ClassPathReaderUtils.getBufferedReader(path).lines() + .filter(l -> l.contains(",http")) .forEach(l -> { - String[] split = l.split(",", 2); - if (split.length != 2) { + List split = StringUtils.reverseSplit(l, ",http", 2); + if (split.size() != 2) { return; } - String name = split[0].trim(); - String url = split[1].trim(); + String name = split.get(0).trim(); + String url = "http" + split.get(1).trim(); String shortUrl = url; if (shortUrl.startsWith("http://em.21dtv")) { shortUrl = shortUrl @@ -278,21 +331,19 @@ public class MvService { try { Document document = new Document(); document.add(new TextField("name", name, Field.Store.YES)); + document.add(new StringField("index", fileName, Field.Store.YES)); document.add(new StringField("url", shortUrl, Field.Store.YES)); mvIndex.indexFile(document); int i = line.incrementAndGet(); if (i % 10000 == 0) { - logger.info("创建索引中....已完成{}万条索引", i / 10000); + logger.info("创建{}索引中....已完成{}万条索引", fileName, i / 10000); } } catch (IOException e) { throw new RuntimeException(e); } }); - mvIndex.close(); - mvFileSizeTxt.createNewFile(); - logger.info("创建索引中....完成, 总共{}条", line.get()); - stopWatch.stop(); - logger.info("重建索引中....完成,耗时 {} 毫秒, 索引位置：{}", stopWatch.getTotalTimeMillis(), file.getPath()); + + logger.info("创建索引中....完成, 源文件:{}, 总共{}条", new File(path).getName(), line.get()); } diff --git a/src/main/java/com/leosam/tvbox/mv/utils/ClassPathReaderUtils.java b/src/main/java/com/leosam/tvbox/mv/utils/ClassPathReaderUtils.java index a3e8882..e31c945 100644 --- a/src/main/java/com/leosam/tvbox/mv/utils/ClassPathReaderUtils.java +++ b/src/main/java/com/leosam/tvbox/mv/utils/ClassPathReaderUtils.java @@ -2,11 +2,19 @@ package com.leosam.tvbox.mv.utils; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; /** * @author admin @@ -14,7 +22,71 @@ import java.nio.charset.StandardCharsets; */ public class ClassPathReaderUtils { - public static int getSize(String path) { + public static final String CLASS_PATH_PREFIX = "classpath:"; + + public static Map getDirectoryFiles(String filePath) throws IOException { + Map pathMap = new LinkedHashMap<>(); + if (StringUtils.isEmpty(filePath) || filePath.startsWith(CLASS_PATH_PREFIX)) { + return pathMap; + } + File data = new File(filePath); + if (data.exists() && data.isFile()) { + pathMap.put(data.getCanonicalPath(), data.getName()); + } + if (data.exists() && data.isDirectory()) { + File[] files = data.listFiles(); + for (File file : files) { + if (file.isFile()) { + pathMap.put(filePath + file.getName(), file.getName()); + } + } + } + return pathMap; + } + + public static String extractFileNameWithoutExtension(String filePath) { + Path path = Paths.get(filePath); + String fileNameWithExtension = path.getFileName().toString(); + int lastDotIndex = fileNameWithExtension.lastIndexOf("."); + String fileNameWithoutExtension = lastDotIndex == -1 ? fileNameWithExtension + : fileNameWithExtension.substring(0, lastDotIndex); + return fileNameWithoutExtension; + } + + public static void deleteAllFile(File file) { + if (file == null) { + return; + } + if (file.isFile()) { + file.delete(); + } + if (file.isDirectory()) { + File[] files = file.listFiles(); + for (File file1 : files) { + if (file1.isFile()) { + file1.delete(); + } + } + } + } + + public static long getAllFileSize(Set dataPath) { + long size = 0; + for (String path : dataPath) { + if (path.startsWith(CLASS_PATH_PREFIX)) { + String pathName = path.replace(CLASS_PATH_PREFIX, ""); + size += getClassPathFileSize(pathName); + } else { + File file = new File(path); + if (file.exists() && file.isFile()) { + size += file.length(); + } + } + } + return size; + } + + public static int getClassPathFileSize(String path) { try { ClassPathResource resource = new ClassPathResource(path); InputStream inputStream = resource.getInputStream(); @@ -38,10 +110,14 @@ public class ClassPathReaderUtils { public static BufferedReader getBufferedReader(String path) { try { - ClassPathResource resource = new ClassPathResource(path); - - - InputStreamReader inputStreamReader = new InputStreamReader(resource.getInputStream(),StandardCharsets.UTF_8); + InputStreamReader inputStreamReader = null; + if (path.startsWith(CLASS_PATH_PREFIX)) { + path = path.replace(CLASS_PATH_PREFIX, ""); + ClassPathResource resource = new ClassPathResource(path); + inputStreamReader = new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8); + } else { + inputStreamReader = new InputStreamReader(new FileInputStream(path), StandardCharsets.UTF_8); + } BufferedReader reader = new BufferedReader(inputStreamReader); return reader; } catch (IOException ex) { @@ -79,6 +155,9 @@ public class ClassPathReaderUtils { private final ClassLoader classLoader; public ClassPathResource(String path) { + if (path.startsWith(CLASS_PATH_PREFIX)) { + path = path.replace(CLASS_PATH_PREFIX, ""); + } this.path = path; this.classLoader = getDefaultClassLoader(); } @@ -96,8 +175,7 @@ public class ClassPathReaderUtils { return is; } - - public static ClassLoader getDefaultClassLoader() { + private static ClassLoader getDefaultClassLoader() { ClassLoader cl = null; try { cl = Thread.currentThread().getContextClassLoader(); diff --git a/src/main/java/com/leosam/tvbox/mv/utils/CollectionUtils.java b/src/main/java/com/leosam/tvbox/mv/utils/CollectionUtils.java index cb77e8a..125f9b3 100644 --- a/src/main/java/com/leosam/tvbox/mv/utils/CollectionUtils.java +++ b/src/main/java/com/leosam/tvbox/mv/utils/CollectionUtils.java @@ -12,4 +12,8 @@ public class CollectionUtils { return (collection == null || collection.isEmpty()); } + public static boolean isNotEmpty(Collection collection) { + return !isEmpty(collection); + } + } diff --git a/src/main/java/com/leosam/tvbox/mv/utils/StringUtils.java b/src/main/java/com/leosam/tvbox/mv/utils/StringUtils.java index 3488d44..bffa3d0 100644 --- a/src/main/java/com/leosam/tvbox/mv/utils/StringUtils.java +++ b/src/main/java/com/leosam/tvbox/mv/utils/StringUtils.java @@ -1,11 +1,16 @@ package com.leosam.tvbox.mv.utils; +import java.util.ArrayList; +import java.util.List; + /** * @author admin * @since 2023/6/12 21:32 */ public class StringUtils { + public static final String EMPTY = ""; + public static boolean isNotEmpty(final CharSequence cs) { return !isEmpty(cs); } @@ -13,4 +18,30 @@ public class StringUtils { public static boolean isEmpty(final CharSequence cs) { return cs == null || cs.length() == 0; } + + public static String defaultIfEmpty(final String str, String defaultStr) { + return isNotEmpty(str) ? str : defaultStr; + } + + public static String trimToEmpty(final String str) { + return str == null ? EMPTY : str.trim(); + } + + public static List reverseSplit(String s, String delimiter, int limit) { + List result = new ArrayList<>(); + if (s == null || delimiter == null || limit <= 0) { + return result; + } + + int lastIndex = s.lastIndexOf(delimiter); + while (limit > 1 && lastIndex != -1) { + result.add(0, s.substring(lastIndex + delimiter.length())); + s = s.substring(0, lastIndex); + lastIndex = s.lastIndexOf(delimiter); + limit--; + } + result.add(0, s); + return result; + } + } diff --git a/src/main/resources/tvbox/home.json b/src/main/resources/config/home.json similarity index 100% rename from src/main/resources/tvbox/home.json rename to src/main/resources/config/home.json diff --git a/src/main/resources/tvbox/singerPic.json b/src/main/resources/config/singerPic.json similarity index 100% rename from src/main/resources/tvbox/singerPic.json rename to src/main/resources/config/singerPic.json diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 189020b..fe16a68 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -5,8 +5,8 @@ - - + + diff --git a/src/main/resources/static/tvbox.json b/src/main/resources/static/tvbox.json index 426dcc0..2d0c3eb 100644 --- a/src/main/resources/static/tvbox.json +++ b/src/main/resources/static/tvbox.json @@ -8,6 +8,14 @@ "searchable": 1, "quickSearch": 1, "filterable": 1 + }, { + "key": "MV_vod_2", + "name": "👀┃DOME┃视频", + "type": 1, + "api": "./vod/dome", + "searchable": 1, + "quickSearch": 1, + "filterable": 1 } ] } \ No newline at end of file