yeyinly/tvbox-mv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Init:初次提交

Browse Source
This commit is contained in:
leosam1024
2023-06-11 13:51:12 +08:00
parent bc1ebd9549
commit f3b147d470
17 changed files with 165624 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/leosam/tvbox/mv/TvboxMvApplication.java
+13
View File
@@ -0,0 +1,13 @@
package com.leosam.tvbox.mv;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class TvboxMvApplication {
public static void main(String[] args) {
SpringApplication.run(TvboxMvApplication.class, args);
}
}
src/main/java/com/leosam/tvbox/mv/controller/IndexController.java
+26
View File
@@ -0,0 +1,26 @@
package com.leosam.tvbox.mv.controller;
import com.leosam.tvbox.mv.data.MvResult;
import com.leosam.tvbox.mv.service.MvService;
import com.leosam.tvbox.mv.utils.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author admin
* @since 2023/6/10 17:34
*/
@RestController
public class IndexController {
@Autowired
private MvService mvService;
@RequestMapping(value = {"/mv/search"})
public MvResult searchMv(String query, String maxCount) throws Exception {
int max = Math.min(Math.max(NumberUtils.toInt(maxCount, 100), 10), 1000);
return mvService.search(query, max);
}
}
src/main/java/com/leosam/tvbox/mv/data/MvContent.java
+59
View File
@@ -0,0 +1,59 @@
package com.leosam.tvbox.mv.data;
/**
* @author admin
* @since 2023/6/10 19:22
*/
public class MvContent {
private String name;
private String songName;
private String songUser;
private String url;
public float getScore() {
return score;
}
public MvContent setScore(float score) {
this.score = score;
return this;
}
private float score;
public String getName() {
return name;
}
public MvContent setName(String name) {
this.name = name;
return this;
}
public String getSongName() {
return songName;
}
public MvContent setSongName(String songName) {
this.songName = songName;
return this;
}
public String getSongUser() {
return songUser;
}
public MvContent setSongUser(String songUser) {
this.songUser = songUser;
return this;
}
public String getUrl() {
return url;
}
public MvContent setUrl(String url) {
this.url = url;
return this;
}
}
src/main/java/com/leosam/tvbox/mv/data/MvResult.java
+41
View File
@@ -0,0 +1,41 @@
package com.leosam.tvbox.mv.data;
import java.util.List;
/**
* @author admin
* @since 2023/6/10 19:20
*/
public class MvResult {
private String query;
private long totalHits;
private List<MvContent> list;
public String getQuery() {
return query;
}
public MvResult setQuery(String query) {
this.query = query;
return this;
}
public long getTotalHits() {
return totalHits;
}
public MvResult setTotalHits(long totalHits) {
this.totalHits = totalHits;
return this;
}
public List<MvContent> getList() {
return list;
}
public MvResult setList(List<MvContent> list) {
this.list = list;
return this;
}
}
src/main/java/com/leosam/tvbox/mv/lucene/MvIndex.java
+53
View File
@@ -0,0 +1,53 @@
package com.leosam.tvbox.mv.lucene;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class MvIndex {
private IndexWriter writer;
public MvIndex(String indexDirectoryPath) throws IOException {
// 使用标准分析器
Analyzer analyzer = new SmartChineseAnalyzer();
// Analyzer analyzer = new StandardAnalyzer();
// 创建索引配置
IndexWriterConfig config = new IndexWriterConfig(analyzer);
// 打开索引目录
Path indexPath = Paths.get(indexDirectoryPath);
Directory indexDirectory = FSDirectory.open(indexPath);
// 创建索引写入器
writer = new IndexWriter(indexDirectory, config);
}
public void close() throws IOException {
writer.commit();
writer.close();
}
public void indexFile(Document document) throws IOException {
writer.addDocument(document);
}
public void indexFile(String name, String url) throws IOException {
Document document = new Document();
document.add(new TextField("name", name, Field.Store.YES));
document.add(new StringField("url", url, Field.Store.YES));
writer.addDocument(document);
}
}
src/main/java/com/leosam/tvbox/mv/lucene/MvSearcher.java
+68
View File
@@ -0,0 +1,68 @@
package com.leosam.tvbox.mv.lucene;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* @author admin
* @since 2023/6/8 21:44
*/
public class MvSearcher {
private IndexReader reader;
private IndexSearcher searcher;
public MvSearcher(String indexDirectoryPath) throws IOException {
Path indexPath = Paths.get(indexDirectoryPath);
Directory directory = FSDirectory.open(indexPath);
reader = DirectoryReader.open(directory);
searcher = new IndexSearcher(reader);
}
public TopDocs searchTopDocs(String field, String queryStr, int maxHit) throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer();
// Analyzer analyzer = new StandardAnalyzer();
QueryParser parser = new QueryParser(field, analyzer);
Query query = parser.parse(queryStr);
TopDocs docs = searcher.search(query, maxHit);
return docs;
}
public void search(String field, String queryStr) throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer();
// Analyzer analyzer = new StandardAnalyzer();
QueryParser parser = new QueryParser(field, analyzer);
Query query = parser.parse(queryStr);
TopDocs docs = searcher.search(query, 200);
// 处理搜索结果
for (ScoreDoc scoreDoc : docs.scoreDocs) {
Document document = searcher.doc(scoreDoc.doc);
System.out.println(document.get("name") + "," + document.get("url") + " score=" + scoreDoc.score);
}
}
public Document getDocument(int docID) throws IOException {
Document document = searcher.doc(docID);
return document;
}
public void close() throws IOException {
reader.close();
}
}
src/main/java/com/leosam/tvbox/mv/service/MvService.java
+154
View File
@@ -0,0 +1,154 @@
package com.leosam.tvbox.mv.service;
import com.leosam.tvbox.mv.data.MvContent;
import com.leosam.tvbox.mv.data.MvResult;
import com.leosam.tvbox.mv.lucene.MvIndex;
import com.leosam.tvbox.mv.lucene.MvSearcher;
import com.leosam.tvbox.mv.utils.ClassPathReaderUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author admin
* @since 2023/6/10 17:35
*/
@Service
public class MvService implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(MvService.class);
private static final String indexDirectoryPath = "index";
private MvSearcher mvSearcher;
public MvResult search(String query, int max) throws Exception {
if (!StringUtils.hasText(query)) {
return new MvResult();
}
StopWatch stopWatch = new StopWatch();
stopWatch.start();
TopDocs topDocs = mvSearcher.searchTopDocs("name", query, max);
MvResult result = new MvResult();
result.setQuery(query);
result.setTotalHits(topDocs.totalHits.value);
result.setList(new LinkedList<>());
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Document document = mvSearcher.getDocument(scoreDoc.doc);
if (document == null) {
continue;
}
String name = document.get("name");
String songName = name;
String songUser = null;
if (name.contains("-")) {
songUser = name.split("-", 2)[0];
songName = name.split("-", 2)[1];
}
String url = document.get("url");
if (!url.startsWith("http")) {
url = "http://em.21dtv.com/songs/" + url + ".mkv";
}
MvContent content = new MvContent();
content.setName(name);
content.setSongName(songName);
content.setSongUser(songUser);
content.setUrl(url);
content.setScore(scoreDoc.score);
result.getList().add(content);
}
stopWatch.stop();
logger.info("查询MV成功，query={}, 命中{}条, 返回={}条, 耗时{}毫秒", query, result.getTotalHits(), result.getList().size(), stopWatch.getTotalTimeMillis());
return result;
}
@Override
public void afterPropertiesSet() throws Exception {
// 重建索引
String indexAbsolutePath = new File(indexDirectoryPath).getAbsoluteFile().getAbsolutePath();
reBuildIndex(indexAbsolutePath);
// 加载索引
mvSearcher = new MvSearcher(indexAbsolutePath);
// 搜索一下，看看是否加载成功
search("五月天", 10);
}
private static void reBuildIndex(String indexDirectoryPath) throws IOException {
StopWatch stopWatch = new StopWatch();
logger.info("重建索引中....");
// 清空以前的索引
stopWatch.start("清空以前索引");
logger.info("清空历史索引....");
File file = new File(indexDirectoryPath).getAbsoluteFile();
if (file.isDirectory()) {
File[] files = file.listFiles();
for (File file1 : files) {
if (file1.isFile()) {
file1.delete();
}
}
}
logger.info("清空历史索引....完成");
stopWatch.stop();
stopWatch.start("创建索引");
logger.info("创建索引中....");
AtomicInteger line = new AtomicInteger();
MvIndex mvIndex = new MvIndex(indexDirectoryPath);
ClassPathReaderUtils.getBufferedReader("tvbox/16wMV.txt").lines()
.filter(l -> l.contains(",h"))
.filter(l -> l.contains("-"))
.forEach(l -> {
String[] split = l.split(",", 2);
if (split.length != 2) {
return;
}
String name = split[0].trim();
String url = split[1].trim();
String shortUrl = url;
if (shortUrl.startsWith("http://em.21dtv")) {
shortUrl = shortUrl
.replace("http://em.21dtv.com/songs/", "")
.replace(".mkv", "");
}
try {
Document document = new Document();
document.add(new TextField("name", name, Field.Store.YES));
document.add(new StringField("url", shortUrl, Field.Store.YES));
mvIndex.indexFile(document);
int i = line.incrementAndGet();
if (i % 10000 == 0) {
logger.info("创建索引中....已完成{}万条索引", i / 10000);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
mvIndex.close();
logger.info("创建索引中....完成, 总共{}条", line.get());
stopWatch.stop();
logger.info("重建索引中....完成,耗时 {} 毫秒, 索引位置：{}", stopWatch.getTotalTimeMillis(), file.getPath());
}
}
src/main/java/com/leosam/tvbox/mv/utils/ClassPathReaderUtils.java
+74
View File
@@ -0,0 +1,74 @@
package com.leosam.tvbox.mv.utils;
import org.springframework.core.io.ClassPathResource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Properties;
/**
* @author admin
* @since 2023/6/10 17:31
*/
public class ClassPathReaderUtils {
public static InputStreamReader getInputStreamReader(String path) {
try {
ClassPathResource resource = new ClassPathResource(path);
InputStreamReader inputStreamReader = new InputStreamReader(resource.getInputStream(),StandardCharsets.UTF_8);
return inputStreamReader;
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
public static BufferedReader getBufferedReader(String path) {
try {
ClassPathResource resource = new ClassPathResource(path);
InputStreamReader inputStreamReader = new InputStreamReader(resource.getInputStream(),StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(inputStreamReader);
return reader;
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
public static Properties getProperties(String path) {
Properties properties = new Properties();
try {
ClassPathResource resource = new ClassPathResource(path);
InputStreamReader inputStreamReader = new InputStreamReader(resource.getInputStream());
properties.load(new BufferedReader(inputStreamReader));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
return properties;
}
public static String getContent(String path) {
StringBuilder content = new StringBuilder();
try {
ClassPathResource resource = new ClassPathResource(path);
InputStreamReader inputStreamReader = new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(inputStreamReader);
String data = null;
while ((data = reader.readLine()) != null) {
content.append(data).append("\n");
}
reader.close();
return content.toString();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
public static void main(String[] args) {
String json = getContent("./tvbox/16wMV.txt");
System.out.println(json);
}
}
src/main/java/com/leosam/tvbox/mv/utils/NumberUtils.java
+20
View File
@@ -0,0 +1,20 @@
package com.leosam.tvbox.mv.utils;
/**
* @author admin
* @since 2023/6/10 19:24
*/
public class NumberUtils {
public static int toInt(final String str, final int defaultValue) {
if (str == null) {
return defaultValue;
}
try {
return Integer.parseInt(str);
} catch (final NumberFormatException nfe) {
return defaultValue;
}
}
}
src/main/resources/application.yaml
+34
View File
@@ -0,0 +1,34 @@
server:
port: 7777
spring:
profiles:
active: test
logging:
file:
name: ./logs/tvbox-mv.log
level:
.: error
org.spring.framework: debug
com.leosam: debug
---
server:
port: 7777
spring:
config:
activate:
on-profile: test
logging:
file:
name: ./logs/tvbox-mv.log
---
server:
port: 7777
spring:
config:
activate:
on-profile: linux
logging:
file:
name: /opt/logs/tvbox-mv.log
src/main/resources/tvbox/16wMV.txt
+164416
View File
File diff suppressed because it is too large Load Diff
src/test/java/com/leosam/tvbox/mv/TvboxMvApplicationTests.java
+13
View File
@@ -0,0 +1,13 @@
package com.leosam.tvbox.mv;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class TvboxMvApplicationTests {
@Test
void contextLoads() {
}
}