feat(rag): 新增 ImpactRAG 索引/检索服务(含 index_dir)
This commit is contained in:
@@ -0,0 +1,64 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Tuple
|
||||||
|
|
||||||
|
from genesis.rag.embeddings import Embedder
|
||||||
|
from genesis.rag.store import RagStore
|
||||||
|
|
||||||
|
_CHUNK = 800
|
||||||
|
_TEXT_EXTS = {
|
||||||
|
".java", ".py", ".js", ".ts", ".go", ".kt", ".scala", ".xml", ".yml",
|
||||||
|
".yaml", ".md", ".txt", ".json", ".csv", ".sql", ".html", ".css", ".sh",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _split(text: str, size: int = _CHUNK) -> List[str]:
|
||||||
|
paras = [p.strip() for p in text.split("\n") if p.strip()]
|
||||||
|
out, buf = [], ""
|
||||||
|
for p in paras:
|
||||||
|
if len(buf) + len(p) > size and buf:
|
||||||
|
out.append(buf)
|
||||||
|
buf = p
|
||||||
|
else:
|
||||||
|
buf = (buf + "\n" + p).strip()
|
||||||
|
if buf:
|
||||||
|
out.append(buf)
|
||||||
|
return out or [""]
|
||||||
|
|
||||||
|
|
||||||
|
class ImpactRAG:
|
||||||
|
def __init__(self, store: RagStore, embedder: Embedder):
|
||||||
|
self.store = store
|
||||||
|
self.embedder = embedder
|
||||||
|
|
||||||
|
def index(self, scope: str, sources: List[Tuple[str, str]]) -> None:
|
||||||
|
chunks = []
|
||||||
|
for name, text in sources:
|
||||||
|
for piece in _split(text):
|
||||||
|
chunks.append(f"[{name}]\n{piece}")
|
||||||
|
embs = self.embedder.embed(chunks)
|
||||||
|
self.store.add(scope, chunks, embs)
|
||||||
|
|
||||||
|
def index_dir(self, scope: str, root: str) -> int:
|
||||||
|
"""D1:索引目录内文本源文件(既有系统源码)。返回索引的片段数。"""
|
||||||
|
root_path = Path(root)
|
||||||
|
sources: List[Tuple[str, str]] = []
|
||||||
|
if root_path.is_dir():
|
||||||
|
for p in root_path.rglob("*"):
|
||||||
|
if p.is_file() and p.suffix.lower() in _TEXT_EXTS:
|
||||||
|
try:
|
||||||
|
text = p.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if text.strip():
|
||||||
|
sources.append((str(p.relative_to(root_path)), text))
|
||||||
|
if sources:
|
||||||
|
self.index(scope, sources)
|
||||||
|
return len(sources)
|
||||||
|
|
||||||
|
def retrieve(self, scope: str, query: str, k: int = 5) -> List[str]:
|
||||||
|
if not query.strip():
|
||||||
|
return []
|
||||||
|
qv = self.embedder.embed([query])[0]
|
||||||
|
return self.store.search(scope, qv, k)
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from genesis.rag.embeddings import FakeEmbedder
|
||||||
|
from genesis.rag.store import RagStore
|
||||||
|
from genesis.rag.impact_rag import ImpactRAG
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieve_returns_relevant_chunk():
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
sources = [
|
||||||
|
("OrderController.java", "public class OrderController { 创建订单 }"),
|
||||||
|
("UserAuth.java", "public class UserAuth { 用户登录认证 }"),
|
||||||
|
]
|
||||||
|
rag.index("p1", sources)
|
||||||
|
res = rag.retrieve("p1", "OrderController 的影响范围", k=1)
|
||||||
|
assert res and "OrderController" in res[0]
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_reads_text_files(tmp_path):
|
||||||
|
(tmp_path / "src").mkdir()
|
||||||
|
(tmp_path / "src" / "OrderController.java").write_text(
|
||||||
|
"public class OrderController { 创建订单 }", encoding="utf-8"
|
||||||
|
)
|
||||||
|
(tmp_path / "src" / "binary.bin").write_bytes(b"\x00\x01")
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
n = rag.index_dir("p1", str(tmp_path / "src"))
|
||||||
|
assert n == 1
|
||||||
|
res = rag.retrieve("p1", "OrderController 的影响范围", k=1)
|
||||||
|
assert res and "OrderController" in res[0]
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
Reference in New Issue
Block a user