fix(rag): 补充 ImpactRAG 分支测试与防御,覆盖率达 100%
This commit is contained in:
@@ -36,12 +36,14 @@ class ImpactRAG:
|
|||||||
chunks = []
|
chunks = []
|
||||||
for name, text in sources:
|
for name, text in sources:
|
||||||
for piece in _split(text):
|
for piece in _split(text):
|
||||||
chunks.append(f"[{name}]\n{piece}")
|
# 跳过空片段,避免产生空向量噪声 chunk
|
||||||
|
if piece.strip():
|
||||||
|
chunks.append(f"[{name}]\n{piece}")
|
||||||
embs = self.embedder.embed(chunks)
|
embs = self.embedder.embed(chunks)
|
||||||
self.store.add(scope, chunks, embs)
|
self.store.add(scope, chunks, embs)
|
||||||
|
|
||||||
def index_dir(self, scope: str, root: str) -> int:
|
def index_dir(self, scope: str, root: str) -> int:
|
||||||
"""D1:索引目录内文本源文件(既有系统源码)。返回索引的片段数。"""
|
"""D1:索引目录内文本源文件(既有系统源码)。返回索引的文件数。"""
|
||||||
root_path = Path(root)
|
root_path = Path(root)
|
||||||
sources: List[Tuple[str, str]] = []
|
sources: List[Tuple[str, str]] = []
|
||||||
if root_path.is_dir():
|
if root_path.is_dir():
|
||||||
@@ -50,9 +52,15 @@ class ImpactRAG:
|
|||||||
try:
|
try:
|
||||||
text = p.read_text(encoding="utf-8", errors="ignore")
|
text = p.read_text(encoding="utf-8", errors="ignore")
|
||||||
except Exception:
|
except Exception:
|
||||||
|
# 读取异常的文件跳过,不中断整体索引
|
||||||
continue
|
continue
|
||||||
if text.strip():
|
# 空内容文件跳过
|
||||||
sources.append((str(p.relative_to(root_path)), text))
|
if not text.strip():
|
||||||
|
continue
|
||||||
|
# 含 NUL 字节的疑似二进制文件(误带白名单扩展名)跳过
|
||||||
|
if "\x00" in text:
|
||||||
|
continue
|
||||||
|
sources.append((str(p.relative_to(root_path)), text))
|
||||||
if sources:
|
if sources:
|
||||||
self.index(scope, sources)
|
self.index(scope, sources)
|
||||||
return len(sources)
|
return len(sources)
|
||||||
|
|||||||
@@ -33,3 +33,124 @@ def test_index_dir_reads_text_files(tmp_path):
|
|||||||
assert res and "OrderController" in res[0]
|
assert res and "OrderController" in res[0]
|
||||||
finally:
|
finally:
|
||||||
store.close()
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieve_empty_query_returns_empty():
|
||||||
|
# 空 query(仅空白)直接返回空列表,不调用向量检索
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
rag.index("p1", [("a.java", "创建订单")])
|
||||||
|
assert rag.retrieve("p1", " ", k=3) == []
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_empty_text_source_not_indexed():
|
||||||
|
# 空文本源:_split 回退为空片段,过滤后无 chunk
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
rag.index("p", [("empty.txt", "")])
|
||||||
|
assert rag.retrieve("p", "x", k=3) == []
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_non_dir_root_returns_zero():
|
||||||
|
# 非目录 root:is_dir() 为 False,返回 0
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
assert rag.index_dir("p", "不存在的路径") == 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_skips_empty_file(tmp_path):
|
||||||
|
# 空内容 .txt 文件不应被索引
|
||||||
|
(tmp_path / "empty.txt").write_text("", encoding="utf-8")
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
assert rag.index_dir("p", str(tmp_path)) == 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_skips_read_error(tmp_path, monkeypatch):
|
||||||
|
# 读取异常的文件被 except 跳过,整体不崩
|
||||||
|
(tmp_path / "a.txt").write_text("创建订单", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
__import__("pathlib").Path,
|
||||||
|
"read_text",
|
||||||
|
lambda *a, **k: (_ for _ in ()).throw(Exception),
|
||||||
|
)
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
assert rag.index_dir("p", str(tmp_path)) == 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_uppercase_ext_indexed(tmp_path):
|
||||||
|
# 大写扩展名(.JAVA)应经 .lower() 命中白名单并被索引
|
||||||
|
(tmp_path / "Foo.JAVA").write_text(
|
||||||
|
"public class Foo { 创建订单 }", encoding="utf-8"
|
||||||
|
)
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
n = rag.index_dir("p", str(tmp_path))
|
||||||
|
assert n == 1
|
||||||
|
res = rag.retrieve("p", "创建订单", k=1)
|
||||||
|
assert res and "Foo" in res[0]
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_empty_dir_returns_zero(tmp_path):
|
||||||
|
# 目录内无文本文件(sources 为空)返回 0
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
assert rag.index_dir("p", str(tmp_path)) == 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_dir_skips_nul_binary_text(tmp_path):
|
||||||
|
# 含 NUL 字节的 .txt 视为二进制误带扩展名,应跳过
|
||||||
|
(tmp_path / "bad.txt").write_bytes(b"hello\x00world")
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
assert rag.index_dir("p", str(tmp_path)) == 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_splits_long_single_paragraph():
|
||||||
|
# 单段落超长:触发 len(buf)+len(p) > size 且 buf 为空分支 -> 整体作为 1 个 chunk
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
rag.index("p", [("long.txt", "丙" * 900)])
|
||||||
|
chunks = rag.retrieve("p", "丙", k=5)
|
||||||
|
assert len(chunks) == 1
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_index_splits_multiple_paragraphs():
|
||||||
|
# 多段落(含空行):触发拆行、空白行过滤、and buf 真分支 -> 2 个 chunk
|
||||||
|
store = RagStore(":memory:")
|
||||||
|
try:
|
||||||
|
rag = ImpactRAG(store, FakeEmbedder())
|
||||||
|
text = "甲" * 500 + "\n\n" + "乙" * 500
|
||||||
|
rag.index("p", [("multi.txt", text)])
|
||||||
|
chunks = rag.retrieve("p", "甲", k=5)
|
||||||
|
assert len(chunks) == 2
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|||||||
Reference in New Issue
Block a user