fix(rag): 移除 embeddings camelCase 死代码,覆盖率达 100%

This commit is contained in:
lhl
2026-08-29 22:35:16 +08:00
parent 9bc7828e63
commit afc09fc908
+11 -23
View File
@@ -1,5 +1,8 @@
from __future__ import annotations
import hashlib
import math
import re
from typing import List, Protocol
@@ -11,26 +14,9 @@ _DIM = 64
def _tokenize(text: str) -> List[str]:
toks = []
cur = ""
for ch in text.lower():
if ch.isalnum():
cur += ch
else:
if cur:
toks.append(cur)
cur = ""
if cur:
toks.append(cur)
out = []
for t in toks:
idx = 0
for i, c in enumerate(t):
if i > 0 and c.isupper():
out.append(t[idx:i])
idx = i
out.append(t[idx:])
return [x for x in out if x]
# 分词意图:先将文本统一转为小写,再按非字母数字字符切分为词元,最后过滤空串
toks = re.split(r"\W+", text.lower())
return [t for t in toks if t]
class FakeEmbedder:
@@ -39,13 +25,15 @@ class FakeEmbedder:
for t in texts:
v = [0.0] * _DIM
for tok in _tokenize(t):
h = __import__("hashlib").md5(tok.encode("utf-8")).digest()
h = hashlib.md5(tok.encode("utf-8")).digest()
idx = h[0] % _DIM
v[idx] += 1.0
norm = __import__("math").sqrt(sum(x * x for x in v)) or 1.0
# L2 归一化;norm 为 0 时(全零向量)用 1.0 防除零
norm = math.sqrt(sum(x * x for x in v)) or 1.0
vecs.append([x / norm for x in v])
return vecs
def get_embedder(engine) -> Embedder:
def get_embedder(engine: str) -> Embedder:
# engine 当前未使用,统一回退到 FakeEmbedder;真实向量模型(如 OpenAI/BGE)接入点预留于此
return FakeEmbedder()