fix(rag): 移除 embeddings camelCase 死代码,覆盖率达 100%
This commit is contained in:
@@ -1,5 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import math
|
||||||
|
import re
|
||||||
from typing import List, Protocol
|
from typing import List, Protocol
|
||||||
|
|
||||||
|
|
||||||
@@ -11,26 +14,9 @@ _DIM = 64
|
|||||||
|
|
||||||
|
|
||||||
def _tokenize(text: str) -> List[str]:
|
def _tokenize(text: str) -> List[str]:
|
||||||
toks = []
|
# 分词意图:先将文本统一转为小写,再按非字母数字字符切分为词元,最后过滤空串
|
||||||
cur = ""
|
toks = re.split(r"\W+", text.lower())
|
||||||
for ch in text.lower():
|
return [t for t in toks if t]
|
||||||
if ch.isalnum():
|
|
||||||
cur += ch
|
|
||||||
else:
|
|
||||||
if cur:
|
|
||||||
toks.append(cur)
|
|
||||||
cur = ""
|
|
||||||
if cur:
|
|
||||||
toks.append(cur)
|
|
||||||
out = []
|
|
||||||
for t in toks:
|
|
||||||
idx = 0
|
|
||||||
for i, c in enumerate(t):
|
|
||||||
if i > 0 and c.isupper():
|
|
||||||
out.append(t[idx:i])
|
|
||||||
idx = i
|
|
||||||
out.append(t[idx:])
|
|
||||||
return [x for x in out if x]
|
|
||||||
|
|
||||||
|
|
||||||
class FakeEmbedder:
|
class FakeEmbedder:
|
||||||
@@ -39,13 +25,15 @@ class FakeEmbedder:
|
|||||||
for t in texts:
|
for t in texts:
|
||||||
v = [0.0] * _DIM
|
v = [0.0] * _DIM
|
||||||
for tok in _tokenize(t):
|
for tok in _tokenize(t):
|
||||||
h = __import__("hashlib").md5(tok.encode("utf-8")).digest()
|
h = hashlib.md5(tok.encode("utf-8")).digest()
|
||||||
idx = h[0] % _DIM
|
idx = h[0] % _DIM
|
||||||
v[idx] += 1.0
|
v[idx] += 1.0
|
||||||
norm = __import__("math").sqrt(sum(x * x for x in v)) or 1.0
|
# L2 归一化;norm 为 0 时(全零向量)用 1.0 防除零
|
||||||
|
norm = math.sqrt(sum(x * x for x in v)) or 1.0
|
||||||
vecs.append([x / norm for x in v])
|
vecs.append([x / norm for x in v])
|
||||||
return vecs
|
return vecs
|
||||||
|
|
||||||
|
|
||||||
def get_embedder(engine) -> Embedder:
|
def get_embedder(engine: str) -> Embedder:
|
||||||
|
# engine 当前未使用,统一回退到 FakeEmbedder;真实向量模型(如 OpenAI/BGE)接入点预留于此
|
||||||
return FakeEmbedder()
|
return FakeEmbedder()
|
||||||
|
|||||||
Reference in New Issue
Block a user