fix(rag): 移除 embeddings camelCase 死代码,覆盖率达 100%
This commit is contained in:
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
from typing import List, Protocol
|
||||
|
||||
|
||||
@@ -11,26 +14,9 @@ _DIM = 64
|
||||
|
||||
|
||||
def _tokenize(text: str) -> List[str]:
|
||||
toks = []
|
||||
cur = ""
|
||||
for ch in text.lower():
|
||||
if ch.isalnum():
|
||||
cur += ch
|
||||
else:
|
||||
if cur:
|
||||
toks.append(cur)
|
||||
cur = ""
|
||||
if cur:
|
||||
toks.append(cur)
|
||||
out = []
|
||||
for t in toks:
|
||||
idx = 0
|
||||
for i, c in enumerate(t):
|
||||
if i > 0 and c.isupper():
|
||||
out.append(t[idx:i])
|
||||
idx = i
|
||||
out.append(t[idx:])
|
||||
return [x for x in out if x]
|
||||
# 分词意图:先将文本统一转为小写,再按非字母数字字符切分为词元,最后过滤空串
|
||||
toks = re.split(r"\W+", text.lower())
|
||||
return [t for t in toks if t]
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
@@ -39,13 +25,15 @@ class FakeEmbedder:
|
||||
for t in texts:
|
||||
v = [0.0] * _DIM
|
||||
for tok in _tokenize(t):
|
||||
h = __import__("hashlib").md5(tok.encode("utf-8")).digest()
|
||||
h = hashlib.md5(tok.encode("utf-8")).digest()
|
||||
idx = h[0] % _DIM
|
||||
v[idx] += 1.0
|
||||
norm = __import__("math").sqrt(sum(x * x for x in v)) or 1.0
|
||||
# L2 归一化;norm 为 0 时(全零向量)用 1.0 防除零
|
||||
norm = math.sqrt(sum(x * x for x in v)) or 1.0
|
||||
vecs.append([x / norm for x in v])
|
||||
return vecs
|
||||
|
||||
|
||||
def get_embedder(engine) -> Embedder:
|
||||
def get_embedder(engine: str) -> Embedder:
|
||||
# engine 当前未使用,统一回退到 FakeEmbedder;真实向量模型(如 OpenAI/BGE)接入点预留于此
|
||||
return FakeEmbedder()
|
||||
|
||||
Reference in New Issue
Block a user