diff --git a/src/genesis/rag/embeddings.py b/src/genesis/rag/embeddings.py index c8ba40e..6c59ff1 100644 --- a/src/genesis/rag/embeddings.py +++ b/src/genesis/rag/embeddings.py @@ -1,5 +1,8 @@ from __future__ import annotations +import hashlib +import math +import re from typing import List, Protocol @@ -11,26 +14,9 @@ _DIM = 64 def _tokenize(text: str) -> List[str]: - toks = [] - cur = "" - for ch in text.lower(): - if ch.isalnum(): - cur += ch - else: - if cur: - toks.append(cur) - cur = "" - if cur: - toks.append(cur) - out = [] - for t in toks: - idx = 0 - for i, c in enumerate(t): - if i > 0 and c.isupper(): - out.append(t[idx:i]) - idx = i - out.append(t[idx:]) - return [x for x in out if x] + # 分词意图:先将文本统一转为小写,再按非字母数字字符切分为词元,最后过滤空串 + toks = re.split(r"\W+", text.lower()) + return [t for t in toks if t] class FakeEmbedder: @@ -39,13 +25,15 @@ class FakeEmbedder: for t in texts: v = [0.0] * _DIM for tok in _tokenize(t): - h = __import__("hashlib").md5(tok.encode("utf-8")).digest() + h = hashlib.md5(tok.encode("utf-8")).digest() idx = h[0] % _DIM v[idx] += 1.0 - norm = __import__("math").sqrt(sum(x * x for x in v)) or 1.0 + # L2 归一化;norm 为 0 时(全零向量)用 1.0 防除零 + norm = math.sqrt(sum(x * x for x in v)) or 1.0 vecs.append([x / norm for x in v]) return vecs -def get_embedder(engine) -> Embedder: +def get_embedder(engine: str) -> Embedder: + # engine 当前未使用,统一回退到 FakeEmbedder;真实向量模型(如 OpenAI/BGE)接入点预留于此 return FakeEmbedder()