From afc09fc908dbbfa7e655d5a27662b3baa562a19d Mon Sep 17 00:00:00 2001
From: lhl <liuhaili1203@126.com>
Date: Sat, 29 Aug 2026 22:35:16 +0800
Subject: [PATCH] =?UTF-8?q?fix(rag):=20=E7=A7=BB=E9=99=A4=20embeddings=20c?=
 =?UTF-8?q?amelCase=20=E6=AD=BB=E4=BB=A3=E7=A0=81=EF=BC=8C=E8=A6=86?=
 =?UTF-8?q?=E7=9B=96=E7=8E=87=E8=BE=BE=20100%?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/genesis/rag/embeddings.py | 34 +++++++++++-----------------------
 1 file changed, 11 insertions(+), 23 deletions(-)

diff --git a/src/genesis/rag/embeddings.py b/src/genesis/rag/embeddings.py
index c8ba40e..6c59ff1 100644
--- a/src/genesis/rag/embeddings.py
+++ b/src/genesis/rag/embeddings.py
@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import hashlib
+import math
+import re
 from typing import List, Protocol
 
 
@@ -11,26 +14,9 @@ _DIM = 64
 
 
 def _tokenize(text: str) -> List[str]:
-    toks = []
-    cur = ""
-    for ch in text.lower():
-        if ch.isalnum():
-            cur += ch
-        else:
-            if cur:
-                toks.append(cur)
-                cur = ""
-    if cur:
-        toks.append(cur)
-    out = []
-    for t in toks:
-        idx = 0
-        for i, c in enumerate(t):
-            if i > 0 and c.isupper():
-                out.append(t[idx:i])
-                idx = i
-        out.append(t[idx:])
-    return [x for x in out if x]
+    # 分词意图：先将文本统一转为小写，再按非字母数字字符切分为词元，最后过滤空串
+    toks = re.split(r"\W+", text.lower())
+    return [t for t in toks if t]
 
 
 class FakeEmbedder:
@@ -39,13 +25,15 @@ class FakeEmbedder:
         for t in texts:
             v = [0.0] * _DIM
             for tok in _tokenize(t):
-                h = __import__("hashlib").md5(tok.encode("utf-8")).digest()
+                h = hashlib.md5(tok.encode("utf-8")).digest()
                 idx = h[0] % _DIM
                 v[idx] += 1.0
-            norm = __import__("math").sqrt(sum(x * x for x in v)) or 1.0
+            # L2 归一化；norm 为 0 时（全零向量）用 1.0 防除零
+            norm = math.sqrt(sum(x * x for x in v)) or 1.0
             vecs.append([x / norm for x in v])
         return vecs
 
 
-def get_embedder(engine) -> Embedder:
+def get_embedder(engine: str) -> Embedder:
+    # engine 当前未使用，统一回退到 FakeEmbedder；真实向量模型（如 OpenAI/BGE）接入点预留于此
     return FakeEmbedder()