lhl
8febc50eb8
- Issue11: approximate_token_count 重写 - 新增 _is_cjk_char（CJK 统一表意/扩展A/假名/韩文/兼容/全角六大 Unicode 范围） - CJK 字符按 1.5 token/字符（旧 4 字符 1 token 严重低估，裁剪失效致 API 超限） - 其余字符仍 4 字符 1 token；最少 1 token - 同步 config-design.md token_estimation 注释 - 新增 4 用例，全量 186 passed / 100.00%（995 stmts/252 br）
103 lines
3.3 KiB
Python
103 lines
3.3 KiB
Python
import pytest
|
||
|
||
from genesis.inference.token import (
|
||
_tiktoken_estimator,
|
||
approximate_token_count,
|
||
make_estimator,
|
||
)
|
||
|
||
|
||
def test_approximate_count_minimum():
|
||
assert approximate_token_count("") >= 1
|
||
assert approximate_token_count("a") == 1
|
||
|
||
|
||
def test_approximate_count_linear():
|
||
# 每 4 字符约 1 token（向上取整）
|
||
assert approximate_token_count("abcd") == 1
|
||
assert approximate_token_count("abcdefgh") == 2
|
||
assert approximate_token_count("abcdefghi") == 3
|
||
|
||
|
||
# ---------- T9: CJK 保守估算 ----------
|
||
|
||
def test_approximate_cjk_conservative():
|
||
"""CJK（中文/日文/韩文）字符按保守 ×1.5/字符 估算，远高于 4 字符 1 token。"""
|
||
cjk = "日本語の設計書" # 7 个 CJK 字符
|
||
assert approximate_token_count(cjk) >= len(cjk) # 至少 1 token/字符
|
||
# 旧逻辑 len//4=1（严重低估）；新逻辑 ≥7
|
||
assert approximate_token_count(cjk) > len(cjk) // 4
|
||
|
||
|
||
def test_approximate_ascii_unchanged():
|
||
"""纯 ASCII 仍按 4 字符 1 token（不回归）。"""
|
||
assert approximate_token_count("abcdefgh") == 2
|
||
|
||
|
||
def test_approximate_mixed_cjk_and_ascii():
|
||
"""混合文本：CJK 部分保守估算 + ASCII 部分按 4 字符 1 token。"""
|
||
# "abc" 3 ASCII → 1 token；"日本語" 3 CJK → ceil(3*1.5)=5；合计 ≥ 6
|
||
total = approximate_token_count("abc日本語")
|
||
assert total >= 1 + 3
|
||
|
||
|
||
def test_approximate_cjk_fullwidth_forms():
|
||
"""全角符号（CJK 兼容/全角区块）同样保守估算（日文文档常见）。"""
|
||
assert approximate_token_count("（設計）") >= 4 # 4 个全角字符，至少 4 token
|
||
|
||
|
||
def test_tiktoken_estimator_missing_falls_back():
|
||
# 未安装 tiktoken 或不可用时返回 None（由 make_estimator 回落 approximate）
|
||
r = _tiktoken_estimator("hello")
|
||
assert r is None or isinstance(r, int)
|
||
|
||
|
||
def test_make_estimator_approximate_backend():
|
||
est = make_estimator("approximate")
|
||
assert est("abcd") == 1
|
||
|
||
|
||
def test_make_estimator_default_without_tiktoken(monkeypatch):
|
||
# 强制模拟 tiktoken 缺失：make_estimator 必须回落 approximate
|
||
import builtins
|
||
real_import = builtins.__import__
|
||
|
||
def fake_import(name, *args, **kwargs):
|
||
if name == "tiktoken":
|
||
raise ImportError("no tiktoken")
|
||
return real_import(name, *args, **kwargs)
|
||
|
||
monkeypatch.setattr(builtins, "__import__", fake_import)
|
||
est = make_estimator("tiktoken")
|
||
assert est("abcd") == 1
|
||
|
||
|
||
def test_tiktoken_estimator_success_path(monkeypatch):
|
||
# 注入假 tiktoken 模块：验证编码成功路径（len(encode(text))）
|
||
import sys
|
||
|
||
class _FakeEncoding:
|
||
def encode(self, s: str):
|
||
return ["t"] * len(s)
|
||
|
||
class _FakeTiktoken:
|
||
@staticmethod
|
||
def get_encoding(name):
|
||
assert name == "cl100k_base"
|
||
return _FakeEncoding()
|
||
|
||
monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken)
|
||
assert _tiktoken_estimator("hello") == 5
|
||
|
||
|
||
def test_tiktoken_estimator_encoding_error(monkeypatch):
|
||
# tiktoken 可用但编码抛异常：返回 None（由 make_estimator 回落 approximate）
|
||
import sys
|
||
|
||
class _FakeTiktoken:
|
||
@staticmethod
|
||
def get_encoding(name):
|
||
raise RuntimeError("boom")
|
||
|
||
monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken)
|
||
assert _tiktoken_estimator("hello") is None |