feat: Token 估算（approximate 内置 / tiktoken 可选回落）

tests/test_inference_token.py

@@ -0,0 +1,76 @@
+import pytest
+
+from genesis.inference.token import (
+    _tiktoken_estimator,
+    approximate_token_count,
+    make_estimator,
+)
+
+
+def test_approximate_count_minimum():
+    assert approximate_token_count("") >= 1
+    assert approximate_token_count("a") == 1
+
+
+def test_approximate_count_linear():
+    # 每 4 字符约 1 token（向上取整）
+    assert approximate_token_count("abcd") == 1
+    assert approximate_token_count("abcdefgh") == 2
+    assert approximate_token_count("abcdefghi") == 3
+
+
+def test_tiktoken_estimator_missing_falls_back():
+    # 未安装 tiktoken 或不可用时返回 None（由 make_estimator 回落 approximate）
+    r = _tiktoken_estimator("hello")
+    assert r is None or isinstance(r, int)
+
+
+def test_make_estimator_approximate_backend():
+    est = make_estimator("approximate")
+    assert est("abcd") == 1
+
+
+def test_make_estimator_default_without_tiktoken(monkeypatch):
+    # 强制模拟 tiktoken 缺失：make_estimator 必须回落 approximate
+    import builtins
+    real_import = builtins.__import__
+
+    def fake_import(name, *args, **kwargs):
+        if name == "tiktoken":
+            raise ImportError("no tiktoken")
+        return real_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", fake_import)
+    est = make_estimator("tiktoken")
+    assert est("abcd") == 1
+
+
+def test_tiktoken_estimator_success_path(monkeypatch):
+    # 注入假 tiktoken 模块：验证编码成功路径（len(encode(text))）
+    import sys
+
+    class _FakeEncoding:
+        def encode(self, s: str):
+            return ["t"] * len(s)
+
+    class _FakeTiktoken:
+        @staticmethod
+        def get_encoding(name):
+            assert name == "cl100k_base"
+            return _FakeEncoding()
+
+    monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken)
+    assert _tiktoken_estimator("hello") == 5
+
+
+def test_tiktoken_estimator_encoding_error(monkeypatch):
+    # tiktoken 可用但编码抛异常：返回 None（由 make_estimator 回落 approximate）
+    import sys
+
+    class _FakeTiktoken:
+        @staticmethod
+        def get_encoding(name):
+            raise RuntimeError("boom")
+
+    monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken)
+    assert _tiktoken_estimator("hello") is None