From 71d38b90bff7aa5d95917508cf0b233d5349e011 Mon Sep 17 00:00:00 2001 From: lhl Date: Sun, 9 Aug 2026 05:58:16 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20Token=20=E4=BC=B0=E7=AE=97=EF=BC=88appr?= =?UTF-8?q?oximate=20=E5=86=85=E7=BD=AE=20/=20tiktoken=20=E5=8F=AF?= =?UTF-8?q?=E9=80=89=E5=9B=9E=E8=90=BD=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _AI_USAGE_LOG.md | 6 ++- src/genesis/inference/token.py | 29 +++++++++++++ tests/test_inference_token.py | 76 ++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 src/genesis/inference/token.py create mode 100644 tests/test_inference_token.py diff --git a/_AI_USAGE_LOG.md b/_AI_USAGE_LOG.md index 0f86b5b..d2bddc1 100644 --- a/_AI_USAGE_LOG.md +++ b/_AI_USAGE_LOG.md @@ -39,4 +39,8 @@ | 2026-08-09 04:40 | 反馈迭代 | 里程碑2.5 Task13 遗留清理：① extraction_method 枚举同源（free_text_extractor→ExtractionMethod.LLM_FROM_FREE_TEXT.value、table_extractor 空/非空两分支→ExtractionMethod.OPENPYXL.value，值不变既有用例兼容）；② table_extractor header_row 越界/负值防御（<0 或 >=len(matrix) 回落 0，keep if not matrix 早返回）；③ merge_fill.forward_fill 完整替换（min_row<1/min_col<1 非法值跳过 + 越界防御）；④ data_models.Provenance.row 语义注释；⑤ 补测试 6 个（sheet_nature MIXED ・/■ 正向 x2、formatting_detector 字体色正向、table_extractor 枚举锁 + forward_fill 防御、free_text_extractor 枚举锁）。TDD：先加测试确认 RED（仅 forward_fill 防御用例失败，其余为已通过回归锁）→ 实现 → GREEN；pytest 全量 58 passed（52 baseline + 6） | src/genesis/parsers/free_text_extractor.py, src/genesis/parsers/table_extractor.py, src/genesis/parsers/merge_fill.py, src/genesis/data_models.py, tests/test_sheet_nature.py, tests/test_formatting_detector.py, tests/test_free_text_extractor.py, tests/test_table_extractor.py, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-09 10:05 | 反馈迭代 | 里程碑2.5 最终评审与交付：整体评审通过（base 81c6b6c..d93cc77，58 passed）；清理 progress.md Task11 重复/编码行；修正 _AI_USAGE_LOG.md Task13 范式步骤用词；产出交付报告 docs/milestone2.5-mixed-parser-review.md（MIXED 遗留#1 闭环） | docs/milestone2.5-mixed-parser-review.md, .superpowers/sdd/progress.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-09 10:12 | 测试验证 | 补齐覆盖率工具：安装 pytest-cov 7.1.0/coverage 7.15.4，pyproject.toml 固化 dev 依赖+addopts（--cov=genesis --cov-report=term-missing）+[tool.coverage.run/report]（source=genesis、branch=true、fail_under=90、skip_empty、sort=miss）；验证 fail_under=99 拦截 exit 1；产出 docs/coverage-report.md 与 htmlcov（已 ignore）；基线覆盖 96.67%（571 stmts，12 miss） | pyproject.toml, docs/coverage-report.md | deepseek-v4-flash-free | -| 2026-08-09 10:20 | 测试验证 | 覆盖率提升至 100%：新增 13 个防御/边界用例（空矩阵/稀疏多列/>表头越界/非法色/RGB访问异常/fill=None/list分值/空sh山 skipped/MXED 段外格式隔离）；pragma 标注 3 处不可达防御分支（classify 恒有表头）；fail_under 90→99；全量 71 passed，覆盖率 100.00%（563 stmts/140 br 全绿）；更新 docs/coverage-report.md | tests/test_sheet_nature.py, tests/test_table_extractor.py, tests/test_formatting_detector.py, tests/test_config.py, tests/test_excel_parser.py, src/genesis/parsers/excel_parser.py, src/genesis/parsers/sheet_nature.py, pyproject.toml, docs/coverage-report.md | deepseek-v4-flash-free | +| 2026-08-09 10:20 | 测试验证 | 覆盖率提升至 100%：新增 13 个防御/边界用例（空矩阵/稀疏多列/表头越界/非法色/RGB访问异常/fill=None/列表类型/空sheet skipped/MIXED 段外格式隔离）；pragma 标注 3 处不可达防御分支（classify 恒有表头）；fail_under 90→99；全量 71 passed，覆盖率 100.00%（563 stmts/140 br 全绿）；更新 docs/coverage-report.md | tests/test_sheet_nature.py, tests/test_table_extractor.py, tests/test_formatting_detector.py, tests/test_config.py, tests/test_excel_parser.py, src/genesis/parsers/excel_parser.py, src/genesis/parsers/sheet_nature.py, pyproject.toml, docs/coverage-report.md | deepseek-v4-flash-free | +| 2026-08-09 下午 | 架构设计 | 里程碑3.1 InferenceEngine 设计完成并批准：头脑风暴三轮定稿（动机/关键决策/架构与关键行为），执行 plan-design-review（无 UI scope，一致性审查全对齐），审查产出 2 项补丁（StructuredResult.status 字段、api-design LLM_NOT_CONFIGURED 错误码补齐）；确定 httpx 客户端选型（pyproject 新增依赖）；落盘 spec 至 docs/superpowers/specs/2026-08-09-inference-engine-design.md（含 3.8 补丁 2/3.9 补丁 1、异常模型、测试策略） | docs/superpowers/specs/2026-08-09-inference-engine-design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | +| 2026-08-09 下午 | Agent 实现 | 里程碑3.1 实施计划生成（writing-plans）：5 任务 TDD 计划落盘 docs/superpowers/plans/2026-08-09-inference-engine.md——Task1 types+exceptions+依赖(httpx/jinja2)、Task2 token(approximate/tiktoken可选)、Task3 PromptRegistry(jinja2)、Task4 HttpLLMClient(重试/退避/超时/鉴权)、Task5 InferenceEngine(chat/chat_structured全流程+注入Fake+文档补丁)；修正 helper 导入、方法名(_call)、结构化状态语义(重试成功仍ok) | docs/superpowers/plans/2026-08-09-inference-engine.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | +| 2026-08-09 05:53 | Agent 实现 | 里程碑3.1 Task1 实现：推理引擎数据模型与异常层。新建 src/genesis/inference/（__init__/types/exceptions：TokenUsage、ChatMessage、ChatResult、StructuredResult、Prompt 五 dataclass 与 LLMError 体系五异常，__init__ 暂不导入 engine/client 防循环）；pyproject.toml 增加 httpx>=0.28/jinja2>=3.1 依赖；tests/test_inference_types.py（6 用例）与 tests/test_inference_errors.py（2 用例）；TDD 验证 RED（ModuleNotFoundError: No module named 'genesis.inference'）→ GREEN（聚焦 8 passed）；pytest 全量 79 passed 覆盖 100.00%（608 stmts/140 br），fail_under=99 达标；提交见 git log | src/genesis/inference/__init__.py, src/genesis/inference/types.py, src/genesis/inference/exceptions.py, pyproject.toml, tests/test_inference_types.py, tests/test_inference_errors.py, _AI_USAGE_LOG.md | deepseek-v4-flash-free | +| 2026-08-09 05:57 | Agent 实现 | 里程碑3.1 Task2 实现：推理引擎 token 估算模块。新建 src/genesis/inference/token.py（approximate_token_count 每 4 字符≈1 token 最少 1、_tiktoken_estimator tiktoken 编码未安装返回 None、make_estimator backend="tiktoken" 默认优先 tiktoken 缺失回落 approximate）；tests/test_inference_token.py 按 brief 5 用例 + 补充 2 用例（假 tiktoken 成功路径、encoding 抛异常回落）覆盖 tiktoken 成功/异常两分支，保证全量覆盖不回落；TDD 验证 RED（ModuleNotFoundError: No module named 'genesis.inference.token'）→ 聚焦 7 passed，token.py 100%；pytest 全量 86 passed 覆盖 100.00%（626 stmts/142 br）；提交见 git log | src/genesis/inference/token.py, tests/test_inference_token.py, _AI_USAGE_LOG.md | deepseek-v4-flash-free | diff --git a/src/genesis/inference/token.py b/src/genesis/inference/token.py new file mode 100644 index 0000000..a43fa1b --- /dev/null +++ b/src/genesis/inference/token.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Callable + + +def approximate_token_count(text: str) -> int: + """内置近似估算：每 4 字符 ≈ 1 token（无外部依赖，可离线）。""" + return max(1, (len(text) + 3) // 4) + + +def _tiktoken_estimator(text: str) -> int | None: + """tiktoken 编码估算；tiktoken 未安装时返回 None。""" + try: + import tiktoken + except ImportError: + return None + try: + enc = tiktoken.get_encoding("cl100k_base") + return len(enc.encode(text)) + except Exception: + return None + + +def make_estimator(backend: str = "tiktoken") -> Callable[[str], int]: + """按配置选择估算器：backend="tiktoken"（默认）优先 tiktoken， + 缺失或异常回落内置 approximate；backend="approximate" 直接用近似。""" + if backend == "approximate": + return approximate_token_count + return lambda text: _tiktoken_estimator(text) or approximate_token_count(text) \ No newline at end of file diff --git a/tests/test_inference_token.py b/tests/test_inference_token.py new file mode 100644 index 0000000..e83e174 --- /dev/null +++ b/tests/test_inference_token.py @@ -0,0 +1,76 @@ +import pytest + +from genesis.inference.token import ( + _tiktoken_estimator, + approximate_token_count, + make_estimator, +) + + +def test_approximate_count_minimum(): + assert approximate_token_count("") >= 1 + assert approximate_token_count("a") == 1 + + +def test_approximate_count_linear(): + # 每 4 字符约 1 token（向上取整） + assert approximate_token_count("abcd") == 1 + assert approximate_token_count("abcdefgh") == 2 + assert approximate_token_count("abcdefghi") == 3 + + +def test_tiktoken_estimator_missing_falls_back(): + # 未安装 tiktoken 或不可用时返回 None（由 make_estimator 回落 approximate） + r = _tiktoken_estimator("hello") + assert r is None or isinstance(r, int) + + +def test_make_estimator_approximate_backend(): + est = make_estimator("approximate") + assert est("abcd") == 1 + + +def test_make_estimator_default_without_tiktoken(monkeypatch): + # 强制模拟 tiktoken 缺失：make_estimator 必须回落 approximate + import builtins + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "tiktoken": + raise ImportError("no tiktoken") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + est = make_estimator("tiktoken") + assert est("abcd") == 1 + + +def test_tiktoken_estimator_success_path(monkeypatch): + # 注入假 tiktoken 模块：验证编码成功路径（len(encode(text))） + import sys + + class _FakeEncoding: + def encode(self, s: str): + return ["t"] * len(s) + + class _FakeTiktoken: + @staticmethod + def get_encoding(name): + assert name == "cl100k_base" + return _FakeEncoding() + + monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken) + assert _tiktoken_estimator("hello") == 5 + + +def test_tiktoken_estimator_encoding_error(monkeypatch): + # tiktoken 可用但编码抛异常：返回 None（由 make_estimator 回落 approximate） + import sys + + class _FakeTiktoken: + @staticmethod + def get_encoding(name): + raise RuntimeError("boom") + + monkeypatch.setitem(sys.modules, "tiktoken", _FakeTiktoken) + assert _tiktoken_estimator("hello") is None \ No newline at end of file