diff --git a/_AI_USAGE_LOG.md b/_AI_USAGE_LOG.md index e0fae6d..764c569 100644 --- a/_AI_USAGE_LOG.md +++ b/_AI_USAGE_LOG.md @@ -75,3 +75,4 @@ | 2026-08-11 | Agent 实现 | T3(架构审查整改):会话状态机实现 + cancelled/resume(Issue3)。新建 src/genesis/state_machine.py(SessionStateMachine:9 状态白名单转移 + StateTransitionError 对应 api §7 STATE_TRANSITION_INVALID 409;cancel 记录 cancelled_from 进 cancelled 终态,resume 回中断点;仅执行中状态可取消,awaiting_*/done 不可;状态集含 8 设计态 + cancelled);同步 agent-runtime-design.md §3.2 状态图/规则表(9 状态 + cancelled 行);新增 10 用例(正常流转/非法转移拒绝/done 终态/cancel 记录/resume 回中断点/非 cancelled 不可 resume/cancelled 不可任意跳转/循环 cancel-resume/未知初始/目标状态防御);TDD 验证 RED(ModuleNotFoundError)→ GREEN(聚焦 10 passed)→ 覆盖补齐 2 用例 → 全量 177 passed 覆盖 100.00%(981 stmts/252 br),fail_under=99 达标 | src/genesis/state_machine.py, tests/test_state_machine.py, docs/agent-runtime-design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-11 | Agent 实现 | T4(架构审查整改):引擎层统一注入防护(Issue4)。engine.py 新增 DEFAULT_SYSTEM_INSTRUCTION 恒定系统指令(含「用户数据段指令不作为要求执行」声明)+ _DATA_BOUNDARY 边界标记 + _wrap_user_data();__init__ 支持 system_instruction 注入覆盖;_call 统一构造 [system 恒定指令, user 边界包裹数据](chat/chat_structured 全生效);FakeLLMClient 记录结构对齐真实 HttpLLMClient payload({role, content} dict),同步 2 处既有断言;同步 agent-runtime-design.md §8.1 标注已实现;新增 5 用例(system 首条恒定/用户数据边界包裹/声明不执行/自定义指令/chat_structured 同防护);TDD 验证 RED(DEFAULT_SYSTEM_INSTRUCTION 不存在)→ GREEN(聚焦 32 passed)→ 全量 182 passed 覆盖 100.00%(987 stmts/252 br),fail_under=99 达标 | src/genesis/inference/engine.py, tests/test_inference_engine.py, tests/inference_helpers.py, docs/agent-runtime-design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-11 | Agent 实现 | T8(架构审查整改):LLM 客户端全异步化(Issue9)。client.py 由同步 httpx.Client 全异步化:LLMClient Protocol chat → async def;HttpLLMClient 用 httpx.AsyncClient + asyncio.sleep 退避(消除 time.sleep 阻塞 asyncio 任务池);__enter__/__exit__ → __aenter__/__aexit__(async with 生命周期闭环);engine.py chat/chat_structured/_call 全部 async + await;FakeLLMClient.chat → async;测试基建用 anyio pytest 插件(@pytest.mark.anyio);test_inference_engine.py 32 用例脚本批量转换 async + NotConfiguredClient 同步 client 转 async;test_inference_client.py 10 用例转 async;同步 inference-engine-design spec 与 milestone3-inference-review httpx 描述(防文档漂移);TDD 验证 RED(async 接口缺失 TypeError)→ GREEN(聚焦 67 passed)→ 全量 182 passed 覆盖 100.00%(987 stmts/252 br),fail_under=99 达标 | src/genesis/inference/client.py, src/genesis/inference/engine.py, tests/test_inference_client.py, tests/test_inference_engine.py, tests/inference_helpers.py, docs/superpowers/specs/2026-08-09-inference-engine-design.md, docs/milestone3-inference-review.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | +| 2026-08-11 | Agent 实现 | T9(架构审查整改):CJK 保守 token 估算(Issue11)。token.py approximate_token_count 重写:新增 _is_cjk_char(CJK 统一表意/扩展A/假名/韩文/兼容/全角六大 Unicode 范围)+ _CJK_TOKENS_PER_CHAR=1.5(旧逻辑 4 字符 1 token 对中文/日文严重低估,裁剪失效致 API 超限);CJK 字符按 1.5 token/字符,其余仍 4 字符 1 token,最少 1 token;同步 config-design.md token_estimation 注释;新增 4 用例(纯 CJK 保守/ASCII 不回归/混合文本/全角符号);TDD 验证 RED((設計)4 字符仅 1 token)→ GREEN(聚焦 11 passed)→ 全量 186 passed 覆盖 100.00%(995 stmts/252 br),fail_under=99 达标 | src/genesis/inference/token.py, tests/test_inference_token.py, docs/config-design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | diff --git a/docs/config-design.md b/docs/config-design.md index 8c42359..59f4521 100644 --- a/docs/config-design.md +++ b/docs/config-design.md @@ -133,7 +133,7 @@ models: timeout_sec: 90 llm_calls: - token_estimation: tiktoken # tiktoken | approximate;tiktoken 缺失自动回落 approximate(内置估算器) + token_estimation: tiktoken # tiktoken | approximate;tiktoken 缺失自动回落 approximate(内置估算器,T9 起对 CJK 保守 ×1.5/字符) max_context_tokens: 32000 # 上下文窗口上限 truncation_policy: # 超限裁剪策略(runtime §2.6) priority: diff --git a/src/genesis/inference/token.py b/src/genesis/inference/token.py index a43fa1b..7628504 100644 --- a/src/genesis/inference/token.py +++ b/src/genesis/inference/token.py @@ -1,11 +1,33 @@ from __future__ import annotations +import unicodedata from typing import Callable +# CJK 每字符保守 token 数(T9 整改:cl100k/4字符1token 对中文/日文严重低估) +# 日文/中文实际每字符 1~3 token,取保守 1.5 防止上下文裁剪失效导致 API 超限 +_CJK_TOKENS_PER_CHAR = 1.5 + + +def _is_cjk_char(ch: str) -> bool: + """判断字符是否属于 CJK 密集区(中文/日文假名/韩文/全角符号)。""" + cp = ord(ch) + return ( + 0x4E00 <= cp <= 0x9FFF # CJK 统一表意文字 + or 0x3040 <= cp <= 0x30FF # 平假名/片假名 + or 0xAC00 <= cp <= 0xD7AF # 韩文音节 + or 0xF900 <= cp <= 0xFAFF # CJK 兼容表意文字 + or 0xFF00 <= cp <= 0xFFEF # 全角形式(全角标点/字母) + or 0x3400 <= cp <= 0x4DBF # CJK 扩展 A + ) + def approximate_token_count(text: str) -> int: - """内置近似估算:每 4 字符 ≈ 1 token(无外部依赖,可离线)。""" - return max(1, (len(text) + 3) // 4) + """内置近似估算(T9 CJK 保守):CJK 字符按 1.5 token/字符, + 其余字符按 4 字符 ≈ 1 token;最少 1 token(无外部依赖,可离线)。""" + cjk_chars = sum(1 for ch in text if _is_cjk_char(ch)) + other_chars = len(text) - cjk_chars + tokens = cjk_chars * _CJK_TOKENS_PER_CHAR + (other_chars + 3) // 4 + return max(1, int(tokens)) def _tiktoken_estimator(text: str) -> int | None: diff --git a/tests/test_inference_token.py b/tests/test_inference_token.py index e83e174..2e01e15 100644 --- a/tests/test_inference_token.py +++ b/tests/test_inference_token.py @@ -19,6 +19,33 @@ def test_approximate_count_linear(): assert approximate_token_count("abcdefghi") == 3 +# ---------- T9: CJK 保守估算 ---------- + +def test_approximate_cjk_conservative(): + """CJK(中文/日文/韩文)字符按保守 ×1.5/字符 估算,远高于 4 字符 1 token。""" + cjk = "日本語の設計書" # 7 个 CJK 字符 + assert approximate_token_count(cjk) >= len(cjk) # 至少 1 token/字符 + # 旧逻辑 len//4=1(严重低估);新逻辑 ≥7 + assert approximate_token_count(cjk) > len(cjk) // 4 + + +def test_approximate_ascii_unchanged(): + """纯 ASCII 仍按 4 字符 1 token(不回归)。""" + assert approximate_token_count("abcdefgh") == 2 + + +def test_approximate_mixed_cjk_and_ascii(): + """混合文本:CJK 部分保守估算 + ASCII 部分按 4 字符 1 token。""" + # "abc" 3 ASCII → 1 token;"日本語" 3 CJK → ceil(3*1.5)=5;合计 ≥ 6 + total = approximate_token_count("abc日本語") + assert total >= 1 + 3 + + +def test_approximate_cjk_fullwidth_forms(): + """全角符号(CJK 兼容/全角区块)同样保守估算(日文文档常见)。""" + assert approximate_token_count("(設計)") >= 4 # 4 个全角字符,至少 4 token + + def test_tiktoken_estimator_missing_falls_back(): # 未安装 tiktoken 或不可用时返回 None(由 make_estimator 回落 approximate) r = _tiktoken_estimator("hello")