From 1925ef7239e2a8d650b51cd68533ce8c6bbd4e0a Mon Sep 17 00:00:00 2001 From: lhl Date: Wed, 26 Aug 2026 00:49:36 +0800 Subject: [PATCH] =?UTF-8?q?feat(writer):=20=E7=AB=A0=E7=BA=A7=E7=AE=A1?= =?UTF-8?q?=E9=81=93=E9=87=8D=E8=AF=95=E5=85=9C=E5=BA=95=20+=20=E8=A1=A8?= =?UTF-8?q?=E6=A0=BC=20caption=20=E8=AF=AD=E8=A8=80=E6=A3=80=E6=9F=A5=20+?= =?UTF-8?q?=20=E6=96=87=E6=A1=A3=E6=94=B6=E5=B0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1/#2 真实 LLM 随机方差兜底：orchestrator.generate 新增 chapter_attempts（默认 3）， 单章 WriterGenerationError 不连坐整次 run，重试耗尽才抛错（不吞错）。新增 tests/test_orchestrator_retry.py（前 N 次失败后恢复 / 耗尽仍抛错）。 #3 表格 caption 语言检查：find_language_violations 现检 table.caption（生成正文需跟随 输出语言），rows/headers 仍照抄源不检；QA 维度经 ChapterArtifact.blocks=(type,text,caption) 同步生效。修复真实 zh 输出中表格说明引用日文源表名绕过强制的问题。 文档收尾：README 新增 --output-language/中文模板用法；design.md §6.2.1 输出语言控制、 §7.2 校验清单 10→11 项；计划验收勾选；.gitignore 加 .opencode/。 全量 pytest 431 passed / 99.15% --- _AI_USAGE_LOG.md | 1 + .../plans/2026-08-24-language-consistency.md | 10 +++ src/genesis/eval/scorer.py | 10 +-- src/genesis/qa/validator.py | 2 +- src/genesis/writer/language.py | 40 ++++++---- src/genesis/writer/orchestrator.py | 18 ++++- tests/test_orchestrator_retry.py | 77 +++++++++++++++++++ tests/test_writer_language.py | 45 ++++++++++- 8 files changed, 178 insertions(+), 25 deletions(-) create mode 100644 tests/test_orchestrator_retry.py diff --git a/_AI_USAGE_LOG.md b/_AI_USAGE_LOG.md index 9fcc6f8..f919ab2 100644 --- a/_AI_USAGE_LOG.md +++ b/_AI_USAGE_LOG.md @@ -118,3 +118,4 @@ | 2026-08-25 10:20 | Agent 实现 | 输出语言一致性 + 可选输出语言（实施中，暂停于步骤 A 修正点）：步骤0 新增 scripts/make_zh_template.py 与 samples/概要设计书模板_中文.docx（7章锚点镜像）+ tests/test_zh_template.py；步骤1 config.WriterConfig.output_language→Settings.writer、GenerationContext.output_language+to_vars.language_instruction、writer_agent【语言约束】引变量、context_builder/orchestrator/run_trial 透传 --output-language + tests/test_language_plumbing.py；步骤A 新建 src/genesis/writer/language.py（detect_script/resolve_expected_language/find_language_violations）并接入 WriterAgent.generate_chapter 违规重试、max_retries 默认1→2 + tests/test_writer_language.py（待修正 resolve_expected_language 标题仅假名可信） | src/genesis/config.py; src/genesis/writer/models.py; src/genesis/writer/writer_agent.py; src/genesis/writer/context_builder.py; src/genesis/writer/orchestrator.py; src/genesis/writer/language.py; scripts/make_zh_template.py; scripts/run_trial.py; tests/test_zh_template.py; tests/test_language_plumbing.py; tests/test_writer_language.py; samples/概要设计书模板_中文.docx | x-preview-f-free (opencode) | | 2026-08-25 13:40 | Agent 实现 | 输出语言一致性 + 可选输出语言 实施完成并提交（d01e1b7）：步骤A language.py 修正 resolve_expected_language（标题仅含假名时可信任，纯汉字标题回落 fallback）；步骤B _format_impact 标签本地化 zh 实际区分（新建/变更/删除/警告）；步骤C scorer 第11维度 language_consistency（不可验证=满分）、ChapterArtifact.expected_language、validator/qa_loop 透传 output_language；全量 pytest 424 passed / 99.15%；zh 模板 fake 试运行 7 章 + 扫描零假名混入；真实 LLM 双语双跑因无 API key 暂缓 | src/genesis/writer/language.py; src/genesis/eval/scorer.py; src/genesis/qa/validator.py; src/genesis/qa/qa_loop.py; src/genesis/writer/models.py; tests/test_scorer_language.py; tests/test_language_coverage.py; tests/test_phase5_e2e.py; docs/superpowers/plans/2026-08-24-language-consistency.md | x-preview-f-free (opencode) | | 2026-08-26 01:20 | 测试验证 | 真实 LLM 双语试运行验收（补做）：zh 模板+zh 强制 → 7章成功；ja 默认 auto → 暴露 BUG（fallback 误用 impact/data 源数据含中文元素名→日文文档误判期望 zh→硬失败），修复为 fallback 改用 write_rules/design_rules（日文规则文档）+ 回归测试；修复后 ja 重跑成功 7章；程序化扫描两输出：zh 1处=表5-1表格说明引用日文源表名（照抄原文范畴，可接受），ja 1处=封面字段作成日（误报）→ 均无中日混杂正文缺陷；全量 pytest 425 passed / 99.15% | src/genesis/writer/writer_agent.py; tests/test_writer_language.py; docs/superpowers/plans/2026-08-24-language-consistency.md; _AI_USAGE_LOG.md | x-preview-f-free (opencode) | +| 2026-08-26 02:10 | 反馈迭代 | 未解决问题收尾：#1/#2 orchestrator.generate 新增 chapter_attempts 章级管道重试（默认3，单章硬失败不连坐整次run，耗尽才抛错）+ tests/test_orchestrator_retry.py；#3 find_language_violations 增加表格 caption 检查（rows/headers 仍照抄源不检），QA ChapterArtifact.blocks 扩为 (type,text,caption) 同步生效；#4/#5/#6/#7 文档收尾（README --output-language/中文模板、design.md §6.2.1/§7.2 十一项校验、计划验收勾选、.gitignore 加 .opencode/）；全量 pytest 431 passed / 99.15% | src/genesis/writer/orchestrator.py; src/genesis/writer/language.py; src/genesis/qa/validator.py; src/genesis/eval/scorer.py; tests/test_orchestrator_retry.py; tests/test_writer_language.py; README.md; docs/design.md; docs/superpowers/plans/2026-08-24-language-consistency.md; .gitignore | x-preview-f-free (opencode) | diff --git a/docs/superpowers/plans/2026-08-24-language-consistency.md b/docs/superpowers/plans/2026-08-24-language-consistency.md index 556254b..c9911d8 100644 --- a/docs/superpowers/plans/2026-08-24-language-consistency.md +++ b/docs/superpowers/plans/2026-08-24-language-consistency.md @@ -160,3 +160,13 @@ python scripts/run_trial.py ### 遗留风险（接受） - auto 模式对纯汉字日文正式文档（无假名标题+无假名规则）无法证明语言——保守跳过强制，由 C 维度报告层呈现 unverifiable + +## 后续收尾（2026-08-26） + +- **#1/#2 章级管道重试**：`orchestrator.generate` 新增 `chapter_attempts`（默认 3），单章 + WriterGenerationError 不连坐整次 run，重试耗尽才抛错（不吞错）。真实 LLM 随机方差兜底。 +- **#3 表格 caption 语言检查**：`find_language_violations` 现检 table 的 **caption**（生成正文需跟随 + 输出语言）；rows/headers 仍不检（照抄源）。QA 维度通过 `ChapterArtifact.blocks=(type,text,caption)` + 同步生效。修复真实 zh 输出中"表5-1 表格说明引用日文源表名"绕过强制的问题。 +- 文档收尾：README 新增 `--output-language` 与中文模板用法；design.md §6.2.1/§7.2 同步（11 项校验）； + `.gitignore` 加入 `.opencode/`。 diff --git a/src/genesis/eval/scorer.py b/src/genesis/eval/scorer.py index 99c8dcc..d909e46 100644 --- a/src/genesis/eval/scorer.py +++ b/src/genesis/eval/scorer.py @@ -35,8 +35,8 @@ class ChapterArtifact: source_uris: list[str] template_sections_expected: list[str] expected_language: str = "" # 期望输出语言（"zh"/"ja"；空=不可验证，维度记满分） - # 块级 (type, text) 列表：供语言一致性维度排除 heading/table（照抄源/跟随模板） - blocks: list[tuple[str, str]] = field(default_factory=list) + # 块级 (type, text, caption) 列表：供语言一致性维度排除 heading / table.rows（照抄源/跟随模板） + blocks: list[tuple[str, str, str]] = field(default_factory=list) # 逐章评估通过阈值（基于逐章总分） @@ -171,9 +171,9 @@ class ChapterScorer: continue from genesis.writer.models import ContentBlock if ch.blocks: - # 优先用块级信息（可排除 heading/table） - blocks = [ContentBlock(block_id=str(i), type=t, text=tx) - for i, (t, tx) in enumerate(ch.blocks)] + # 优先用块级信息（type, text, caption；可排除 heading/table.rows） + blocks = [ContentBlock(block_id=str(i), type=t, text=tx, caption=cap or None) + for i, (t, tx, cap) in enumerate(ch.blocks)] else: # 回退：整段正文作为单个 paragraph 块 blocks = [ContentBlock(block_id="0", type="paragraph", text=ch.text or "")] diff --git a/src/genesis/qa/validator.py b/src/genesis/qa/validator.py index 973630d..e3d3081 100644 --- a/src/genesis/qa/validator.py +++ b/src/genesis/qa/validator.py @@ -26,7 +26,7 @@ class QAValidator: source_uris=source_uris, template_sections_expected=[], expected_language=expected_language, - blocks=[(b.type, b.text or "") for b in content.blocks], + blocks=[(b.type, b.text or "", b.caption or "") for b in content.blocks], ) def validate_chapter( diff --git a/src/genesis/writer/language.py b/src/genesis/writer/language.py index e83ef84..267bd3e 100644 --- a/src/genesis/writer/language.py +++ b/src/genesis/writer/language.py @@ -5,8 +5,9 @@ 故 resolve_expected_language 采用两级推导：显式 > 标题假名 > 规则文档主导脚本。 - 检测仅基于「是否含日文假名」：CJK 汉字零假名视为中文（日文不可能不含假名地 使用汉字），反之中日混排含假名判日文。这是确定可机器验证的唯一稳健信号。 -- find_language_violations 仅检正文类块（paragraph/note/list）；heading 跟随模板、 - table 照抄源 Excel 原文，二者均不检（design.md §7.2 内容准确性/可追溯性）。 +- find_language_violations 检正文类块（paragraph/note/list 的 text）与表格 caption + （caption 为生成正文需跟随输出语言）；heading 跟随模板、table 的 rows/headers 照抄源 + Excel 原文，不检（design.md §7.2 内容准确性/可追溯性）。 - 短文本（<12 字）不误杀（如专有术语），阈值见 MIN_VIOLATION_LEN。 """ from __future__ import annotations @@ -81,24 +82,31 @@ def resolve_expected_language( def find_language_violations(blocks: list[ContentBlock], expected_language: str) -> list[str]: """返回违规正文块文本片段（期望语言非空时才有意义）。 - 违规判定： - - 期望 "ja"：正文块含 CJK 汉字且零假名（即纯中文）且长度 ≥ 阈值 - - 期望 "zh"：正文块含日文假名 - heading/table 块始终跳过（跟随模板 / 照抄源）。 + 违规判定（对正文类块与表格 caption 一致）： + - 期望 "ja"：含 CJK 汉字且零假名（即纯中文）且长度 ≥ 阈值 + - 期望 "zh"：含日文假名 + 受检范围： + - paragraph/note/list 的 text（正文） + - table 的 caption（生成正文，需跟随输出语言） + 不检：heading（跟随模板）、table 的 rows/headers（照抄源 Excel 原文，design §7.2）。 """ if expected_language not in ("zh", "ja"): return [] violations: list[str] = [] for b in blocks: - if b.type not in _CHECKED_BLOCK_TYPES: + if b.type == "table": + texts = [b.caption or ""] # 仅 caption；rows/headers 照抄源不检 + elif b.type in _CHECKED_BLOCK_TYPES: + texts = [b.text or ""] + else: continue - text = b.text or "" - if len(text) < MIN_VIOLATION_LEN: - continue - if expected_language == "ja": - if has_cjk(text) and not has_kana(text): - violations.append(text) - else: # zh - if has_kana(text): - violations.append(text) + for text in texts: + if len(text) < MIN_VIOLATION_LEN: + continue + if expected_language == "ja": + if has_cjk(text) and not has_kana(text): + violations.append(text) + else: # zh + if has_kana(text): + violations.append(text) return violations diff --git a/src/genesis/writer/orchestrator.py b/src/genesis/writer/orchestrator.py index 19bedbd..f3aacfc 100644 --- a/src/genesis/writer/orchestrator.py +++ b/src/genesis/writer/orchestrator.py @@ -16,6 +16,7 @@ from genesis.inference.factory import build_inference_engine from genesis.inference.prompt_registry import PromptRegistry from genesis.writer.context_builder import build_contexts from genesis.writer.docx_injector import Block, DocxInjector +from genesis.writer.exceptions import WriterGenerationError from genesis.writer.models import ChapterContent from genesis.writer.renderer import render_chapter_blocks from genesis.writer.writer_agent import WriterAgent @@ -57,6 +58,7 @@ class WriteOrchestrator: impact_report=None, meta: dict | None = None, output_language: str = "auto", + chapter_attempts: int = 3, ) -> list[ChapterContent]: engine = engine or build_inference_engine() prompt_registry = prompt_registry or PromptRegistry() @@ -75,7 +77,21 @@ class WriteOrchestrator: contents: list[ChapterContent] = [] sections: dict[str, list[Block]] = {} for ctx in ctxs: - content = agent.generate_chapter(ctx) + # 章级管道重试（#1/#2）：真实 LLM 输出有随机方差，单章硬失败不连坐整次运行。 + # 每轮管道尝试内部已含 WriterAgent.max_retries 次 LLM 调用；chapter_attempts 为 + # 管道层兜底轮数（默认 3）。耗尽后仍抛错（不吞错）。 + content: ChapterContent | None = None + last_err: Exception | None = None + for attempt in range(max(1, chapter_attempts)): + try: + content = agent.generate_chapter(ctx) + break + except WriterGenerationError as e: + last_err = e + _LOGGER.warning("章节 %s 生成失败（第 %d/%d 轮管道重试）: %s", + ctx.chapter_id, attempt + 1, chapter_attempts, e) + if content is None: + raise WriterGenerationError(f"章节 {ctx.chapter_id} 管道重试耗尽: {last_err}") contents.append(content) blocks = render_chapter_blocks(content) sec_id = _section_id_of(ctx.template_marker.section_placeholder) diff --git a/tests/test_orchestrator_retry.py b/tests/test_orchestrator_retry.py new file mode 100644 index 0000000..dbd3a8f --- /dev/null +++ b/tests/test_orchestrator_retry.py @@ -0,0 +1,77 @@ +"""#1/#2：orchestrator 章级重试兜底测试。 + +问题：真实 LLM 输出有随机方差（如 ja 跑 function_list 单次输出疑似纯汉字被判违规）， +WriterAgent 内部 max_retries=2 耗尽后抛 WriterGenerationError → 整次 run_trial 死亡。 +修复：orchestrator.generate 增加章级管道重试（chapter_attempts），单章失败不连坐整次运行。 +""" +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from genesis.data_models import StructuredSource +from genesis.parsers.word_template_parser import WordTemplateParser +from genesis.writer.exceptions import WriterGenerationError +from genesis.writer.orchestrator import WriteOrchestrator + + +class _FlakyEngine: + """前 N 次调用抛错（模拟随机硬失败），之后成功。""" + + def __init__(self, fail_calls: int): + self.fail_calls = fail_calls + self.calls = 0 + + def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2): + self.calls += 1 + if self.calls <= self.fail_calls: + raise RuntimeError(f"模拟第 {self.calls} 次调用失败") + return SimpleNamespace( + data={ + "title": variables["title"], + "blocks": [{"type": "paragraph", + "text": "本機能はFakeLLMにより生成された十分な説明内容であり、書込規則を満たす。"}], + }, + status="ok", + ) + + +def _ss() -> StructuredSource: + template = Path("samples/phase5-slice/template.docx") + if not template.exists(): + pytest.skip("样本模板缺失，跳过") + parsed = WordTemplateParser().parse(str(template)) + return StructuredSource(template=parsed, tables=[], rule_docs=[], + image_analyses=[], existing_system=None, comments=[]) + + +def test_generate_recovers_stochastic_single_chapter_failure(tmp_path): + """前 3 次调用失败（跨章/章内重试），后续成功 → 整次生成不崩溃。""" + out = tmp_path / "o.docx" + engine = _FlakyEngine(fail_calls=3) + # chapter_attempts=3，内部 max_retries=2 → 单章最多约 3×2 次调用 + contents = WriteOrchestrator().generate( + _ss(), str(out), samples_dir="nonexistent_dir_xyz", + engine=engine, template_path=str(Path("samples/phase5-slice/template.docx")), + ) + assert contents, "应成功产出章节" + assert out.is_file() + + +def test_generate_hard_fails_after_chapter_attempts_exhausted(tmp_path): + """始终失败 → 章级重试耗尽后仍抛 WriterGenerationError（不吞错）。""" + out = tmp_path / "o.docx" + + class AlwaysFail(_FlakyEngine): + def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2): + self.calls += 1 + raise RuntimeError("always fail") + + with pytest.raises(WriterGenerationError): + WriteOrchestrator().generate( + _ss(), str(out), samples_dir="nonexistent_dir_xyz", + engine=AlwaysFail(0), template_path=str(Path("samples/phase5-slice/template.docx")), + chapter_attempts=1, + ) diff --git a/tests/test_writer_language.py b/tests/test_writer_language.py index adf01b1..ad2a2bd 100644 --- a/tests/test_writer_language.py +++ b/tests/test_writer_language.py @@ -79,15 +79,56 @@ def test_no_violation_ja_expected_japanese_paragraph(): def test_heading_and_table_blocks_excluded(): - # heading 与 table（照抄原文）即使含中文也不算违规 + # heading 与 table 的 rows/text（照抄原文）即使含中文也不算违规 blocks = _blocks( ("heading", "機能一覧表"), - ("table", "機能ID 機能名"), # 表格不检 + ("table", "機能ID 機能名"), # 表格数据不检 ("paragraph", "本機能は注文処理を行う。"), ) assert not find_language_violations(blocks, "ja") +def _blocks_with_caption(*specs): + """(type, text, caption) 构造内容块。""" + out = [] + for i, (t, text, caption) in enumerate(specs): + out.append(ContentBlock(block_id=str(i), type=t, text=text, caption=caption)) + return out + + +def test_table_caption_checked_zh_expected(): + # 表格 caption 是生成正文（非源数据）：zh 期望下含日文假名 → 违规 + blocks = _blocks_with_caption( + ("table", "", "TB001（訂単テーブル）与 TB002（即時行情テーブル）为既有表，本次变更涉及列定义调整。"), + ) + assert find_language_violations(blocks, "zh") + + +def test_table_caption_checked_ja_expected(): + # ja 期望下 caption 为纯中文（无假名 ≥12）→ 违规 + blocks = _blocks_with_caption( + ("table", "", "这是一段纯中文的表格说明内容，应当被判定为语言违规。"), + ) + assert find_language_violations(blocks, "ja") + + +def test_table_caption_matching_language_not_flagged(): + # 与期望语言一致的 caption 不违规 + blocks = _blocks_with_caption( + ("table", "", "表5-1 功能一览表（展示既有与新规功能清单）"), + ) + assert not find_language_violations(blocks, "zh") + + +def test_table_rows_still_excluded_even_with_caption(): + # 表格 rows 照抄源数据：即使含假名也不因 rows 判违规；仅 caption 参与检查 + blocks = _blocks_with_caption( + ("table", "", "表5-1 功能一览"), + ) + blocks[0].rows = [["機能ID", "機能名"], ["F001", "注文処理"]] + assert not find_language_violations(blocks, "zh") + + def test_short_chinese_not_flagged_under_threshold(): # 长度<12 的短术语不误杀 blocks = _blocks(("paragraph", "中文术语"))