feat(writer): 输出语言一致性保障 + 用户可选输出语言

- 步骤0: 新增中文镜像模板 scripts/make_zh_template.py 与 samples/概要设计书模板_中文.docx(7章锚点原样保留)
- 步骤1: config.WriterConfig.output_language→Settings.writer;GenerationContext.output_language + to_vars.language_instruction(zh/ja/auto);writer_agent【语言约束】改引变量;context_builder/orchestrator/run_trial 透传 --output-language
- 步骤A: 新建 src/genesis/writer/language.py(detect_script/resolve_expected_language/find_language_violations);WriterAgent.generate_chapter 按期望语言强制、违规重试、耗尽硬失败;max_retries 默认 1→2
- 步骤B: _format_impact 影响调查标签按 output_language 本地化(zh 新建/变更/删除/警告)
- 步骤C: eval scorer 第 11 维度 language_consistency(不可验证=满分,不拉低总分);ChapterArtifact.expected_language;QAValidator.validate_doc 透传;QALoop.run 透传 output_language
- 测试: test_zh_template/test_language_plumbing/test_writer_language/test_scorer_language/test_language_coverage,并更新 test_phase5_e2e
- 全量 pytest 424 passed / 99.15%(覆盖率门槛 99% 达标)
This commit is contained in:
lhl
2026-08-25 23:30:24 +08:00
parent 9d0d3409c2
commit d01e1b720f
20 changed files with 1049 additions and 31 deletions
+102
View File
@@ -0,0 +1,102 @@
"""步骤 A/C 边界覆盖测试(恢复覆盖率至 ≥99%)。"""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from genesis.writer.language import find_language_violations
from genesis.writer.models import ChapterContent, ContentBlock, GenerationContext, ChapterSpec
from genesis.writer.writer_agent import WriterAgent
from genesis.writer.writer_state import WriterState
from genesis.writer.exceptions import WriterGenerationError
from genesis.qa.qa_loop import QALoop
from genesis.data_models import StructuredSource
from genesis.parsers.word_template_parser import WordTemplateParser
class _Reg:
@staticmethod
def get_or_create(name, template):
return SimpleNamespace(name=name, version="1", template=template)
def test_find_violations_unexpected_language_code_returns_empty():
blocks = [ContentBlock(block_id="0", type="paragraph", text="中文正文内容充分长度足够")]
# 非 zh/ja 的期望语言 → 不强制,返回空
assert find_language_violations(blocks, "xx") == []
def test_writer_agent_zh_expected_rejects_japanese():
class JpEngine:
def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2):
return SimpleNamespace(
data={"title": variables["title"],
"blocks": [{"type": "paragraph",
"text": "本機能は注文処理を行う画面であり、詳細は以下の通り。"}]},
status="ok",
)
agent = WriterAgent(session_id="s", engine=JpEngine(), prompt_registry=_Reg(),
state=WriterState(["db_design"]), max_retries=1)
ctx = GenerationContext(chapter_id="db_design", title="DB 設計",
template_marker=ChapterSpec(chapter_id="db_design", title="DB 設計"),
structured_source=None, write_rules=[], design_rules=[],
template_styles=set(), output_language="zh")
try:
agent.generate_chapter(ctx)
assert False, "应抛 WriterGenerationError"
except WriterGenerationError:
pass
def test_qa_loop_passes_with_explicit_zh_on_chinese_fake(tmp_path):
"""显式 zh + 中文 Fake → 语言维度通过(验证透传链路)。"""
template = Path("samples/phase5-slice/template.docx")
if not template.exists():
import pytest
pytest.skip("样本模板缺失")
parsed = WordTemplateParser().parse(str(template))
ss = StructuredSource(template=parsed, tables=[], rule_docs=[], image_analyses=[],
existing_system=None, comments=[])
out = tmp_path / "o.docx"
class ZhEngine:
def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2):
return SimpleNamespace(
data={"title": variables["title"],
"blocks": [{"type": "paragraph",
"text": "由 FakeLLM 生成的充分说明内容,满足写入规则要求。"}]},
status="ok",
)
loop = QALoop(max_rounds=1)
report = loop.run(ss, str(out), samples_dir="nonexistent_dir_xyz",
engine=ZhEngine(), template_path=str(template), output_language="zh")
assert report.passed is True
def test_qa_loop_fails_with_explicit_ja_on_chinese_fake(tmp_path):
"""显式 ja + 中文 Fake → 语言维度判失败。"""
template = Path("samples/phase5-slice/template.docx")
if not template.exists():
import pytest
pytest.skip("样本模板缺失")
parsed = WordTemplateParser().parse(str(template))
ss = StructuredSource(template=parsed, tables=[], rule_docs=[], image_analyses=[],
existing_system=None, comments=[])
out = tmp_path / "o.docx"
class ZhEngine:
def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2):
return SimpleNamespace(
data={"title": variables["title"],
"blocks": [{"type": "paragraph",
"text": "由 FakeLLM 生成的充分说明内容,满足写入规则要求。"}]},
status="ok",
)
loop = QALoop(max_rounds=1)
# 显式 ja + 中文 Fake:writer 生成阶段即因语言不一致硬失败(设计语义)
try:
loop.run(ss, str(out), samples_dir="nonexistent_dir_xyz",
engine=ZhEngine(), template_path=str(template), output_language="ja")
assert False, "应抛出 WriterGenerationError"
except WriterGenerationError:
pass
+93
View File
@@ -0,0 +1,93 @@
"""步骤 1:语言参数贯通测试(config → GenerationContext → to_vars → context_builder → run_trial)。"""
from __future__ import annotations
from genesis.data_models import (
CellValue, ExcelTable, ParsedTemplate, Provenance, SheetType, StructuredSource,
)
from genesis.writer.context_builder import build_contexts
from genesis.writer.models import ChapterSpec, GenerationContext
def _src() -> StructuredSource:
return StructuredSource(
tables=[],
template=ParsedTemplate("t.docx", [__import__("genesis.data_models", fromlist=["ChapterMarker"]).ChapterMarker("heading", "1. はじめに", 1)], {}, {"used": []}),
rule_docs=[], image_analyses=[], existing_system=None, comments=[],
)
def test_generation_context_default_output_language_is_auto():
ctx = GenerationContext(
chapter_id="db_design", title="DB 設計",
template_marker=ChapterSpec(chapter_id="db_design", title="DB 設計"),
structured_source=None, write_rules=[], design_rules=[], template_styles=set(),
)
assert ctx.output_language == "auto"
def test_to_vars_language_instruction_explicit_zh():
ctx = GenerationContext(
chapter_id="db_design", title="DB 设計",
template_marker=ChapterSpec(chapter_id="db_design", title="DB 设計"),
structured_source=None, write_rules=[], design_rules=[], template_styles=set(),
output_language="zh",
)
v = ctx.to_vars()
assert "language_instruction" in v
assert "简体中文" in v["language_instruction"]
def test_to_vars_language_instruction_explicit_ja():
ctx = GenerationContext(
chapter_id="db_design", title="DB 設計",
template_marker=ChapterSpec(chapter_id="db_design", title="DB 設計"),
structured_source=None, write_rules=[], design_rules=[], template_styles=set(),
output_language="ja",
)
v = ctx.to_vars()
assert "日本語" in v["language_instruction"]
def test_to_vars_language_instruction_auto_follows_title():
ctx = GenerationContext(
chapter_id="db_design", title="DB 設計",
template_marker=ChapterSpec(chapter_id="db_design", title="DB 設計"),
structured_source=None, write_rules=[], design_rules=[], template_styles=set(),
output_language="auto",
)
v = ctx.to_vars()
assert "标题" in v["language_instruction"] and "一致" in v["language_instruction"]
def test_context_builder_passes_output_language():
src = _src()
ctxs = build_contexts(src, output_language="zh")
assert all(c.output_language == "zh" for c in ctxs)
# 回归:默认仍为 auto
ctxs2 = build_contexts(src)
assert all(c.output_language == "auto" for c in ctxs2)
def test_format_impact_zh_labels_localized():
"""步骤 Boutput_language=zh 时影响调查标签本地化为中文,且不影响 ja 默认。"""
from genesis.data_models import (
ChangeAnalysis, ChangeElement, ChangeType, ImpactReport, ImpactWarning,
)
from genesis.writer.models import _format_impact
ca = ChangeAnalysis(
project_type="enhancement",
new_elements=[ChangeElement("F001", "機能", "止损风控", ChangeType.NEW)],
modified_elements=[ChangeElement("F002", "機能", "下单流程", ChangeType.MODIFIED)],
deleted_elements=[ChangeElement("F030", "機能", "旧功能", ChangeType.DELETED)],
unchanged_elements=[],
warnings=[ImpactWarning("F030", "缺少既存対応")],
)
report = ImpactReport(metadata={}, change_analysis=ca, summary={})
zh = _format_impact(report, None, "zh")
assert "[新建]" in zh and "[变更]" in zh and "[删除]" in zh and "[警告]" in zh
assert "受影响" in zh
# ja / auto 回落仍用日文标签
ja = _format_impact(report, None, "ja")
assert "[新規]" in ja and "[削除]" in ja
+5 -4
View File
@@ -12,13 +12,14 @@ from genesis.qa.qa_loop import QALoop
class FakeEngine:
def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2):
cid = variables.get("title", "x")
# 日文模板章节(标题含假名)→ 返回日文正文,满足语言一致性强制
return SimpleNamespace(
data={
"title": cid,
"blocks": [
{"type": "heading", "level": 2, "text": f"{cid}"},
{"type": "paragraph", "text": "FakeLLM 生成的充分说明内容,满足写入规则要求"},
{"type": "table", "caption": "示例表", "rows": [["列1", "列2"], ["1", "2"]]},
{"type": "heading", "level": 2, "text": f"{cid}"},
{"type": "paragraph", "text": "本機能はFakeLLMにより生成された十分な説明内容であり、書込規則を満たす"},
{"type": "table", "caption": "示例表", "rows": [["列1", "列2"], ["1", "2"]]},
],
},
status="ok",
@@ -50,5 +51,5 @@ def test_phase5_e2e_fake_llm(tmp_path):
assert report.passed is True
loaded = Document(str(out))
text = _all_text(loaded)
assert "FakeLLM 生成的充分说明内容" in text
assert "本機能はFakeLLMにより生成された十分な説明内容" in text
assert "示例表" in text
+92
View File
@@ -0,0 +1,92 @@
"""步骤 CQA 第 11 维度 language_consistency 测试。
评审 R1:不可验证(expected_language 为空/auto)→ 维度满分 1.0 通过,不拉低总分,
使既有 test_eval_scorer / test_phase5_scorer 的 total_score==1.0 断言继续成立。
"""
from __future__ import annotations
from genesis.data_models import (
CellValue, ExcelTable, ParsedTemplate, Provenance, SheetType, StructuredSource,
)
from genesis.eval.scorer import ChapterArtifact, ChapterScorer, EvalReport
def _dim(report: EvalReport, name: str):
for d in report.dimensions:
if d.name == name:
return d
raise AssertionError(f"维度未找到: {name}")
def _source() -> StructuredSource:
cell = CellValue(
value="登録",
provenance=Provenance(file_name="f.xlsx", sheet_name="機能一覧", row=3, column="C", column_header="x"),
)
table = ExcelTable(
name="機能一覧", detected_type=SheetType.FUNCTION,
extraction_method="structured", headers=["v"], rows=[{"v": cell}],
)
return StructuredSource(
tables=[table],
template=ParsedTemplate(file_name="t.docx", sections=[], placeholders={}, styles={}),
rule_docs=[], image_analyses=[], existing_system=None, comments=[],
)
def test_language_dim_unverifiable_is_full_score():
"""expected_language 为空(auto/不可验证)→ 维度满分通过,不拉低总分。"""
src = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="本機能は注文処理を行う画面である。",
source_uris=[], template_sections_expected=["ch3"],
expected_language="",
)
report = ChapterScorer().score([artifact], src)
dim = _dim(report, "language_consistency")
assert dim.score == 1.0
assert dim.passed is True
# 既有总分断言仍成立(全部确定性维度满分)
assert report.total_score == 1.0
def test_language_dim_ja_expected_but_chinese_fails():
src = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="这是一段纯中文的章节正文内容,应当判定为语言违规。",
source_uris=[], template_sections_expected=["ch3"],
expected_language="ja",
)
report = ChapterScorer().score([artifact], src)
dim = _dim(report, "language_consistency")
assert dim.score == 0.0
assert dim.passed is False
assert report.total_score < 1.0
def test_language_dim_zh_expected_japanese_fails():
src = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="本機能は注文処理を行う画面であり、詳細は以下の通り。",
source_uris=[], template_sections_expected=["ch3"],
expected_language="zh",
)
report = ChapterScorer().score([artifact], src)
dim = _dim(report, "language_consistency")
assert dim.score == 0.0
def test_language_dim_threshold_in_defaults():
assert ChapterScorer().thresholds["language_consistency"] == 1.0
def test_dimension_count_increases_by_one():
src = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="正常内容", source_uris=[], template_sections_expected=["ch3"],
)
before = len(ChapterScorer(thresholds={}, llm_evaluators={}).score([artifact], src).dimensions)
# 默认 scorer 含 language_consistency;对比一个不含该维度的基线不可行,
# 此处仅断言维度中包含 language_consistency 名称
report = ChapterScorer().score([artifact], src)
assert any(d.name == "language_consistency" for d in report.dimensions)
+172
View File
@@ -0,0 +1,172 @@
"""步骤 A:确定性语言检测 + 输出语言强制测试。
- detect_script:含假名→jaCJK 零假名→zh,否则 None
- resolve_expected_language:显式 > 标题假名 > 规则文档主导脚本 > ""(不可验证)
- find_language_violations:仅检正文块(heading/table 不检),长度≥12 防误杀
- WriterAgent.generate_chapter:期望语言非空时违规即重试,耗尽→硬失败
"""
from __future__ import annotations
from genesis.writer.language import (
detect_script,
find_language_violations,
resolve_expected_language,
)
from genesis.writer.models import ChapterContent, ContentBlock, GenerationContext, ChapterSpec
from genesis.writer.writer_agent import WriterAgent
from genesis.writer.writer_state import WriterState
from genesis.writer.exceptions import WriterGenerationError
# ---------- detect_script ----------
def test_detect_script_kana_is_ja():
assert detect_script("機能一覧の説明を記述します。") == "ja"
def test_detect_script_cjk_no_kana_is_zh():
assert detect_script("功能一览的说明内容。") == "zh"
def test_detect_script_ascii_is_none():
assert detect_script("Hello world 123") is None
def test_detect_script_mixed_cjk_kana_is_ja():
# 含假名即判 ja,即使混有汉字(中文不可能含日文假名)
assert detect_script("DB設計の概要を説明する。") == "ja"
# ---------- resolve_expected_language ----------
def test_resolve_explicit_overrides():
assert resolve_expected_language("zh", "DB 設計", ["日本語の影響"]) == "zh"
assert resolve_expected_language("ja", "DB 设計", ["中文影响"]) == "ja"
def test_resolve_falls_back_to_title_kana():
# 标题含假名→ja(注意:纯汉字日文标题如「DB設計」无假名,会落入 fallback)
assert resolve_expected_language("auto", "機能一覧の説明", []) == "ja"
def test_resolve_falls_back_to_fallback_text():
# 标题纯汉字无假名(中文式)→ 看 fallback(日文影响书)→ ja
assert resolve_expected_language("auto", "DB 設計", ["影響調査の結果(日本語)"]) == "ja"
def test_resolve_unverifiable_when_no_hint():
assert resolve_expected_language("auto", "DB 設計", ["123 abc"]) == ""
# ---------- find_language_violations ----------
def _blocks(*specs):
out = []
for i, (t, text) in enumerate(specs):
out.append(ContentBlock(block_id=str(i), type=t, text=text))
return out
def test_violation_ja_expected_but_pure_chinese_paragraph():
blocks = _blocks(("paragraph", "这是一段纯中文的章节正文内容。"))
viol = find_language_violations(blocks, "ja")
assert viol # 期望日文却为中文 → 违规
def test_no_violation_ja_expected_japanese_paragraph():
blocks = _blocks(("paragraph", "本機能は注文処理を行う画面であり、詳細は以下の通り。"))
assert not find_language_violations(blocks, "ja")
def test_heading_and_table_blocks_excluded():
# heading 与 table(照抄原文)即使含中文也不算违规
blocks = _blocks(
("heading", "機能一覧表"),
("table", "機能ID 機能名"), # 表格不检
("paragraph", "本機能は注文処理を行う。"),
)
assert not find_language_violations(blocks, "ja")
def test_short_chinese_not_flagged_under_threshold():
# 长度<12 的短术语不误杀
blocks = _blocks(("paragraph", "中文术语"))
assert not find_language_violations(blocks, "ja")
def test_violation_zh_expected_but_kana_present():
blocks = _blocks(("paragraph", "本機能は注文処理を行う画面である。"))
assert find_language_violations(blocks, "zh")
# ---------- WriterAgent 强制 ----------
class MixedLanguageEngine:
"""返回中英混杂(期望日文时违规)的章节内容。"""
def __init__(self, texts):
self._texts = list(texts)
self._i = 0
def chat_structured(self, *, session_id, prompt, variables, schema, retry_count=2):
from types import SimpleNamespace
text = self._texts[min(self._i, len(self._texts) - 1)]
self._i += 1
return SimpleNamespace(
data={"title": variables["title"], "blocks": [{"type": "paragraph", "text": text}]},
status="ok",
)
class FakePromptRegistry:
@staticmethod
def get_or_create(name, template):
from types import SimpleNamespace
return SimpleNamespace(name=name, version="1", template=template)
def _ctx_with_lang(cid, title, lang):
return GenerationContext(
chapter_id=cid, title=title,
template_marker=ChapterSpec(chapter_id=cid, title=title),
structured_source=None, write_rules=[], design_rules=[],
template_styles=set(), output_language=lang,
)
def test_generate_chapter_retries_on_language_violation_then_passes():
# 第一次返回中文(ja 期望→违规),第二次返回日文(通过)
engine = MixedLanguageEngine([
"这是一段纯中文的章节正文内容,应当被判定为语言违规。",
"本機能は注文処理を行う画面であり、詳細は以下の通り記述する。",
])
agent = WriterAgent(session_id="s", engine=engine,
prompt_registry=FakePromptRegistry(),
state=WriterState(["db_design"]), max_retries=3)
content = agent.generate_chapter(_ctx_with_lang("db_design", "DB 設計", "ja"))
assert content is not None
assert "本機能" in content.blocks[0].text
def test_generate_chapter_hard_fails_when_violation_persists():
# 始终返回中文(ja 期望),重试耗尽→硬失败
engine = MixedLanguageEngine([
"这是一段纯中文的章节正文内容,应当被判定为语言违规。",
"还是一段纯中文的章节正文内容,依旧违规。",
])
agent = WriterAgent(session_id="s", engine=engine,
prompt_registry=FakePromptRegistry(),
state=WriterState(["db_design"]), max_retries=2)
try:
agent.generate_chapter(_ctx_with_lang("db_design", "DB 設計", "ja"))
assert False, "应抛出 WriterGenerationError"
except WriterGenerationError as e:
assert "语言" in str(e) or "language" in str(e).lower()
def test_generate_chapter_auto_unverifiable_not_enforced():
# auto 且无法推导期望语言(标题纯汉字无假名、无日文 fallback)→ 不强制,直接通过
engine = MixedLanguageEngine(["这是中文正文但 auto 不可验证所以放行。"])
agent = WriterAgent(session_id="s", engine=engine,
prompt_registry=FakePromptRegistry(),
state=WriterState(["db_design"]), max_retries=1)
content = agent.generate_chapter(_ctx_with_lang("db_design", "DB 設計", "auto"))
assert content is not None
+77
View File
@@ -0,0 +1,77 @@
"""中文样本模板镜像测试(步骤 0)。
断言 zh 模板与 ja 模板章节结构一致:7 个 H1 章、锚点 id(section:xxx)逐一对应,
且锚点段落文本原样保留(映射/注入器零改动)。
"""
from __future__ import annotations
import shutil
from pathlib import Path
import pytest
from docx import Document
from genesis.parsers.word_template_parser import WordTemplateParser
from genesis.writer.template_mapper import map_template
_SAMPLES = Path(r"D:\00_project\Genesis\samples")
_JA = _SAMPLES / "概要設計書テンプレート.docx"
_ZH = _SAMPLES / "概要设计书模板_中文.docx"
EXPECTED_ANCHORS = [
"introduction", "function_list", "screen_list", "report_list",
"db_design", "if_definition", "batch_list",
]
@pytest.fixture
def zh_template(tmp_path):
"""生成 zh 模板到临时目录并返回路径(不污染 samples/)。"""
from scripts.make_zh_template import build_zh_template
dst = tmp_path / "概要设计书模板_中文.docx"
build_zh_template(str(_JA), str(dst))
return dst
def _parse(path: Path):
return WordTemplateParser().parse(str(path))
def test_zh_template_has_same_seven_chapter_anchors(zh_template):
ja = _parse(_JA)
zh = _parse(zh_template)
ja_ids = [ph for ph in ja.placeholders if ph.startswith("section:")]
zh_ids = [ph for ph in zh.placeholders if ph.startswith("section:")]
assert sorted(zh_ids) == sorted(ja_ids)
assert zh_ids == [f"section:{a}" for a in EXPECTED_ANCHORS]
def test_zh_template_anchor_paragraphs_unchanged(zh_template):
"""锚点段落文本在 zh 模板中必须与 ja 完全一致(映射/注入器依赖它)。"""
ja_doc = Document(str(_JA))
zh_doc = Document(str(zh_template))
ja_anchors = [p.text for p in ja_doc.paragraphs if p.text.startswith("{{section:")]
zh_anchors = [p.text for p in zh_doc.paragraphs if p.text.startswith("{{section:")]
assert zh_anchors == ja_anchors
def test_zh_template_headings_translated(zh_template):
zh_doc = Document(str(zh_template))
texts = {p.text for p in zh_doc.paragraphs if p.style.name.startswith("Heading")}
assert "1. 前言" in texts
assert "2. 功能一览" in texts
assert "5. DB设计" in texts
assert "7. 批处理一览" in texts
def test_zh_template_maps_to_same_seven_chapters(zh_template):
"""template_mapper 对 zh 模板产出 7 个章节,锚点与 ja 对应。"""
pt = _parse(zh_template)
specs = map_template(pt)
assert len(specs) == 7
assert [s.section_placeholder for s in specs] == [
f"section:{a}" for a in EXPECTED_ANCHORS
]