lhl
d01e1b720f
- 步骤0: 新增中文镜像模板 scripts/make_zh_template.py 与 samples/概要设计书模板_中文.docx（7章锚点原样保留） - 步骤1: config.WriterConfig.output_language→Settings.writer；GenerationContext.output_language + to_vars.language_instruction（zh/ja/auto）；writer_agent【语言约束】改引变量；context_builder/orchestrator/run_trial 透传 --output-language - 步骤A: 新建 src/genesis/writer/language.py（detect_script/resolve_expected_language/find_language_violations）；WriterAgent.generate_chapter 按期望语言强制、违规重试、耗尽硬失败；max_retries 默认 1→2 - 步骤B: _format_impact 影响调查标签按 output_language 本地化（zh 新建/变更/删除/警告） - 步骤C: eval scorer 第 11 维度 language_consistency（不可验证=满分，不拉低总分）；ChapterArtifact.expected_language；QAValidator.validate_doc 透传；QALoop.run 透传 output_language - 测试: test_zh_template/test_language_plumbing/test_writer_language/test_scorer_language/test_language_coverage，并更新 test_phase5_e2e - 全量 pytest 424 passed / 99.15%（覆盖率门槛 99% 达标）
93 lines
3.5 KiB
Python
93 lines
3.5 KiB
Python
"""步骤 C：QA 第 11 维度 language_consistency 测试。
|
||
|
||
评审 R1：不可验证（expected_language 为空/auto）→ 维度满分 1.0 通过，不拉低总分，
|
||
使既有 test_eval_scorer / test_phase5_scorer 的 total_score==1.0 断言继续成立。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from genesis.data_models import (
|
||
CellValue, ExcelTable, ParsedTemplate, Provenance, SheetType, StructuredSource,
|
||
)
|
||
from genesis.eval.scorer import ChapterArtifact, ChapterScorer, EvalReport
|
||
|
||
|
||
def _dim(report: EvalReport, name: str):
|
||
for d in report.dimensions:
|
||
if d.name == name:
|
||
return d
|
||
raise AssertionError(f"维度未找到: {name}")
|
||
|
||
|
||
def _source() -> StructuredSource:
|
||
cell = CellValue(
|
||
value="登録",
|
||
provenance=Provenance(file_name="f.xlsx", sheet_name="機能一覧", row=3, column="C", column_header="x"),
|
||
)
|
||
table = ExcelTable(
|
||
name="機能一覧", detected_type=SheetType.FUNCTION,
|
||
extraction_method="structured", headers=["v"], rows=[{"v": cell}],
|
||
)
|
||
return StructuredSource(
|
||
tables=[table],
|
||
template=ParsedTemplate(file_name="t.docx", sections=[], placeholders={}, styles={}),
|
||
rule_docs=[], image_analyses=[], existing_system=None, comments=[],
|
||
)
|
||
|
||
|
||
def test_language_dim_unverifiable_is_full_score():
|
||
"""expected_language 为空（auto/不可验证）→ 维度满分通过，不拉低总分。"""
|
||
src = _source()
|
||
artifact = ChapterArtifact(
|
||
chapter_id="ch3", text="本機能は注文処理を行う画面である。",
|
||
source_uris=[], template_sections_expected=["ch3"],
|
||
expected_language="",
|
||
)
|
||
report = ChapterScorer().score([artifact], src)
|
||
dim = _dim(report, "language_consistency")
|
||
assert dim.score == 1.0
|
||
assert dim.passed is True
|
||
# 既有总分断言仍成立（全部确定性维度满分）
|
||
assert report.total_score == 1.0
|
||
|
||
|
||
def test_language_dim_ja_expected_but_chinese_fails():
|
||
src = _source()
|
||
artifact = ChapterArtifact(
|
||
chapter_id="ch3", text="这是一段纯中文的章节正文内容，应当判定为语言违规。",
|
||
source_uris=[], template_sections_expected=["ch3"],
|
||
expected_language="ja",
|
||
)
|
||
report = ChapterScorer().score([artifact], src)
|
||
dim = _dim(report, "language_consistency")
|
||
assert dim.score == 0.0
|
||
assert dim.passed is False
|
||
assert report.total_score < 1.0
|
||
|
||
|
||
def test_language_dim_zh_expected_japanese_fails():
|
||
src = _source()
|
||
artifact = ChapterArtifact(
|
||
chapter_id="ch3", text="本機能は注文処理を行う画面であり、詳細は以下の通り。",
|
||
source_uris=[], template_sections_expected=["ch3"],
|
||
expected_language="zh",
|
||
)
|
||
report = ChapterScorer().score([artifact], src)
|
||
dim = _dim(report, "language_consistency")
|
||
assert dim.score == 0.0
|
||
|
||
|
||
def test_language_dim_threshold_in_defaults():
|
||
assert ChapterScorer().thresholds["language_consistency"] == 1.0
|
||
|
||
|
||
def test_dimension_count_increases_by_one():
|
||
src = _source()
|
||
artifact = ChapterArtifact(
|
||
chapter_id="ch3", text="正常内容", source_uris=[], template_sections_expected=["ch3"],
|
||
)
|
||
before = len(ChapterScorer(thresholds={}, llm_evaluators={}).score([artifact], src).dimensions)
|
||
# 默认 scorer 含 language_consistency；对比一个不含该维度的基线不可行，
|
||
# 此处仅断言维度中包含 language_consistency 名称
|
||
report = ChapterScorer().score([artifact], src)
|
||
assert any(d.name == "language_consistency" for d in report.dimensions)
|