feat(eval): 黄金集 + 评分器(T13 架构审查整改,OV4)

- T13 (OV4, P1): 新建 src/genesis/eval/ 包
  - golden_set.py: GoldenCase/GoldenSet(YAML 加载,samples/ 真实脱敏样本作 input_ref)
  - scorer.py: ChapterScorer 按 §7.2 指标体系打分
    - 确定性维度: traceability(resolver 验证 source_uri 可解析率)/
      placeholder_residue(无 {{...}} 残留)/chapter_completeness(章节覆盖)
    - LLM 语义维度: llm_evaluators 钩子(默认中性分,待 Phase5)
  - tests/fixtures/eval/golden_set.yaml 示例黄金集
- 新增 test_eval_scorer.py(9 用例)
- 同步 design.md §7.5 黄金集与评分器(定位 CI 质量门禁)
- TDD: RED(模块缺失)→ GREEN(聚焦 8 passed)→ 全量 240 passed / 100.00%(1279 stmts/308 br)
This commit is contained in:
lhl
2026-08-12 22:44:47 +08:00
parent 69aec7716c
commit d2e651bad0
7 changed files with 376 additions and 0 deletions
+9
View File
@@ -0,0 +1,9 @@
cases:
- id: g1
input_ref: samples/要件定義.xlsx
expected_min_score: 0.7
note: 脱敏真实样本回归基线(Phase5 Writer 实现后填充实际评分)
- id: g2
input_ref: samples/概要設計書_template.docx
expected_min_score: 0.7
note: 模板结构合规基线
+180
View File
@@ -0,0 +1,180 @@
"""评分器测试(T13OV4)。
OV4 裁定:成功标准无量度(无黄金集/评分器)→ 建立黄金集 + 评分器。
本文件测试确定性可机器验证维度(可追溯性/占位符残留/章节完整性),
LLM 语义维度预留钩子;并测试黄金集加载。
"""
from __future__ import annotations
import pytest
from genesis.data_models import (
CellValue,
ExcelTable,
ParsedTemplate,
Provenance,
SheetType,
StructuredSource,
)
from genesis.eval.scorer import (
ChapterArtifact,
ChapterScorer,
DimensionScore,
EvalReport,
)
from genesis.eval.golden_set import GoldenCase, GoldenSet
# ---------- 小源(供可追溯性维度定位) ----------
def _source() -> StructuredSource:
cell = CellValue(
value="登録",
provenance=Provenance(file_name="f.xlsx", sheet_name="機能一覧", row=3, column="C", column_header="x"),
)
table = ExcelTable(
name="機能一覧", detected_type=SheetType.FUNCTION,
extraction_method="structured", headers=["v"], rows=[{"v": cell}],
)
return StructuredSource(
tables=[table],
template=ParsedTemplate(file_name="t.docx", sections=[], placeholders={}, styles={}),
rule_docs=[], image_analyses=[], existing_system=None, comments=[],
)
# ---------- 可追溯性维度 ----------
def test_traceability_full_when_all_uris_resolvable():
source = _source()
artifact = ChapterArtifact(
chapter_id="ch3",
text="機能一覧(出典: f.xlsx#機能一覧!C3",
source_uris=["f.xlsx#機能一覧!C3"],
template_sections_expected=["ch3"],
)
report = ChapterScorer().score([artifact], source)
trace = _dim(report, "traceability")
assert trace.score == 1.0
assert trace.passed is True
def test_traceability_zero_when_uris_fake():
source = _source()
artifact = ChapterArtifact(
chapter_id="ch3",
text="機能(出典: fake.xlsx#X!Z9",
source_uris=["fake.xlsx#X!Z9"],
template_sections_expected=["ch3"],
)
report = ChapterScorer().score([artifact], source)
trace = _dim(report, "traceability")
assert trace.score == 0.0
assert trace.passed is False
# ---------- 占位符残留维度 ----------
def test_placeholder_residue_fails():
source = _source()
artifact = ChapterArtifact(
chapter_id="ch3",
text="未替换占位符 {{section:db_tables}}",
source_uris=["f.xlsx#機能一覧!C3"],
template_sections_expected=["ch3"],
)
report = ChapterScorer().score([artifact], source)
dim = _dim(report, "placeholder_residue")
assert dim.score == 0.0
assert dim.passed is False
def test_placeholder_residue_ok_when_clean():
source = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="正常生成内容", source_uris=[], template_sections_expected=["ch3"],
)
report = ChapterScorer().score([artifact], source)
assert _dim(report, "placeholder_residue").score == 1.0
# ---------- 章节完整性维度 ----------
def test_completeness_fails_when_section_missing():
source = _source()
# 期望 ch3/ch4 两章,但只生成 ch3
artifacts = [ChapterArtifact(
chapter_id="ch3", text="a", source_uris=[], template_sections_expected=["ch3", "ch4"],
)]
report = ChapterScorer().score(artifacts, source)
dim = _dim(report, "chapter_completeness")
assert dim.score == 0.5
assert dim.passed is False
# ---------- LLM 维度钩子 ----------
def test_llm_dimension_hook_invoked():
source = _source()
called = {}
def fake_llm(chapter: ChapterArtifact) -> DimensionScore:
called["hit"] = True
return DimensionScore(name="llm_accuracy", score=0.8, passed=True, detail="stub")
artifact = ChapterArtifact(
chapter_id="ch3", text="x", source_uris=[], template_sections_expected=["ch3"],
)
scorer = ChapterScorer(llm_evaluators={"llm_accuracy": fake_llm})
report = scorer.score([artifact], source)
assert called.get("hit") is True
assert _dim(report, "llm_accuracy").score == 0.8
# ---------- 总分聚合 + 通过判定 ----------
def test_total_score_aggregation():
source = _source()
artifact = ChapterArtifact(
chapter_id="ch3", text="正常(出典: f.xlsx#機能一覧!C3",
source_uris=["f.xlsx#機能一覧!C3"], template_sections_expected=["ch3"],
)
report = ChapterScorer().score([artifact], source)
assert isinstance(report, EvalReport)
assert 0.0 <= report.total_score <= 1.0
# 全部确定性维度满分 → 总分接近 1.0(仅 llm 维度默认中性 0.5)
assert report.total_score >= 0.8
def test_empty_chapters_does_not_crash():
"""空章节输入:无引用/无约束 → 确定性维度中性满分,不应抛错。"""
source = _source()
report = ChapterScorer().score([], source)
assert report.total_score == 1.0
assert report.passed is True
# ---------- 黄金集加载 ----------
def test_golden_set_load(tmp_path):
yaml_text = """
cases:
- id: g1
input_ref: samples/要件定義.xlsx
expected_min_score: 0.7
note: 脱敏真实样本回归基线
"""
p = tmp_path / "golden_set.yaml"
p.write_text(yaml_text, encoding="utf-8")
gs = GoldenSet.load(p)
assert len(gs.cases) == 1
assert gs.cases[0].id == "g1"
assert gs.cases[0].expected_min_score == 0.7
def _dim(report: EvalReport, name: str) -> DimensionScore:
for d in report.dimensions:
if d.name == name:
return d
raise AssertionError(f"维度未找到: {name}")