feat(eval): 黄金集 + 评分器(T13 架构审查整改,OV4)
- T13 (OV4, P1): 新建 src/genesis/eval/ 包
- golden_set.py: GoldenCase/GoldenSet(YAML 加载,samples/ 真实脱敏样本作 input_ref)
- scorer.py: ChapterScorer 按 §7.2 指标体系打分
- 确定性维度: traceability(resolver 验证 source_uri 可解析率)/
placeholder_residue(无 {{...}} 残留)/chapter_completeness(章节覆盖)
- LLM 语义维度: llm_evaluators 钩子(默认中性分,待 Phase5)
- tests/fixtures/eval/golden_set.yaml 示例黄金集
- 新增 test_eval_scorer.py(9 用例)
- 同步 design.md §7.5 黄金集与评分器(定位 CI 质量门禁)
- TDD: RED(模块缺失)→ GREEN(聚焦 8 passed)→ 全量 240 passed / 100.00%(1279 stmts/308 br)
This commit is contained in:
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
cases:
|
||||
- id: g1
|
||||
input_ref: samples/要件定義.xlsx
|
||||
expected_min_score: 0.7
|
||||
note: 脱敏真实样本回归基线(Phase5 Writer 实现后填充实际评分)
|
||||
- id: g2
|
||||
input_ref: samples/概要設計書_template.docx
|
||||
expected_min_score: 0.7
|
||||
note: 模板结构合规基线
|
||||
@@ -0,0 +1,180 @@
|
||||
"""评分器测试(T13,OV4)。
|
||||
|
||||
OV4 裁定:成功标准无量度(无黄金集/评分器)→ 建立黄金集 + 评分器。
|
||||
本文件测试确定性可机器验证维度(可追溯性/占位符残留/章节完整性),
|
||||
LLM 语义维度预留钩子;并测试黄金集加载。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from genesis.data_models import (
|
||||
CellValue,
|
||||
ExcelTable,
|
||||
ParsedTemplate,
|
||||
Provenance,
|
||||
SheetType,
|
||||
StructuredSource,
|
||||
)
|
||||
from genesis.eval.scorer import (
|
||||
ChapterArtifact,
|
||||
ChapterScorer,
|
||||
DimensionScore,
|
||||
EvalReport,
|
||||
)
|
||||
from genesis.eval.golden_set import GoldenCase, GoldenSet
|
||||
|
||||
|
||||
# ---------- 小源(供可追溯性维度定位) ----------
|
||||
|
||||
def _source() -> StructuredSource:
|
||||
cell = CellValue(
|
||||
value="登録",
|
||||
provenance=Provenance(file_name="f.xlsx", sheet_name="機能一覧", row=3, column="C", column_header="x"),
|
||||
)
|
||||
table = ExcelTable(
|
||||
name="機能一覧", detected_type=SheetType.FUNCTION,
|
||||
extraction_method="structured", headers=["v"], rows=[{"v": cell}],
|
||||
)
|
||||
return StructuredSource(
|
||||
tables=[table],
|
||||
template=ParsedTemplate(file_name="t.docx", sections=[], placeholders={}, styles={}),
|
||||
rule_docs=[], image_analyses=[], existing_system=None, comments=[],
|
||||
)
|
||||
|
||||
|
||||
# ---------- 可追溯性维度 ----------
|
||||
|
||||
def test_traceability_full_when_all_uris_resolvable():
|
||||
source = _source()
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3",
|
||||
text="機能一覧(出典: f.xlsx#機能一覧!C3)",
|
||||
source_uris=["f.xlsx#機能一覧!C3"],
|
||||
template_sections_expected=["ch3"],
|
||||
)
|
||||
report = ChapterScorer().score([artifact], source)
|
||||
trace = _dim(report, "traceability")
|
||||
assert trace.score == 1.0
|
||||
assert trace.passed is True
|
||||
|
||||
|
||||
def test_traceability_zero_when_uris_fake():
|
||||
source = _source()
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3",
|
||||
text="機能(出典: fake.xlsx#X!Z9)",
|
||||
source_uris=["fake.xlsx#X!Z9"],
|
||||
template_sections_expected=["ch3"],
|
||||
)
|
||||
report = ChapterScorer().score([artifact], source)
|
||||
trace = _dim(report, "traceability")
|
||||
assert trace.score == 0.0
|
||||
assert trace.passed is False
|
||||
|
||||
|
||||
# ---------- 占位符残留维度 ----------
|
||||
|
||||
def test_placeholder_residue_fails():
|
||||
source = _source()
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3",
|
||||
text="未替换占位符 {{section:db_tables}}",
|
||||
source_uris=["f.xlsx#機能一覧!C3"],
|
||||
template_sections_expected=["ch3"],
|
||||
)
|
||||
report = ChapterScorer().score([artifact], source)
|
||||
dim = _dim(report, "placeholder_residue")
|
||||
assert dim.score == 0.0
|
||||
assert dim.passed is False
|
||||
|
||||
|
||||
def test_placeholder_residue_ok_when_clean():
|
||||
source = _source()
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3", text="正常生成内容", source_uris=[], template_sections_expected=["ch3"],
|
||||
)
|
||||
report = ChapterScorer().score([artifact], source)
|
||||
assert _dim(report, "placeholder_residue").score == 1.0
|
||||
|
||||
|
||||
# ---------- 章节完整性维度 ----------
|
||||
|
||||
def test_completeness_fails_when_section_missing():
|
||||
source = _source()
|
||||
# 期望 ch3/ch4 两章,但只生成 ch3
|
||||
artifacts = [ChapterArtifact(
|
||||
chapter_id="ch3", text="a", source_uris=[], template_sections_expected=["ch3", "ch4"],
|
||||
)]
|
||||
report = ChapterScorer().score(artifacts, source)
|
||||
dim = _dim(report, "chapter_completeness")
|
||||
assert dim.score == 0.5
|
||||
assert dim.passed is False
|
||||
|
||||
|
||||
# ---------- LLM 维度钩子 ----------
|
||||
|
||||
def test_llm_dimension_hook_invoked():
|
||||
source = _source()
|
||||
called = {}
|
||||
|
||||
def fake_llm(chapter: ChapterArtifact) -> DimensionScore:
|
||||
called["hit"] = True
|
||||
return DimensionScore(name="llm_accuracy", score=0.8, passed=True, detail="stub")
|
||||
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3", text="x", source_uris=[], template_sections_expected=["ch3"],
|
||||
)
|
||||
scorer = ChapterScorer(llm_evaluators={"llm_accuracy": fake_llm})
|
||||
report = scorer.score([artifact], source)
|
||||
assert called.get("hit") is True
|
||||
assert _dim(report, "llm_accuracy").score == 0.8
|
||||
|
||||
|
||||
# ---------- 总分聚合 + 通过判定 ----------
|
||||
|
||||
def test_total_score_aggregation():
|
||||
source = _source()
|
||||
artifact = ChapterArtifact(
|
||||
chapter_id="ch3", text="正常(出典: f.xlsx#機能一覧!C3)",
|
||||
source_uris=["f.xlsx#機能一覧!C3"], template_sections_expected=["ch3"],
|
||||
)
|
||||
report = ChapterScorer().score([artifact], source)
|
||||
assert isinstance(report, EvalReport)
|
||||
assert 0.0 <= report.total_score <= 1.0
|
||||
# 全部确定性维度满分 → 总分接近 1.0(仅 llm 维度默认中性 0.5)
|
||||
assert report.total_score >= 0.8
|
||||
|
||||
|
||||
def test_empty_chapters_does_not_crash():
|
||||
"""空章节输入:无引用/无约束 → 确定性维度中性满分,不应抛错。"""
|
||||
source = _source()
|
||||
report = ChapterScorer().score([], source)
|
||||
assert report.total_score == 1.0
|
||||
assert report.passed is True
|
||||
|
||||
|
||||
# ---------- 黄金集加载 ----------
|
||||
|
||||
def test_golden_set_load(tmp_path):
|
||||
yaml_text = """
|
||||
cases:
|
||||
- id: g1
|
||||
input_ref: samples/要件定義.xlsx
|
||||
expected_min_score: 0.7
|
||||
note: 脱敏真实样本回归基线
|
||||
"""
|
||||
p = tmp_path / "golden_set.yaml"
|
||||
p.write_text(yaml_text, encoding="utf-8")
|
||||
gs = GoldenSet.load(p)
|
||||
assert len(gs.cases) == 1
|
||||
assert gs.cases[0].id == "g1"
|
||||
assert gs.cases[0].expected_min_score == 0.7
|
||||
|
||||
|
||||
def _dim(report: EvalReport, name: str) -> DimensionScore:
|
||||
for d in report.dimensions:
|
||||
if d.name == name:
|
||||
return d
|
||||
raise AssertionError(f"维度未找到: {name}")
|
||||
Reference in New Issue
Block a user