feat(eval): 黄金集 + 评分器(T13 架构审查整改,OV4)

- T13 (OV4, P1): 新建 src/genesis/eval/ 包
  - golden_set.py: GoldenCase/GoldenSet(YAML 加载,samples/ 真实脱敏样本作 input_ref)
  - scorer.py: ChapterScorer 按 §7.2 指标体系打分
    - 确定性维度: traceability(resolver 验证 source_uri 可解析率)/
      placeholder_residue(无 {{...}} 残留)/chapter_completeness(章节覆盖)
    - LLM 语义维度: llm_evaluators 钩子(默认中性分,待 Phase5)
  - tests/fixtures/eval/golden_set.yaml 示例黄金集
- 新增 test_eval_scorer.py(9 用例)
- 同步 design.md §7.5 黄金集与评分器(定位 CI 质量门禁)
- TDD: RED(模块缺失)→ GREEN(聚焦 8 passed)→ 全量 240 passed / 100.00%(1279 stmts/308 br)
This commit is contained in:
lhl
2026-08-12 22:44:47 +08:00
parent 69aec7716c
commit d2e651bad0
7 changed files with 376 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
"""eval 包:生成质量评估(T13,OV4)。
提供:黄金集(GoldenSet)结构 + 评分器(ChapterScorer)。
评分器实现 §7.2 中确定性可机器验证维度(可追溯性/占位符残留/章节完整性),
LLM 语义维度(内容准确性/幻觉)通过注入钩子扩展,默认返回中性分。
"""
from genesis.eval.golden_set import GoldenCase, GoldenSet
from genesis.eval.scorer import ChapterArtifact, ChapterScorer, DimensionScore, EvalReport
__all__ = [
"GoldenCase",
"GoldenSet",
"ChapterArtifact",
"ChapterScorer",
"DimensionScore",
"EvalReport",
]
+31
View File
@@ -0,0 +1,31 @@
"""黄金集(T13OV4)。
GoldenCase:一条黄金样例(输入样本引用 + 期望最低评分 + 备注)。
GoldenSet:从 YAML 加载回归基线(samples/ 真实脱敏样本作为 input_ref 基础)。
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import yaml
from pydantic import BaseModel, Field
class GoldenCase(BaseModel):
id: str
input_ref: str
expected_min_score: float = Field(default=0.7, ge=0.0, le=1.0)
note: str = ""
class GoldenSet:
def __init__(self, cases: list[GoldenCase]) -> None:
self.cases = cases
@classmethod
def load(cls, path: Path | str) -> "GoldenSet":
data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
cases = [GoldenCase(**c) for c in data.get("cases", [])]
return cls(cases=cases)
+124
View File
@@ -0,0 +1,124 @@
"""评分器(T13OV4)。
对生成章节按 §7.2 指标体系评分。确定性维度:
- traceability(可追溯性):source_uri 全部能在源中定位 → 1.0,否则按可解析比例
- placeholder_residue(占位符残留):文本无 {{...}} → 1.0,否则 0.0
- chapter_completeness(章节完整性):生成章节覆盖期望集合 → 覆盖率
LLM 语义维度(内容准确性/幻觉/规则遵守)通过 llm_evaluators 钩子注入,
默认返回中性分 0.5(标记未启用),待 Phase5 接入真实 LLM 校验。
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from genesis.data_models import StructuredSource
from genesis.parsers.resolver import validate_source_uris
_PLACEHOLDER_RE = re.compile(r"\{\{.*?\}\}")
@dataclass
class DimensionScore:
name: str
score: float # 0.0 ~ 1.0
passed: bool
detail: str = ""
@dataclass
class ChapterArtifact:
chapter_id: str
text: str
source_uris: list[str]
template_sections_expected: list[str]
@dataclass
class EvalReport:
dimensions: list[DimensionScore]
total_score: float
passed: bool
# 维度默认通过阈值
DEFAULT_THRESHOLDS: dict[str, float] = {
"traceability": 1.0,
"placeholder_residue": 1.0,
"chapter_completeness": 1.0,
}
class ChapterScorer:
"""章节生成质量评分器(确定性维度 + LLM 钩子)。"""
def __init__(
self,
thresholds: dict[str, float] | None = None,
llm_evaluators: dict[str, "callable"] | None = None,
) -> None:
self.thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
self.llm_evaluators = llm_evaluators or {}
def score(self, chapters: list[ChapterArtifact], source: StructuredSource) -> EvalReport:
dimensions: list[DimensionScore] = []
dimensions.append(self._traceability(chapters, source))
dimensions.append(self._placeholder_residue(chapters))
dimensions.append(self._completeness(chapters))
# LLM 语义维度钩子(每个章节独立评,取该维度平均)
for name, fn in self.llm_evaluators.items():
dimensions.append(self._run_llm_dimension(name, fn, chapters))
total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0
passed = all(d.passed for d in dimensions)
return EvalReport(dimensions=dimensions, total_score=round(total, 4), passed=passed)
# ---------- 确定性维度 ----------
def _traceability(self, chapters: list[ChapterArtifact], source: StructuredSource) -> DimensionScore:
all_uris: list[str] = []
for ch in chapters:
all_uris.extend(ch.source_uris)
if not all_uris:
# 无引用则视为满分(不扣分;可追溯性仅约束「有引用时须可解析」)
return DimensionScore("traceability", 1.0, True, "无 source_uri 引用")
result = validate_source_uris(all_uris, source)
ratio = len(result.resolved) / len(all_uris)
passed = ratio >= self.thresholds["traceability"]
return DimensionScore(
"traceability", round(ratio, 4), passed,
f"resolved {len(result.resolved)}/{len(all_uris)}unresolved: {result.unresolved}",
)
def _placeholder_residue(self, chapters: list[ChapterArtifact]) -> DimensionScore:
bad = [ch.chapter_id for ch in chapters if _PLACEHOLDER_RE.search(ch.text)]
score = 0.0 if bad else 1.0
return DimensionScore(
"placeholder_residue", score, not bad,
"残留占位符: " + (", ".join(bad) if bad else ""),
)
def _completeness(self, chapters: list[ChapterArtifact]) -> DimensionScore:
expected = set()
for ch in chapters:
expected.update(ch.template_sections_expected)
if not expected:
return DimensionScore("chapter_completeness", 1.0, True, "无章节期望约束")
got = {ch.chapter_id for ch in chapters}
coverage = len(got & expected) / len(expected)
passed = coverage >= self.thresholds["chapter_completeness"]
return DimensionScore(
"chapter_completeness", round(coverage, 4), passed,
f"覆盖率 {len(got & expected)}/{len(expected)}",
)
# ---------- LLM 维度 ----------
def _run_llm_dimension(self, name: str, fn, chapters: list[ChapterArtifact]) -> DimensionScore:
scores = [fn(ch) for ch in chapters]
avg = sum(s.score for s in scores) / len(scores) if scores else 0.5
detail = " | ".join(s.detail for s in scores) if scores else "no chapters"
return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail)