feat(eval): 黄金集 + 评分器(T13 架构审查整改,OV4)
- T13 (OV4, P1): 新建 src/genesis/eval/ 包
- golden_set.py: GoldenCase/GoldenSet(YAML 加载,samples/ 真实脱敏样本作 input_ref)
- scorer.py: ChapterScorer 按 §7.2 指标体系打分
- 确定性维度: traceability(resolver 验证 source_uri 可解析率)/
placeholder_residue(无 {{...}} 残留)/chapter_completeness(章节覆盖)
- LLM 语义维度: llm_evaluators 钩子(默认中性分,待 Phase5)
- tests/fixtures/eval/golden_set.yaml 示例黄金集
- 新增 test_eval_scorer.py(9 用例)
- 同步 design.md §7.5 黄金集与评分器(定位 CI 质量门禁)
- TDD: RED(模块缺失)→ GREEN(聚焦 8 passed)→ 全量 240 passed / 100.00%(1279 stmts/308 br)
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
"""eval 包:生成质量评估(T13,OV4)。
|
||||
|
||||
提供:黄金集(GoldenSet)结构 + 评分器(ChapterScorer)。
|
||||
评分器实现 §7.2 中确定性可机器验证维度(可追溯性/占位符残留/章节完整性),
|
||||
LLM 语义维度(内容准确性/幻觉)通过注入钩子扩展,默认返回中性分。
|
||||
"""
|
||||
|
||||
from genesis.eval.golden_set import GoldenCase, GoldenSet
|
||||
from genesis.eval.scorer import ChapterArtifact, ChapterScorer, DimensionScore, EvalReport
|
||||
|
||||
__all__ = [
|
||||
"GoldenCase",
|
||||
"GoldenSet",
|
||||
"ChapterArtifact",
|
||||
"ChapterScorer",
|
||||
"DimensionScore",
|
||||
"EvalReport",
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""黄金集(T13,OV4)。
|
||||
|
||||
GoldenCase:一条黄金样例(输入样本引用 + 期望最低评分 + 备注)。
|
||||
GoldenSet:从 YAML 加载回归基线(samples/ 真实脱敏样本作为 input_ref 基础)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class GoldenCase(BaseModel):
|
||||
id: str
|
||||
input_ref: str
|
||||
expected_min_score: float = Field(default=0.7, ge=0.0, le=1.0)
|
||||
note: str = ""
|
||||
|
||||
|
||||
class GoldenSet:
|
||||
def __init__(self, cases: list[GoldenCase]) -> None:
|
||||
self.cases = cases
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path | str) -> "GoldenSet":
|
||||
data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
|
||||
cases = [GoldenCase(**c) for c in data.get("cases", [])]
|
||||
return cls(cases=cases)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""评分器(T13,OV4)。
|
||||
|
||||
对生成章节按 §7.2 指标体系评分。确定性维度:
|
||||
- traceability(可追溯性):source_uri 全部能在源中定位 → 1.0,否则按可解析比例
|
||||
- placeholder_residue(占位符残留):文本无 {{...}} → 1.0,否则 0.0
|
||||
- chapter_completeness(章节完整性):生成章节覆盖期望集合 → 覆盖率
|
||||
|
||||
LLM 语义维度(内容准确性/幻觉/规则遵守)通过 llm_evaluators 钩子注入,
|
||||
默认返回中性分 0.5(标记未启用),待 Phase5 接入真实 LLM 校验。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from genesis.data_models import StructuredSource
|
||||
from genesis.parsers.resolver import validate_source_uris
|
||||
|
||||
_PLACEHOLDER_RE = re.compile(r"\{\{.*?\}\}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DimensionScore:
|
||||
name: str
|
||||
score: float # 0.0 ~ 1.0
|
||||
passed: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChapterArtifact:
|
||||
chapter_id: str
|
||||
text: str
|
||||
source_uris: list[str]
|
||||
template_sections_expected: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalReport:
|
||||
dimensions: list[DimensionScore]
|
||||
total_score: float
|
||||
passed: bool
|
||||
|
||||
|
||||
# 维度默认通过阈值
|
||||
DEFAULT_THRESHOLDS: dict[str, float] = {
|
||||
"traceability": 1.0,
|
||||
"placeholder_residue": 1.0,
|
||||
"chapter_completeness": 1.0,
|
||||
}
|
||||
|
||||
|
||||
class ChapterScorer:
|
||||
"""章节生成质量评分器(确定性维度 + LLM 钩子)。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
thresholds: dict[str, float] | None = None,
|
||||
llm_evaluators: dict[str, "callable"] | None = None,
|
||||
) -> None:
|
||||
self.thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
|
||||
self.llm_evaluators = llm_evaluators or {}
|
||||
|
||||
def score(self, chapters: list[ChapterArtifact], source: StructuredSource) -> EvalReport:
|
||||
dimensions: list[DimensionScore] = []
|
||||
dimensions.append(self._traceability(chapters, source))
|
||||
dimensions.append(self._placeholder_residue(chapters))
|
||||
dimensions.append(self._completeness(chapters))
|
||||
|
||||
# LLM 语义维度钩子(每个章节独立评,取该维度平均)
|
||||
for name, fn in self.llm_evaluators.items():
|
||||
dimensions.append(self._run_llm_dimension(name, fn, chapters))
|
||||
|
||||
total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0
|
||||
passed = all(d.passed for d in dimensions)
|
||||
return EvalReport(dimensions=dimensions, total_score=round(total, 4), passed=passed)
|
||||
|
||||
# ---------- 确定性维度 ----------
|
||||
|
||||
def _traceability(self, chapters: list[ChapterArtifact], source: StructuredSource) -> DimensionScore:
|
||||
all_uris: list[str] = []
|
||||
for ch in chapters:
|
||||
all_uris.extend(ch.source_uris)
|
||||
if not all_uris:
|
||||
# 无引用则视为满分(不扣分;可追溯性仅约束「有引用时须可解析」)
|
||||
return DimensionScore("traceability", 1.0, True, "无 source_uri 引用")
|
||||
result = validate_source_uris(all_uris, source)
|
||||
ratio = len(result.resolved) / len(all_uris)
|
||||
passed = ratio >= self.thresholds["traceability"]
|
||||
return DimensionScore(
|
||||
"traceability", round(ratio, 4), passed,
|
||||
f"resolved {len(result.resolved)}/{len(all_uris)}(unresolved: {result.unresolved})",
|
||||
)
|
||||
|
||||
def _placeholder_residue(self, chapters: list[ChapterArtifact]) -> DimensionScore:
|
||||
bad = [ch.chapter_id for ch in chapters if _PLACEHOLDER_RE.search(ch.text)]
|
||||
score = 0.0 if bad else 1.0
|
||||
return DimensionScore(
|
||||
"placeholder_residue", score, not bad,
|
||||
"残留占位符: " + (", ".join(bad) if bad else "无"),
|
||||
)
|
||||
|
||||
def _completeness(self, chapters: list[ChapterArtifact]) -> DimensionScore:
|
||||
expected = set()
|
||||
for ch in chapters:
|
||||
expected.update(ch.template_sections_expected)
|
||||
if not expected:
|
||||
return DimensionScore("chapter_completeness", 1.0, True, "无章节期望约束")
|
||||
got = {ch.chapter_id for ch in chapters}
|
||||
coverage = len(got & expected) / len(expected)
|
||||
passed = coverage >= self.thresholds["chapter_completeness"]
|
||||
return DimensionScore(
|
||||
"chapter_completeness", round(coverage, 4), passed,
|
||||
f"覆盖率 {len(got & expected)}/{len(expected)}",
|
||||
)
|
||||
|
||||
# ---------- LLM 维度 ----------
|
||||
|
||||
def _run_llm_dimension(self, name: str, fn, chapters: list[ChapterArtifact]) -> DimensionScore:
|
||||
scores = [fn(ch) for ch in chapters]
|
||||
avg = sum(s.score for s in scores) / len(scores) if scores else 0.5
|
||||
detail = " | ".join(s.detail for s in scores) if scores else "no chapters"
|
||||
return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail)
|
||||
Reference in New Issue
Block a user