feat(eval): populate EvalReport.per_chapter + failed_chapters in score()

This commit is contained in:
lhl
2026-08-13 10:44:35 +08:00
parent 08e311b0a1
commit d29cb78167
2 changed files with 135 additions and 1 deletions
+53 -1
View File
@@ -36,12 +36,19 @@ class ChapterArtifact:
template_sections_expected: list[str]
# 逐章评估通过阈值(基于逐章总分)
PASS_THRESHOLD: float = 0.6
# 内容充分性维度:文本长度达到该值即视为充分
ADEQUACY_MIN_LEN: int = 15
@dataclass
class EvalReport:
dimensions: list[DimensionScore]
total_score: float
passed: bool
failed_chapters: list[str] = field(default_factory=list)
per_chapter: list["EvalReport"] = field(default_factory=list)
# 维度默认通过阈值
@@ -75,7 +82,20 @@ class ChapterScorer:
total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0
passed = all(d.passed for d in dimensions)
return EvalReport(dimensions=dimensions, total_score=round(total, 4), passed=passed)
# 逐章评估:对每一章节独立评分,收集 per_chapter,并标记未达标章节
per_chapter = [self._score_chapter(ch, source) for ch in chapters]
failed_chapters = [
ch.chapter_id for ch, r in zip(chapters, per_chapter) if r.failed_chapters
]
return EvalReport(
dimensions=dimensions,
total_score=round(total, 4),
passed=passed,
failed_chapters=failed_chapters,
per_chapter=per_chapter,
)
# ---------- 确定性维度 ----------
@@ -123,3 +143,35 @@ class ChapterScorer:
avg = sum(s.score for s in scores) / len(scores) if scores else 0.5
detail = " | ".join(s.detail for s in scores) if scores else "no chapters"
return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail)
# ---------- 逐章评估 ----------
def _score_chapter(self, chapter: ChapterArtifact, source: StructuredSource) -> EvalReport:
"""对单个章节独立评分,返回该章节的 EvalReport。
复用确定性维度逻辑(可追溯性/占位符残留)并计算内容充分性,
总分低于 PASS_THRESHOLD 即判该章节失败。
"""
dims = [
self._traceability([chapter], source),
self._placeholder_residue([chapter]),
self._adequacy(chapter),
]
total = sum(d.score for d in dims) / len(dims) if dims else 0.0
failed = (not all(d.passed for d in dims)) or total < PASS_THRESHOLD
return EvalReport(
dimensions=dims,
total_score=round(total, 4),
passed=not failed,
failed_chapters=[chapter.chapter_id] if failed else [],
per_chapter=[],
)
def _adequacy(self, chapter: ChapterArtifact) -> DimensionScore:
"""内容充分性:基于正文字本长度启发式判定章节是否充分。"""
text = chapter.text or ""
score = 1.0 if len(text) >= ADEQUACY_MIN_LEN else 0.3
passed = score >= PASS_THRESHOLD
return DimensionScore(
"adequacy", score, passed, f"内容长度 {len(text)}(充分阈值 {ADEQUACY_MIN_LEN}"
)