feat(eval): populate EvalReport.per_chapter + failed_chapters in score()
This commit is contained in:
@@ -36,12 +36,19 @@ class ChapterArtifact:
|
||||
template_sections_expected: list[str]
|
||||
|
||||
|
||||
# 逐章评估通过阈值(基于逐章总分)
|
||||
PASS_THRESHOLD: float = 0.6
|
||||
# 内容充分性维度:文本长度达到该值即视为充分
|
||||
ADEQUACY_MIN_LEN: int = 15
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalReport:
|
||||
dimensions: list[DimensionScore]
|
||||
total_score: float
|
||||
passed: bool
|
||||
failed_chapters: list[str] = field(default_factory=list)
|
||||
per_chapter: list["EvalReport"] = field(default_factory=list)
|
||||
|
||||
|
||||
# 维度默认通过阈值
|
||||
@@ -75,7 +82,20 @@ class ChapterScorer:
|
||||
|
||||
total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0
|
||||
passed = all(d.passed for d in dimensions)
|
||||
return EvalReport(dimensions=dimensions, total_score=round(total, 4), passed=passed)
|
||||
|
||||
# 逐章评估:对每一章节独立评分,收集 per_chapter,并标记未达标章节
|
||||
per_chapter = [self._score_chapter(ch, source) for ch in chapters]
|
||||
failed_chapters = [
|
||||
ch.chapter_id for ch, r in zip(chapters, per_chapter) if r.failed_chapters
|
||||
]
|
||||
|
||||
return EvalReport(
|
||||
dimensions=dimensions,
|
||||
total_score=round(total, 4),
|
||||
passed=passed,
|
||||
failed_chapters=failed_chapters,
|
||||
per_chapter=per_chapter,
|
||||
)
|
||||
|
||||
# ---------- 确定性维度 ----------
|
||||
|
||||
@@ -123,3 +143,35 @@ class ChapterScorer:
|
||||
avg = sum(s.score for s in scores) / len(scores) if scores else 0.5
|
||||
detail = " | ".join(s.detail for s in scores) if scores else "no chapters"
|
||||
return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail)
|
||||
|
||||
# ---------- 逐章评估 ----------
|
||||
|
||||
def _score_chapter(self, chapter: ChapterArtifact, source: StructuredSource) -> EvalReport:
|
||||
"""对单个章节独立评分,返回该章节的 EvalReport。
|
||||
|
||||
复用确定性维度逻辑(可追溯性/占位符残留)并计算内容充分性,
|
||||
总分低于 PASS_THRESHOLD 即判该章节失败。
|
||||
"""
|
||||
dims = [
|
||||
self._traceability([chapter], source),
|
||||
self._placeholder_residue([chapter]),
|
||||
self._adequacy(chapter),
|
||||
]
|
||||
total = sum(d.score for d in dims) / len(dims) if dims else 0.0
|
||||
failed = (not all(d.passed for d in dims)) or total < PASS_THRESHOLD
|
||||
return EvalReport(
|
||||
dimensions=dims,
|
||||
total_score=round(total, 4),
|
||||
passed=not failed,
|
||||
failed_chapters=[chapter.chapter_id] if failed else [],
|
||||
per_chapter=[],
|
||||
)
|
||||
|
||||
def _adequacy(self, chapter: ChapterArtifact) -> DimensionScore:
|
||||
"""内容充分性:基于正文字本长度启发式判定章节是否充分。"""
|
||||
text = chapter.text or ""
|
||||
score = 1.0 if len(text) >= ADEQUACY_MIN_LEN else 0.3
|
||||
passed = score >= PASS_THRESHOLD
|
||||
return DimensionScore(
|
||||
"adequacy", score, passed, f"内容长度 {len(text)}(充分阈值 {ADEQUACY_MIN_LEN})"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user