Coverage for src\genesis\eval\scorer.py: 100%

1"""评分器（T13，OV4）。 

2 

3对生成章节按 §7.2 指标体系评分。确定性维度： 

4 - traceability（可追溯性）：source_uri 全部能在源中定位 → 1.0，否则按可解析比例 

5 - placeholder_residue（占位符残留）：文本无 {{...}} → 1.0，否则 0.0 

6 - chapter_completeness（章节完整性）：生成章节覆盖期望集合 → 覆盖率 

7 

8LLM 语义维度（内容准确性/幻觉/规则遵守）通过 llm_evaluators 钩子注入， 

9默认返回中性分 0.5（标记未启用），待 Phase5 接入真实 LLM 校验。 

10""" 

11 

12from __future__ import annotations 

13 

14import re 

15from dataclasses import dataclass, field 

16 

17from genesis.data_models import StructuredSource 

18from genesis.parsers.resolver import validate_source_uris 

19 

20_PLACEHOLDER_RE = re.compile(r"\{\{.*?\}\}") 

21 

22 

23@dataclass 

24class DimensionScore: 

25 name: str 

26 score: float # 0.0 ~ 1.0 

27 passed: bool 

28 detail: str = "" 

29 

30 

31@dataclass 

32class ChapterArtifact: 

33 chapter_id: str 

34 text: str 

35 source_uris: list[str] 

36 template_sections_expected: list[str] 

37 expected_language: str = "" # 期望输出语言（"zh"/"ja"；空=不可验证，维度记满分） 

38 # 块级 (type, text, caption) 列表：供语言一致性维度排除 heading / table.rows（照抄源/跟随模板） 

39 blocks: list[tuple[str, str, str]] = field(default_factory=list) 

40 

41 

42# 逐章评估通过阈值（基于逐章总分） 

43PASS_THRESHOLD: float = 0.6 

44# 内容充分性维度：文本长度达到该值即视为充分 

45ADEQUACY_MIN_LEN: int = 15 

46 

47 

48@dataclass 

49class EvalReport: 

50 dimensions: list[DimensionScore] 

51 total_score: float 

52 passed: bool 

53 failed_chapters: list[str] = field(default_factory=list) 

54 per_chapter: list["EvalReport"] = field(default_factory=list) 

55 

56 

57# 维度默认通过阈值 

58DEFAULT_THRESHOLDS: dict[str, float] = { 

59 "traceability": 1.0, 

60 "placeholder_residue": 1.0, 

61 "chapter_completeness": 1.0, 

62 "language_consistency": 1.0, 

63} 

64 

65 

66class ChapterScorer: 

67 """章节生成质量评分器（确定性维度 + LLM 钩子）。""" 

68 

69 def __init__( 

70 self, 

71 thresholds: dict[str, float] | None = None, 

72 llm_evaluators: dict[str, "callable"] | None = None, 

73 ) -> None: 

74 self.thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})} 

75 self.llm_evaluators = llm_evaluators or {} 

76 

77 def score(self, chapters: list[ChapterArtifact], source: StructuredSource) -> EvalReport: 

78 dimensions: list[DimensionScore] = [] 

79 dimensions.append(self._traceability(chapters, source)) 

80 dimensions.append(self._placeholder_residue(chapters)) 

81 dimensions.append(self._completeness(chapters)) 

82 dimensions.append(self._language_consistency(chapters)) 

83 

84 # LLM 语义维度钩子（每个章节独立评，取该维度平均） 

85 for name, fn in self.llm_evaluators.items(): 

86 dimensions.append(self._run_llm_dimension(name, fn, chapters)) 

87 

88 total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0 

89 passed = all(d.passed for d in dimensions) 

90 

91 # 逐章评估：对每一章节独立评分，收集 per_chapter，并标记未达标章节 

92 per_chapter = [self._score_chapter(ch, source) for ch in chapters] 

93 failed_chapters = [ 

94 ch.chapter_id for ch, r in zip(chapters, per_chapter) if r.failed_chapters 

95 ] 

96 

97 return EvalReport( 

98 dimensions=dimensions, 

99 total_score=round(total, 4), 

100 passed=passed, 

101 failed_chapters=failed_chapters, 

102 per_chapter=per_chapter, 

103 ) 

104 

105 # ---------- 确定性维度 ---------- 

106 

107 def _traceability(self, chapters: list[ChapterArtifact], source: StructuredSource) -> DimensionScore: 

108 all_uris: list[str] = [] 

109 for ch in chapters: 

110 all_uris.extend(ch.source_uris) 

111 if not all_uris: 

112 # 无引用则视为满分（不扣分；可追溯性仅约束「有引用时须可解析」） 

113 return DimensionScore("traceability", 1.0, True, "无 source_uri 引用") 

114 result = validate_source_uris(all_uris, source) 

115 ratio = len(result.resolved) / len(all_uris) 

116 passed = ratio >= self.thresholds["traceability"] 

117 return DimensionScore( 

118 "traceability", round(ratio, 4), passed, 

119 f"resolved {len(result.resolved)}/{len(all_uris)}（unresolved: {result.unresolved}）", 

120 ) 

121 

122 def _placeholder_residue(self, chapters: list[ChapterArtifact]) -> DimensionScore: 

123 bad = [ch.chapter_id for ch in chapters if _PLACEHOLDER_RE.search(ch.text)] 

124 score = 0.0 if bad else 1.0 

125 return DimensionScore( 

126 "placeholder_residue", score, not bad, 

127 "残留占位符: " + (", ".join(bad) if bad else "无"), 

128 ) 

129 

130 def _completeness(self, chapters: list[ChapterArtifact]) -> DimensionScore: 

131 expected = set() 

132 for ch in chapters: 

133 expected.update(ch.template_sections_expected) 

134 if not expected: 

135 return DimensionScore("chapter_completeness", 1.0, True, "无章节期望约束") 

136 got = {ch.chapter_id for ch in chapters} 

137 coverage = len(got & expected) / len(expected) 

138 passed = coverage >= self.thresholds["chapter_completeness"] 

139 return DimensionScore( 

140 "chapter_completeness", round(coverage, 4), passed, 

141 f"覆盖率 {len(got & expected)}/{len(expected)}", 

142 ) 

143 

144 # ---------- LLM 维度 ---------- 

145 

146 def _run_llm_dimension(self, name: str, fn, chapters: list[ChapterArtifact]) -> DimensionScore: 

147 scores = [fn(ch) for ch in chapters] 

148 avg = sum(s.score for s in scores) / len(scores) if scores else 0.5 

149 detail = " | ".join(s.detail for s in scores) if scores else "no chapters" 

150 return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail) 

151 

152 # ---------- 语言一致性维度（步骤 C，确定性）---------- 

153 

154 def _language_consistency(self, chapters: list[ChapterArtifact]) -> DimensionScore: 

155 """第 11 维度：输出语言与期望语言一致（确定性，脚本可验证）。 

156 

157 与 writer.language 共用单一检测事实来源。期望语言为空（auto/不可验证） 

158 → 记满分 1.0 通过（评审 R1：不拉低总分，避免误伤既有断言）。 

159 仅检正文块（heading/table 不检，表格照抄源、标题跟随模板）。 

160 """ 

161 from genesis.writer.language import find_language_violations 

162 

163 if not chapters: 

164 return DimensionScore("language_consistency", 1.0, True, "no chapters") 

165 per: list[DimensionScore] = [] 

166 for ch in chapters: 

167 expected = ch.expected_language 

168 if not expected: 

169 per.append(DimensionScore( 

170 "language_consistency", 1.0, True, "unverifiable (no expected language)")) 

171 continue 

172 from genesis.writer.models import ContentBlock 

173 if ch.blocks: 

174 # 优先用块级信息（type, text, caption；可排除 heading/table.rows） 

175 blocks = [ContentBlock(block_id=str(i), type=t, text=tx, caption=cap or None) 

176 for i, (t, tx, cap) in enumerate(ch.blocks)] 

177 else: 

178 # 回退：整段正文作为单个 paragraph 块 

179 blocks = [ContentBlock(block_id="0", type="paragraph", text=ch.text or "")] 

180 viol = find_language_violations(blocks, expected) 

181 score = 0.0 if viol else 1.0 

182 passed = score >= self.thresholds["language_consistency"] 

183 per.append(DimensionScore( 

184 "language_consistency", score, passed, 

185 f"期望 {expected}，违规 {len(viol)} 处" if viol else f"期望 {expected}，一致")) 

186 avg = sum(p.score for p in per) / len(per) 

187 passed = all(p.passed for p in per) 

188 detail = " | ".join(p.detail for p in per) 

189 return DimensionScore("language_consistency", round(avg, 4), passed, detail) 

190 

191 # ---------- 逐章评估 ---------- 

192 

193 def _score_chapter(self, chapter: ChapterArtifact, source: StructuredSource) -> EvalReport: 

194 """对单个章节独立评分，返回该章节的 EvalReport。 

195 

196 复用确定性维度逻辑（可追溯性/占位符残留）并计算内容充分性， 

197 总分低于 PASS_THRESHOLD 即判该章节失败。 

198 """ 

199 dims = [ 

200 self._traceability([chapter], source), 

201 self._placeholder_residue([chapter]), 

202 self._adequacy(chapter), 

203 self._language_consistency([chapter]), 

204 ] 

205 total = sum(d.score for d in dims) / len(dims) if dims else 0.0 

206 failed = (not all(d.passed for d in dims)) or total < PASS_THRESHOLD 

207 return EvalReport( 

208 dimensions=dims, 

209 total_score=round(total, 4), 

210 passed=not failed, 

211 failed_chapters=[chapter.chapter_id] if failed else [], 

212 per_chapter=[], 

213 ) 

214 

215 def _adequacy(self, chapter: ChapterArtifact) -> DimensionScore: 

216 """内容充分性：基于正文字本长度启发式判定章节是否充分。""" 

217 text = chapter.text or "" 

218 score = 1.0 if len(text) >= ADEQUACY_MIN_LEN else 0.3 

219 passed = score >= PASS_THRESHOLD 

220 return DimensionScore( 

221 "adequacy", score, passed, f"内容长度 {len(text)}（充分阈值 {ADEQUACY_MIN_LEN}）" 

222 )