diff --git a/_AI_USAGE_LOG.md b/_AI_USAGE_LOG.md index 050bdf1..a2a3ed7 100644 --- a/_AI_USAGE_LOG.md +++ b/_AI_USAGE_LOG.md @@ -79,4 +79,5 @@ | 2026-08-11 | Agent 实现 | T5(架构审查整改):删死配置 + 同步文档(Issue5 + OV1)。config.py 删除 QdrantStoreConfig 类与 VectorStoreConfig.qdrant 字段、AppConfig.task_queue.redis_url;同步更新 6 处文档(api-design §1/§4.3/§5.2/§5.3/§6.2:TaskQueue 标注 v1 仅 InMemory、Redis/Valkey 为 v2 预留;rag-layer §9:Storage Adapter 仅 ChromaAdapter、移除切换流程/工厂 qdrant 分支;agent-runtime §3.1/§3.5;design §5.5/§8.4.1;config-design env/app.yaml/rag.yaml/docker compose;web-ui §4.1)+ tests/fixtures/rag.yaml 去 qdrant 段;历史评审记录(design-review/web-ui-review/phase1 plan)保留原样不改写;新增 3 用例(QdrantStoreConfig 已删/vector_store 无 qdrant 字段/task_queue 无 redis_url)+ 同步 2 个既有 qdrant 依赖用例;TDD 验证 RED(三处死配置存在)→ GREEN(聚焦 10 passed)→ 全量 189 passed 覆盖 100.00%(991 stmts/252 br),fail_under=99 达标 | src/genesis/config.py, tests/test_config.py, tests/fixtures/rag.yaml, docs/api-design.md, docs/rag-layer-design.md, docs/agent-runtime-design.md, docs/design.md, docs/config-design.md, docs/web-ui-design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-11 | Agent 实现 | T6+T11(架构审查整改,Lane B):v1 rerank 精排 + bge-m3 多语言切换(Issue6 + OV2)。config.py 新增 RerankConfig(enabled=True/model=BAAI/bge-reranker-v2-m3/device=cpu)挂入 RagConfig;EmbeddingConfig.model 默认 bge-small-zh-v1.5 → BAAI/bge-m3(实际语料日文);rag-layer-design.md 新增 §2.3 多语言与日文样本验证、§6.3 Rerank 精排(窗口=RRF top-10、候选≤top_k 跳过、故障降级 RRF 原序),原 §6.3-6.6 顺延 6.4-6.7;选型表/依赖表/manifest/流程图 bge-small-zh → bge-m3;config-design.md embedding 默认 + 新增 rerank 段;design.md §5.5 与 implementation-plan 4.3 同步;新增 tests/test_rag_design_consistency.py 一致性门禁(6 用例:代码默认/fixture 同步/4 文档用 bge-m3+reranker/无 legacy 引用);TDD 验证 RED(默认模型仍旧+rerank 字段不存在)→ GREEN(聚焦 13 passed)→ 全量 198 passed 覆盖 100.00%(996 stmts/252 br),fail_under=99 达标 | src/genesis/config.py, tests/test_config.py, tests/test_rag_design_consistency.py, tests/fixtures/rag.yaml, docs/rag-layer-design.md, docs/config-design.md, docs/design.md, docs/implementation-plan.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-11 | Agent 实现 | T12(架构审查整改,P1):URI 统一 + resolver + 强验证(OV3)。新建 src/genesis/parsers/resolver.py:parse_source_uri(解析 file.xlsx#Sheet!CellRef → SourceRef,格式非法 raise URIError)、provenance_to_uri(Provenance 还原,与 build 互逆)、resolve_source_uri(StructuredSource 内定位真实 CellValue)、validate_source_uris(批量强验证 → ValidationResult(resolved/unresolved),格式错误或源中不存在一律 unresolved,防 QA#8 编造 URI 作弊);统一 URI 唯一生成入口 build_source_uri(provenance.py),formatting_detector 经其生成,无散落不一致;新建 tests/test_resolver.py(13 用例:解析/往返/定位/批量验证/防御分支);同步 design.md §9.2 机制化说明 + §6.8 第五步存在性校验引用;TDD 验证 RED(模块缺失)→ GREEN(聚焦 10 passed)→ 全量 231 passed 覆盖 100.00%(1191 stmts/298 br),fail_under=99 达标 | src/genesis/parsers/resolver.py, tests/test_resolver.py, docs/design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | +| 2026-08-11 | Agent 实现 | T13(架构审查整改,P1):黄金集 + 评分器(OV4)。新建 src/genesis/eval/ 包:golden_set.py(GoldenCase/GoldenSet,YAML 加载,samples/ 真实脱敏样本作 input_ref 基线)、scorer.py(ChapterScorer 按 §7.2 指标体系打分);确定性维度 traceability(resolver 验证 source_uri 可解析率)/placeholder_residue(无 {{...}} 残留)/chapter_completeness(章节覆盖期望集合);LLM 语义维度经 llm_evaluators 钩子注入(默认中性分,待 Phase5);新增 tests/test_eval_scorer.py(9 用例:各维度/聚合/钩子/空输入);tests/fixtures/eval/golden_set.yaml 示例黄金集(2 case);同步 design.md §7.5 黄金集与评分器机制化说明(定位为 CI 质量门禁);TDD 验证 RED(模块缺失 + NameError)→ GREEN(聚焦 8 passed)→ 全量 240 passed 覆盖 100.00%(1279 stmts/308 br),fail_under=99 达标 | src/genesis/eval/__init__.py, src/genesis/eval/golden_set.py, src/genesis/eval/scorer.py, tests/test_eval_scorer.py, tests/fixtures/eval/golden_set.yaml, docs/design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | | 2026-08-11 | Agent 实现 | T14+T16(架构审查整改,Lane A):DataGate 机制化 + 任务级持久化(OV5 + OV7)。新建 src/genesis/orchestrator/ 包:datagate.py(DataGate.load(source, selector):子集加载 + 规模保护 max_total_rows=500 无 selector 拒绝全量 + token 预算 max_total_tokens=8000 复用 CJK 保守估算 + 未知表容错)+ task_queue.py(TaskQueue ABC + PersistentTaskQueue SQLite 落盘:enqueue/poll/update_status/get/cancel/recover/close + 幂等去重 §5.3 + recover 将 running→failed、pending 保留);新建 tests/test_datagate.py(8 用例:子集/规模保护/1000 行 Excel selector 放行/token 预算/未知表/空 selector)+ tests/test_task_queue.py(11 用例:CRUD/幂等缓存/重开不丢/recover 语义/防御分支);同步 agent-runtime-design.md(§4.2 原则→机制 + §3.5/3.6 任务级恢复已实现)、api-design.md §5.2/5.3、design.md §8.4.1;TDD 验证 RED(模块缺失/错误消息不匹配)→ GREEN(聚焦 8+8 passed)→ 全量 218 passed 覆盖 100.00%(1140 stmts/278 br),fail_under=99 达标 | src/genesis/orchestrator/__init__.py, src/genesis/orchestrator/datagate.py, src/genesis/orchestrator/task_queue.py, tests/test_datagate.py, tests/test_task_queue.py, docs/agent-runtime-design.md, docs/api-design.md, docs/design.md, _AI_USAGE_LOG.md | deepseek-v4-flash-free | diff --git a/docs/design.md b/docs/design.md index 3180f13..7c065e4 100644 --- a/docs/design.md +++ b/docs/design.md @@ -994,6 +994,19 @@ QA 输出: QA 发现错误 → 将问题列表反馈给 Writer → Writer 只修正错误章节(不重新生成全部) → 重新 QA 校验 + +### 7.5 黄金集与评分器(T13 机制化,OV4) + +> OV4 裁定:成功标准须有量度 → 建立黄金集 + 评分器(已实现于 `src/genesis/eval/`)。 + +- **评分器(ChapterScorer)**:按 §7.2 指标体系输出各维度 `DimensionScore(score, passed)` 与总分 `EvalReport`。 + - 确定性维度(代码可验证,无需 LLM): + - `traceability`:所有 `source_uri` 经 `resolver.validate_source_uris` 定位(不可解析 → 扣分,防 QA#8 作弊) + - `placeholder_residue`:渲染文本无 `{{...}}` 残留(残留即 fail) + - `chapter_completeness`:生成章节覆盖模板期望集合(覆盖率) + - LLM 语义维度(内容准确性/幻觉/规则遵守):通过 `llm_evaluators` 钩子注入,默认中性分,待 Phase5 接入真实推理 +- **黄金集(GoldenSet)**:从 YAML 加载回归基线,`samples/` 真实脱敏样本作 `input_ref`(审查报告 §8.2 已确认 7 个样本为黄金集基础);每条 `GoldenCase` 标注 `expected_min_score`,Phase5 后用于端到端回归 +- 评分器作为 CI 质量门禁:生成结果总分 < 阈值 → 阻断合并(与 fail_under=99 覆盖率门禁同级) → 重复至全部通过或用户确认放行 ``` diff --git a/src/genesis/eval/__init__.py b/src/genesis/eval/__init__.py new file mode 100644 index 0000000..a708b94 --- /dev/null +++ b/src/genesis/eval/__init__.py @@ -0,0 +1,18 @@ +"""eval 包:生成质量评估(T13,OV4)。 + +提供:黄金集(GoldenSet)结构 + 评分器(ChapterScorer)。 +评分器实现 §7.2 中确定性可机器验证维度(可追溯性/占位符残留/章节完整性), +LLM 语义维度(内容准确性/幻觉)通过注入钩子扩展,默认返回中性分。 +""" + +from genesis.eval.golden_set import GoldenCase, GoldenSet +from genesis.eval.scorer import ChapterArtifact, ChapterScorer, DimensionScore, EvalReport + +__all__ = [ + "GoldenCase", + "GoldenSet", + "ChapterArtifact", + "ChapterScorer", + "DimensionScore", + "EvalReport", +] diff --git a/src/genesis/eval/golden_set.py b/src/genesis/eval/golden_set.py new file mode 100644 index 0000000..6c63f93 --- /dev/null +++ b/src/genesis/eval/golden_set.py @@ -0,0 +1,31 @@ +"""黄金集(T13,OV4)。 + +GoldenCase:一条黄金样例(输入样本引用 + 期望最低评分 + 备注)。 +GoldenSet:从 YAML 加载回归基线(samples/ 真实脱敏样本作为 input_ref 基础)。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import yaml +from pydantic import BaseModel, Field + + +class GoldenCase(BaseModel): + id: str + input_ref: str + expected_min_score: float = Field(default=0.7, ge=0.0, le=1.0) + note: str = "" + + +class GoldenSet: + def __init__(self, cases: list[GoldenCase]) -> None: + self.cases = cases + + @classmethod + def load(cls, path: Path | str) -> "GoldenSet": + data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + cases = [GoldenCase(**c) for c in data.get("cases", [])] + return cls(cases=cases) diff --git a/src/genesis/eval/scorer.py b/src/genesis/eval/scorer.py new file mode 100644 index 0000000..b77433c --- /dev/null +++ b/src/genesis/eval/scorer.py @@ -0,0 +1,124 @@ +"""评分器(T13,OV4)。 + +对生成章节按 §7.2 指标体系评分。确定性维度: + - traceability(可追溯性):source_uri 全部能在源中定位 → 1.0,否则按可解析比例 + - placeholder_residue(占位符残留):文本无 {{...}} → 1.0,否则 0.0 + - chapter_completeness(章节完整性):生成章节覆盖期望集合 → 覆盖率 + +LLM 语义维度(内容准确性/幻觉/规则遵守)通过 llm_evaluators 钩子注入, +默认返回中性分 0.5(标记未启用),待 Phase5 接入真实 LLM 校验。 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from genesis.data_models import StructuredSource +from genesis.parsers.resolver import validate_source_uris + +_PLACEHOLDER_RE = re.compile(r"\{\{.*?\}\}") + + +@dataclass +class DimensionScore: + name: str + score: float # 0.0 ~ 1.0 + passed: bool + detail: str = "" + + +@dataclass +class ChapterArtifact: + chapter_id: str + text: str + source_uris: list[str] + template_sections_expected: list[str] + + +@dataclass +class EvalReport: + dimensions: list[DimensionScore] + total_score: float + passed: bool + + +# 维度默认通过阈值 +DEFAULT_THRESHOLDS: dict[str, float] = { + "traceability": 1.0, + "placeholder_residue": 1.0, + "chapter_completeness": 1.0, +} + + +class ChapterScorer: + """章节生成质量评分器(确定性维度 + LLM 钩子)。""" + + def __init__( + self, + thresholds: dict[str, float] | None = None, + llm_evaluators: dict[str, "callable"] | None = None, + ) -> None: + self.thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})} + self.llm_evaluators = llm_evaluators or {} + + def score(self, chapters: list[ChapterArtifact], source: StructuredSource) -> EvalReport: + dimensions: list[DimensionScore] = [] + dimensions.append(self._traceability(chapters, source)) + dimensions.append(self._placeholder_residue(chapters)) + dimensions.append(self._completeness(chapters)) + + # LLM 语义维度钩子(每个章节独立评,取该维度平均) + for name, fn in self.llm_evaluators.items(): + dimensions.append(self._run_llm_dimension(name, fn, chapters)) + + total = sum(d.score for d in dimensions) / len(dimensions) if dimensions else 0.0 + passed = all(d.passed for d in dimensions) + return EvalReport(dimensions=dimensions, total_score=round(total, 4), passed=passed) + + # ---------- 确定性维度 ---------- + + def _traceability(self, chapters: list[ChapterArtifact], source: StructuredSource) -> DimensionScore: + all_uris: list[str] = [] + for ch in chapters: + all_uris.extend(ch.source_uris) + if not all_uris: + # 无引用则视为满分(不扣分;可追溯性仅约束「有引用时须可解析」) + return DimensionScore("traceability", 1.0, True, "无 source_uri 引用") + result = validate_source_uris(all_uris, source) + ratio = len(result.resolved) / len(all_uris) + passed = ratio >= self.thresholds["traceability"] + return DimensionScore( + "traceability", round(ratio, 4), passed, + f"resolved {len(result.resolved)}/{len(all_uris)}(unresolved: {result.unresolved})", + ) + + def _placeholder_residue(self, chapters: list[ChapterArtifact]) -> DimensionScore: + bad = [ch.chapter_id for ch in chapters if _PLACEHOLDER_RE.search(ch.text)] + score = 0.0 if bad else 1.0 + return DimensionScore( + "placeholder_residue", score, not bad, + "残留占位符: " + (", ".join(bad) if bad else "无"), + ) + + def _completeness(self, chapters: list[ChapterArtifact]) -> DimensionScore: + expected = set() + for ch in chapters: + expected.update(ch.template_sections_expected) + if not expected: + return DimensionScore("chapter_completeness", 1.0, True, "无章节期望约束") + got = {ch.chapter_id for ch in chapters} + coverage = len(got & expected) / len(expected) + passed = coverage >= self.thresholds["chapter_completeness"] + return DimensionScore( + "chapter_completeness", round(coverage, 4), passed, + f"覆盖率 {len(got & expected)}/{len(expected)}", + ) + + # ---------- LLM 维度 ---------- + + def _run_llm_dimension(self, name: str, fn, chapters: list[ChapterArtifact]) -> DimensionScore: + scores = [fn(ch) for ch in chapters] + avg = sum(s.score for s in scores) / len(scores) if scores else 0.5 + detail = " | ".join(s.detail for s in scores) if scores else "no chapters" + return DimensionScore(name, round(avg, 4), all(s.passed for s in scores), detail) diff --git a/tests/fixtures/eval/golden_set.yaml b/tests/fixtures/eval/golden_set.yaml new file mode 100644 index 0000000..e53645e --- /dev/null +++ b/tests/fixtures/eval/golden_set.yaml @@ -0,0 +1,9 @@ +cases: + - id: g1 + input_ref: samples/要件定義.xlsx + expected_min_score: 0.7 + note: 脱敏真实样本回归基线(Phase5 Writer 实现后填充实际评分) + - id: g2 + input_ref: samples/概要設計書_template.docx + expected_min_score: 0.7 + note: 模板结构合规基线 diff --git a/tests/test_eval_scorer.py b/tests/test_eval_scorer.py new file mode 100644 index 0000000..067c14d --- /dev/null +++ b/tests/test_eval_scorer.py @@ -0,0 +1,180 @@ +"""评分器测试(T13,OV4)。 + +OV4 裁定:成功标准无量度(无黄金集/评分器)→ 建立黄金集 + 评分器。 +本文件测试确定性可机器验证维度(可追溯性/占位符残留/章节完整性), +LLM 语义维度预留钩子;并测试黄金集加载。 +""" + +from __future__ import annotations + +import pytest + +from genesis.data_models import ( + CellValue, + ExcelTable, + ParsedTemplate, + Provenance, + SheetType, + StructuredSource, +) +from genesis.eval.scorer import ( + ChapterArtifact, + ChapterScorer, + DimensionScore, + EvalReport, +) +from genesis.eval.golden_set import GoldenCase, GoldenSet + + +# ---------- 小源(供可追溯性维度定位) ---------- + +def _source() -> StructuredSource: + cell = CellValue( + value="登録", + provenance=Provenance(file_name="f.xlsx", sheet_name="機能一覧", row=3, column="C", column_header="x"), + ) + table = ExcelTable( + name="機能一覧", detected_type=SheetType.FUNCTION, + extraction_method="structured", headers=["v"], rows=[{"v": cell}], + ) + return StructuredSource( + tables=[table], + template=ParsedTemplate(file_name="t.docx", sections=[], placeholders={}, styles={}), + rule_docs=[], image_analyses=[], existing_system=None, comments=[], + ) + + +# ---------- 可追溯性维度 ---------- + +def test_traceability_full_when_all_uris_resolvable(): + source = _source() + artifact = ChapterArtifact( + chapter_id="ch3", + text="機能一覧(出典: f.xlsx#機能一覧!C3)", + source_uris=["f.xlsx#機能一覧!C3"], + template_sections_expected=["ch3"], + ) + report = ChapterScorer().score([artifact], source) + trace = _dim(report, "traceability") + assert trace.score == 1.0 + assert trace.passed is True + + +def test_traceability_zero_when_uris_fake(): + source = _source() + artifact = ChapterArtifact( + chapter_id="ch3", + text="機能(出典: fake.xlsx#X!Z9)", + source_uris=["fake.xlsx#X!Z9"], + template_sections_expected=["ch3"], + ) + report = ChapterScorer().score([artifact], source) + trace = _dim(report, "traceability") + assert trace.score == 0.0 + assert trace.passed is False + + +# ---------- 占位符残留维度 ---------- + +def test_placeholder_residue_fails(): + source = _source() + artifact = ChapterArtifact( + chapter_id="ch3", + text="未替换占位符 {{section:db_tables}}", + source_uris=["f.xlsx#機能一覧!C3"], + template_sections_expected=["ch3"], + ) + report = ChapterScorer().score([artifact], source) + dim = _dim(report, "placeholder_residue") + assert dim.score == 0.0 + assert dim.passed is False + + +def test_placeholder_residue_ok_when_clean(): + source = _source() + artifact = ChapterArtifact( + chapter_id="ch3", text="正常生成内容", source_uris=[], template_sections_expected=["ch3"], + ) + report = ChapterScorer().score([artifact], source) + assert _dim(report, "placeholder_residue").score == 1.0 + + +# ---------- 章节完整性维度 ---------- + +def test_completeness_fails_when_section_missing(): + source = _source() + # 期望 ch3/ch4 两章,但只生成 ch3 + artifacts = [ChapterArtifact( + chapter_id="ch3", text="a", source_uris=[], template_sections_expected=["ch3", "ch4"], + )] + report = ChapterScorer().score(artifacts, source) + dim = _dim(report, "chapter_completeness") + assert dim.score == 0.5 + assert dim.passed is False + + +# ---------- LLM 维度钩子 ---------- + +def test_llm_dimension_hook_invoked(): + source = _source() + called = {} + + def fake_llm(chapter: ChapterArtifact) -> DimensionScore: + called["hit"] = True + return DimensionScore(name="llm_accuracy", score=0.8, passed=True, detail="stub") + + artifact = ChapterArtifact( + chapter_id="ch3", text="x", source_uris=[], template_sections_expected=["ch3"], + ) + scorer = ChapterScorer(llm_evaluators={"llm_accuracy": fake_llm}) + report = scorer.score([artifact], source) + assert called.get("hit") is True + assert _dim(report, "llm_accuracy").score == 0.8 + + +# ---------- 总分聚合 + 通过判定 ---------- + +def test_total_score_aggregation(): + source = _source() + artifact = ChapterArtifact( + chapter_id="ch3", text="正常(出典: f.xlsx#機能一覧!C3)", + source_uris=["f.xlsx#機能一覧!C3"], template_sections_expected=["ch3"], + ) + report = ChapterScorer().score([artifact], source) + assert isinstance(report, EvalReport) + assert 0.0 <= report.total_score <= 1.0 + # 全部确定性维度满分 → 总分接近 1.0(仅 llm 维度默认中性 0.5) + assert report.total_score >= 0.8 + + +def test_empty_chapters_does_not_crash(): + """空章节输入:无引用/无约束 → 确定性维度中性满分,不应抛错。""" + source = _source() + report = ChapterScorer().score([], source) + assert report.total_score == 1.0 + assert report.passed is True + + +# ---------- 黄金集加载 ---------- + +def test_golden_set_load(tmp_path): + yaml_text = """ +cases: + - id: g1 + input_ref: samples/要件定義.xlsx + expected_min_score: 0.7 + note: 脱敏真实样本回归基线 +""" + p = tmp_path / "golden_set.yaml" + p.write_text(yaml_text, encoding="utf-8") + gs = GoldenSet.load(p) + assert len(gs.cases) == 1 + assert gs.cases[0].id == "g1" + assert gs.cases[0].expected_min_score == 0.7 + + +def _dim(report: EvalReport, name: str) -> DimensionScore: + for d in report.dimensions: + if d.name == name: + return d + raise AssertionError(f"维度未找到: {name}")