feat(orchestrator): DataGate 机制化 + 任务级持久化（T14/T16 架构审查整改）

- T14 (OV5): 新建 src/genesis/orchestrator/datagate.py - DataGate.load(source, selector) 机制化：子集加载 + 规模保护（超 500 行无 selector 拒绝全量，1000 行 Excel 防上下文爆炸）+ token 预算（8000，复用 CJK 保守估算）+ 未知表容错 - agent-runtime §4.2 原则→机制说明 - T16 (OV7): 新建 src/genesis/orchestrator/task_queue.py - TaskQueue ABC + PersistentTaskQueue（SQLite 落盘） - enqueue/poll/update_status/get/cancel/recover/close - 幂等去重（§5.3 缓存结果）+ recover 将 running→failed、pending 保留 - api-design §5.2/5.3、agent-runtime §3.5/3.6、design §8.4.1 同步 - 新增 test_datagate.py（8 用例）+ test_task_queue.py（11 用例） - TDD: RED（模块缺失）→ GREEN（聚焦 16 passed）→ 全量 218 passed / 100.00%（1140 stmts/278 br）
2026-08-12 15:43:10 +08:00
parent 3decdfc31c
commit 6a54580ca4
9 changed files with 734 additions and 5 deletions
@@ -0,0 +1,137 @@
+"""DataGate 机制化测试（T14，OV5）。
+
+OV5 裁定：DataGate 是原则非机制，1000 行 Excel 上下文爆炸
+→ 机制化：规模保护（超阈值无 selector 拒绝全量）+ token 预算控制 + 子集加载。
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from genesis.data_models import CellValue, ExcelTable, Provenance, SheetType
+from genesis.orchestrator.datagate import (
+    DataGate,
+    DataGateError,
+    DataSelector,
+    DataGateResult,
+)
+
+
+def _table(name: str, n_rows: int) -> ExcelTable:
+    rows: list[dict[str, CellValue]] = []
+    for i in range(n_rows):
+        rows.append({
+            "ID": CellValue(
+                value=f"{name}-{i}",
+                provenance=Provenance(file_name="要件定義.xlsx", sheet_name=name, row=i + 2, column="A", column_header="ID"),
+            )
+        })
+    return ExcelTable(
+        name=name,
+        detected_type=SheetType.FUNCTION,
+        extraction_method="structured",
+        headers=["ID"],
+        rows=rows,
+    )
+
+
+# ---------- 子集加载 ----------
+
+def test_load_returns_selector_subset():
+    gate = DataGate()
+    source = _source_with_tables(["機能一覧", "画面一覧"], rows_each=5)
+
+    result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
+
+    assert isinstance(result, DataGateResult)
+    assert result.loaded_tables == ["画面一覧"]
+    assert result.loaded_rows == 5
+
+
+# ---------- 规模保护（OV5 核心：1000 行 Excel 防上下文爆炸） ----------
+
+def test_load_rejects_huge_source_without_selector():
+    gate = DataGate(max_total_rows=500)
+    source = _source_with_tables(["機能一覧"], rows_each=1000)
+
+    with pytest.raises(DataGateError, match="selector"):
+        gate.load(source)
+
+
+def test_load_allows_small_source_without_selector():
+    gate = DataGate(max_total_rows=500)
+    source = _source_with_tables(["機能一覧"], rows_each=10)
+
+    result = gate.load(source)
+
+    assert result.loaded_rows == 10
+
+
+def test_huge_source_with_selector_is_allowed():
+    """1000 行 Excel + selector 限定单表 → 不触发全量防护。"""
+    gate = DataGate(max_total_rows=500)
+    source = _source_with_tables(["機能一覧", "画面一覧", "帳票一覧"], rows_each=1000)
+
+    result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
+
+    assert result.loaded_tables == ["画面一覧"]
+    assert result.loaded_rows == 1000
+
+
+# ---------- token 预算控制 ----------
+
+def test_token_budget_rejects_huge_load():
+    gate = DataGate(max_total_tokens=100)
+    source = _source_with_tables(["機能一覧"], rows_each=200)
+
+    with pytest.raises(DataGateError, match="token"):
+        gate.load(source, DataSelector(table_ids=["機能一覧"]))
+
+
+def test_token_budget_ok_within_limit():
+    gate = DataGate(max_total_tokens=10_000)
+    source = _source_with_tables(["機能一覧"], rows_each=20)
+
+    result = gate.load(source, DataSelector(table_ids=["機能一覧"]))
+
+    assert result.token_estimate <= 10_000
+
+
+# ---------- 未知表容错 ----------
+
+def test_unknown_table_id_returns_empty():
+    gate = DataGate()
+    source = _source_with_tables(["機能一覧"], rows_each=5)
+
+    result = gate.load(source, DataSelector(table_ids=["不存在"]))
+
+    assert result.loaded_tables == []
+    assert result.loaded_rows == 0
+
+
+def test_empty_selector_equals_no_selector():
+    """selector 未指定任何表 → 等同无 selector，触发规模保护。"""
+    gate = DataGate(max_total_rows=10)
+    source = _source_with_tables(["機能一覧"], rows_each=100)
+
+    with pytest.raises(DataGateError, match="selector"):
+        gate.load(source, DataSelector())
+
+
+def _source_with_tables(names: list[str], rows_each: int):
+    from genesis.data_models import (
+        ExistingSystemInfo,
+        ImageAnalysis,
+        ParsedTemplate,
+        RuleDocument,
+        StructuredSource,
+    )
+
+    return StructuredSource(
+        tables=[_table(n, rows_each) for n in names],
+        template=ParsedTemplate(file_name="template.docx", sections=[], placeholders={}, styles={}),
+        rule_docs=[],
+        image_analyses=[],
+        existing_system=None,
+        comments=[],
+    )
@@ -0,0 +1,177 @@
+"""任务级持久化测试（T16，OV7）。
+
+OV7 裁定：崩溃恢复只到会话级，任务层丢数据 → 任务级持久化。
+实现：api-design §5 TaskQueue 抽象 + PersistentTaskQueue（SQLite 落盘），
+重启后可 recover 未完成任务（running → failed，pending 保留）。
+"""
+
+from __future__ import annotations
+
+from genesis.orchestrator.task_queue import (
+    PersistentTaskQueue,
+    TaskSpec,
+    TaskStatus,
+)
+
+
+def _spec(**overrides) -> TaskSpec:
+    base = dict(
+        task_id="t-1",
+        session_id="s-1",
+        step="generate",
+        chapter_id="ch-3",
+        idempotency_key="s-1|generate|ch-3",
+        payload={"chapter_id": "ch-3"},
+    )
+    base.update(overrides)
+    return TaskSpec(**base)
+
+
+def test_enqueue_and_get(tmp_path):
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    handle = q.enqueue(_spec())
+
+    got = q.get("t-1")
+    assert got is not None
+    assert got.status == TaskStatus.PENDING
+    assert got.payload == {"chapter_id": "ch-3"}
+    q.close()
+
+
+def test_poll_returns_session_tasks(tmp_path):
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    q.enqueue(_spec(task_id="t-1"))
+    q.enqueue(_spec(task_id="t-2", chapter_id="ch-4", idempotency_key="s-1|generate|ch-4"))
+
+    tasks = q.poll("s-1")
+    assert {t.task_id for t in tasks} == {"t-1", "t-2"}
+    q.close()
+
+
+def test_update_status_with_result(tmp_path):
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    q.enqueue(_spec())
+    q.update_status("t-1", TaskStatus.COMPLETED, result={"html": "<h1>章</h1>"})
+
+    got = q.get("t-1")
+    assert got.status == TaskStatus.COMPLETED
+    assert got.result == {"html": "<h1>章</h1>"}
+    q.close()
+
+
+def test_cancel(tmp_path):
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    q.enqueue(_spec())
+    assert q.cancel("t-1") is True
+    assert q.get("t-1").status == TaskStatus.CANCELLED
+    assert q.cancel("t-1") is False  # 已终态不可再取消
+    q.close()
+
+
+def test_idempotent_enqueue_returns_cached_result(tmp_path):
+    """§5.3 幂等去重：同 idempotency_key 已完成 → 返回缓存结果，不重复入队。"""
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    q.enqueue(_spec())
+    q.update_status("t-1", TaskStatus.COMPLETED, result={"html": "cached"})
+
+    second = q.enqueue(_spec(task_id="t-999"))
+
+    assert second.task_id == "t-1"  # 返回原 handle
+    assert second.result == {"html": "cached"}
+    assert q.get("t-999") is None
+    q.close()
+
+
+def test_persistence_survives_reopen(tmp_path):
+    """OV7 核心：任务写入 SQLite，重启（重建实例）后数据不丢。"""
+    db = tmp_path / "tasks.db"
+    q1 = PersistentTaskQueue(db_path=db)
+    q1.enqueue(_spec())
+    q1.update_status("t-1", TaskStatus.RUNNING)
+    q1.close()
+
+    q2 = PersistentTaskQueue(db_path=db)
+    got = q2.get("t-1")
+    assert got is not None
+    assert got.status == TaskStatus.RUNNING
+    q2.close()
+
+
+def test_recover_marks_running_as_failed_keeps_pending(tmp_path):
+    """崩溃恢复：running → failed（中断标记），pending 保留待执行，completed 不动。"""
+    db = tmp_path / "tasks.db"
+    q1 = PersistentTaskQueue(db_path=db)
+    q1.enqueue(_spec(task_id="t-running"))
+    q1.update_status("t-running", TaskStatus.RUNNING)
+    q1.enqueue(_spec(task_id="t-pending", chapter_id="ch-5", idempotency_key="s-1|generate|ch-5"))
+    q1.enqueue(_spec(task_id="t-done", chapter_id="ch-6", idempotency_key="s-1|generate|ch-6"))
+    q1.update_status("t-done", TaskStatus.COMPLETED, result={"html": "ok"})
+    q1.close()
+
+    q2 = PersistentTaskQueue(db_path=db)
+    recovered = q2.recover()
+
+    assert q2.get("t-running").status == TaskStatus.FAILED
+    assert q2.get("t-pending").status == TaskStatus.PENDING
+    assert q2.get("t-done").status == TaskStatus.COMPLETED
+    assert {t.task_id for t in recovered} == {"t-running", "t-pending"}  # 未完成待处理
+    q2.close()
+
+
+def test_poll_only_incomplete_after_recover(tmp_path):
+    db = tmp_path / "tasks.db"
+    q1 = PersistentTaskQueue(db_path=db)
+    q1.enqueue(_spec(task_id="t-running"))
+    q1.update_status("t-running", TaskStatus.RUNNING)
+    q1.close()
+
+    q2 = PersistentTaskQueue(db_path=db)
+    q2.recover()
+    pending = q2.poll("s-1")
+    assert {t.task_id for t in pending} == {"t-running"}  # 已标记 failed，仍可重试
+    q2.close()
+
+
+# ---------- 防御分支（覆盖率 100% 基线） ----------
+
+def test_update_status_unknown_task_raises(tmp_path):
+    import pytest
+
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    with pytest.raises(KeyError, match="t-404"):
+        q.update_status("t-404", TaskStatus.RUNNING)
+    q.close()
+
+
+def test_update_status_terminal_rejected(tmp_path):
+    import pytest
+
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    q.enqueue(_spec())
+    q.update_status("t-1", TaskStatus.COMPLETED, result={"html": "done"})
+    with pytest.raises(ValueError, match="终态"):
+        q.update_status("t-1", TaskStatus.RUNNING)
+    q.close()
+
+
+def test_idem_lookup_with_null_chapter_id(tmp_path):
+    """幂等查找：chapter_id 为 None（非章级任务）时仍命中已有任务。"""
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    spec = _spec(chapter_id=None, task_id="t-impact", step="impact",
+                 idempotency_key="s-1|impact|")
+    q.enqueue(spec)
+
+    second = q.enqueue(_spec(chapter_id=None, task_id="t-impact-2", step="impact",
+                             idempotency_key="s-1|impact|"))
+    assert second.task_id == "t-impact"  # 命中已有，未重复入队
+    q.close()
+
+
+def test_row_to_handle_unknown_task_raises(tmp_path):
+    """白盒：_row_to_handle 无行时抛 KeyError（内部防御分支）。"""
+    import pytest
+
+    q = PersistentTaskQueue(db_path=tmp_path / "tasks.db")
+    with pytest.raises(KeyError, match="t-404"):
+        q._row_to_handle("t-404")
+    q.close()