feat(orchestrator): DataGate 机制化 + 任务级持久化(T14/T16 架构审查整改)

- T14 (OV5): 新建 src/genesis/orchestrator/datagate.py
  - DataGate.load(source, selector) 机制化:子集加载 + 规模保护(超 500 行
    无 selector 拒绝全量,1000 行 Excel 防上下文爆炸)+ token 预算(8000,
    复用 CJK 保守估算)+ 未知表容错
  - agent-runtime §4.2 原则→机制说明
- T16 (OV7): 新建 src/genesis/orchestrator/task_queue.py
  - TaskQueue ABC + PersistentTaskQueue(SQLite 落盘)
  - enqueue/poll/update_status/get/cancel/recover/close
  - 幂等去重(§5.3 缓存结果)+ recover 将 running→failed、pending 保留
  - api-design §5.2/5.3、agent-runtime §3.5/3.6、design §8.4.1 同步
- 新增 test_datagate.py(8 用例)+ test_task_queue.py(11 用例)
- TDD: RED(模块缺失)→ GREEN(聚焦 16 passed)→ 全量 218 passed / 100.00%(1140 stmts/278 br)
This commit is contained in:
lhl
2026-08-12 15:43:10 +08:00
parent 3decdfc31c
commit 6a54580ca4
9 changed files with 734 additions and 5 deletions
+137
View File
@@ -0,0 +1,137 @@
"""DataGate 机制化测试(T14OV5)。
OV5 裁定:DataGate 是原则非机制,1000 行 Excel 上下文爆炸
→ 机制化:规模保护(超阈值无 selector 拒绝全量)+ token 预算控制 + 子集加载。
"""
from __future__ import annotations
import pytest
from genesis.data_models import CellValue, ExcelTable, Provenance, SheetType
from genesis.orchestrator.datagate import (
DataGate,
DataGateError,
DataSelector,
DataGateResult,
)
def _table(name: str, n_rows: int) -> ExcelTable:
rows: list[dict[str, CellValue]] = []
for i in range(n_rows):
rows.append({
"ID": CellValue(
value=f"{name}-{i}",
provenance=Provenance(file_name="要件定義.xlsx", sheet_name=name, row=i + 2, column="A", column_header="ID"),
)
})
return ExcelTable(
name=name,
detected_type=SheetType.FUNCTION,
extraction_method="structured",
headers=["ID"],
rows=rows,
)
# ---------- 子集加载 ----------
def test_load_returns_selector_subset():
gate = DataGate()
source = _source_with_tables(["機能一覧", "画面一覧"], rows_each=5)
result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
assert isinstance(result, DataGateResult)
assert result.loaded_tables == ["画面一覧"]
assert result.loaded_rows == 5
# ---------- 规模保护(OV5 核心:1000 行 Excel 防上下文爆炸) ----------
def test_load_rejects_huge_source_without_selector():
gate = DataGate(max_total_rows=500)
source = _source_with_tables(["機能一覧"], rows_each=1000)
with pytest.raises(DataGateError, match="selector"):
gate.load(source)
def test_load_allows_small_source_without_selector():
gate = DataGate(max_total_rows=500)
source = _source_with_tables(["機能一覧"], rows_each=10)
result = gate.load(source)
assert result.loaded_rows == 10
def test_huge_source_with_selector_is_allowed():
"""1000 行 Excel + selector 限定单表 → 不触发全量防护。"""
gate = DataGate(max_total_rows=500)
source = _source_with_tables(["機能一覧", "画面一覧", "帳票一覧"], rows_each=1000)
result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
assert result.loaded_tables == ["画面一覧"]
assert result.loaded_rows == 1000
# ---------- token 预算控制 ----------
def test_token_budget_rejects_huge_load():
gate = DataGate(max_total_tokens=100)
source = _source_with_tables(["機能一覧"], rows_each=200)
with pytest.raises(DataGateError, match="token"):
gate.load(source, DataSelector(table_ids=["機能一覧"]))
def test_token_budget_ok_within_limit():
gate = DataGate(max_total_tokens=10_000)
source = _source_with_tables(["機能一覧"], rows_each=20)
result = gate.load(source, DataSelector(table_ids=["機能一覧"]))
assert result.token_estimate <= 10_000
# ---------- 未知表容错 ----------
def test_unknown_table_id_returns_empty():
gate = DataGate()
source = _source_with_tables(["機能一覧"], rows_each=5)
result = gate.load(source, DataSelector(table_ids=["不存在"]))
assert result.loaded_tables == []
assert result.loaded_rows == 0
def test_empty_selector_equals_no_selector():
"""selector 未指定任何表 → 等同无 selector,触发规模保护。"""
gate = DataGate(max_total_rows=10)
source = _source_with_tables(["機能一覧"], rows_each=100)
with pytest.raises(DataGateError, match="selector"):
gate.load(source, DataSelector())
def _source_with_tables(names: list[str], rows_each: int):
from genesis.data_models import (
ExistingSystemInfo,
ImageAnalysis,
ParsedTemplate,
RuleDocument,
StructuredSource,
)
return StructuredSource(
tables=[_table(n, rows_each) for n in names],
template=ParsedTemplate(file_name="template.docx", sections=[], placeholders={}, styles={}),
rule_docs=[],
image_analyses=[],
existing_system=None,
comments=[],
)