lhl
6a54580ca4
- T14 (OV5): 新建 src/genesis/orchestrator/datagate.py
- DataGate.load(source, selector) 机制化：子集加载 + 规模保护（超 500 行
无 selector 拒绝全量，1000 行 Excel 防上下文爆炸）+ token 预算（8000，
复用 CJK 保守估算）+ 未知表容错
- agent-runtime §4.2 原则→机制说明
- T16 (OV7): 新建 src/genesis/orchestrator/task_queue.py
- TaskQueue ABC + PersistentTaskQueue（SQLite 落盘）
- enqueue/poll/update_status/get/cancel/recover/close
- 幂等去重（§5.3 缓存结果）+ recover 将 running→failed、pending 保留
- api-design §5.2/5.3、agent-runtime §3.5/3.6、design §8.4.1 同步
- 新增 test_datagate.py（8 用例）+ test_task_queue.py（11 用例）
- TDD: RED（模块缺失）→ GREEN（聚焦 16 passed）→ 全量 218 passed / 100.00%（1140 stmts/278 br）
138 lines
4.1 KiB
Python
138 lines
4.1 KiB
Python
"""DataGate 机制化测试（T14，OV5）。
|
||
|
||
OV5 裁定：DataGate 是原则非机制，1000 行 Excel 上下文爆炸
|
||
→ 机制化：规模保护（超阈值无 selector 拒绝全量）+ token 预算控制 + 子集加载。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from genesis.data_models import CellValue, ExcelTable, Provenance, SheetType
|
||
from genesis.orchestrator.datagate import (
|
||
DataGate,
|
||
DataGateError,
|
||
DataSelector,
|
||
DataGateResult,
|
||
)
|
||
|
||
|
||
def _table(name: str, n_rows: int) -> ExcelTable:
|
||
rows: list[dict[str, CellValue]] = []
|
||
for i in range(n_rows):
|
||
rows.append({
|
||
"ID": CellValue(
|
||
value=f"{name}-{i}",
|
||
provenance=Provenance(file_name="要件定義.xlsx", sheet_name=name, row=i + 2, column="A", column_header="ID"),
|
||
)
|
||
})
|
||
return ExcelTable(
|
||
name=name,
|
||
detected_type=SheetType.FUNCTION,
|
||
extraction_method="structured",
|
||
headers=["ID"],
|
||
rows=rows,
|
||
)
|
||
|
||
|
||
# ---------- 子集加载 ----------
|
||
|
||
def test_load_returns_selector_subset():
|
||
gate = DataGate()
|
||
source = _source_with_tables(["機能一覧", "画面一覧"], rows_each=5)
|
||
|
||
result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
|
||
|
||
assert isinstance(result, DataGateResult)
|
||
assert result.loaded_tables == ["画面一覧"]
|
||
assert result.loaded_rows == 5
|
||
|
||
|
||
# ---------- 规模保护（OV5 核心：1000 行 Excel 防上下文爆炸） ----------
|
||
|
||
def test_load_rejects_huge_source_without_selector():
|
||
gate = DataGate(max_total_rows=500)
|
||
source = _source_with_tables(["機能一覧"], rows_each=1000)
|
||
|
||
with pytest.raises(DataGateError, match="selector"):
|
||
gate.load(source)
|
||
|
||
|
||
def test_load_allows_small_source_without_selector():
|
||
gate = DataGate(max_total_rows=500)
|
||
source = _source_with_tables(["機能一覧"], rows_each=10)
|
||
|
||
result = gate.load(source)
|
||
|
||
assert result.loaded_rows == 10
|
||
|
||
|
||
def test_huge_source_with_selector_is_allowed():
|
||
"""1000 行 Excel + selector 限定单表 → 不触发全量防护。"""
|
||
gate = DataGate(max_total_rows=500)
|
||
source = _source_with_tables(["機能一覧", "画面一覧", "帳票一覧"], rows_each=1000)
|
||
|
||
result = gate.load(source, DataSelector(table_ids=["画面一覧"]))
|
||
|
||
assert result.loaded_tables == ["画面一覧"]
|
||
assert result.loaded_rows == 1000
|
||
|
||
|
||
# ---------- token 预算控制 ----------
|
||
|
||
def test_token_budget_rejects_huge_load():
|
||
gate = DataGate(max_total_tokens=100)
|
||
source = _source_with_tables(["機能一覧"], rows_each=200)
|
||
|
||
with pytest.raises(DataGateError, match="token"):
|
||
gate.load(source, DataSelector(table_ids=["機能一覧"]))
|
||
|
||
|
||
def test_token_budget_ok_within_limit():
|
||
gate = DataGate(max_total_tokens=10_000)
|
||
source = _source_with_tables(["機能一覧"], rows_each=20)
|
||
|
||
result = gate.load(source, DataSelector(table_ids=["機能一覧"]))
|
||
|
||
assert result.token_estimate <= 10_000
|
||
|
||
|
||
# ---------- 未知表容错 ----------
|
||
|
||
def test_unknown_table_id_returns_empty():
|
||
gate = DataGate()
|
||
source = _source_with_tables(["機能一覧"], rows_each=5)
|
||
|
||
result = gate.load(source, DataSelector(table_ids=["不存在"]))
|
||
|
||
assert result.loaded_tables == []
|
||
assert result.loaded_rows == 0
|
||
|
||
|
||
def test_empty_selector_equals_no_selector():
|
||
"""selector 未指定任何表 → 等同无 selector，触发规模保护。"""
|
||
gate = DataGate(max_total_rows=10)
|
||
source = _source_with_tables(["機能一覧"], rows_each=100)
|
||
|
||
with pytest.raises(DataGateError, match="selector"):
|
||
gate.load(source, DataSelector())
|
||
|
||
|
||
def _source_with_tables(names: list[str], rows_each: int):
|
||
from genesis.data_models import (
|
||
ExistingSystemInfo,
|
||
ImageAnalysis,
|
||
ParsedTemplate,
|
||
RuleDocument,
|
||
StructuredSource,
|
||
)
|
||
|
||
return StructuredSource(
|
||
tables=[_table(n, rows_each) for n in names],
|
||
template=ParsedTemplate(file_name="template.docx", sections=[], placeholders={}, styles={}),
|
||
rule_docs=[],
|
||
image_analyses=[],
|
||
existing_system=None,
|
||
comments=[],
|
||
)
|