docs: add db-pipeline-agent2data integration plan
This commit is contained in:
hangshuo652
@@ -0,0 +1,281 @@
|
||||
# DB 管道接入 Agent2Data — 实现计划
|
||||
|
||||
## 问题
|
||||
|
||||
DB 管道 (`orchestrator_db.py`) 从未调用 Agent2Data，而 Agent2Data 的 LLM 场景/边界数据是 cobol_testgen 覆盖率基线的补充，应作为第 4 个数据源合并。
|
||||
|
||||
同时，非 DB 管道存在两条重复入口（`orchestrator.py` + `__init__.py:main()` 行内管线），导致：
|
||||
- 行内管线不经过我们刚修的 Agent2Data 合并逻辑
|
||||
- 行内管线不走质量门/hina 分类/Agent3 诊断
|
||||
- 两条管线功能不一致
|
||||
|
||||
---
|
||||
|
||||
## 阶段 A：DB 管道接入 Agent2Data
|
||||
|
||||
### 目标
|
||||
|
||||
在 `data_merger.py:generate_all_data()` 中新增 Agent2Data 作为第 4 个数据源，使其在 DB 管道 (`orchestrator_db.py`) 中生效。
|
||||
|
||||
### 修改文件
|
||||
|
||||
#### A1. `data_merger.py` — `generate_all_data()` 新增 Agent2Data 源
|
||||
|
||||
**现状**：三路合并（白盒 + 功能 + 策略），参数不含 FieldTree
|
||||
|
||||
**改动**：
|
||||
|
||||
```python
|
||||
def generate_all_data(
|
||||
program_id: str,
|
||||
src_text: str,
|
||||
st: dict,
|
||||
copybook_dirs: list = None,
|
||||
design_doc_dir: str = None,
|
||||
llm_client=None,
|
||||
config: dict = None,
|
||||
merge_strategy: str = "default",
|
||||
field_tree=None, # ← NEW: Agent2Data 需要的 FieldTree
|
||||
) -> list[dict]:
|
||||
```
|
||||
|
||||
在原有 `_dedup()` 链后新增第 4 个源：
|
||||
|
||||
```python
|
||||
# ④ Agent2Data LLM 场景数据 (NEW)
|
||||
agent2_records = []
|
||||
if field_tree is not None and llm_client is not None:
|
||||
try:
|
||||
from agents import Agent2Data
|
||||
suite = Agent2Data(llm_client).design(
|
||||
field_tree,
|
||||
target=config.get("coverage_default", "boundary"),
|
||||
spark_mode=False,
|
||||
)
|
||||
for tc in suite.test_cases:
|
||||
rec = dict(tc.fields)
|
||||
rec["_source"] = "agent2"
|
||||
agent2_records.append(rec)
|
||||
logger.info(f" Agent2Data records: {len(agent2_records)}")
|
||||
except Exception as e:
|
||||
logger.warning(f" Agent2Data failed, skipped: {e}")
|
||||
|
||||
# 合并：additional(agent2) 优先，再追加 whitebox+func 中不重复的
|
||||
all_records = _dedup(addition=agent2_records, main=all_records)
|
||||
```
|
||||
|
||||
**优雅降级**：`field_tree=None` 或 `llm_client=None` 时不调用，不影响现有管线。
|
||||
|
||||
#### A2. `orchestrator_db.py:step2_generate_inputs()` — 构造 FieldTree 并传入
|
||||
|
||||
**现状**：调用 `generate_all_data()` 时未传 field_tree 参数
|
||||
|
||||
**改动**（在 `generate_all_data` 调用附近）：
|
||||
|
||||
```python
|
||||
from data.field_tree import FieldTree, Field
|
||||
|
||||
# 从 structure['data_fields'] 构造 FieldTree
|
||||
data_fields = st.get("data_fields", [])
|
||||
field_tree = None
|
||||
if data_fields:
|
||||
try:
|
||||
fields = []
|
||||
for f in data_fields:
|
||||
if f.get("is_88"):
|
||||
continue
|
||||
pi = f.get("pic_info", {})
|
||||
fields.append(Field(
|
||||
name=f["name"],
|
||||
level=f.get("level", 0),
|
||||
pic=f.get("pic", ""),
|
||||
usage=f.get("usage", "DISPLAY"),
|
||||
length=pi.get("length", 0) or pi.get("digits", 0) + pi.get("decimal", 0),
|
||||
decimal=pi.get("decimal", 0),
|
||||
signed=pi.get("signed", False),
|
||||
redefines=f.get("redefines"),
|
||||
occurs=f.get("occurs"),
|
||||
))
|
||||
field_tree = FieldTree(fields=fields, copybook_name=self.program_id)
|
||||
except Exception as e:
|
||||
logger.warning(f" FieldTree construction failed (Agent2Data will be skipped): {e}")
|
||||
|
||||
# 传入 generate_all_data
|
||||
recs = generate_all_data(
|
||||
...,
|
||||
field_tree=field_tree, # ← NEW
|
||||
)
|
||||
```
|
||||
|
||||
#### A3. 前置：`data_merger.py` 导入补充
|
||||
|
||||
```python
|
||||
# 文件头部新增 import (紧接现有 import)
|
||||
try:
|
||||
from agents import Agent2Data
|
||||
except ImportError:
|
||||
Agent2Data = None
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 验证方法
|
||||
|
||||
```bash
|
||||
cd C:\Users\marye\Desktop\2026技术大赛\cobol-java-v3
|
||||
set COB_LIBRARY_PATH=<tna>\bin;<v3>\gixsql\lib
|
||||
set GIXSQL_DB_PATH=C:\Temp\gix
|
||||
python -m cobol_testgen --gcov <tna>\src\ZAN06UPD.cbl runtime\
|
||||
```
|
||||
|
||||
检查日志输出是否包含 `Agent2Data records: N`。
|
||||
|
||||
---
|
||||
|
||||
## 阶段 B：统一非 DB 入口
|
||||
|
||||
### 目标
|
||||
|
||||
消除 `__init__.py:main()` 中的 ~350 行行内管线代码，非 DB 程序统一走 `orchestrator.py:run_pipeline()`，同时将行内管线的 gcov + HTML 覆盖率报告功能搬迁到 `orchestrator.py`。
|
||||
|
||||
### 修改文件
|
||||
|
||||
#### B1. `orchestrator.py:run_pipeline()` — 增加 gcov 支持 + HTML 覆盖率报告
|
||||
|
||||
**现状**：`orchestrator.py` 用 `CobolRunner.compile()` + `CobolRunner.run()`（旧 API，stdin/stdout 管道），没有 gcov，没有 HTML 覆盖率报告。
|
||||
|
||||
**改动**：
|
||||
|
||||
```python
|
||||
def run_pipeline(cfg: Config, cpath: str, cbl: str, java: str, map_path: str) -> VerificationRun:
|
||||
t0 = time.time()
|
||||
vr = VerificationRun(program=Path(java).stem, runner=cfg.runner_mode)
|
||||
|
||||
try:
|
||||
text = Path(cpath).read_text()
|
||||
# ... 保持 Agent1Parser + cobol_testgen + hina + Agent2Data + DataWriter 不变 ...
|
||||
|
||||
# COBOL 编译 (改为新 API + gcov 支持)
|
||||
cob = CobolRunner()
|
||||
source_dir = str(Path(cbl).parent)
|
||||
work_dir = Path("temp_work").resolve()
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
copybook_dirs = [str(Path(cbl).parent.parent / "cpy")] # 从 config/copybook_paths 获取
|
||||
sub_o = compile_sub_modules(source_dir, work_dir, copybook_dirs)
|
||||
exe_path = compile_program(
|
||||
program_name=Path(cbl).stem,
|
||||
source_dir=source_dir,
|
||||
work_dir=work_dir,
|
||||
sub_objects=sub_o,
|
||||
cpy_dir=copybook_dirs,
|
||||
gcov=cfg.gcov_enabled,
|
||||
)
|
||||
|
||||
# COBOL 运行 (文件 I/O)
|
||||
skip_expected = ...
|
||||
run_all(
|
||||
program_name=Path(cbl).stem,
|
||||
outdir=str(output_dir),
|
||||
temp_dir=work_dir,
|
||||
fields_dict=..., fd_fields=...,
|
||||
select_info=..., open_dir=...,
|
||||
term_types=..., records=...,
|
||||
expected_records=...,
|
||||
source_dir=source_dir,
|
||||
path_infos=...,
|
||||
multi_write_fds=...,
|
||||
skip_records=...,
|
||||
skip_term_types=...,
|
||||
)
|
||||
|
||||
# HTML 覆盖率报告
|
||||
if cfg.gcov_enabled:
|
||||
run_coverage(...)
|
||||
generate_coverage_index(...)
|
||||
|
||||
# ... 保持 Java 编译运行 + 比对 + Agent3 + ReportGenerator 不变 ...
|
||||
```
|
||||
|
||||
#### B2. `__init__.py:main()` 非 DB 分支 — 委托给 `orchestrator.py`
|
||||
|
||||
**现状**：~350 行行内管线代码
|
||||
|
||||
**改动**：
|
||||
|
||||
```python
|
||||
# 非 DB 程序
|
||||
for filepath in non_db_files:
|
||||
try:
|
||||
from orchestrator import run_pipeline
|
||||
from config import Config
|
||||
cfg = Config()
|
||||
cfg.gcov_enabled = gcov_mode
|
||||
# 复制相关路径配置
|
||||
cfg.copybook_paths = [str(filepath.parent / ".." / "cpy")]
|
||||
|
||||
vr = run_pipeline(
|
||||
cfg=cfg,
|
||||
cpath=str(filepath), # 用作 COPYBOOK
|
||||
cbl=str(filepath),
|
||||
java="", # 无双版本验证时跳过
|
||||
map_path="",
|
||||
)
|
||||
# 输出处理（拷贝到 prog_outdir，生成覆盖率索引等）
|
||||
...
|
||||
except Exception as e:
|
||||
logger.error(f" orchestrator pipeline failed: {e}")
|
||||
# 回退：行内管线
|
||||
...
|
||||
```
|
||||
|
||||
**回退策略**：如果 `orchestrator.py` 调用失败，自动回退到行内管线（保留原有行为）。
|
||||
|
||||
---
|
||||
|
||||
### 验证方法
|
||||
|
||||
统一前后结果应完全一致：
|
||||
|
||||
```bash
|
||||
# 统一前
|
||||
python -m cobol_testgen --gcov <tna>\src\ZAN01CHK.cbl runtime-new\
|
||||
|
||||
# 统一后
|
||||
python -m cobol_testgen --gcov <tna>\src\ZAN01CHK.cbl runtime-unified\
|
||||
|
||||
diff -r runtime-new runtime-unified
|
||||
```
|
||||
|
||||
两条命令的输出目录结构、覆盖率数据应一致。
|
||||
|
||||
---
|
||||
|
||||
## 实施顺序与依赖
|
||||
|
||||
```
|
||||
阶段 A (DB + Agent2Data) ──────────── 独立，无外部依赖
|
||||
├── A1: data_merger.py ─── 新增 field_tree 参数 + Agent2Data 调用
|
||||
└── A2: orchestrator_db.py ─── 构造 FieldTree 并传入
|
||||
|
||||
阶段 B (入口统一) ─────────────────── 依赖阶段 A 完成后可开始
|
||||
├── B1: orchestrator.py ─── 增加 gcov + HTML 覆盖率能力
|
||||
├── B2: __init__.py:main() ─── 非 DB 委托给 orchestrator.py
|
||||
└── B3: 验证一致性 + 清理旧代码
|
||||
```
|
||||
|
||||
### 主要风险
|
||||
|
||||
| 风险 | 等级 | 缓解 |
|
||||
|------|:----:|------|
|
||||
| Agent2Data 字段值不符合 PIC 格式（阶段 A） | 🟡 | Agent2Data 数据排在末尾，不破坏基线；后续可加 PIC 格式化 |
|
||||
| FieldTree 构造精度不足（阶段 A） | 🟡 | `st["data_fields"]` 来自 `extract_structure()`，与 Agent1Parser 的 FieldTree 结构等价但来源不同 |
|
||||
| `orchestrator.py` 搬入 gcov 后与行内管线行为不一致（阶段 B） | 🔴 | 并排运行对比，保持回退机制 |
|
||||
| 行内管线的 C01 特定逻辑 (`_inject_c01_coverage_records` 等) 在 orchestrator.py 中缺失（阶段 B） | 🔴 | 搬迁时确认 orchestrator.py 是否也需要这些逻辑；若不需要则跳过（行内管线专门为 KIN/ZAN 程序集优化） |
|
||||
|
||||
---
|
||||
|
||||
## 不纳入范围
|
||||
|
||||
- Agent2Data 输出值的 PIC 格式化（后续优化）
|
||||
- 统一后删除 `__init__.py` 行内管线死代码（阶段 B 完成后清理）
|
||||
- `main.py` 废弃（阶段 B 后单独处理）
|
||||
Reference in New Issue
Block a user