feat: DB管线补全 + 新增orchestrator_db/program_schema/to_sql + 清理临时脚本
This commit is contained in:
hangshuo652
+178
-17
@@ -10,6 +10,7 @@ import os
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -281,6 +282,7 @@ def main():
|
||||
|
||||
do_run = False
|
||||
gcov_mode = False
|
||||
gixsql_mode = False
|
||||
temp_dir = None
|
||||
if '--run' in args:
|
||||
do_run = True
|
||||
@@ -291,6 +293,9 @@ def main():
|
||||
if not _HAVE_RUNNER:
|
||||
logger.warning("--gcov: runner.py not found. Compile/run will be skipped. "
|
||||
"Use --gcov without runner only generates test data + static coverage.")
|
||||
if '--gixsql' in args:
|
||||
gixsql_mode = True
|
||||
args.remove('--gixsql')
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == '--temp-dir':
|
||||
@@ -342,6 +347,67 @@ def main():
|
||||
|
||||
programs = []
|
||||
|
||||
if gixsql_mode:
|
||||
# DB pipeline: GixsqlOrchestrator
|
||||
import sys as _sys
|
||||
_v3_root = str(Path(__file__).parent.parent)
|
||||
if _v3_root not in _sys.path:
|
||||
_sys.path.insert(0, _v3_root)
|
||||
from orchestrator_db import GixsqlOrchestrator
|
||||
from config import Config
|
||||
|
||||
config = Config()
|
||||
src_dir = cobol_files[0].parent if cobol_files else Path.cwd()
|
||||
cpy_dirs = [src_dir / '..' / 'cpy']
|
||||
|
||||
for filepath in cobol_files:
|
||||
pid = filepath.stem
|
||||
prog_outdir = outdir / pid
|
||||
prog_outdir.mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'logs').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'input').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'output').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'json').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"\n========== DB: {pid} ==========")
|
||||
orch = GixsqlOrchestrator(
|
||||
config=config, program_id=pid,
|
||||
cobol_src_dir=str(src_dir),
|
||||
copybook_dirs=[str(d) for d in cpy_dirs],
|
||||
skip_jvm=True,
|
||||
)
|
||||
vr = orch.run_all(generate_coverage=False)
|
||||
|
||||
# Copy output files to outdir
|
||||
if orch.runtime_dir.exists():
|
||||
for item in orch.runtime_dir.iterdir():
|
||||
if item.is_file():
|
||||
shutil.copy2(str(item), str(prog_outdir / item.name))
|
||||
|
||||
logger.info(f" {pid}: rc={vr.exit_code} status={vr.status}")
|
||||
|
||||
# Coverage report (only once, with correct output_dir)
|
||||
if '--coverage' in getattr(config, 'gixsql_compile_flags', ''):
|
||||
cov_result = orch.generate_coverage_report(output_dir=str(prog_outdir / 'coverage'))
|
||||
if cov_result.success:
|
||||
cv = cov_result.data.get("coverage", "unknown")
|
||||
logger.info(f" Coverage: {cv}")
|
||||
cov_dict = cov_result.data.get("_cov_dict")
|
||||
if cov_dict:
|
||||
# Fix detail_relpath relative to top-level index
|
||||
rel = Path(prog_outdir / 'coverage' / f"{pid}_coverage.html")
|
||||
cov_dict['detail_relpath'] = str(rel.relative_to(outdir).as_posix())
|
||||
programs.append(cov_dict)
|
||||
else:
|
||||
logger.warning(" --coverage not in gixsql_compile_flags; skipping coverage")
|
||||
|
||||
if programs:
|
||||
from cobol_testgen.coverage import generate_coverage_index as _gen_idx
|
||||
_gen_idx(programs, outdir / 'coverage')
|
||||
logger.info(f"\n覆盖率总览：{outdir / 'coverage' / 'index.html'}")
|
||||
return
|
||||
|
||||
for filepath in cobol_files:
|
||||
if not filepath.exists():
|
||||
logger.error(f"错误：文件不存在 {filepath}")
|
||||
@@ -432,6 +498,15 @@ def main():
|
||||
for child in fds:
|
||||
field_to_fd[child] = fd_name
|
||||
|
||||
# Per-program output directory (always)
|
||||
prog_outdir = outdir / filepath.stem
|
||||
prog_outdir.mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'logs').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'input').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'output').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'json').mkdir(parents=True, exist_ok=True)
|
||||
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"\n========== {filepath.name} ==========")
|
||||
logger.info(f"\n字段列表:")
|
||||
logger.info(f"{'层级':<6} {'名称':<25} {'PIC':<15} {'类型':<12} {'长度':<5}")
|
||||
@@ -479,10 +554,11 @@ def main():
|
||||
other += 1
|
||||
return eq1_true > 0 and other == 0
|
||||
|
||||
before = len(path_infos)
|
||||
path_infos = [p for p in path_infos if not _is_skip(p[0])]
|
||||
after = len(path_infos)
|
||||
logger.info(f" SKIP 过滤: {before} -> {after} 条路径（预期减少 1）")
|
||||
skip_path_infos = [p for p in path_infos if _is_skip(p[0])]
|
||||
main_path_infos = [p for p in path_infos if not _is_skip(p[0])]
|
||||
path_infos = main_path_infos
|
||||
if skip_path_infos:
|
||||
logger.info(f" Skip 路径: {len(skip_path_infos)} 条（将单独生成数据集）")
|
||||
|
||||
open_dir = scan_open_statements(proc_div) if proc_div else {}
|
||||
|
||||
@@ -539,8 +615,7 @@ def main():
|
||||
else:
|
||||
db_input = None
|
||||
|
||||
(outdir / 'json').mkdir(parents=True, exist_ok=True)
|
||||
outpath = outdir / 'json' / (filepath.stem + '.json')
|
||||
outpath = prog_outdir / 'json' / (filepath.stem + '.json')
|
||||
output_json(records, outpath, roles,
|
||||
fd_fields=fd_fields, field_to_fd=field_to_fd,
|
||||
open_dir=open_dir,
|
||||
@@ -550,14 +625,54 @@ def main():
|
||||
|
||||
select_info = parse_file_control(preprocessed)
|
||||
|
||||
output_input_files(records, outdir / 'input', filepath.stem, roles,
|
||||
output_input_files(records, prog_outdir / 'input', filepath.stem, roles,
|
||||
fd_fields, field_to_fd, open_dir,
|
||||
term_types=term_types,
|
||||
data_fields=fields_dict, select_info=select_info)
|
||||
|
||||
# ── Skip 数据集（主 FD 空文件触发 PERFORM UNTIL 条件即时满足）──
|
||||
if skip_path_infos:
|
||||
skip_records, _, skip_term_types = generate_records(
|
||||
skip_path_infos, fields_dict, assignments, file_sec=file_sec)
|
||||
# 剥离主 FD 的输入字段（记录不写入输入文件 → 文件为空）
|
||||
eof_fd = 'R01INNFIL'
|
||||
eof_fd_fields = set(fd_fields.get(eof_fd, []))
|
||||
eof_fd_dir = (open_dir or {}).get(eof_fd, '')
|
||||
for rec in skip_records:
|
||||
for fname in list(rec.keys()):
|
||||
if fname in eof_fd_fields:
|
||||
r = roles.get(fname, 'unused')
|
||||
if eof_fd_dir in ('INPUT', 'I-O') and r in ('input', 'inout'):
|
||||
del rec[fname]
|
||||
# 写 Skip JSON
|
||||
skip_outpath = prog_outdir / 'json' / (filepath.stem + '_skip.json')
|
||||
output_json(skip_records, skip_outpath, roles,
|
||||
fd_fields=fd_fields, field_to_fd=field_to_fd,
|
||||
open_dir=open_dir, term_types=skip_term_types,
|
||||
data_fields=fields_dict)
|
||||
# 写 Skip 输入文件（主 FD 因字段已剥离而不输出）
|
||||
skip_input_dir = prog_outdir / 'input_skip'
|
||||
output_input_files(skip_records, skip_input_dir,
|
||||
filepath.stem + '_skip', roles,
|
||||
fd_fields, field_to_fd, open_dir,
|
||||
term_types=skip_term_types,
|
||||
data_fields=fields_dict, select_info=select_info)
|
||||
# 强制写空主 FD 输入文件（0 条记录，COBOL 运行时需要文件存在）
|
||||
eof_input_path = skip_input_dir / f'{filepath.stem}_skip_{eof_fd}.json'
|
||||
eof_input_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(eof_input_path, 'w', encoding='utf-8') as f:
|
||||
json.dump([], f)
|
||||
# 空二进制文件（COBOL INPUT 模式需要物理文件存在）
|
||||
eof_assign = select_info.get(eof_fd, {}).get('assign', '')
|
||||
if eof_assign:
|
||||
bin_path = skip_input_dir / eof_assign
|
||||
bin_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
bin_path.write_bytes(b'')
|
||||
logger.info(f" Skip 数据集: {skip_outpath}（空 {eof_fd}）")
|
||||
|
||||
gcov_data = None
|
||||
if gcov_mode and proc_div and _HAVE_GCOV and _HAVE_RUNNER:
|
||||
_temp = temp_dir or str(outdir / '.gcov_cache')
|
||||
_temp = temp_dir or str(prog_outdir / '.gcov_cache')
|
||||
source_dir = str(filepath.parent)
|
||||
expected_records: list[dict] = [{}] * len(records)
|
||||
if file_sec and os.path.exists(outpath):
|
||||
@@ -575,7 +690,7 @@ def main():
|
||||
expected_records[i] = exp
|
||||
|
||||
group_results = run_all(
|
||||
filepath.stem, str(outdir), _temp,
|
||||
filepath.stem, str(prog_outdir), _temp,
|
||||
fields_dict, fd_fields, select_info, open_dir,
|
||||
term_types, records, expected_records=expected_records,
|
||||
source_dir=source_dir, path_infos=path_infos,
|
||||
@@ -596,7 +711,7 @@ def main():
|
||||
|
||||
if do_run and proc_div and _HAVE_RUNNER:
|
||||
run_and_compare(
|
||||
filepath.stem, str(outdir), fields_dict,
|
||||
filepath.stem, str(prog_outdir), fields_dict,
|
||||
fd_fields, select_info, open_dir,
|
||||
term_types, records,
|
||||
)
|
||||
@@ -611,14 +726,44 @@ def main():
|
||||
vals.append(f"{marker}{f['name']}={rec.get(f['name'], '?')}")
|
||||
logger.debug(f" 记录 {i}: {' | '.join(vals)}")
|
||||
|
||||
(outdir / 'coverage').mkdir(parents=True, exist_ok=True)
|
||||
cov_prefix = str(outdir / 'coverage' / filepath.stem)
|
||||
index_relpath = 'index.html'
|
||||
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
|
||||
cov_prefix = str(prog_outdir / 'coverage' / filepath.stem)
|
||||
# DEBUG: check DP#3 constraints
|
||||
dp3_t_count = 0
|
||||
dp3_f_count = 0
|
||||
dp3_t_paths = 0
|
||||
dp3_f_paths = 0
|
||||
dp3_sample = set()
|
||||
for cons, _ in branch_paths_with_assigns:
|
||||
has_t = False
|
||||
has_f = False
|
||||
for c in cons:
|
||||
if len(c) == 4:
|
||||
c0 = str(c[0]).strip()
|
||||
c1 = str(c[1]).strip()
|
||||
c2 = str(c[2]).strip()
|
||||
c3 = c[3]
|
||||
if c0 == 'WRK-R02KEY' and c1 == '>=' and c2 == 'WRK-R01KEY':
|
||||
if c3:
|
||||
dp3_t_count += 1
|
||||
has_t = True
|
||||
else:
|
||||
dp3_f_count += 1
|
||||
has_f = True
|
||||
elif c0 == 'WRK-R02KEY':
|
||||
dp3_sample.add(f"({c0},{c1},{c2},{c3})")
|
||||
if has_t:
|
||||
dp3_t_paths += 1
|
||||
if has_f:
|
||||
dp3_f_paths += 1
|
||||
logger.info(f"DEBUG DP#3: T={dp3_t_count}/{dp3_t_paths}paths, F={dp3_f_count}/{dp3_f_paths}paths (total={len(branch_paths_with_assigns)})")
|
||||
if dp3_sample:
|
||||
logger.info(f"DEBUG DP#3 other constraints: {sorted(dp3_sample)[:5]}")
|
||||
cov_result = run_coverage(branch_tree, branch_paths_with_assigns, fields_dict,
|
||||
source, cov_prefix, index_relpath=index_relpath,
|
||||
source, cov_prefix, index_relpath='index.html',
|
||||
gcov_data=gcov_data)
|
||||
|
||||
programs.append(cov_result)
|
||||
programs[-1]['detail_relpath'] = f'{filepath.stem}/coverage/{filepath.stem}_coverage.html'
|
||||
|
||||
if programs:
|
||||
generate_coverage_index(programs, outdir / 'coverage')
|
||||
@@ -630,15 +775,19 @@ def main():
|
||||
# ════════════════════════════════════════════
|
||||
|
||||
|
||||
def extract_structure(cobol_source: str) -> dict:
|
||||
def extract_structure(cobol_source: str, copybook_dirs: list = None) -> dict:
|
||||
"""分析 COBOL 源码的结构，返回结构摘要。不生成测试数据，只做静态分析。
|
||||
|
||||
Args:
|
||||
cobol_source: COBOL source text.
|
||||
copybook_dirs: Optional list of COPYBOOK search paths.
|
||||
|
||||
Returns:
|
||||
dict with: paragraphs, decision_points, branch_tree, file_count,
|
||||
open_directions, has_search_all, has_evaluate,
|
||||
has_call, has_break, total_branches, total_paragraphs
|
||||
"""
|
||||
preprocessed = preprocess(cobol_source)
|
||||
preprocessed = preprocess(cobol_source, extra_search_paths=copybook_dirs)
|
||||
data_div = extract_data_division(preprocessed)
|
||||
data_fields = parse_data_division(data_div) if data_div else []
|
||||
|
||||
@@ -998,11 +1147,23 @@ def generate_data(cobol_source: str, structure: dict = None,
|
||||
proc_div = extract_procedure_division(preprocessed)
|
||||
_, assignments = build_branch_tree_fallback(proc_div, fields_dict)
|
||||
|
||||
# EXEC SQL ブロックは preprocess で除去されるため、
|
||||
# 原ソースから直接抽出して assignments にマージする
|
||||
from .core import extract_sql_assignments
|
||||
sql_assigns = extract_sql_assignments(cobol_source)
|
||||
for tgt, asgn_list in sql_assigns.items():
|
||||
for asgn in asgn_list:
|
||||
assignments.setdefault(tgt, []).append(asgn)
|
||||
|
||||
file_sec = parse_file_section(preprocessed)
|
||||
|
||||
branch_paths_unfiltered = mcdc_enum_paths(branch_tree, fields_dict)
|
||||
path_infos = []
|
||||
for c, a in branch_paths_unfiltered:
|
||||
for cc in c:
|
||||
if len(cc) >= 4 and str(cc[0]) in ('WS-STATUS', 'WS-APPL-ID'):
|
||||
print(f" PATH-DEBUG: {cc}", flush=True)
|
||||
break
|
||||
filtered_c, term = get_term_type(c)
|
||||
path_infos.append((filtered_c, a, term))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user