feat: DB管线补全 + 新增orchestrator_db/program_schema/to_sql + 清理临时脚本

This commit is contained in:
hangshuo652
2026-07-11 14:55:52 +08:00
parent 40e8a50ab4
commit af37e33b98
32 changed files with 3232 additions and 255 deletions
+178 -17
View File
@@ -10,6 +10,7 @@ import os
import sys
import json
import re
import shutil
import logging
from datetime import datetime
from pathlib import Path
@@ -281,6 +282,7 @@ def main():
do_run = False
gcov_mode = False
gixsql_mode = False
temp_dir = None
if '--run' in args:
do_run = True
@@ -291,6 +293,9 @@ def main():
if not _HAVE_RUNNER:
logger.warning("--gcov: runner.py not found. Compile/run will be skipped. "
"Use --gcov without runner only generates test data + static coverage.")
if '--gixsql' in args:
gixsql_mode = True
args.remove('--gixsql')
i = 0
while i < len(args):
if args[i] == '--temp-dir':
@@ -342,6 +347,67 @@ def main():
programs = []
if gixsql_mode:
# DB pipeline: GixsqlOrchestrator
import sys as _sys
_v3_root = str(Path(__file__).parent.parent)
if _v3_root not in _sys.path:
_sys.path.insert(0, _v3_root)
from orchestrator_db import GixsqlOrchestrator
from config import Config
config = Config()
src_dir = cobol_files[0].parent if cobol_files else Path.cwd()
cpy_dirs = [src_dir / '..' / 'cpy']
for filepath in cobol_files:
pid = filepath.stem
prog_outdir = outdir / pid
prog_outdir.mkdir(parents=True, exist_ok=True)
(prog_outdir / 'logs').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'input').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'output').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'json').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
logger.info(f"\n========== DB: {pid} ==========")
orch = GixsqlOrchestrator(
config=config, program_id=pid,
cobol_src_dir=str(src_dir),
copybook_dirs=[str(d) for d in cpy_dirs],
skip_jvm=True,
)
vr = orch.run_all(generate_coverage=False)
# Copy output files to outdir
if orch.runtime_dir.exists():
for item in orch.runtime_dir.iterdir():
if item.is_file():
shutil.copy2(str(item), str(prog_outdir / item.name))
logger.info(f" {pid}: rc={vr.exit_code} status={vr.status}")
# Coverage report (only once, with correct output_dir)
if '--coverage' in getattr(config, 'gixsql_compile_flags', ''):
cov_result = orch.generate_coverage_report(output_dir=str(prog_outdir / 'coverage'))
if cov_result.success:
cv = cov_result.data.get("coverage", "unknown")
logger.info(f" Coverage: {cv}")
cov_dict = cov_result.data.get("_cov_dict")
if cov_dict:
# Fix detail_relpath relative to top-level index
rel = Path(prog_outdir / 'coverage' / f"{pid}_coverage.html")
cov_dict['detail_relpath'] = str(rel.relative_to(outdir).as_posix())
programs.append(cov_dict)
else:
logger.warning(" --coverage not in gixsql_compile_flags; skipping coverage")
if programs:
from cobol_testgen.coverage import generate_coverage_index as _gen_idx
_gen_idx(programs, outdir / 'coverage')
logger.info(f"\n覆盖率总览:{outdir / 'coverage' / 'index.html'}")
return
for filepath in cobol_files:
if not filepath.exists():
logger.error(f"错误:文件不存在 {filepath}")
@@ -432,6 +498,15 @@ def main():
for child in fds:
field_to_fd[child] = fd_name
# Per-program output directory (always)
prog_outdir = outdir / filepath.stem
prog_outdir.mkdir(parents=True, exist_ok=True)
(prog_outdir / 'logs').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'input').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'output').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'json').mkdir(parents=True, exist_ok=True)
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
logger.info(f"\n========== {filepath.name} ==========")
logger.info(f"\n字段列表:")
logger.info(f"{'层级':<6} {'名称':<25} {'PIC':<15} {'类型':<12} {'长度':<5}")
@@ -479,10 +554,11 @@ def main():
other += 1
return eq1_true > 0 and other == 0
before = len(path_infos)
path_infos = [p for p in path_infos if not _is_skip(p[0])]
after = len(path_infos)
logger.info(f" SKIP 过滤: {before} -> {after} 条路径(预期减少 1")
skip_path_infos = [p for p in path_infos if _is_skip(p[0])]
main_path_infos = [p for p in path_infos if not _is_skip(p[0])]
path_infos = main_path_infos
if skip_path_infos:
logger.info(f" Skip 路径: {len(skip_path_infos)} 条(将单独生成数据集)")
open_dir = scan_open_statements(proc_div) if proc_div else {}
@@ -539,8 +615,7 @@ def main():
else:
db_input = None
(outdir / 'json').mkdir(parents=True, exist_ok=True)
outpath = outdir / 'json' / (filepath.stem + '.json')
outpath = prog_outdir / 'json' / (filepath.stem + '.json')
output_json(records, outpath, roles,
fd_fields=fd_fields, field_to_fd=field_to_fd,
open_dir=open_dir,
@@ -550,14 +625,54 @@ def main():
select_info = parse_file_control(preprocessed)
output_input_files(records, outdir / 'input', filepath.stem, roles,
output_input_files(records, prog_outdir / 'input', filepath.stem, roles,
fd_fields, field_to_fd, open_dir,
term_types=term_types,
data_fields=fields_dict, select_info=select_info)
# ── Skip 数据集(主 FD 空文件触发 PERFORM UNTIL 条件即时满足)──
if skip_path_infos:
skip_records, _, skip_term_types = generate_records(
skip_path_infos, fields_dict, assignments, file_sec=file_sec)
# 剥离主 FD 的输入字段(记录不写入输入文件 → 文件为空)
eof_fd = 'R01INNFIL'
eof_fd_fields = set(fd_fields.get(eof_fd, []))
eof_fd_dir = (open_dir or {}).get(eof_fd, '')
for rec in skip_records:
for fname in list(rec.keys()):
if fname in eof_fd_fields:
r = roles.get(fname, 'unused')
if eof_fd_dir in ('INPUT', 'I-O') and r in ('input', 'inout'):
del rec[fname]
# 写 Skip JSON
skip_outpath = prog_outdir / 'json' / (filepath.stem + '_skip.json')
output_json(skip_records, skip_outpath, roles,
fd_fields=fd_fields, field_to_fd=field_to_fd,
open_dir=open_dir, term_types=skip_term_types,
data_fields=fields_dict)
# 写 Skip 输入文件(主 FD 因字段已剥离而不输出)
skip_input_dir = prog_outdir / 'input_skip'
output_input_files(skip_records, skip_input_dir,
filepath.stem + '_skip', roles,
fd_fields, field_to_fd, open_dir,
term_types=skip_term_types,
data_fields=fields_dict, select_info=select_info)
# 强制写空主 FD 输入文件(0 条记录,COBOL 运行时需要文件存在)
eof_input_path = skip_input_dir / f'{filepath.stem}_skip_{eof_fd}.json'
eof_input_path.parent.mkdir(parents=True, exist_ok=True)
with open(eof_input_path, 'w', encoding='utf-8') as f:
json.dump([], f)
# 空二进制文件(COBOL INPUT 模式需要物理文件存在)
eof_assign = select_info.get(eof_fd, {}).get('assign', '')
if eof_assign:
bin_path = skip_input_dir / eof_assign
bin_path.parent.mkdir(parents=True, exist_ok=True)
bin_path.write_bytes(b'')
logger.info(f" Skip 数据集: {skip_outpath}(空 {eof_fd}")
gcov_data = None
if gcov_mode and proc_div and _HAVE_GCOV and _HAVE_RUNNER:
_temp = temp_dir or str(outdir / '.gcov_cache')
_temp = temp_dir or str(prog_outdir / '.gcov_cache')
source_dir = str(filepath.parent)
expected_records: list[dict] = [{}] * len(records)
if file_sec and os.path.exists(outpath):
@@ -575,7 +690,7 @@ def main():
expected_records[i] = exp
group_results = run_all(
filepath.stem, str(outdir), _temp,
filepath.stem, str(prog_outdir), _temp,
fields_dict, fd_fields, select_info, open_dir,
term_types, records, expected_records=expected_records,
source_dir=source_dir, path_infos=path_infos,
@@ -596,7 +711,7 @@ def main():
if do_run and proc_div and _HAVE_RUNNER:
run_and_compare(
filepath.stem, str(outdir), fields_dict,
filepath.stem, str(prog_outdir), fields_dict,
fd_fields, select_info, open_dir,
term_types, records,
)
@@ -611,14 +726,44 @@ def main():
vals.append(f"{marker}{f['name']}={rec.get(f['name'], '?')}")
logger.debug(f" 记录 {i}: {' | '.join(vals)}")
(outdir / 'coverage').mkdir(parents=True, exist_ok=True)
cov_prefix = str(outdir / 'coverage' / filepath.stem)
index_relpath = 'index.html'
(prog_outdir / 'coverage').mkdir(parents=True, exist_ok=True)
cov_prefix = str(prog_outdir / 'coverage' / filepath.stem)
# DEBUG: check DP#3 constraints
dp3_t_count = 0
dp3_f_count = 0
dp3_t_paths = 0
dp3_f_paths = 0
dp3_sample = set()
for cons, _ in branch_paths_with_assigns:
has_t = False
has_f = False
for c in cons:
if len(c) == 4:
c0 = str(c[0]).strip()
c1 = str(c[1]).strip()
c2 = str(c[2]).strip()
c3 = c[3]
if c0 == 'WRK-R02KEY' and c1 == '>=' and c2 == 'WRK-R01KEY':
if c3:
dp3_t_count += 1
has_t = True
else:
dp3_f_count += 1
has_f = True
elif c0 == 'WRK-R02KEY':
dp3_sample.add(f"({c0},{c1},{c2},{c3})")
if has_t:
dp3_t_paths += 1
if has_f:
dp3_f_paths += 1
logger.info(f"DEBUG DP#3: T={dp3_t_count}/{dp3_t_paths}paths, F={dp3_f_count}/{dp3_f_paths}paths (total={len(branch_paths_with_assigns)})")
if dp3_sample:
logger.info(f"DEBUG DP#3 other constraints: {sorted(dp3_sample)[:5]}")
cov_result = run_coverage(branch_tree, branch_paths_with_assigns, fields_dict,
source, cov_prefix, index_relpath=index_relpath,
source, cov_prefix, index_relpath='index.html',
gcov_data=gcov_data)
programs.append(cov_result)
programs[-1]['detail_relpath'] = f'{filepath.stem}/coverage/{filepath.stem}_coverage.html'
if programs:
generate_coverage_index(programs, outdir / 'coverage')
@@ -630,15 +775,19 @@ def main():
# ════════════════════════════════════════════
def extract_structure(cobol_source: str) -> dict:
def extract_structure(cobol_source: str, copybook_dirs: list = None) -> dict:
"""分析 COBOL 源码的结构,返回结构摘要。不生成测试数据,只做静态分析。
Args:
cobol_source: COBOL source text.
copybook_dirs: Optional list of COPYBOOK search paths.
Returns:
dict with: paragraphs, decision_points, branch_tree, file_count,
open_directions, has_search_all, has_evaluate,
has_call, has_break, total_branches, total_paragraphs
"""
preprocessed = preprocess(cobol_source)
preprocessed = preprocess(cobol_source, extra_search_paths=copybook_dirs)
data_div = extract_data_division(preprocessed)
data_fields = parse_data_division(data_div) if data_div else []
@@ -998,11 +1147,23 @@ def generate_data(cobol_source: str, structure: dict = None,
proc_div = extract_procedure_division(preprocessed)
_, assignments = build_branch_tree_fallback(proc_div, fields_dict)
# EXEC SQL ブロックは preprocess で除去されるため、
# 原ソースから直接抽出して assignments にマージする
from .core import extract_sql_assignments
sql_assigns = extract_sql_assignments(cobol_source)
for tgt, asgn_list in sql_assigns.items():
for asgn in asgn_list:
assignments.setdefault(tgt, []).append(asgn)
file_sec = parse_file_section(preprocessed)
branch_paths_unfiltered = mcdc_enum_paths(branch_tree, fields_dict)
path_infos = []
for c, a in branch_paths_unfiltered:
for cc in c:
if len(cc) >= 4 and str(cc[0]) in ('WS-STATUS', 'WS-APPL-ID'):
print(f" PATH-DEBUG: {cc}", flush=True)
break
filtered_c, term = get_term_type(c)
path_infos.append((filtered_c, a, term))
+12 -7
View File
@@ -83,11 +83,16 @@ def parse_single_condition(text, fields=None):
# Resolve 88-level condition names
if fields:
for f in fields:
if f.get('is_88') and f['name'] == text.upper():
return (f.get('parent', ''), '=', f.get('value', ''))
# NOT 88-level → invert operator
if f.get('is_88') and text.upper().startswith('NOT ') and f['name'] == text[4:].strip().upper():
return (f.get('parent', ''), '<>', f.get('value', ''))
if isinstance(f, dict):
if f.get('is_88') and f['name'] == text.upper():
return (f.get('parent', ''), '=', f.get('value', ''))
if f.get('is_88') and text.upper().startswith('NOT ') and f['name'] == text[4:].strip().upper():
return (f.get('parent', ''), '<>', f.get('value', ''))
else:
if f.is_88 and f.name == text.upper():
return (f.parent or '', '=', f.value or '')
if f.is_88 and text.upper().startswith('NOT ') and f.name == text[4:].strip().upper():
return (f.parent or '', '<>', f.value or '')
# Strip OF qualifier: "STD-KEY OF MASTER-REC" → "STD-KEY"
if ' OF ' in text.upper():
@@ -268,10 +273,10 @@ def evaluate_tree(tree, assignment):
def is_field(name, fields):
# Strip subscript: WS-ITEM-STATUS(WS-INDEX-VAR) -> WS-ITEM-STATUS
bare = re.sub(r'\s*\(.*?\)\s*$', '', name).strip()
for f in fields:
if f['name'] == bare.upper():
fname = f['name'] if isinstance(f, dict) else f.name
if fname == bare.upper():
return True
return False
+161 -32
View File
@@ -705,6 +705,12 @@ class _BrParser:
if m_when:
cond_upper = m_when.group(1).strip()
self.advance()
# Continuation: next line may be = VALUE (COBOL multi-line WHEN)
if self.pos < len(self.lines):
peek = self.clean()
if peek and not re.match(r'^(WHEN|AT\s+END|END-SEARCH)', peek, re.IGNORECASE):
cond_upper += ' ' + peek
self.advance()
cond_tree = parse_compound_condition(cond_upper, self.fields)
body_seq = self.parse_seq(
end_check=lambda l: re.match(r'^(WHEN|AT\s+END)\b', l) or l in ('END-SEARCH',)
@@ -761,7 +767,7 @@ class _BrParser:
return node
m = re.match(r'^WHEN\s+(.+?)\s*$', line)
if m:
raw_val = m.group(1).strip().strip("'").strip('"')
raw_val = m.group(1).strip()
self.advance()
# Capture multi-line WHEN conditions (AND/OR continuation)
while self.pos < len(self.lines):
@@ -777,7 +783,7 @@ class _BrParser:
else:
case_seq = self.parse_seq(end_check=lambda l: l.startswith('WHEN') or l == 'END-EVALUATE')
if node.subjects:
vals = [v.strip().strip("'").strip('"')
vals = [v.strip()
for v in re.split(r'\s+ALSO\s+', raw_val)]
node.when_list.append((vals, case_seq))
else:
@@ -1188,6 +1194,21 @@ class _BrParser:
_RE_WHERE = re.compile(r'\bWHERE\b\s+(.*)', re.IGNORECASE)
_RE_SQL_INSERT = re.compile(
r'INSERT\s+INTO\s+(\w[\w-]*)\s*\(([^)]+)\)\s+VALUES\s*\(([^)]+)\)',
re.IGNORECASE
)
_RE_SQL_DELETE = re.compile(
r'DELETE\s+FROM\s+(\w[\w-]*)(?:\s+WHERE\s+(.+))?',
re.IGNORECASE
)
_RE_SQL_UPDATE = re.compile(
r'UPDATE\s+(\w[\w-]*)\s+SET\s+(.+?)(?:\s+WHERE\s+(.+))?$',
re.IGNORECASE
)
def _parse_sql_block(self) -> str:
"""Consume lines from EXEC SQL until END-EXEC. Returns SQL text."""
texts = []
@@ -1209,43 +1230,119 @@ class _BrParser:
def _parse_sql(self, sql_text: str):
"""Parse SQL text from EXEC SQL block. Returns Assign node or None."""
# 1) SELECT ... INTO ... FROM
m = self._RE_SELECT_INTO.search(sql_text)
if not m:
return None
if m:
select_list = m.group(1).strip()
into_raw = m.group(2).strip()
from_table = m.group(3).strip().upper()
remaining = sql_text[m.end():].strip()
select_list = m.group(1).strip()
into_raw = m.group(2).strip()
from_table = m.group(3).strip().upper()
remaining = sql_text[m.end():].strip()
into_vars = []
for v in re.split(r'\s*,\s*', into_raw):
v = v.strip().lstrip(':')
parts = v.split(':')
into_vars.append(parts[0].upper())
if len(parts) > 1:
into_vars.append(parts[1].upper())
# Parse INTO variables (handle indicator vars: :host:indicator)
into_vars = []
for v in re.split(r'\s*,\s*', into_raw):
v = v.strip().lstrip(':')
parts = v.split(':')
into_vars.append(parts[0].upper())
if len(parts) > 1:
into_vars.append(parts[1].upper())
where_clause = ''
wm = self._RE_WHERE.search(remaining)
if wm:
where_clause = wm.group(1).strip()
# Extract WHERE clause
where_clause = ''
wm = self._RE_WHERE.search(remaining)
if wm:
where_clause = wm.group(1).strip()
info = {
'type': 'exec_sql_select',
'table': from_table,
'select_list': select_list,
'into_vars': into_vars,
'where': where_clause,
'sql_text': sql_text,
}
info = {
'type': 'exec_sql_select',
'table': from_table,
'select_list': select_list,
'into_vars': into_vars,
'where': where_clause,
'sql_text': sql_text,
}
for var in into_vars:
self.assignments.setdefault(var, []).append(info)
for var in into_vars:
self.assignments.setdefault(var, []).append(info)
return Assign(into_vars[0], info)
return Assign(into_vars[0], info)
# 2) INSERT INTO table (...) VALUES (...)
m = self._RE_SQL_INSERT.search(sql_text)
if m:
table = m.group(1).strip().upper()
columns_str = m.group(2).strip()
values_str = m.group(3).strip()
host_vars = []
for v in re.split(r'\s*,\s*', values_str):
v = v.strip()
if v.startswith(':'):
v = v.lstrip(':')
parts = v.split(':')
host_vars.append(parts[0].upper())
info = {
'type': 'exec_sql_insert',
'table': table,
'columns': [c.strip() for c in columns_str.split(',')],
'raw_values': values_str,
'host_vars': host_vars,
'sql_text': sql_text,
}
synthetic = f'__SQL_INSERT_{table}'
self.assignments.setdefault(synthetic, []).append(info)
return Assign(synthetic, info)
# 3) DELETE FROM table WHERE ...
m = self._RE_SQL_DELETE.search(sql_text)
if m:
table = m.group(1).strip().upper()
where_clause = m.group(2).strip() if m.group(2) else ''
host_vars = re.findall(r':(\w[\w-]*)', where_clause)
info = {
'type': 'exec_sql_delete',
'table': table,
'where': where_clause,
'host_vars': [h.upper() for h in host_vars],
'sql_text': sql_text,
}
synthetic = f'__SQL_DELETE_{table}'
self.assignments.setdefault(synthetic, []).append(info)
return Assign(synthetic, info)
# 4) UPDATE table SET ... WHERE ...
m = self._RE_SQL_UPDATE.search(sql_text)
if m:
table = m.group(1).strip().upper()
set_clause = m.group(2).strip()
where_clause = m.group(3).strip() if m.group(3) else ''
host_vars = []
for part in re.split(r'\s*,\s*', set_clause):
sm = re.match(r'\w[\w-]*\s*=\s*(:\w[\w-]*(?::\w[\w-]*)?)', part, re.IGNORECASE)
if sm:
hv = sm.group(1).lstrip(':')
parts = hv.split(':')
host_vars.append(parts[0].upper())
for wv in re.findall(r':(\w[\w-]*)', where_clause):
wvu = wv.upper()
if wvu not in host_vars:
host_vars.append(wvu)
info = {
'type': 'exec_sql_update',
'table': table,
'set_clause': set_clause,
'where': where_clause,
'host_vars': host_vars,
'sql_text': sql_text,
}
synthetic = f'__SQL_UPDATE_{table}'
self.assignments.setdefault(synthetic, []).append(info)
return Assign(synthetic, info)
return None
# ── 工具函数 ──
@@ -1527,6 +1624,12 @@ def propagate_assignments(rec, assignments, fields, file_sec=None):
start = asgn['refmod_start'] - 1
end = start + asgn['refmod_length']
src_val = src_val[start:end]
# Type-safe MOVE: alphanumeric→numeric → strip non-digit chars
_tgt_pi = next((f.get('pic_info', {}) for f in fields if f['name'] == resolved_tgt), {})
if _tgt_pi.get('type') == 'numeric' and not src_val.lstrip('-').replace('.', '').isdigit():
digits = _tgt_pi.get('digits', 0) + _tgt_pi.get('decimal', 0)
src_val = ''.join(c for c in src_val if c.isdigit())[:max(digits, 1)] or '0'
src_val = src_val.zfill(max(digits, 1))
rec[resolved_tgt] = src_val
# Pass 2: literal MOVE
@@ -1979,3 +2082,29 @@ def _find_multi_write_fds(tree, field_to_fd):
loop_write = set()
_collect_write_fds(tree.children[main_loop_idx], loop_write, field_to_fd)
return pre_write & loop_write
# ── EXEC SQL ブロック抽出(preprocess で除去される前の生ソースから)──
_RE_EXEC_SQL = re.compile(
r'EXEC\s+SQL\s+(.*?)\s+END-EXEC\.?',
re.DOTALL | re.IGNORECASE
)
def extract_sql_assignments(source: str) -> dict:
"""原ソースから EXEC SQL ブロックを抽出し Assign 情報を返す。
preprocess() が全 EXEC SQL ブロックを除去するため、その前に
生ソースから直接抽出する。戻り値は assignments dict と互換。
"""
from collections import defaultdict
parser = _BrParser([])
parser.assignments = defaultdict(list)
for m in _RE_EXEC_SQL.finditer(source):
sql_text = re.sub(r'\s+', ' ', m.group(1).strip())
parser._parse_sql(sql_text)
return dict(parser.assignments)
+51 -11
View File
@@ -176,7 +176,30 @@ def mark_coverage(decision_points, leaf_stats, branch_paths, fields):
leaf.covered_false = True
for dp in decision_points:
dp.implied_branches = set(dp.active_branches)
missing = None
if dp.kind == 'IF':
parsed = getattr(dp, 'parsed', None)
if parsed and is_field(parsed[0], fields):
has_T = 'T' in dp.active_branches
has_F = 'F' in dp.active_branches
if has_T and not has_F:
missing = 'F'
elif has_F and not has_T:
missing = 'T'
elif dp.kind == 'PERFORM':
parsed = getattr(dp, 'parsed', None)
if parsed and is_field(parsed[0], fields):
has_E = 'Enter' in dp.active_branches
has_S = 'Skip' in dp.active_branches
if has_E and not has_S:
missing = 'Skip'
elif has_S and not has_E:
missing = 'Enter'
if missing:
dp.implied_branches = {missing}
else:
dp.implied_branches = set(dp.active_branches)
def _match_constraint(c, parsed):
@@ -232,6 +255,14 @@ def _mark_if(dp, cons):
dp.active_branches.add('F')
except KeyError:
pass
else:
# All leaves are synthetic (e.g. FUNCTION MOD → _FUNC_MOD): can't match
# but path generator traversed both branches — mark both covered
all_synthetic = all(
not is_field(ls.field, []) for ls in dp.leaves
)
if all_synthetic:
dp.active_branches.update(['T', 'F'])
else:
matched = 0
for leaf in dp.leaves:
@@ -336,11 +367,10 @@ def _mark_search(dp, cons, fields=None):
continue
if isinstance(cond_tree, CondLeaf):
for c in cons:
if len(c) == 4:
if len(c) == 4 and c[3]:
base_c = re.sub(r'\s*\(.*?\)\s*$', '', c[0])
base_cond = re.sub(r'\s*\(.*?\)\s*$', '', cond_tree.field)
if base_c == base_cond and c[1] == cond_tree.op \
and str(c[2]) == str(cond_tree.value) and c[3]:
if base_c == base_cond:
branch_masks[i] = True
break
else:
@@ -424,12 +454,16 @@ def _get_fields_in_cond(cond_text):
def locate_decision_lines(decision_points, raw_source):
lines = raw_source.upper().splitlines()
used_indices = {} # label → last matched 0-indexed line number
for dp in decision_points:
patterns = _build_search_patterns(dp)
for i, line in enumerate(lines):
start = used_indices.get(dp.label, -1) + 1
for i in range(start, len(lines)):
line = lines[i]
for pat in patterns:
if re.search(pat, line):
dp.source_line = i + 1
used_indices[dp.label] = i
break
if dp.source_line:
break
@@ -1192,14 +1226,23 @@ def _find_proc_range(raw_source: str):
def run_coverage(branch_tree, branch_paths_with_assigns, fields,
raw_source, output_prefix, index_relpath=None,
gcov_data=None):
gcov_data=None, gcov_source=None):
decision_points, leaf_stats = collect_decision_points(branch_tree, fields)
mark_coverage(decision_points, leaf_stats, branch_paths_with_assigns, fields)
# Use gcov_source (preprocessed) for line location if available (matches gcov_data line numbers)
source_for_lines = gcov_source or raw_source
if source_for_lines:
locate_decision_lines(decision_points, source_for_lines)
if gcov_data:
mark_from_gcov(decision_points, gcov_data, branch_tree)
# leaf_stats 保留静态分析结果(gcov 无 -b 时不提供叶条件级别的分支数据)
mark_from_gcov(decision_points, gcov_data, branch_tree,
gcov_source or raw_source)
for dp in decision_points:
ln = dp.source_line
if ln > 0 and ln in gcov_data and gcov_data[ln] == 0:
dp.implied_branches.clear()
_source_note = ''
if gcov_data:
@@ -1210,9 +1253,6 @@ def run_coverage(branch_tree, branch_paths_with_assigns, fields,
'</div>'
)
if raw_source:
locate_decision_lines(decision_points, raw_source)
total = sum(len(dp.branch_names) for dp in decision_points)
covered = sum(len(dp.active_branches) for dp in decision_points)
implied = sum(len(dp.implied_branches) for dp in decision_points)
+265 -12
View File
@@ -101,10 +101,51 @@ def _cap_paths_fair(new_active, child_paths):
# ── 路径枚举 ──
def eval_true_branch_constraints(when_value: str, fields: list) -> tuple:
"""解析 EVALUATE TRUE 的 WHEN 条件,返回 (true_set, false_sets)。
true_set: list[Constraint] — 使此 WHEN 为 True 的一组约束
false_sets: list[list[Constraint]] — 使此 WHEN 为 False 的 MC/DC 倒集
适用于 EVALUATE TRUE 的所有 WHEN 类型:
- 简单条件: WS-STATUS = '9' → 直接产生 (T, [F])
- CondNot: NOT WS-STATUS = '1' → (翻转, [翻转倒])
- 复合条件: WS-STATUS = '1' AND WS-APPL-ID = 0 → MC/DC 约束集
"""
cond = parse_compound_condition(when_value, fields)
if cond and isinstance(cond, CondLeaf) and is_field(cond.field, fields):
t = [(cond.field, cond.op, cond.value, True)]
f = [[(cond.field, cond.op, cond.value, False)]]
return t, f
if cond and isinstance(cond, CondNot) and isinstance(cond.child, CondLeaf) and is_field(cond.child.field, fields):
leaf = cond.child
t = [(leaf.field, leaf.op, leaf.value, False)]
f = [[(leaf.field, leaf.op, leaf.value, True)]]
return t, f
leaves = collect_leaves(cond) if cond else []
if leaves and all(is_field(l.field, fields) for l in leaves):
sets = mcdc_sets(cond, fields)
if sets:
true_sets = [list(cs) for cs, decision in sets if decision]
false_sets = [list(cs) for cs, decision in sets if not decision]
if true_sets:
return true_sets[0], false_sets
return [], []
_enum_counter = 0
def enum_paths(node, fields):
global _enum_counter
_enum_counter += 1
"""枚举路径,每条路径返回 (constraints, assignments).
返回 list[tuple[list[tuple], dict]].
"""
pass
if isinstance(node, Assign):
return [([], {node.target: [node.source_info]})]
@@ -199,6 +240,16 @@ def enum_paths(node, fields):
for fp_cons, fp_assign in (false_sub or [([], {})]):
paths.append(([(field, op, val, False)] + fp_cons, fp_assign))
return paths
# Fallback: unparseable condition (e.g. FUNCTION MOD) — still traverse both branches
if node.true_seq or node.false_seq:
paths = []
ts = enum_paths(node.true_seq, fields)
for sp_cons, sp_assign in (ts or [([], {})]):
paths.append((sp_cons, sp_assign))
fs = enum_paths(node.false_seq, fields)
for fp_cons, fp_assign in (fs or [([], {})]):
paths.append((fp_cons, fp_assign))
return paths if paths else [([], {})]
return [([], {})]
elif isinstance(node, BrEval):
@@ -260,11 +311,14 @@ def enum_paths(node, fields):
if not new_false_sets:
prior_false_sets = []
break
combined = []
for pf_set in prior_false_sets:
for nf_set in new_false_sets:
combined.append(list(pf_set) + list(nf_set))
prior_false_sets = combined
if not prior_false_sets:
prior_false_sets = list(new_false_sets)
else:
combined = []
for pf_set in prior_false_sets:
for nf_set in new_false_sets:
combined.append(list(pf_set) + list(nf_set))
prior_false_sets = combined
else:
prior_false_sets = []
break
@@ -334,6 +388,8 @@ def enum_paths(node, fields):
if parsed and is_field(parsed[0], fields):
field, op, val = parsed
paths = []
# Skip 路径放在首位,确保不被 _cap_paths 截断丢失
paths.append(([(field, op, val, True)], {}))
false_sub = _cap_paths(enum_paths(node.body_seq, fields))
false_sub = [([c for c in cons if c is not _STOP_EXIT_PERFORM], a) for cons, a in false_sub]
for sp_cons, sp_assign in (false_sub or [([], {})]):
@@ -383,7 +439,6 @@ def enum_paths(node, fields):
paths.append((the_cons + sp_cons, merged_max))
except (ValueError, TypeError):
pass
paths.append(([(field, op, val, True)], {}))
return paths
# 尝试复合条件(AND/OR
cond_tree = parse_compound_condition(node.condition, fields)
@@ -393,6 +448,10 @@ def enum_paths(node, fields):
sets = mcdc_sets(cond_tree, fields)
if sets:
paths = []
# Skip (True) 路径放在首位,确保不被 _cap_paths 截断丢失
for constraints, decision in sets:
if decision:
paths.append((list(constraints), {}))
false_sub = _cap_paths(enum_paths(node.body_seq, fields))
false_sub = [([c for c in cons if c is not _STOP_EXIT_PERFORM], a) for cons, a in false_sub]
for sp_cons, sp_assign in (false_sub or [([], {})]):
@@ -409,11 +468,27 @@ def enum_paths(node, fields):
for constraints, decision in sets:
if not decision:
paths.append((list(constraints) + sp_cons, sp_assign))
for constraints, decision in sets:
if decision:
paths.append((list(constraints), {}))
if paths:
return paths
# 单叶子 fallback(不可识别/未知字段)
if len(leaves) == 1:
leaf = leaves[0]
paths = []
# Skip 路径放在首位,确保不被 _cap_paths 截断丢失
paths.append(([(leaf.field, leaf.op, leaf.value, True)], {}))
body_paths = _cap_paths(enum_paths(node.body_seq, fields))
body_paths = [([c for c in cons if c is not _STOP_EXIT_PERFORM], a) for cons, a in body_paths]
for sp_cons, sp_assign in (body_paths or [([], {})]):
if node.varying_from and node.varying_var:
from_asgn = {'type': 'move_literal', 'literal': node.varying_from}
from_assign = {node.varying_var: [from_asgn]}
merged = {}
for d in (from_assign, sp_assign):
for k, v in d.items():
merged.setdefault(k, []).extend(v if isinstance(v, list) else [v])
sp_assign = merged
paths.append(([(leaf.field, leaf.op, leaf.value, False)] + sp_cons, sp_assign))
return paths
return [([], {})]
elif isinstance(node, CallNode):
@@ -649,6 +724,116 @@ def make_base_record(seq_num: int, fields: list) -> dict:
return rec
def _resolve_field_value(field_name, rec, fields):
"""将字段名解析为当前记录值。
对组项目(无 PIC)拼接其基本子字段的值。
返回字符串值,或在无法解析时返回 None。
"""
for f in fields:
if f['name'] == field_name:
if f.get('pic'):
return str(rec.get(field_name, ''))
else:
children = _children_of(field_name, fields)
parts = []
for c in children:
if c.get('pic'):
parts.append(str(rec.get(c['name'], '')))
return ''.join(parts) if parts else None
return None
def _expand_group_constraint(rec, field_name, operator, value, want_true, fields, assignments=None, path_assign=None):
"""将组项目间的比较约束展开为子字段约束。
COBOL 组项目比较 = 逐子字段字典序比较(先比较第一个子字段,
若相等则继续比较下一个)。
策略:
- >= True: 让第一个子字段 > 对应右侧子字段
- >= False (<): 让第一个子字段 < 对应右侧子字段
- = True: 让所有子字段逐个相等
- = False (<>): 让第一个子字段 != 对应右侧子字段
"""
field_children = _children_of(field_name, fields)
elementary = [c for c in field_children if c.get('pic')]
if not elementary:
return False
# 解析右侧值:如果是字段名,找到其子字段或值
right_children = []
if any(f['name'] == value for f in fields):
for f in fields:
if f['name'] == value:
if f.get('pic'):
# 基本字段:直接用其值
right_val = _resolve_field_value(value, rec, fields)
if right_val is not None:
if operator in ('>=', '>') and want_true:
apply_constraint(rec, elementary[0]['name'], '>', right_val, True, fields, assignments, path_assign)
return True
elif operator in ('>=', '>') and not want_true:
apply_constraint(rec, elementary[0]['name'], '<', right_val, True, fields, assignments, path_assign)
return True
elif operator == '=' and want_true:
apply_constraint(rec, elementary[0]['name'], '=', right_val, True, fields, assignments, path_assign)
return True
elif operator == '=' and not want_true:
apply_constraint(rec, elementary[0]['name'], '<>', right_val, True, fields, assignments, path_assign)
return True
else:
# 组项目:找对应子字段
right_children = [c for c in _children_of(value, fields) if c.get('pic')]
break
if not right_children:
# value 不是字段名(字面量)或无法解析,直接用值
right_children = elementary # 使用同样的子字段结构,各自对比值
min_len = min(len(elementary), len(right_children))
if min_len == 0:
return False
if operator in ('>=', '>') and want_true:
first = elementary[0]
# 取第一个右侧子字段的值
right_val = _resolve_field_value(right_children[0]['name'], rec, fields)
if right_val:
apply_constraint(rec, first['name'], '>', right_val, True, fields, assignments, path_assign)
else:
apply_constraint(rec, first['name'], '>=', str(right_children[0]['name']), True, fields, assignments, path_assign)
return True
elif operator in ('>=', '>') and not want_true:
first = elementary[0]
right_val = _resolve_field_value(right_children[0]['name'], rec, fields)
if right_val:
apply_constraint(rec, first['name'], '<', right_val, True, fields, assignments, path_assign)
else:
apply_constraint(rec, first['name'], '<', str(right_children[0]['name']), True, fields, assignments, path_assign)
return True
elif operator == '=' and want_true:
for i in range(min_len):
right_val = _resolve_field_value(right_children[i]['name'], rec, fields)
if right_val:
apply_constraint(rec, elementary[i]['name'], '=', right_val, True, fields, assignments, path_assign)
else:
apply_constraint(rec, elementary[i]['name'], '=', str(right_children[i]['name']), True, fields, assignments, path_assign)
return True
elif operator == '=' and not want_true:
first = elementary[0]
right_val = _resolve_field_value(right_children[0]['name'], rec, fields)
if right_val:
apply_constraint(rec, first['name'], '<>', right_val, True, fields, assignments, path_assign)
else:
apply_constraint(rec, first['name'], '<>', str(right_children[0]['name']), True, fields, assignments, path_assign)
return True
return False
# ── 约束应用 ──
def _check_constraint_satisfied(rec, field_name, operator, value, want_true, fields):
@@ -862,6 +1047,46 @@ def _reconcile_unstring_fields(rec, left_field, operator, right_field, want_true
logger.debug(f"字段间比较协调:{left_field}={left_val} {operator} {right_field} -> {right_root}={rec[right_root]} (want={want_true})")
def _apply_redefines_child_constraint(rec, field_name, operator, value, want_true, fields, parent_name):
"""Apply constraint on a group REDEFINES child by computing parent value."""
# Find the child's pic_info and compute satisfying value
child_pi = None
offset = 0
total_len = 0
child_len = 0
for f in fields:
if f.get('redefines') and not f.get('pic') and f['redefines'] == parent_name:
redef_children = _children_of(f['name'], fields)
for c in redef_children:
c_len = (c.get('pic_info', {}).get('digits', 0) + c.get('pic_info', {}).get('decimal', 0)
or c.get('pic_info', {}).get('length', 0))
if c['name'] == field_name:
child_pi = c.get('pic_info', {})
child_len = c_len
break
offset += c_len
total_len = offset + sum(
(cc.get('pic_info', {}).get('digits', 0) + cc.get('pic_info', {}).get('decimal', 0)
or cc.get('pic_info', {}).get('length', 0))
for cc in redef_children[redef_children.index(c):]
) if child_pi else 0
break
if not child_pi:
return
val = satisfying_value(child_pi, operator, value, want_true)
val = val.zfill(child_len)[:child_len]
# Merge into parent's current value
parent_val = str(rec.get(parent_name, ''))
if len(parent_val) < offset + child_len:
parent_val = parent_val.ljust(offset + child_len, '0')
parent_val = parent_val[:offset] + val + parent_val[offset + child_len:]
# Apply the combined constraint to the parent
apply_constraint(rec, parent_name, '=', f'"{parent_val}"', True, fields)
def apply_constraint(rec, field_name, operator, value, want_true, fields, assignments=None, path_assign=None):
# 标准化字段名:去除括号内空格(WS-CELL ( 1, 1 ) → WS-CELL(1,1)
field_name = re.sub(r'\s*([(),])\s*', r'\1', field_name)
@@ -896,6 +1121,17 @@ def apply_constraint(rec, field_name, operator, value, want_true, fields, assign
apply_constraint(rec, parent_name, operator, value, want_true, fields, assignments, path_assign)
return
break
# 组 REDEFINES 子字段:通过父字段传播约束
for f in fields:
if f.get('redefines') and not f.get('pic'):
redef_children = _children_of(f['name'], fields)
if any(c['name'] == field_name for c in redef_children):
parent_name = f['redefines']
logger.debug(f"组 REDEFINES 子字段约束: {field_name}{parent_name}")
_apply_redefines_child_constraint(rec, field_name, operator, value, want_true, fields, parent_name)
return
chain = None
if assignments:
root_var, chain = trace_to_root(field_name, assignments, fields, path_assign)
@@ -904,6 +1140,14 @@ def apply_constraint(rec, field_name, operator, value, want_true, fields, assign
if any(f['name'] == new_field_name for f in fields):
field_name, operator, value = new_field_name, new_op, new_val
# 组项目展开:当 field_name 是组项目(无 PIC)时,展开为子字段约束
field_def = next((f for f in fields if f['name'] == field_name), None)
if field_def and not field_def.get('pic'):
expanded = _expand_group_constraint(rec, field_name, operator, value, want_true,
fields, assignments, path_assign)
if expanded:
return
# 字段间比较:在 satisfied check 前解析/处理
if any(f['name'] == value for f in fields):
resolved_literal = None
@@ -921,8 +1165,13 @@ def apply_constraint(rec, field_name, operator, value, want_true, fields, assign
_apply_arith_constraint(rec, field_name, operator, value, want_true, fields)
return
else:
logger.debug(f"字段间比较约束跳过:{field_name} {operator} {value}")
return
# 尝试将字段名值解析为记录值
resolved_val = _resolve_field_value(value, rec, fields)
if resolved_val is not None:
value = resolved_val
else:
logger.debug(f"字段间比较约束跳过:{field_name} {operator} {value}")
return
# 如果当前值已满足该约束,跳过覆盖(保持先前约束的一致性)
# 但零值时强制使用边界值(非 0/非 min)
@@ -1114,7 +1363,11 @@ def _enum_search_paths(node, fields):
for k, v in sp_assign.items():
merged_assign.setdefault(k, []).extend(v if isinstance(v, list) else [v])
if cond_tree and isinstance(cond_tree, CondLeaf):
paths.append(([(elem_key, cond_tree.op, matching_val, True)] + sp_cons, merged_assign))
# Also set the subject field (right side of comparison) to match
subj = cond_tree.value
if any(f['name'] == subj for f in fields):
merged_assign[subj] = [{'type': 'move_literal', 'literal': matching_val}]
paths.append(([(elem_key, cond_tree.op, matching_val.rstrip(), True)] + sp_cons, merged_assign))
else:
paths.append((sp_cons, merged_assign))
+20 -2
View File
@@ -14,6 +14,7 @@ import re
import logging
from .models import BrSeq, BrIf, BrEval, BrPerform, BrSearch, Assign, CallNode, CondNot, CondLeaf, ExitNode, GoTo
from .cond import parse_single_condition, parse_compound_condition, is_field, collect_leaves, mcdc_sets
from .design import eval_true_branch_constraints
logger = logging.getLogger(__name__)
@@ -183,6 +184,18 @@ def _make_path_for_branch(dp, branch_idx, fields):
node = dp["node"]
n_when = len(node.when_list)
dp_id = dp.get("id", 0)
if node.subject == 'TRUE':
for prev_idx in range(branch_idx):
prev_value, _ = node.when_list[prev_idx]
_, prev_false_sets = eval_true_branch_constraints(prev_value, fields)
if prev_false_sets:
constraints.extend(prev_false_sets[0])
if branch_idx < n_when:
value, seq = node.when_list[branch_idx]
true_set, _ = eval_true_branch_constraints(value, fields)
constraints.extend(true_set)
print(f" MCDC-DEBUG: EVALUATE TRUE branch={branch_idx} subject={node.subject} constraints={constraints}", flush=True)
return (constraints, {})
if branch_idx < n_when:
value, seq = node.when_list[branch_idx]
if is_field(node.subject, fields):
@@ -246,8 +259,13 @@ def enum_paths(node, fields):
if bp: paths.append(bp)
if node.has_other:
other_cons = list(dp.get("access_constraints", []))
for v, _ in node.when_list:
if is_field(node.subject, fields):
if node.subject == 'TRUE':
for v, _ in node.when_list:
_, false_sets = eval_true_branch_constraints(v, fields)
if false_sets:
other_cons.extend(false_sets[0])
elif is_field(node.subject, fields):
for v, _ in node.when_list:
other_cons.append((node.subject, '<>', v, True))
paths.append((other_cons, {}))
+30
View File
@@ -236,3 +236,33 @@ def _unpack_record(data: bytes, fd_field_dicts: list[dict]) -> dict:
record[field_dict['name']] = unpack_value(data[offset:offset + slen], field_dict)
offset += slen
return record
def write_variable_file(file_path: str, fd_field_dicts: list[dict],
records: list[dict]) -> int:
"""写入 RECORDING MODE V 文件(带 4 字节 RDW 前缀)。
RDW: Little-Endian unsigned short 记录长度含自身4字节
Args:
file_path: 输出路径
fd_field_dicts: FD 字段定义列表
records: 记录列表
Returns:
int: 写入的记录数
"""
with open(file_path, 'wb') as f:
for record in records:
data = bytearray()
for field_dict in fd_field_dicts:
val = record.get(field_dict['name'], '')
packed = pack_value(val, field_dict)
data.extend(packed)
record_len = len(data) + 4
rdw = struct.pack('<H', record_len)
f.write(rdw)
f.write(b'\x00\x00')
f.write(data)
logger.info(f" wrote {len(records)} records to {file_path}")
return len(records)
+116 -16
View File
@@ -3,14 +3,15 @@ import re, struct
from pathlib import Path
from typing import Any
def analyze_fd_layout(source_text: str) -> dict[str, dict]:
"""From preprocessed COBOL source, extract FD file layouts."""
from .read import parse_file_control, parse_file_section, parse_data_division, extract_data_division, scan_open_statements
def analyze_fd_layout(source_text: str, copybook_dirs: list[str] = None) -> dict[str, dict]:
"""From COBOL source, extract FD file layouts."""
from .read import preprocess, parse_file_control, parse_file_section, parse_data_division, extract_data_division, scan_open_statements
fc = parse_file_control(source_text) if source_text else {}
fs = parse_file_section(source_text) if source_text else {}
ops = scan_open_statements(source_text) if source_text else {}
dd = extract_data_division(source_text)
pp = preprocess(source_text, extra_search_paths=copybook_dirs)
fc = parse_file_control(pp) if pp else {}
fs = parse_file_section(pp) if pp else {}
ops = scan_open_statements(pp) if pp else {}
dd = extract_data_division(pp)
all_fields = parse_data_division(dd) if dd else []
layouts = {}
@@ -25,6 +26,7 @@ def analyze_fd_layout(source_text: str) -> dict[str, dict]:
if f.name == rec_name:
found = True
rec_level = f.level
rec_field_obj = f
continue
if found:
if f.level is not None and f.level <= rec_level:
@@ -37,14 +39,45 @@ def analyze_fd_layout(source_text: str) -> dict[str, dict]:
else:
length = 0
ftype = pi.type if pi else "unknown"
usage = f.usage if f.usage else None
children.append({
"name": f.name, "pic": str(f.pic or ""),
"type": ftype, "length": length, "offset": offset,
"usage": usage,
"pic_info": {
"type": f.pic_info.type if f.pic_info else "unknown",
"digits": f.pic_info.digits if f.pic_info else 0,
"decimal": f.pic_info.decimal if f.pic_info else 0,
"length": f.pic_info.length if f.pic_info else 0,
"signed": f.pic_info.signed if f.pic_info else False,
} if f.pic_info else None,
})
offset += length
# If record has no elementary children but the 01-level has PIC info
# (e.g. 01 SYSINREC PIC X(080)), use it as a single opaque field
if not children and rec_field_obj and rec_field_obj.pic_info:
pi = rec_field_obj.pic_info
length = pi.length or 0
if length > 0:
children.append({
"name": rec_field_obj.name,
"pic": str(rec_field_obj.pic or ""),
"type": "alphanumeric",
"length": length,
"offset": 0,
"usage": None,
"pic_info": {
"type": "alphanumeric",
"digits": 0,
"decimal": 0,
"length": length,
"signed": False,
},
})
offset = length
records.append({"record_name": rec_name, "fields": children, "record_length": offset})
assign_to = fc.get(fd_name, {}).get("assign_to", fd_name)
assign_to = fc.get(fd_name, {}).get("assign", fd_name)
layouts[assign_to] = {
"fd_name": fd_name, "records": records,
"direction": ops.get(fd_name, "INPUT"),
@@ -70,6 +103,23 @@ def select_records_for_file(records: list[dict], layout: dict) -> list[dict]:
def _format_value(value: Any, field: dict) -> bytes:
"""Format a value for COBOL fixed-length storage."""
from . import file_io
usage = field.get("usage")
if usage in ("COMP", "COMP-3", "BINARY", "PACKED-DECIMAL"):
pic_info = field.get("pic_info") or {}
packed = file_io.pack_value(str(value) if value is not None else "", {
"usage": usage,
"pic_info": pic_info,
})
want_len = file_io.get_storage_length({
"usage": usage,
"pic_info": pic_info,
})
if len(packed) < want_len:
packed = packed.rjust(want_len, b'\x00')
return packed[:want_len]
ftype = field["type"]
length = field["length"]
val = str(value) if value is not None else ""
@@ -80,7 +130,6 @@ def _format_value(value: Any, field: dict) -> bytes:
except (ValueError, TypeError):
num = 0
num = abs(num)
# Truncate to fit PIC digits
max_val = 10 ** length - 1
if num > max_val:
num = max_val
@@ -104,8 +153,7 @@ def write_flat_file(records: list[dict], layout: dict, outpath: Path, field_filt
return
# Pick the record with the most fields (best coverage for multi-record FDs)
rec = max(layout["records"], key=lambda r: (len(r["fields"]), r["record_length"]))
rec_len = rec["record_length"]
if rec_len == 0:
if rec["record_length"] == 0:
return
rec_fields = rec["fields"]
@@ -114,23 +162,25 @@ def write_flat_file(records: list[dict], layout: dict, outpath: Path, field_filt
with open(outpath, "wb") as f:
for row in records:
buf = bytearray(rec_len)
buf = bytearray()
for field in rec_fields:
val = row.get(field["name"], "")
formatted = _format_value(val, field)
end = min(field["offset"] + len(formatted), rec_len)
buf[field["offset"]:end] = formatted[:end - field["offset"]]
buf.extend(formatted)
f.write(buf)
def write_all_files(records: list[dict], source_text: str, outdir: Path, prefix: str = ""):
def write_all_files(records: list[dict], source_text: str, outdir: Path, prefix: str = "", copybook_dirs: list[str] = None):
"""Analyze source, write flat files for all INPUT FDs."""
outdir = Path(outdir)
layouts = analyze_fd_layout(source_text)
layouts = analyze_fd_layout(source_text, copybook_dirs=copybook_dirs)
written = []
for filename, layout in layouts.items():
if layout["direction"] == "OUTPUT":
continue
# Skip SYSIN files — handled separately by write_sysin_file
if layout["fd_name"] == "SYSINFILE":
continue
fnames = set()
for rec in layout["records"]:
for f in rec["fields"]:
@@ -152,3 +202,53 @@ def write_all_files(records: list[dict], source_text: str, outdir: Path, prefix:
write_flat_file(filtered, layout, outpath)
written.append((filename, outpath, len(filtered)))
return written
def write_sysin_file(records: list[dict], source_text: str, outdir: Path, prefix: str = "", copybook_dirs: list[str] = None):
"""Generate SYSIN configuration card file from FD layout + generated records."""
outdir = Path(outdir)
layouts = analyze_fd_layout(source_text, copybook_dirs=copybook_dirs)
sysin_filename = None
sysin_layout = None
for filename, layout in layouts.items():
if layout["fd_name"] == "SYSINFILE":
sysin_filename = filename
sysin_layout = layout
break
if not sysin_layout:
return None
# Determine record length from layout
rec_length = 0
for rec in sysin_layout["records"]:
if rec["record_length"] > rec_length:
rec_length = rec["record_length"]
if rec_length == 0:
rec_length = 80
# Extract unique employee IDs from records (skip sentinel '00000000')
emp_ids = sorted(set(
r.get("R01EMP-ID", "") for r in records
if r.get("R01EMP-ID") and r["R01EMP-ID"] != "00000000"
))
# Limit to 8 per T card (78 chars of data: 8 * (8+1) = 72 fits)
emp_ids = emp_ids[:8]
# Build SYSIN card records
# Card format: position 1 = type, position 2 = space (ignored), position 3+ = data
lines = [
f"P YEAR-MONTH=202607", # Period card
f"M MODE=NORMAL", # Mode card
]
if emp_ids:
lines.append(f"T {','.join(emp_ids)}") # Target card
# Write as fixed-length flat file
outpath = outdir / (prefix + sysin_filename)
with open(outpath, "wb") as f:
for line in lines:
buf = line.encode("ascii", errors="replace")
if len(buf) < rec_length:
buf = buf.ljust(rec_length, b" ")
f.write(buf[:rec_length])
return outpath
+61 -8
View File
@@ -82,15 +82,49 @@ def _wsl_path(windows_path: str) -> str:
return f'/mnt/{drive}/{rest}'
def _find_if_body_lines(source_lines: list[str], if_lineno_1: int):
"""在源码中定位 IF 语句的 THEN/ELSE 体行范围(行号 1-indexed)。
Returns (then_lines, else_lines):
then_lines: list[int] THEN 体的行号1-indexed
else_lines: list[int] ELSE 体的行号1-indexed ELSE 则为空
"""
start = if_lineno_1 # 0-indexed, start AFTER the IF line
depth = 1
else_start_0 = None
end_if_0 = None
n = len(source_lines)
for i in range(start, n):
line = source_lines[i].upper().strip()
if re.match(r'\bIF\b', line) and not re.match(r'ELSE\s+IF', line, re.IGNORECASE):
depth += 1
if re.match(r'END-IF', line):
depth -= 1
if depth == 0:
end_if_0 = i
break
if depth == 1 and re.match(r'ELSE\b', line):
else_start_0 = i
then_start_1 = if_lineno_1 + 1
if else_start_0 is not None:
then_1 = list(range(then_start_1, else_start_0 + 1))
else_1 = list(range(else_start_0 + 2, (end_if_0 or start) + 2))
else:
then_1 = list(range(then_start_1, (end_if_0 or start) + 2))
else_1 = []
return then_1, else_1
def mark_from_gcov(decision_points: list, gcov_data: dict[int, int],
branch_tree) -> None:
branch_tree, source_text: str | None = None) -> None:
"""用 gcov 行执行计数推断决策点分支覆盖,直接修改 decision_points 的 active_branches。
推断规则简化版先覆盖主要场景
source_text 提供时预处理源码IF 分支使用体行计数精确判断 T/F
IF (条件行 L):
- 条件行 L gcov count == 0 不可到达不标记
- 条件行 L gcov count > 0 标记 T F 都覆盖
- 体行计数 > 0 对应分支覆盖T=THEN体F=ELSE体
- 无体行数据时回退count==0 跳过count>0 标记 T/F
EVALUATE:
- subject count > 0 标记所有 WHEN 为已覆盖
@@ -100,6 +134,8 @@ def mark_from_gcov(decision_points: list, gcov_data: dict[int, int],
- count > 1 循环体至少进入一次 Enter 覆盖
- Skip 总视为覆盖无论进入与否最终都会跳出
"""
source_lines = source_text.splitlines() if source_text else None
for dp in decision_points:
ln = dp.source_line
if ln <= 0 or ln not in gcov_data:
@@ -110,10 +146,27 @@ def mark_from_gcov(decision_points: list, gcov_data: dict[int, int],
continue
if dp.kind == 'IF':
if count == 0:
continue
dp.active_branches.add('T')
dp.active_branches.add('F')
# 清除静态分析的 IF 标记,用 gcov 运行时数据重新判断
dp.active_branches.discard('T')
dp.active_branches.discard('F')
if source_lines and ln <= len(source_lines):
then_lines, else_lines = _find_if_body_lines(source_lines, ln)
then_cov = any(gcov_data.get(tl, 0) > 0 for tl in then_lines)
else_cov = any(gcov_data.get(el, 0) > 0 for el in else_lines)
if then_cov:
dp.active_branches.add('T')
if else_cov:
dp.active_branches.add('F')
# 如果体行范围为空或无法判断,回退到基于 IF 行计数
if not then_lines and not else_lines:
if count > 0:
dp.active_branches.add('T')
dp.active_branches.add('F')
else:
# 无源码文本回退到原逻辑
if count > 0:
dp.active_branches.add('T')
dp.active_branches.add('F')
elif dp.kind == 'EVALUATE':
if count == 0:
+9 -2
View File
@@ -90,8 +90,15 @@ def _convert_node(node: BranchNode, parent: BrSeq):
if c.kind == "WHEN":
cond = (c.branch_names or [""])[0]
cond = cond[5:-1] if cond.startswith("WHEN(") and cond.endswith(")") else cond
# Strip trailing body text (everything after first COBOL verb)
cond = cond.split()[0] if cond.split() else cond
# Strip trailing body text at first COBOL verb
for verb in ('DISPLAY', 'MOVE', 'ADD', 'SUBTRACT', 'MULTIPLY', 'DIVIDE', 'COMPUTE',
'STRING', 'UNSTRING', 'SET', 'INSPECT', 'INITIALIZE', 'CONTINUE',
'PERFORM', 'CALL', 'EXIT', 'GOBACK', 'STOP',
'READ', 'WRITE', 'DELETE', 'REWRITE', 'ACCEPT', 'OPEN', 'CLOSE'):
idx = cond.upper().find(f' {verb} ')
if idx >= 0:
cond = cond[:idx].strip()
break
ws = BrSeq()
for wc in c.children: _convert_node(wc, ws)
if cond.upper() == "OTHER":
+21 -1
View File
@@ -221,10 +221,30 @@ def extract_branch_tree(source: str, data_fields: list = None) -> tuple[Any, lis
while len(stack) > 1 and stack[-1].kind == "WHEN":
stack.pop()
cond = m.group(1).strip().rstrip('.')
# Peek ahead for multi-line condition continuations (AND/OR)
j = i + 1
while j < len(lines):
next_raw = lines[j]
next_line = _clean_line(next_raw)
if not next_line:
j += 1
continue
if any(next_line.startswith(kw) for kw in (
'IF', 'ELSE', 'WHEN', 'OTHER', 'EVALUATE',
'END-IF', 'END-EVALUATE', 'END-PERFORM', 'END-READ', 'END-CALL',
'DISPLAY', 'MOVE', 'ADD', 'SUBTRACT', 'MULTIPLY', 'DIVIDE', 'COMPUTE',
'STRING', 'UNSTRING', 'SET', 'INSPECT', 'INITIALIZE', 'CONTINUE',
'PERFORM', 'CALL', 'EXIT', 'GOBACK', 'STOP', 'THEN',
'READ', 'WRITE', 'DELETE', 'REWRITE', 'ACCEPT', 'OPEN', 'CLOSE',
'EXEC', 'END-EXEC',
)):
break
cond += ' ' + next_line
j += 1
i = j
when_node = BranchNode("WHEN", branch_names=[f"WHEN({cond})"])
stack[-1].children.append(when_node)
stack.append(when_node)
i += 1
continue
# WHEN OTHER
+2 -2
View File
@@ -29,10 +29,10 @@ def _is_fixed_format(source: str) -> bool:
return fixed_hits >= free_hits if (fixed_hits + free_hits) > 0 else True
def preprocess(source: str) -> str:
def preprocess(source: str, extra_search_paths: list[str] = None) -> str:
# COPY 预处理:展开或移除 COPY 语句
# Lark 语法不支持 COPY(这是预处理指令),必须在解析前处理
source = resolve_copybooks(source, '.')
source = resolve_copybooks(source, '.', extra_search_paths=extra_search_paths)
# Strip EXEC ... END-EXEC blocks (CICS/SQL) before Lark parsing
source = re.sub(
+435
View File
@@ -0,0 +1,435 @@
"""SQL层:WHERE约束解析 + DB输入行生成"""
import re
import logging
import itertools
logger = logging.getLogger(__name__)
# ── String literal protection ──
def _protect_strings(text: str) -> (str, list):
"""Replace string literals with placeholders. Returns (clean_text, replacements)."""
replacements = []
def _repl(m):
idx = len(replacements)
replacements.append(m.group(0))
return f"__STR{idx}__"
cleaned = re.sub(r"'[^']*'|\"[^\"]*\"", _repl, text)
return cleaned, replacements
def _restore_strings(text: str, replacements: list) -> str:
for i, s in enumerate(replacements):
text = text.replace(f"__STR{i}__", s)
return text
# ── Bracket-aware AND splitting ──
def _split_on_AND(text: str) -> list[str]:
"""Split WHERE clause on AND, respecting parentheses."""
parts = []
current = []
depth = 0
tokens = re.split(r'(\bAND\b|\bOR\b|[()])', text, flags=re.IGNORECASE)
for token in tokens:
if not token.strip():
continue
if token == '(':
depth += 1
current.append(token)
elif token == ')':
depth -= 1
current.append(token)
elif token.upper() == 'AND' and depth == 0:
parts.append(' '.join(current).strip())
current = []
elif token.upper() == 'OR' and depth == 0:
current.append(token) # OR stays as inner condition text
else:
current.append(token)
if current:
parts.append(' '.join(current).strip())
return parts
# ── WHERE condition parsing ──
_COL_OP_PAT = re.compile(
r'(\w[\w.-]*)\s*' # column name (with optional alias prefix)
r'(=|>|<|>=|<=|<>|!=|NOT\s*=)\s*'
r'(:\w[\w-]*(?::\w[\w-]*)?|__STR\d+__|[\w\d.-]+)',
re.IGNORECASE
)
_RE_IN_CLAUSE = re.compile(
r'(\w[\w.-]*)\s+(NOT\s+)?IN\s*\((.+?)\)',
re.IGNORECASE
)
_RE_BETWEEN = re.compile(
r'(\w[\w.-]*)\s+(NOT\s+)?BETWEEN\s+(.+?)\s+AND\s+(.+)',
re.IGNORECASE
)
_RE_LIKE = re.compile(
r'(\w[\w.-]*)\s+(NOT\s+)?LIKE\s+(__STR\d+__)',
re.IGNORECASE
)
_RE_IS_NULL = re.compile(
r'(\w[\w.-]*)\s+IS\s+(NOT\s+)?NULL',
re.IGNORECASE
)
def _parse_where_condition(part: str, replacements: list) -> dict | None:
"""Parse a single WHERE condition (after AND split)."""
part = part.strip()
if not part:
return None
# IS NULL
m = _RE_IS_NULL.match(part)
if m:
col = m.group(1).upper()
neg = bool(m.group(2))
return {'col': col, 'type': 'is_null', 'neg': neg, 'op': 'IS NULL' if not neg else 'IS NOT NULL'}
# IN
m = _RE_IN_CLAUSE.match(part)
if m:
col = m.group(1).upper()
neg = bool(m.group(2))
vals_text = m.group(3)
# Parse values from IN list
vals = []
for v in re.split(r'\s*,\s*', vals_text):
v = v.strip()
if v.startswith('__STR') and v.endswith('__'):
idx = int(v[5:-2])
vals.append(replacements[idx].strip("'\""))
elif v.startswith(':'):
vals.append({'type': 'host_var', 'host_var': v[1:].upper()})
else:
vals.append(v.strip())
return {
'col': col, 'type': 'in', 'neg': neg,
'op': 'NOT IN' if neg else 'IN',
'values': vals,
}
# BETWEEN
m = _RE_BETWEEN.match(part)
if m:
col = m.group(1).upper()
neg = bool(m.group(2))
lo = m.group(3).strip()
hi = m.group(4).strip()
return {
'col': col, 'type': 'between', 'neg': neg,
'op': 'BETWEEN',
'lo': lo.strip("'\""), 'hi': hi.strip("'\""),
}
# LIKE
m = _RE_LIKE.match(part)
if m:
col = m.group(1).upper()
neg = bool(m.group(2))
pat_ph = m.group(3)
idx = int(pat_ph[5:-2])
pattern = replacements[idx].strip("'\"") if idx < len(replacements) else pat_ph
return {
'col': col, 'type': 'like', 'neg': neg,
'op': 'NOT LIKE' if neg else 'LIKE',
'pattern': pattern,
}
# col op value
m = _COL_OP_PAT.match(part)
if m:
col = m.group(1).upper()
op = m.group(2).upper().strip()
val = m.group(3).strip()
# Normalize NOT = to <>
if op == 'NOT =' or op == 'NOT=':
op = '<>'
if val.startswith(':'):
host_var = val[1:].upper()
if ':' in host_var:
host_var = host_var.split(':')[0]
return {'col': col, 'type': 'host_var', 'host_var': host_var, 'op': op, 'literal': None}
elif val.startswith('__STR') and val.endswith('__'):
idx = int(val[5:-2])
_quotes = "'\""
literal = replacements[idx].strip(_quotes) if idx < len(replacements) else val
return {'col': col, 'type': 'literal', 'host_var': None, 'op': op, 'literal': literal}
else:
return {'col': col, 'type': 'literal', 'host_var': None, 'op': op, 'literal': val}
return None
# ── Column name → COBOL field name ──
_COLUMN_MAP = {}
def guess_cobol_field(col_name: str, table: str,
declared_columns: dict,
column_map: dict = None) -> str:
"""Map SQL column name to COBOL field name.
Priority: 1. DECLARE TABLE PIC alias 2. column_map 3. naming conv 4. as-is
"""
if column_map is None:
column_map = _COLUMN_MAP
# 1. DECLARE TABLE explicit PIC mapping
if table in declared_columns:
for c in declared_columns[table]:
if c['name'] == col_name and c.get('db_type') == 'PIC':
return c.get('pic', col_name)
# 2. User map
key = f"{table}.{col_name}"
if key in column_map:
return column_map[key]
# 3. Naming convention: CUST_ID → CUST-ID
candidate = col_name.replace('_', '-')
# 4. Strip table alias prefix: A.ID → ID
if '.' in candidate:
candidate = candidate.split('.')[1]
return candidate
# ── Main constraint extraction ──
def sql_extract_constraints(where_clause: str, table: str,
host_vars: dict[str, str],
column_map: dict[str, str],
declared_columns: dict) -> list[dict]:
"""Parse WHERE clause into constraint list."""
if not where_clause:
return []
# Protect string literals
cleaned, replacements = _protect_strings(where_clause)
# Split on AND
and_parts = _split_on_AND(cleaned)
constraints = []
for part in and_parts:
part = part.strip()
if not part:
continue
cond = _parse_where_condition(part, replacements)
if cond:
# Map column to COBOL field
cobol_field = guess_cobol_field(cond['col'], table, declared_columns, column_map)
cond['cobol_field'] = cobol_field
constraints.append(cond)
else:
logger.warning(f"Unparseable WHERE condition: {_restore_strings(part, replacements)}")
return constraints
# ── DB input row generation ──
_COLUMN_DEFAULTS = {
'CHAR': lambda size: ' ' * (size or 1),
'VARCHAR': lambda size: ' ' * (size or 1),
'INTEGER': lambda _: '000000000',
'SMALLINT': lambda _: '0000',
'DECIMAL': lambda _: '000000',
'DATE': lambda _: '20260603',
'PIC': lambda _: '?',
}
def _format_db_value(col_info: dict, raw_val: str) -> str:
db_type = col_info.get('db_type', 'CHAR')
formatter = _COLUMN_DEFAULTS.get(db_type, lambda _: str(raw_val)[:10])
default = formatter(0)
if raw_val is None:
return default
if db_type in ('INTEGER', 'SMALLINT', 'DECIMAL'):
try:
return str(int(raw_val)).zfill(len(default))
except ValueError:
return default
return str(raw_val).ljust(len(default))[:len(default)]
def _make_key_unique(key_val: str, path_index: int, seen_keys: set) -> str:
unique = f"{path_index:03d}{key_val[:5]}"
while unique in seen_keys:
unique = f"{path_index:03d}{hash(key_val) % 100000:05d}"
seen_keys.add(unique)
return unique
def collect_sql_meta(assignments: dict, declared_columns: dict,
column_map: dict = None) -> list[dict]:
"""Collect SQL metadata from assignments. Returns list of SQL info dicts."""
sql_meta = []
seen = set()
for tgt, asgn_list in assignments.items():
if isinstance(asgn_list, dict):
asgn_list = [asgn_list]
for asgn in asgn_list:
atype = asgn.get('type', '')
if not atype.startswith('exec_sql_'):
continue
key = asgn.get('sql_text', '')
if key in seen:
continue
seen.add(key)
where = asgn.get('where', '')
table = asgn.get('table', '')
where_constraints = sql_extract_constraints(
where, table, {}, column_map or {}, declared_columns
)
meta = dict(asgn)
meta['where_constraints'] = where_constraints
sql_meta.append(meta)
return sql_meta
def _path_has_sql_ok(path_cons: list) -> bool:
"""Check if a path requires SQLCODE = 0 (SQL succeeded)."""
sql_ok = True # default: no SQLCODE constraint, assume success
for pc in path_cons:
if len(pc) >= 4 and pc[0] == 'SQLCODE':
if pc[1] == '<>' and pc[3]:
sql_ok = False
if pc[1] == '=' and not pc[3]:
sql_ok = False
if pc[1] == '>' and pc[3]:
sql_ok = False
break
return sql_ok
def _infer_columns_from_where(where_cons: list) -> list[dict]:
"""Infer column definitions from WHERE constraints when DECLARE TABLE is missing."""
seen = {}
for wc in where_cons:
col_name = wc.get('col', '').split('.')[-1]
if col_name and col_name not in seen:
seen[col_name] = {'name': col_name, 'db_type': 'CHAR', 'size': 10}
return list(seen.values())
def build_db_input(
branch_paths: list[tuple[list, dict]],
fields_dict: list[dict],
assignments: dict,
sql_meta: list[dict],
declared_columns: dict,
records: list[dict] = None,
) -> dict:
"""Generate DB input rows per branch path.
Returns {table: [{col: val, ...}, ...]}.
"""
if not sql_meta:
return {}
db_input = {}
seen_keys = {}
seq_counter = itertools.count(1)
# Collect all SQL meta per path
for path_idx, (path_cons, path_assign) in enumerate(branch_paths):
# Skip paths where SQL fails (SQLCODE <> 0)
if not _path_has_sql_ok(path_cons):
continue
rec = records[path_idx] if records and path_idx < len(records) else {}
for sql in sql_meta:
atype = sql.get('type', '')
table = sql['table']
where_cons = sql.get('where_constraints', [])
if table not in db_input:
db_input[table] = []
seen_keys[table] = set()
if atype == 'exec_sql_insert':
# INSERT creates rows at runtime; no initial rows needed
continue
if atype in ('exec_sql_delete', 'exec_sql_update'):
# DELETE/UPDATE needs existing rows to act on
col_infos = declared_columns.get(table, [])
if not col_infos:
col_infos = _infer_columns_from_where(where_cons)
row = {}
for ci in col_infos:
col_name = ci['name'].upper()
val = None
for wc in where_cons:
wc_col = wc.get('col', '').upper().split('.')[-1]
if wc_col != col_name:
continue
if wc['type'] == 'literal':
val = wc.get('literal', '')
break
elif wc['type'] == 'host_var':
hv = wc.get('host_var', '').upper()
val = str(rec.get(hv, ''))
break
if val is None or not val.strip():
val = str(rec.get(ci['name'], ''))
if val and val.strip():
row[ci['name']] = _format_db_value(ci, val)
if not row:
row['_path'] = str(path_idx)
db_input[table].append(row)
continue
# exec_sql_select (and any future read-only types)
row = {}
col_infos = declared_columns.get(table, [])
if not col_infos:
col_infos = _infer_columns_from_where(where_cons)
into_vars = sql.get('into_vars', [])
for iv in into_vars:
if iv not in [c['name'] for c in col_infos]:
col_infos.append({'name': iv, 'db_type': 'CHAR', 'size': 20})
for col_info in col_infos:
col_name = col_info['name']
val = None
for wc in where_cons:
if wc['type'] == 'literal' and wc.get('col', '').upper() == col_name:
val = wc.get('literal', '')
break
if wc['type'] == 'host_var':
hv = wc.get('host_var', '').upper()
for pc_field, pc_op, pc_val, pc_want in path_cons:
if pc_field == hv:
val = pc_val if pc_want else ''
break
if val is None and hv in rec:
val = str(rec[hv])
if val is not None:
row[col_name] = _format_db_value(col_info, val)
else:
row[col_name] = _format_db_value(col_info, str(next(seq_counter)))
if not row:
row['_path'] = str(path_idx)
if col_infos:
first_col = col_infos[0]['name']
if first_col in row:
row[first_col] = _make_key_unique(row[first_col], path_idx, seen_keys[table])
db_input[table].append(row)
return db_input