307 lines
13 KiB
Python
307 lines
13 KiB
Python
"""S18: ALL benchmark programs — full E2E: parse → generate → flatfile → compile → run → verify
|
|
|
|
Run: cd <project-root> && python test-data/s18_all_benchmark_e2e.py
|
|
"""
|
|
import sys, os, subprocess, re, json
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
|
P=0;F=0
|
|
def ck(v,m=""): global P,F; (P:=P+1) if v else (F:=F+1, print(f" FAIL {m}"))
|
|
def sec(n): print(f"\n{'='*60}\n{n}\n{'='*60}")
|
|
|
|
from _paths import BENCH_ROOT as ROOT
|
|
COPYBOOKS = os.path.join(ROOT, "common", "copybooks")
|
|
COBC = "cobc"
|
|
|
|
from cobol_testgen import extract_structure, generate_data
|
|
from cobol_testgen.read import preprocess, resolve_copybooks, extract_data_division, extract_procedure_division, parse_data_division
|
|
from cobol_testgen.flatfile import analyze_fd_layout, write_all_files
|
|
|
|
def find_main_cbl(directory):
|
|
"""Return the 'main' .cbl file in a benchmark directory."""
|
|
cbls = [f for f in os.listdir(directory) if f.endswith('.cbl') and not f.startswith('.')]
|
|
if not cbls: return None
|
|
wrappers = [f for f in cbls if re.match(r'main-\d{2}-', f, re.IGNORECASE)]
|
|
if wrappers:
|
|
return max(wrappers, key=lambda f: os.path.getsize(os.path.join(directory, f)))
|
|
return max(cbls, key=lambda f: os.path.getsize(os.path.join(directory, f)))
|
|
|
|
def detect_sort_using(source: str) -> list[str]:
|
|
"""Detect SORT ... USING filename GIVING filename patterns.
|
|
Returns list of filenames used as INPUT in SORT statements.
|
|
"""
|
|
using_files = []
|
|
for m in re.finditer(
|
|
r'SORT\s+\w[\w-]*\s+.*?USING\s+(\w[\w-]*).*?GIVING\s+(\w[\w-]*)',
|
|
source, re.IGNORECASE | re.DOTALL
|
|
):
|
|
using_files.append(m.group(1).upper())
|
|
return using_files
|
|
|
|
def guess_input_files(source: str) -> set:
|
|
"""Detect all files that are likely INPUT but missed by OPEN parsing."""
|
|
names_in = set()
|
|
# SORT USING
|
|
for m in re.finditer(
|
|
r'USING\s+(\w[\w-]*)', source, re.IGNORECASE
|
|
):
|
|
names_in.add(m.group(1).upper())
|
|
# READ statements (imply INPUT)
|
|
for m in re.finditer(
|
|
r'READ\s+(\w[\w-]*)', source, re.IGNORECASE
|
|
):
|
|
names_in.add(m.group(1).upper())
|
|
return names_in
|
|
|
|
# ─────────────────────────────────────────────────
|
|
# Scan all programs
|
|
# ─────────────────────────────────────────────────
|
|
sec("SCAN: Finding all benchmark programs")
|
|
|
|
programs = []
|
|
for d in sorted(os.listdir(ROOT)):
|
|
dp = os.path.join(ROOT, d)
|
|
if not os.path.isdir(dp) or d in ('common', 'docs', 'cross-cutting'):
|
|
continue
|
|
fname = find_main_cbl(dp)
|
|
if not fname: continue
|
|
fpath = os.path.join(dp, fname)
|
|
try:
|
|
src = open(fpath, encoding='utf-8').read()
|
|
except Exception:
|
|
print(f" {d:<30} UNREADABLE"); continue
|
|
has_cics = bool(re.search(r'EXEC\s+(CICS|SQL)\b', src, re.IGNORECASE))
|
|
sort_using = detect_sort_using(src)
|
|
programs.append({
|
|
'dir': d, 'file': fname, 'path': fpath,
|
|
'source': src, 'cics': has_cics, 'sort_using': sort_using,
|
|
})
|
|
print(f" {d:<30} {fname:<30} cics={int(has_cics)} sort={len(sort_using)}")
|
|
|
|
print(f"\nTotal programs: {len(programs)}")
|
|
|
|
# ─────────────────────────────────────────────────
|
|
# Phase 1: Parse + Generate + Flat files
|
|
# ─────────────────────────────────────────────────
|
|
sec("PHASE 1: Parse → Generate → Flat files")
|
|
|
|
parse_ok = 0; gen_ok = 0; flat_ok = 0
|
|
for prog in programs:
|
|
d, fname, fpath = prog['dir'], prog['file'], prog['path']
|
|
src = prog['source']
|
|
dp = os.path.join(ROOT, d)
|
|
|
|
# Clean old output files (skip if already compiled .exe — subprocess handles it)
|
|
for f in os.listdir(dp):
|
|
fp = os.path.join(dp, f)
|
|
if not os.path.isfile(fp):
|
|
continue
|
|
if f.endswith(('.gcno', '.gcda', '.gcov')) or (f.endswith('.exe') and f.startswith('test-')):
|
|
try: os.remove(fp)
|
|
except OSError: pass
|
|
|
|
try:
|
|
st = extract_structure(src)
|
|
pp = resolve_copybooks(src, dp, extra_search_paths=[COPYBOOKS])
|
|
pp = preprocess(pp)
|
|
recs = generate_data(pp, st)
|
|
gen_ok += 1
|
|
parse_ok += 1
|
|
|
|
# Build FD layouts
|
|
layouts = analyze_fd_layout(pp)
|
|
|
|
# For SORT programs or programs where OPEN is not used,
|
|
# mark the USING/READ files as INPUT
|
|
guess_input = guess_input_files(src)
|
|
for lname in list(layouts.keys()):
|
|
lname_upper = lname.upper()
|
|
# Find matching FD key
|
|
fd_key = None
|
|
for lk in list(layouts.keys()):
|
|
if lk.upper() == lname_upper:
|
|
fd_key = lk; break
|
|
fname_upper = re.sub(r'\..*$', '', lk).upper()
|
|
if fname_upper and fname_upper in guess_input:
|
|
fd_key = lk; break
|
|
assign_to = layouts[lk].get('fd_name', '').upper()
|
|
if assign_to in guess_input:
|
|
fd_key = lk; break
|
|
|
|
if fd_key and layouts[fd_key]['direction'] != 'INPUT':
|
|
# Check if this file matches any guessed input name
|
|
lk_up = layouts[fd_key]['fd_name'].upper()
|
|
assign_up = os.path.splitext(lk)[0].upper()
|
|
if lk_up in guess_input or assign_up in guess_input:
|
|
layouts[fd_key]['direction'] = 'INPUT'
|
|
|
|
# Redo: just directly detect USING files and mark
|
|
using_in_source = detect_sort_using(src)
|
|
for lname, layout in list(layouts.items()):
|
|
fd_name = layout['fd_name'].upper()
|
|
if fd_name in using_in_source:
|
|
layout['direction'] = 'INPUT'
|
|
# Also match by filename stem
|
|
fstem = re.sub(r'\..*$', '', lname).upper()
|
|
if fstem in using_in_source:
|
|
layout['direction'] = 'INPUT'
|
|
|
|
# Write flat files
|
|
written = write_all_files(recs, pp, dp) if layouts else []
|
|
flat_ok += len(written)
|
|
|
|
# Check what files we actually wrote
|
|
prog['recs'] = len(recs)
|
|
prog['branches'] = st.get('total_branches', 0)
|
|
prog['layouts'] = len(layouts)
|
|
prog['written'] = [(fn, os.path.getsize(os.path.join(dp, fn)) if os.path.exists(os.path.join(dp, fn)) else 0)
|
|
for fn, _, _ in written]
|
|
prog['pp'] = pp
|
|
prog['st'] = st
|
|
print(f" {d:<30} branches={st.get('total_branches',0):3d} recs={len(recs):3d} layouts={len(layouts)} flats={len(written)}")
|
|
except Exception as e:
|
|
print(f" {d:<30} FAIL: {str(e)[:80]}")
|
|
prog['status'] = 'fail'
|
|
prog['error'] = str(e)[:100]
|
|
|
|
ck(parse_ok == len(programs), f"Parse OK: {parse_ok}/{len(programs)}")
|
|
ck(gen_ok >= len(programs) - 3, f"Generate OK: {gen_ok}/{len(programs)}")
|
|
|
|
# ─────────────────────────────────────────────────
|
|
# Phase 2: Compile
|
|
# ─────────────────────────────────────────────────
|
|
sec("PHASE 2: Compile with GnuCOBOL")
|
|
compile_ok = 0; compile_fail = 0; compile_skip = 0
|
|
for prog in programs:
|
|
d, fname, fpath = prog['dir'], prog['file'], prog['path']
|
|
dp = os.path.join(ROOT, d)
|
|
exe = os.path.join(dp, fname.replace('.cbl', '.exe'))
|
|
|
|
if prog.get('cics'):
|
|
compile_skip += 1
|
|
prog['compile'] = 'skip(cics)'
|
|
print(f" {d:<30} SKIP (CICS)")
|
|
continue
|
|
if prog.get('status') == 'fail':
|
|
compile_skip += 1
|
|
prog['compile'] = 'skip(fail)'
|
|
continue
|
|
|
|
cmd = [COBC, '-x', '-Wall', fpath, '-o', exe, '-I', COPYBOOKS, '-I', dp]
|
|
try:
|
|
p = subprocess.run(cmd, capture_output=True, timeout=45, cwd=dp)
|
|
out = p.stdout.decode('utf-8', errors='replace') if p.stdout else ''
|
|
err = p.stderr.decode('utf-8', errors='replace') if p.stderr else ''
|
|
if p.returncode == 0:
|
|
compile_ok += 1
|
|
prog['compile'] = 'ok'
|
|
prog['exe'] = exe
|
|
prog['exe_size'] = os.path.getsize(exe) if os.path.exists(exe) else 0
|
|
print(f" {d:<30} OK {prog['exe_size']:>6}B")
|
|
else:
|
|
compile_fail += 1
|
|
prog['compile'] = 'fail'
|
|
prog['compile_err'] = (err or out or '')[:150]
|
|
print(f" {d:<30} FAIL: {prog['compile_err'][:80]}")
|
|
except subprocess.TimeoutExpired:
|
|
compile_fail += 1
|
|
prog['compile'] = 'timeout'
|
|
print(f" {d:<30} TIMEOUT")
|
|
|
|
print(f"\nCompile: {compile_ok} OK / {compile_fail} FAIL / {compile_skip} skip")
|
|
ck(compile_fail < 10, f"Compile: {compile_fail} failures")
|
|
|
|
# ─────────────────────────────────────────────────
|
|
# Phase 3: Run
|
|
# ─────────────────────────────────────────────────
|
|
sec("PHASE 3: Run")
|
|
run_ok = 0; run_fail = 0; run_timeout = 0; run_skip = 0
|
|
|
|
for prog in programs:
|
|
if prog.get('compile') != 'ok' or 'exe' not in prog:
|
|
run_skip += 1
|
|
prog['run'] = 'skip'; continue
|
|
|
|
d, exe = prog['dir'], prog['exe']
|
|
dp = os.path.join(ROOT, d)
|
|
try:
|
|
p = subprocess.run([exe], capture_output=True, timeout=10, cwd=dp, shell=True)
|
|
if p.returncode == 0:
|
|
run_ok += 1
|
|
prog['run'] = 'ok'
|
|
# Collect output files (dat, txt, tmp)
|
|
out_files = []
|
|
for fn in sorted(os.listdir(dp)):
|
|
fp = os.path.join(dp, fn)
|
|
if os.path.isfile(fp) and os.path.getsize(fp) > 0:
|
|
ext = os.path.splitext(fn)[1].lower()
|
|
if ext in ('.dat', '.txt', '.tmp', '.out', '.rpt'):
|
|
if not (fname := prog.get('file', '')).replace('.cbl','') in fn:
|
|
out_files.append((fn, os.path.getsize(fp)))
|
|
prog['out_files'] = out_files
|
|
print(f" {d:<30} OK ({len(out_files)} out files)")
|
|
else:
|
|
run_fail += 1
|
|
err = p.stderr.decode('utf-8', errors='replace')[:100] if p.stderr else ''
|
|
prog['run'] = f'fail({p.returncode})'
|
|
prog['run_stderr'] = err
|
|
print(f" {d:<30} FAIL rc={p.returncode} {err[:60]}")
|
|
except subprocess.TimeoutExpired:
|
|
run_timeout += 1
|
|
prog['run'] = 'timeout'
|
|
print(f" {d:<30} TIMEOUT")
|
|
|
|
print(f"\nRun: {run_ok} OK / {run_fail} FAIL / {run_timeout} timeout / {run_skip} skip")
|
|
ck(run_fail + run_timeout < compile_ok * 0.5, f"Run failures: {run_fail} fail + {run_timeout} timeout")
|
|
|
|
# ─────────────────────────────────────────────────
|
|
# Summary
|
|
# ─────────────────────────────────────────────────
|
|
sec("FINAL SUMMARY")
|
|
|
|
total = len(programs)
|
|
print(f"{'Program':<28} {'Br':>3} {'Recs':>4} {'Compile':<10} {'Run':<10} {'OutFiles':>8} {'FlatFiles':>9}")
|
|
print(f"{'─'*78}")
|
|
for prog in programs:
|
|
d = prog['dir']
|
|
br = prog.get('branches', 0)
|
|
recs = prog.get('recs', 0)
|
|
comp = prog.get('compile', '-')
|
|
run_st = prog.get('run', '-')
|
|
outf = len(prog.get('out_files', []))
|
|
flat_written = len(prog.get('written', []))
|
|
if prog.get('status') == 'fail':
|
|
print(f" {d:<28} FAIL {prog.get('error','')[:40]}")
|
|
else:
|
|
print(f" {d:<28} {br:>3} {recs:>4} {comp:<10} {run_st:<10} {outf:>3} files {flat_written:>3} flats")
|
|
|
|
# Aggregate counts
|
|
print(f"\n{'─'*78}")
|
|
print(f"{'TOTAL':<28} {sum(p.get('branches',0) for p in programs):>3} ")
|
|
print(f"\n Total programs: {total}")
|
|
print(f" Parse OK: {parse_ok}")
|
|
print(f" Generate OK: {gen_ok}")
|
|
print(f" Compile OK: {compile_ok}")
|
|
print(f" Run OK: {run_ok}")
|
|
print(f" Run FAIL: {run_fail}")
|
|
print(f" Run TIMEOUT: {run_timeout}")
|
|
print(f" Run SKIP: {run_skip}")
|
|
print(f" Flat files: {flat_ok}")
|
|
|
|
# Failures list
|
|
failures = []
|
|
for prog in programs:
|
|
if prog.get('compile') == 'fail':
|
|
failures.append(f" {prog['dir']}: COMPILE {prog.get('compile_err','')[:60]}")
|
|
if prog.get('run', '').startswith('fail'):
|
|
failures.append(f" {prog['dir']}: RUN {prog.get('run_stderr','')[:40]}")
|
|
|
|
if failures:
|
|
print(f"\nFailures ({len(failures)}):")
|
|
for f in failures:
|
|
print(f" {f}")
|
|
|
|
print(f"\n{'='*55}")
|
|
print(f"S18: {P} PASS / {F} FAIL")
|
|
print(f"{'='*55}")
|
|
if F > 0: sys.exit(1)
|