cobol-java-v3/test-data/s16_benchmark_e2e.py

"""S16: External benchmark E2E — focused on parse → generate → compile"""
import sys, os, subprocess, re
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
P=0;F=0
def ck(v,m=""): global P,F; (P:=P+1) if v else (F:=F+1, print(f"  FAIL {m}"))
def sec(n): print(f"\n{'='*60}\n{n}\n{'='*60}")

ROOT = "D:/cobol-java/cobol-test-programs/"
COPYBOOKS = os.path.join(ROOT, "common", "copybooks")
COBC = "cobc"

from cobol_testgen import extract_structure, generate_data
from cobol_testgen.read import preprocess, resolve_copybooks
from cobol_testgen.flatfile import analyze_fd_layout, write_all_files

def find_main(directory):
    cbls = [f for f in os.listdir(directory) if f.endswith('.cbl') and not f.startswith('.')]
    wrappers = [f for f in cbls if re.match(r'main-\d{2}-', f, re.IGNORECASE)]
    if wrappers:
        best = max(wrappers, key=lambda f: os.path.getsize(os.path.join(directory, f)))
        return best
    return max(cbls, key=lambda f: os.path.getsize(os.path.join(directory, f))) if cbls else None

progs = []; all_results = {}
for d in sorted(os.listdir(ROOT)):
    dp = os.path.join(ROOT, d)
    if os.path.isdir(dp) and d not in ('common','docs','cross-cutting') and (fname := find_main(dp)):
        progs.append((d, fname, os.path.join(dp, fname)))
print(f"Found {len(progs)} programs")

# ── PHASE 1: Parse + Generate + Flatfiles ──
sec("PHASE 1: Parse → Generate → Flat files")
parse_ok=0; gen_ok=0; flat_written=0
for dirname, fname, fpath in progs:
    dp = os.path.join(ROOT, dirname)
    try:
        src = open(fpath, encoding='utf-8').read()
        st = extract_structure(src)
        pp_path = resolve_copybooks(src, dp, extra_search_paths=[COPYBOOKS])
        pp = preprocess(pp_path)
        recs = generate_data(pp, st)
        layouts = analyze_fd_layout(pp)
        flats = write_all_files(recs, pp, dp) if layouts else []
        parse_ok += 1
        gen_ok += 1
        flat_written += len(flats)
        all_results[dirname] = {"recs": len(recs), "fds": len(layouts), "flats": len(flats)}
        print(f"  {dirname:30s} {len(recs):3d} recs  {len(layouts)} FDs  {len(flats)} files")
    except Exception as e:
        all_results[dirname] = {"status": "fail", "error": str(e)[:80]}
        print(f"  {dirname:30s} FAIL: {str(e)[:60]}")

ck(parse_ok == len(progs), f"Parse: {parse_ok}/{len(progs)}")
ck(gen_ok >= len(progs) - 3, f"Generate: {gen_ok}/{len(progs)}")
print(f"\n  Flat files written: {flat_written} total")

# ── PHASE 2: Compile ──
sec("PHASE 2: Compile with GnuCOBOL")
compile_ok=0; compile_fail=0; skipped=[]
for dirname, fname, fpath in progs:
    dp = os.path.join(ROOT, dirname)
    exe = os.path.join(dp, fname.replace('.cbl', '.exe'))
    if dirname in ('14-online-cics',):
        skipped.append(dirname); continue
    cmd = [COBC, '-x', '-Wall', fpath, '-o', exe, '-I', COPYBOOKS, '-I', dp]
    try:
        p = subprocess.run(cmd, capture_output=True, timeout=45, cwd=dp)
        out = p.stdout.decode('utf-8', errors='replace') if p.stdout else ''
        err = p.stderr.decode('utf-8', errors='replace') if p.stderr else ''
        if p.returncode == 0:
            compile_ok += 1; all_results[dirname]["compile"] = "ok"
            all_results[dirname]["exe_size"] = os.path.getsize(exe) if os.path.exists(exe) else 0
        else:
            compile_fail += 1; all_results[dirname]["compile"] = "fail"
            all_results[dirname]["compile_err"] = (err or out or "")[:120]
    except subprocess.TimeoutExpired:
        compile_fail += 1; all_results[dirname]["compile"] = "timeout"
    print(f"  {dirname:30s} {all_results[dirname].get('compile','N/A'):>5}  {all_results[dirname].get('exe_size',0):>6}B")

print(f"\nCompile: {compile_ok} OK / {compile_fail} FAIL / {len(skipped)} skipped")
ck(compile_fail < 10, f"Compile: {compile_fail} failures")

# ── PHASE 3: Run ──
sec("PHASE 3: Run (compiled programs)")
run_ok=0; run_fail=0; run_timeout=0
for dirname, fname, _ in progs:
    dp = os.path.join(ROOT, dirname)
    exe = os.path.join(dp, fname.replace('.cbl', '.exe'))
    if dirname in ('14-online-cics',) or not os.path.exists(exe):
        continue
    try:
        p = subprocess.run([exe], capture_output=True, timeout=10, cwd=dp, shell=True)
        if p.returncode == 0:
            run_ok += 1; all_results[dirname]["run"] = "ok"
            out_files = [fn for fn in os.listdir(dp) if fn.endswith('.dat')
                         and os.path.getsize(os.path.join(dp, fn)) > 0
                         and not any(x in fn.lower() for x in ['file-in'])]
            all_results[dirname]["out_files"] = out_files
        else:
            run_fail += 1; all_results[dirname]["run"] = f"fail({p.returncode})"
    except subprocess.TimeoutExpired:
        run_timeout += 1; all_results[dirname]["run"] = "timeout"

print(f"  Run: {run_ok} OK / {run_fail} FAIL / {run_timeout} timeout")
ck(run_fail + run_timeout < compile_ok, f"Run failures: {run_fail} + {run_timeout} timeout")

# ── Summary ──
sec("SUMMARY")
print(f"Programs:           {len(progs)}")
print(f"Parse OK:           {parse_ok}")
print(f"Generate OK:        {gen_ok}")
print(f"Compile OK:         {compile_ok}")
print(f"Compile FAIL:       {compile_fail}")
print(f"Run OK:             {run_ok}")
print(f"Run FAIL:           {run_fail}")
print(f"Run TIMEOUT:        {run_timeout}")
print()
for dirname, r in all_results.items():
    status = r.get("status", "")
    if status == "fail":
        print(f"  {dirname:<28} FAIL: {r.get('error','')[:50]}")
        continue
    recs = r.get("recs", 0)
    comp = r.get("compile", "-")
    run_st = r.get("run", "-")
    outs = len(r.get("out_files", []))
    flats = r.get("flats", 0)
    print(f"  {dirname:<28} {recs:3d} rec  C={comp:<5} R={run_st:<8} {flats}fl/{outs}out")

print(f"\nS16: {P} PASS / {F} FAIL")
if F > 0: sys.exit(1)