cobol-java-v3/test-data/s18_all_benchmark_e2e.py

"""S18: ALL benchmark programs — full E2E: parse → generate → flatfile → compile → run → verify

Run: cd D:/cobol-java/cobol-java-v3 && python test-data/s18_all_benchmark_e2e.py
"""
import sys, os, subprocess, re, json
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
P=0;F=0
def ck(v,m=""): global P,F; (P:=P+1) if v else (F:=F+1, print(f"  FAIL {m}"))
def sec(n): print(f"\n{'='*60}\n{n}\n{'='*60}")

ROOT = "D:/cobol-java/cobol-test-programs/"
COPYBOOKS = os.path.join(ROOT, "common", "copybooks")
COBC = "cobc"

from cobol_testgen import extract_structure, generate_data
from cobol_testgen.read import preprocess, resolve_copybooks, extract_data_division, extract_procedure_division, parse_data_division
from cobol_testgen.flatfile import analyze_fd_layout, write_all_files

def find_main_cbl(directory):
    """Return the 'main' .cbl file in a benchmark directory."""
    cbls = [f for f in os.listdir(directory) if f.endswith('.cbl') and not f.startswith('.')]
    if not cbls: return None
    wrappers = [f for f in cbls if re.match(r'main-\d{2}-', f, re.IGNORECASE)]
    if wrappers:
        return max(wrappers, key=lambda f: os.path.getsize(os.path.join(directory, f)))
    return max(cbls, key=lambda f: os.path.getsize(os.path.join(directory, f)))

def detect_sort_using(source: str) -> list[str]:
    """Detect SORT ... USING filename GIVING filename patterns.
    Returns list of filenames used as INPUT in SORT statements.
    """
    using_files = []
    for m in re.finditer(
        r'SORT\s+\w[\w-]*\s+.*?USING\s+(\w[\w-]*).*?GIVING\s+(\w[\w-]*)',
        source, re.IGNORECASE | re.DOTALL
    ):
        using_files.append(m.group(1).upper())
    return using_files

def guess_input_files(source: str) -> set:
    """Detect all files that are likely INPUT but missed by OPEN parsing."""
    names_in = set()
    # SORT USING
    for m in re.finditer(
        r'USING\s+(\w[\w-]*)', source, re.IGNORECASE
    ):
        names_in.add(m.group(1).upper())
    # READ statements (imply INPUT)
    for m in re.finditer(
        r'READ\s+(\w[\w-]*)', source, re.IGNORECASE
    ):
        names_in.add(m.group(1).upper())
    return names_in

# ─────────────────────────────────────────────────
# Scan all programs
# ─────────────────────────────────────────────────
sec("SCAN: Finding all benchmark programs")

programs = []
for d in sorted(os.listdir(ROOT)):
    dp = os.path.join(ROOT, d)
    if not os.path.isdir(dp) or d in ('common', 'docs', 'cross-cutting'):
        continue
    fname = find_main_cbl(dp)
    if not fname: continue
    fpath = os.path.join(dp, fname)
    try:
        src = open(fpath, encoding='utf-8').read()
    except Exception:
        print(f"  {d:<30} UNREADABLE"); continue
    has_cics = bool(re.search(r'EXEC\s+(CICS|SQL)\b', src, re.IGNORECASE))
    sort_using = detect_sort_using(src)
    programs.append({
        'dir': d, 'file': fname, 'path': fpath,
        'source': src, 'cics': has_cics, 'sort_using': sort_using,
    })
    print(f"  {d:<30} {fname:<30} cics={int(has_cics)} sort={len(sort_using)}")

print(f"\nTotal programs: {len(programs)}")

# ─────────────────────────────────────────────────
# Phase 1: Parse + Generate + Flat files
# ─────────────────────────────────────────────────
sec("PHASE 1: Parse → Generate → Flat files")

parse_ok = 0; gen_ok = 0; flat_ok = 0
for prog in programs:
    d, fname, fpath = prog['dir'], prog['file'], prog['path']
    src = prog['source']
    dp = os.path.join(ROOT, d)

    # Clean old output files (skip if already compiled .exe — subprocess handles it)
    for f in os.listdir(dp):
        fp = os.path.join(dp, f)
        if not os.path.isfile(fp):
            continue
        if f.endswith(('.gcno', '.gcda', '.gcov')) or (f.endswith('.exe') and f.startswith('test-')):
            try: os.remove(fp)
            except OSError: pass

    try:
        st = extract_structure(src)
        pp = resolve_copybooks(src, dp, extra_search_paths=[COPYBOOKS])
        pp = preprocess(pp)
        recs = generate_data(pp, st)
        gen_ok += 1
        parse_ok += 1

        # Build FD layouts
        layouts = analyze_fd_layout(pp)

        # For SORT programs or programs where OPEN is not used,
        # mark the USING/READ files as INPUT
        guess_input = guess_input_files(src)
        for lname in list(layouts.keys()):
            lname_upper = lname.upper()
            # Find matching FD key
            fd_key = None
            for lk in list(layouts.keys()):
                if lk.upper() == lname_upper:
                    fd_key = lk; break
                fname_upper = re.sub(r'\..*$', '', lk).upper()
                if fname_upper and fname_upper in guess_input:
                    fd_key = lk; break
                assign_to = layouts[lk].get('fd_name', '').upper()
                if assign_to in guess_input:
                    fd_key = lk; break

            if fd_key and layouts[fd_key]['direction'] != 'INPUT':
                # Check if this file matches any guessed input name
                lk_up = layouts[fd_key]['fd_name'].upper()
                assign_up = os.path.splitext(lk)[0].upper()
                if lk_up in guess_input or assign_up in guess_input:
                    layouts[fd_key]['direction'] = 'INPUT'

        # Redo: just directly detect USING files and mark
        using_in_source = detect_sort_using(src)
        for lname, layout in list(layouts.items()):
            fd_name = layout['fd_name'].upper()
            if fd_name in using_in_source:
                layout['direction'] = 'INPUT'
            # Also match by filename stem
            fstem = re.sub(r'\..*$', '', lname).upper()
            if fstem in using_in_source:
                layout['direction'] = 'INPUT'

        # Write flat files
        written = write_all_files(recs, pp, dp) if layouts else []
        flat_ok += len(written)

        # Check what files we actually wrote
        prog['recs'] = len(recs)
        prog['branches'] = st.get('total_branches', 0)
        prog['layouts'] = len(layouts)
        prog['written'] = [(fn, os.path.getsize(os.path.join(dp, fn)) if os.path.exists(os.path.join(dp, fn)) else 0)
                          for fn, _, _ in written]
        prog['pp'] = pp
        prog['st'] = st
        print(f"  {d:<30} branches={st.get('total_branches',0):3d} recs={len(recs):3d} layouts={len(layouts)} flats={len(written)}")
    except Exception as e:
        print(f"  {d:<30} FAIL: {str(e)[:80]}")
        prog['status'] = 'fail'
        prog['error'] = str(e)[:100]

ck(parse_ok == len(programs), f"Parse OK: {parse_ok}/{len(programs)}")
ck(gen_ok >= len(programs) - 3, f"Generate OK: {gen_ok}/{len(programs)}")

# ─────────────────────────────────────────────────
# Phase 2: Compile
# ─────────────────────────────────────────────────
sec("PHASE 2: Compile with GnuCOBOL")
compile_ok = 0; compile_fail = 0; compile_skip = 0
for prog in programs:
    d, fname, fpath = prog['dir'], prog['file'], prog['path']
    dp = os.path.join(ROOT, d)
    exe = os.path.join(dp, fname.replace('.cbl', '.exe'))

    if prog.get('cics'):
        compile_skip += 1
        prog['compile'] = 'skip(cics)'
        print(f"  {d:<30} SKIP (CICS)")
        continue
    if prog.get('status') == 'fail':
        compile_skip += 1
        prog['compile'] = 'skip(fail)'
        continue

    cmd = [COBC, '-x', '-Wall', fpath, '-o', exe, '-I', COPYBOOKS, '-I', dp]
    try:
        p = subprocess.run(cmd, capture_output=True, timeout=45, cwd=dp)
        out = p.stdout.decode('utf-8', errors='replace') if p.stdout else ''
        err = p.stderr.decode('utf-8', errors='replace') if p.stderr else ''
        if p.returncode == 0:
            compile_ok += 1
            prog['compile'] = 'ok'
            prog['exe'] = exe
            prog['exe_size'] = os.path.getsize(exe) if os.path.exists(exe) else 0
            print(f"  {d:<30} OK  {prog['exe_size']:>6}B")
        else:
            compile_fail += 1
            prog['compile'] = 'fail'
            prog['compile_err'] = (err or out or '')[:150]
            print(f"  {d:<30} FAIL: {prog['compile_err'][:80]}")
    except subprocess.TimeoutExpired:
        compile_fail += 1
        prog['compile'] = 'timeout'
        print(f"  {d:<30} TIMEOUT")

print(f"\nCompile: {compile_ok} OK / {compile_fail} FAIL / {compile_skip} skip")
ck(compile_fail < 10, f"Compile: {compile_fail} failures")

# ─────────────────────────────────────────────────
# Phase 3: Run
# ─────────────────────────────────────────────────
sec("PHASE 3: Run")
run_ok = 0; run_fail = 0; run_timeout = 0; run_skip = 0

for prog in programs:
    if prog.get('compile') != 'ok' or 'exe' not in prog:
        run_skip += 1
        prog['run'] = 'skip'; continue

    d, exe = prog['dir'], prog['exe']
    dp = os.path.join(ROOT, d)
    try:
        p = subprocess.run([exe], capture_output=True, timeout=10, cwd=dp, shell=True)
        if p.returncode == 0:
            run_ok += 1
            prog['run'] = 'ok'
            # Collect output files (dat, txt, tmp)
            out_files = []
            for fn in sorted(os.listdir(dp)):
                fp = os.path.join(dp, fn)
                if os.path.isfile(fp) and os.path.getsize(fp) > 0:
                    ext = os.path.splitext(fn)[1].lower()
                    if ext in ('.dat', '.txt', '.tmp', '.out', '.rpt'):
                        if not (fname := prog.get('file', '')).replace('.cbl','') in fn:
                            out_files.append((fn, os.path.getsize(fp)))
            prog['out_files'] = out_files
            print(f"  {d:<30} OK  ({len(out_files)} out files)")
        else:
            run_fail += 1
            err = p.stderr.decode('utf-8', errors='replace')[:100] if p.stderr else ''
            prog['run'] = f'fail({p.returncode})'
            prog['run_stderr'] = err
            print(f"  {d:<30} FAIL rc={p.returncode} {err[:60]}")
    except subprocess.TimeoutExpired:
        run_timeout += 1
        prog['run'] = 'timeout'
        print(f"  {d:<30} TIMEOUT")

print(f"\nRun: {run_ok} OK / {run_fail} FAIL / {run_timeout} timeout / {run_skip} skip")
ck(run_fail + run_timeout < compile_ok * 0.5, f"Run failures: {run_fail} fail + {run_timeout} timeout")

# ─────────────────────────────────────────────────
# Summary
# ─────────────────────────────────────────────────
sec("FINAL SUMMARY")

total = len(programs)
print(f"{'Program':<28} {'Br':>3} {'Recs':>4} {'Compile':<10} {'Run':<10} {'OutFiles':>8} {'FlatFiles':>9}")
print(f"{'─'*78}")
for prog in programs:
    d = prog['dir']
    br = prog.get('branches', 0)
    recs = prog.get('recs', 0)
    comp = prog.get('compile', '-')
    run_st = prog.get('run', '-')
    outf = len(prog.get('out_files', []))
    flat_written = len(prog.get('written', []))
    if prog.get('status') == 'fail':
        print(f"  {d:<28} FAIL {prog.get('error','')[:40]}")
    else:
        print(f"  {d:<28} {br:>3} {recs:>4} {comp:<10} {run_st:<10} {outf:>3} files  {flat_written:>3} flats")

# Aggregate counts
print(f"\n{'─'*78}")
print(f"{'TOTAL':<28} {sum(p.get('branches',0) for p in programs):>3}        ")
print(f"\n  Total programs:  {total}")
print(f"  Parse OK:        {parse_ok}")
print(f"  Generate OK:     {gen_ok}")
print(f"  Compile OK:      {compile_ok}")
print(f"  Run OK:          {run_ok}")
print(f"  Run FAIL:        {run_fail}")
print(f"  Run TIMEOUT:     {run_timeout}")
print(f"  Run SKIP:        {run_skip}")
print(f"  Flat files:      {flat_ok}")

# Failures list
failures = []
for prog in programs:
    if prog.get('compile') == 'fail':
        failures.append(f"  {prog['dir']}: COMPILE {prog.get('compile_err','')[:60]}")
    if prog.get('run', '').startswith('fail'):
        failures.append(f"  {prog['dir']}: RUN {prog.get('run_stderr','')[:40]}")

if failures:
    print(f"\nFailures ({len(failures)}):")
    for f in failures:
        print(f"  {f}")

print(f"\n{'='*55}")
print(f"S18: {P} PASS / {F} FAIL")
print(f"{'='*55}")
if F > 0: sys.exit(1)