"""S18: ALL benchmark programs — full E2E: parse → generate → flatfile → compile → run → verify Run: cd D:/cobol-java/cobol-java-v3 && python test-data/s18_all_benchmark_e2e.py """ import sys, os, subprocess, re, json sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) P=0;F=0 def ck(v,m=""): global P,F; (P:=P+1) if v else (F:=F+1, print(f" FAIL {m}")) def sec(n): print(f"\n{'='*60}\n{n}\n{'='*60}") ROOT = "D:/cobol-java/cobol-test-programs/" COPYBOOKS = os.path.join(ROOT, "common", "copybooks") COBC = "cobc" from cobol_testgen import extract_structure, generate_data from cobol_testgen.read import preprocess, resolve_copybooks, extract_data_division, extract_procedure_division, parse_data_division from cobol_testgen.flatfile import analyze_fd_layout, write_all_files def find_main_cbl(directory): """Return the 'main' .cbl file in a benchmark directory.""" cbls = [f for f in os.listdir(directory) if f.endswith('.cbl') and not f.startswith('.')] if not cbls: return None wrappers = [f for f in cbls if re.match(r'main-\d{2}-', f, re.IGNORECASE)] if wrappers: return max(wrappers, key=lambda f: os.path.getsize(os.path.join(directory, f))) return max(cbls, key=lambda f: os.path.getsize(os.path.join(directory, f))) def detect_sort_using(source: str) -> list[str]: """Detect SORT ... USING filename GIVING filename patterns. Returns list of filenames used as INPUT in SORT statements. """ using_files = [] for m in re.finditer( r'SORT\s+\w[\w-]*\s+.*?USING\s+(\w[\w-]*).*?GIVING\s+(\w[\w-]*)', source, re.IGNORECASE | re.DOTALL ): using_files.append(m.group(1).upper()) return using_files def guess_input_files(source: str) -> set: """Detect all files that are likely INPUT but missed by OPEN parsing.""" names_in = set() # SORT USING for m in re.finditer( r'USING\s+(\w[\w-]*)', source, re.IGNORECASE ): names_in.add(m.group(1).upper()) # READ statements (imply INPUT) for m in re.finditer( r'READ\s+(\w[\w-]*)', source, re.IGNORECASE ): names_in.add(m.group(1).upper()) return names_in # ───────────────────────────────────────────────── # Scan all programs # ───────────────────────────────────────────────── sec("SCAN: Finding all benchmark programs") programs = [] for d in sorted(os.listdir(ROOT)): dp = os.path.join(ROOT, d) if not os.path.isdir(dp) or d in ('common', 'docs', 'cross-cutting'): continue fname = find_main_cbl(dp) if not fname: continue fpath = os.path.join(dp, fname) try: src = open(fpath, encoding='utf-8').read() except Exception: print(f" {d:<30} UNREADABLE"); continue has_cics = bool(re.search(r'EXEC\s+(CICS|SQL)\b', src, re.IGNORECASE)) sort_using = detect_sort_using(src) programs.append({ 'dir': d, 'file': fname, 'path': fpath, 'source': src, 'cics': has_cics, 'sort_using': sort_using, }) print(f" {d:<30} {fname:<30} cics={int(has_cics)} sort={len(sort_using)}") print(f"\nTotal programs: {len(programs)}") # ───────────────────────────────────────────────── # Phase 1: Parse + Generate + Flat files # ───────────────────────────────────────────────── sec("PHASE 1: Parse → Generate → Flat files") parse_ok = 0; gen_ok = 0; flat_ok = 0 for prog in programs: d, fname, fpath = prog['dir'], prog['file'], prog['path'] src = prog['source'] dp = os.path.join(ROOT, d) # Clean old output files (skip if already compiled .exe — subprocess handles it) for f in os.listdir(dp): fp = os.path.join(dp, f) if not os.path.isfile(fp): continue if f.endswith(('.gcno', '.gcda', '.gcov')) or (f.endswith('.exe') and f.startswith('test-')): try: os.remove(fp) except OSError: pass try: st = extract_structure(src) pp = resolve_copybooks(src, dp, extra_search_paths=[COPYBOOKS]) pp = preprocess(pp) recs = generate_data(pp, st) gen_ok += 1 parse_ok += 1 # Build FD layouts layouts = analyze_fd_layout(pp) # For SORT programs or programs where OPEN is not used, # mark the USING/READ files as INPUT guess_input = guess_input_files(src) for lname in list(layouts.keys()): lname_upper = lname.upper() # Find matching FD key fd_key = None for lk in list(layouts.keys()): if lk.upper() == lname_upper: fd_key = lk; break fname_upper = re.sub(r'\..*$', '', lk).upper() if fname_upper and fname_upper in guess_input: fd_key = lk; break assign_to = layouts[lk].get('fd_name', '').upper() if assign_to in guess_input: fd_key = lk; break if fd_key and layouts[fd_key]['direction'] != 'INPUT': # Check if this file matches any guessed input name lk_up = layouts[fd_key]['fd_name'].upper() assign_up = os.path.splitext(lk)[0].upper() if lk_up in guess_input or assign_up in guess_input: layouts[fd_key]['direction'] = 'INPUT' # Redo: just directly detect USING files and mark using_in_source = detect_sort_using(src) for lname, layout in list(layouts.items()): fd_name = layout['fd_name'].upper() if fd_name in using_in_source: layout['direction'] = 'INPUT' # Also match by filename stem fstem = re.sub(r'\..*$', '', lname).upper() if fstem in using_in_source: layout['direction'] = 'INPUT' # Write flat files written = write_all_files(recs, pp, dp) if layouts else [] flat_ok += len(written) # Check what files we actually wrote prog['recs'] = len(recs) prog['branches'] = st.get('total_branches', 0) prog['layouts'] = len(layouts) prog['written'] = [(fn, os.path.getsize(os.path.join(dp, fn)) if os.path.exists(os.path.join(dp, fn)) else 0) for fn, _, _ in written] prog['pp'] = pp prog['st'] = st print(f" {d:<30} branches={st.get('total_branches',0):3d} recs={len(recs):3d} layouts={len(layouts)} flats={len(written)}") except Exception as e: print(f" {d:<30} FAIL: {str(e)[:80]}") prog['status'] = 'fail' prog['error'] = str(e)[:100] ck(parse_ok == len(programs), f"Parse OK: {parse_ok}/{len(programs)}") ck(gen_ok >= len(programs) - 3, f"Generate OK: {gen_ok}/{len(programs)}") # ───────────────────────────────────────────────── # Phase 2: Compile # ───────────────────────────────────────────────── sec("PHASE 2: Compile with GnuCOBOL") compile_ok = 0; compile_fail = 0; compile_skip = 0 for prog in programs: d, fname, fpath = prog['dir'], prog['file'], prog['path'] dp = os.path.join(ROOT, d) exe = os.path.join(dp, fname.replace('.cbl', '.exe')) if prog.get('cics'): compile_skip += 1 prog['compile'] = 'skip(cics)' print(f" {d:<30} SKIP (CICS)") continue if prog.get('status') == 'fail': compile_skip += 1 prog['compile'] = 'skip(fail)' continue cmd = [COBC, '-x', '-Wall', fpath, '-o', exe, '-I', COPYBOOKS, '-I', dp] try: p = subprocess.run(cmd, capture_output=True, timeout=45, cwd=dp) out = p.stdout.decode('utf-8', errors='replace') if p.stdout else '' err = p.stderr.decode('utf-8', errors='replace') if p.stderr else '' if p.returncode == 0: compile_ok += 1 prog['compile'] = 'ok' prog['exe'] = exe prog['exe_size'] = os.path.getsize(exe) if os.path.exists(exe) else 0 print(f" {d:<30} OK {prog['exe_size']:>6}B") else: compile_fail += 1 prog['compile'] = 'fail' prog['compile_err'] = (err or out or '')[:150] print(f" {d:<30} FAIL: {prog['compile_err'][:80]}") except subprocess.TimeoutExpired: compile_fail += 1 prog['compile'] = 'timeout' print(f" {d:<30} TIMEOUT") print(f"\nCompile: {compile_ok} OK / {compile_fail} FAIL / {compile_skip} skip") ck(compile_fail < 10, f"Compile: {compile_fail} failures") # ───────────────────────────────────────────────── # Phase 3: Run # ───────────────────────────────────────────────── sec("PHASE 3: Run") run_ok = 0; run_fail = 0; run_timeout = 0; run_skip = 0 for prog in programs: if prog.get('compile') != 'ok' or 'exe' not in prog: run_skip += 1 prog['run'] = 'skip'; continue d, exe = prog['dir'], prog['exe'] dp = os.path.join(ROOT, d) try: p = subprocess.run([exe], capture_output=True, timeout=10, cwd=dp, shell=True) if p.returncode == 0: run_ok += 1 prog['run'] = 'ok' # Collect output files (dat, txt, tmp) out_files = [] for fn in sorted(os.listdir(dp)): fp = os.path.join(dp, fn) if os.path.isfile(fp) and os.path.getsize(fp) > 0: ext = os.path.splitext(fn)[1].lower() if ext in ('.dat', '.txt', '.tmp', '.out', '.rpt'): if not (fname := prog.get('file', '')).replace('.cbl','') in fn: out_files.append((fn, os.path.getsize(fp))) prog['out_files'] = out_files print(f" {d:<30} OK ({len(out_files)} out files)") else: run_fail += 1 err = p.stderr.decode('utf-8', errors='replace')[:100] if p.stderr else '' prog['run'] = f'fail({p.returncode})' prog['run_stderr'] = err print(f" {d:<30} FAIL rc={p.returncode} {err[:60]}") except subprocess.TimeoutExpired: run_timeout += 1 prog['run'] = 'timeout' print(f" {d:<30} TIMEOUT") print(f"\nRun: {run_ok} OK / {run_fail} FAIL / {run_timeout} timeout / {run_skip} skip") ck(run_fail + run_timeout < compile_ok * 0.5, f"Run failures: {run_fail} fail + {run_timeout} timeout") # ───────────────────────────────────────────────── # Summary # ───────────────────────────────────────────────── sec("FINAL SUMMARY") total = len(programs) print(f"{'Program':<28} {'Br':>3} {'Recs':>4} {'Compile':<10} {'Run':<10} {'OutFiles':>8} {'FlatFiles':>9}") print(f"{'─'*78}") for prog in programs: d = prog['dir'] br = prog.get('branches', 0) recs = prog.get('recs', 0) comp = prog.get('compile', '-') run_st = prog.get('run', '-') outf = len(prog.get('out_files', [])) flat_written = len(prog.get('written', [])) if prog.get('status') == 'fail': print(f" {d:<28} FAIL {prog.get('error','')[:40]}") else: print(f" {d:<28} {br:>3} {recs:>4} {comp:<10} {run_st:<10} {outf:>3} files {flat_written:>3} flats") # Aggregate counts print(f"\n{'─'*78}") print(f"{'TOTAL':<28} {sum(p.get('branches',0) for p in programs):>3} ") print(f"\n Total programs: {total}") print(f" Parse OK: {parse_ok}") print(f" Generate OK: {gen_ok}") print(f" Compile OK: {compile_ok}") print(f" Run OK: {run_ok}") print(f" Run FAIL: {run_fail}") print(f" Run TIMEOUT: {run_timeout}") print(f" Run SKIP: {run_skip}") print(f" Flat files: {flat_ok}") # Failures list failures = [] for prog in programs: if prog.get('compile') == 'fail': failures.append(f" {prog['dir']}: COMPILE {prog.get('compile_err','')[:60]}") if prog.get('run', '').startswith('fail'): failures.append(f" {prog['dir']}: RUN {prog.get('run_stderr','')[:40]}") if failures: print(f"\nFailures ({len(failures)}):") for f in failures: print(f" {f}") print(f"\n{'='*55}") print(f"S18: {P} PASS / {F} FAIL") print(f"{'='*55}") if F > 0: sys.exit(1)