"""Gixsql CBL Runner — gixpp + cobc pipeline for DB COBOL programs.""" from __future__ import annotations import logging import os import re import sqlite3 import subprocess from dataclasses import dataclass, field from pathlib import Path from typing import Optional logger = logging.getLogger(__name__) def _strip_schema_qualifiers(text: str) -> str: """DB2 schema-qualified table refs (SCHEMA.TABLE) → TABLE. SQLite has no schema objects, so gixsql-generated SQL such as `FROM SALARYDB.EMP_MASTER` fails at OPEN time ('no such table: SALARYDB.EMP_MASTER', SQLCODE != 0) and the program ABENDs before its main loop. Strip the qualifier from table positions only (after FROM/INTO/UPDATE/JOIN). Column aliases (E.EMP-ID, B.DEPT-CODE) and host-var INTO (:DBV-X) are untouched. Program-agnostic: handles ANY ., incl. multi-part (DB2INST1.PAYROLL.TIMESHEET). """ def _fix(m: re.Match) -> str: kw = m.group(1) table = m.group(2).rsplit('.', 1)[-1] return f"{kw} {table}" return re.sub( r'\b(FROM|INTO|UPDATE|JOIN)\s+([\w-]+(?:\.[\w-]+)+)', _fix, text, flags=re.IGNORECASE, ) def _normalize_schema_qualifiers(text: str) -> str: """Strip schema qualifiers inside EXEC SQL blocks (before gixpp). Applied to the source BEFORE gixpp so the generated SQL string literals reference plain table names. Only EXEC SQL blocks are touched; other text (data literals, comments) is left unchanged. Mirrors `_normalize_current_timestamp`. """ def _fix_block(m: re.Match) -> str: return _strip_schema_qualifiers(m.group(0)) return re.sub(r'(?is)EXEC SQL\b.*?\bEND-EXEC', _fix_block, text) def _normalize_current_timestamp(text: str) -> str: """SQLite backend: DB2 `CURRENT TIMESTAMP` → `CURRENT_TIMESTAMP` inside EXEC SQL. Applied to the source BEFORE gixpp. gixpp wraps long SQL across continuation lines and can split the token (`CURRENT TIMES` / `TAMP`), which defeats a post-gixpp same-line regex; normalizing the source first makes the generated SQL string literals concatenate to the SQLite-valid `CURRENT_TIMESTAMP`. Only EXEC SQL blocks are touched; other text (data literals, comments) is left unchanged. """ def _fix_sql(m: re.Match) -> str: block = m.group(0) return re.sub(r'\bCURRENT\s+TIMESTAMP\b', 'CURRENT_TIMESTAMP', block, flags=re.IGNORECASE) return re.sub(r'(?is)EXEC SQL\b.*?\bEND-EXEC', _fix_sql, text) @dataclass class GixsqlBuildResult: success: bool exe_path: str = "" log: str = "" @dataclass class GixsqlRunResult: success: bool returncode: int = -1 db_path: str = "" log: str = "" @dataclass class GixsqlTableData: table_name: str = "" rows: list[dict] = field(default_factory=list) class GixsqlCobolRunner: """gixpp + cobc 管线:预处理 → 编译 → 运行 → DB读取""" def __init__(self, gixpp_path: str | Path, lib_path: str | Path, cobc_cmd: str = "cobc", compile_flags: str = ""): self.gixpp_path = Path(gixpp_path) self.lib_path = Path(lib_path) self.cobc_cmd = cobc_cmd self.compile_flags = compile_flags def _build_env(self) -> dict: env = os.environ.copy() env["LD_LIBRARY_PATH"] = str(self.lib_path) if "PATH" in env: env["PATH"] = str(self.lib_path) + ";" + env["PATH"] else: env["PATH"] = str(self.lib_path) return env def _expand_copy_replacing(self, text: str, search_dirs: list[Path]) -> str: """Expand COPY ... REPLACING statements inline (Python-side). gixpp's ESQL parser chokes on COPY with REPLACING pseudo-text (==...==). """ def _resolve_copy(m: re.Match) -> str: name = m.group(1) for d in search_dirs: for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""): cp = d / f"{name}{ext}" if cp.exists(): cpy_text = cp.read_text(encoding="utf-8-sig") replaces_text = m.group(2) if replaces_text: pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replaces_text) for old_txt, new_txt in pairs: cpy_text = cpy_text.replace(old_txt, new_txt) return cpy_text logger.warning(f" COPY {name} not found in {search_dirs}") return f" * COPY {name} NOT FOUND" text = re.sub( r'^ {6,}COPY\s+(\w+)\s+REPLACING\s+(.+?)\.$', _resolve_copy, text, flags=re.MULTILINE | re.IGNORECASE, ) return text def _expand_all_copies(self, text: str, search_dirs: list[Path]) -> str: """Expand ALL COPY statements (including COPY SQLCA) — replaces cobc -E.""" def _resolve_copy_cb(m: re.Match) -> str: name = m.group(1).strip().upper() replacing_text = m.group(2) # Try each search dir for d in search_dirs: for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""): cp = d / f"{name}{ext}" if cp.exists(): cpy_text = cp.read_text(encoding="utf-8-sig") if replacing_text: pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replacing_text) for old_txt, new_txt in pairs: cpy_text = cpy_text.replace(old_txt, new_txt) return cpy_text # If SQLCA not found in copybook dirs, provide inline definition if name == "SQLCA": return ( " 01 SQLCA.\n" " 05 SQLCAID PIC X(8).\n" " 05 SQLCABC PIC S9(9) COMP-5.\n" " 05 SQLCODE PIC S9(9) COMP-5.\n" " 05 SQLERRM.\n" " 49 SQLERRML PIC S9(4) COMP-5.\n" " 49 SQLERRMC PIC X(256).\n" " 05 SQLERRP PIC X(8).\n" " 05 SQLERRD PIC S9(9) COMP-5 OCCURS 6.\n" " 05 SQLWARN.\n" " 10 SQLWARN0 PIC X(1).\n" " 10 SQLWARN1 PIC X(1).\n" " 10 SQLWARN2 PIC X(1).\n" " 10 SQLWARN3 PIC X(1).\n" " 10 SQLWARN4 PIC X(1).\n" " 10 SQLWARN5 PIC X(1).\n" " 10 SQLWARN6 PIC X(1).\n" " 10 SQLWARN7 PIC X(1).\n" " 05 SQLEXT PIC X(8).\n" ) logger.warning(f" COPY {name} not found in {search_dirs}") return f" * COPY {name} NOT FOUND\n" # Expand COPY name. and COPY name REPLACING ... . text = re.sub( r'^ {6,}COPY\s+(\w+(?:-\w+)*)\s*(REPLACING\s+.+?)?\.$', _resolve_copy_cb, text, flags=re.MULTILINE | re.IGNORECASE, ) return text def _normalize_source(self, src_path: Path) -> tuple[Path, Path]: """Pre-process COBOL source for gixpp: expand COPY, strip comments, fix indentation. Returns (pre_path, norm_path) where both point to the same gixpp-ready source. """ text = src_path.read_text(encoding="utf-8-sig") # 0. Strip ALL comment lines BEFORE any EXEC SQL transforms (regex may match # 'EXEC SQL' inside Japanese comments, pulling comment text into SQL strings) text = re.sub(r'^[ \t]{0,10}\*.*\n?', '', text, flags=re.MULTILINE) # 1. Replace EXEC SQL INCLUDE SQLCA → COPY SQLCA text = re.sub(r'EXEC SQL INCLUDE SQLCA END-EXEC\.', ' COPY SQLCA.', text, flags=re.IGNORECASE) # Transform EXEC SQL CONNECT TO 'literal' → use WS variables (gixpp requires :variable not literal) def _transform_connect(m): inner = m.group(1) m_lit = re.search(r"CONNECT\s+TO\s+'([^']*)'", inner, re.IGNORECASE) if not m_lit: return m.group(0) conn_var = "WS-GIX-CONN" usr_var = "WS-GIX-USR" new_inner = re.sub( r"CONNECT\s+TO\s+'[^']*'", f"CONNECT TO :{conn_var} USER :{usr_var}", inner, flags=re.IGNORECASE | re.DOTALL ) # Orchestrator copies DB to CWD/kin. return (f"MOVE 'sqlite://kin' TO {conn_var}\n" f" MOVE 'gix' TO {usr_var}\n" f" EXEC SQL\n" f" {new_inner.strip()}\n" f" END-EXEC.") text = re.sub(r'(?is)EXEC SQL(.*?CONNECT\s+TO.*?)END-EXEC\.', _transform_connect, text) # Add WS-GIX-* after COPY SQLCA text = re.sub( r"^(\s*COPY SQLCA\.)", r"\1\n" r" 01 WS-GIX-VARS.\n" r" 03 WS-GIX-CONN PIC X(256).\n" r" 03 WS-GIX-USR PIC X(16).", text, flags=re.MULTILINE | re.IGNORECASE ) # 2. Expand ALL COPY statements in Python (replaces cobc -E) if hasattr(self, '_copybook_dirs') and self._copybook_dirs: text = self._expand_all_copies(text, self._copybook_dirs) # 4. Collapse multiple spaces between keywords lines = [] for line in text.splitlines(keepends=True): line = re.sub(r'(\b\w+)\s{2,}(\b\w+\b)', lambda m: f'{m.group(1)} {m.group(2)}', line) lines.append(line) text = ''.join(lines) # 5. Normalize DIVISION/SECTION headers to start at column 8 (7-space indent) # gixpp fixed-format scanner requires headers in Area A (columns 8-11). def _fix_header(m): return ' ' + m.group(1).lstrip() text = re.sub( r'^ {6,12}((?:IDENTIFICATION|ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION\.' r'|(?:FILE|WORKING-STORAGE|LINKAGE|CONFIGURATION|INPUT-OUTPUT)\s+SECTION\.' r'|(?:FILE-CONTROL|I-O-CONTROL)\.' r'|SOURCE-COMPUTER\.|OBJECT-COMPUTER\.|SPECIAL-NAMES\.' r')', _fix_header, text, flags=re.MULTILINE | re.IGNORECASE ) pre_path = src_path.parent / f"{src_path.stem}_pre.cbl" pre_path.write_text(text, encoding="utf-8") # 6. No cobc -E — all COPY expansions done in Python above. # Use the pre_path directly as the norm_path (gixpp input). # 6. Fix SQL clause ordering in EXEC SQL blocks: # gixpp expects SELECT ... INTO ... FROM ... (INTO before FROM), # but some programs have SELECT ... FROM ... INTO ... def _fix_sql_from_into(m): block = m.group(0) # Only touch SELECT ... FROM ... INTO (not INSERT/DELETE) if re.match(r'\s*EXEC SQL\s+SELECT\b', block, re.MULTILINE | re.IGNORECASE): # Swap FROM line and INTO line within SELECT blocks block = re.sub( r'^(\s+)(FROM\b.*)\n(\s+)(INTO\b.*)$', r'\1\4\n\1\2', block, flags=re.MULTILINE | re.IGNORECASE ) return block text = re.sub(r'(\s*EXEC SQL\n.*?\n\s*END-EXEC\.)', _fix_sql_from_into, text, flags=re.DOTALL | re.IGNORECASE) # 7. Add missing 01-level variables used in PROCEDURE DIVISION but # not defined in DATA DIVISION (source program defects). missing_vars = { 'KIN06CLD': ' 01 WS-COL-IDX PIC 9(002).\n', } pid = src_path.stem.upper() if pid in missing_vars and not re.search(r'\b01\s+WS-COL-IDX\b', text): text = re.sub( r'^(\s*)(PROCEDURE\s+DIVISION)', lambda m: m.group(1) + missing_vars[pid] + '\n' + m.group(1) + m.group(2), text, count=1, flags=re.MULTILINE | re.IGNORECASE ) # 8. SQLite backend: DB2 `CURRENT TIMESTAMP` → `CURRENT_TIMESTAMP` inside # EXEC SQL. Must run before gixpp (gixpp can split the token across # continuation lines, defeating the post-gixpp same-line patch). text = _normalize_current_timestamp(text) # 9. SQLite backend: DB2 schema-qualified table refs (SCHEMA.TABLE) → # TABLE inside EXEC SQL. gixsql would otherwise emit # `FROM SALARYDB.EMP_MASTER`, which SQLite cannot resolve. text = _normalize_schema_qualifiers(text) norm_path = src_path.parent / f"{src_path.stem}_norm.cbl" norm_path.write_text(text, encoding="utf-8") # Save a copy in runtime for diagnosis try: debug_dir = Path(__file__).parent.parent / "runtime" / src_path.stem / "pre_src" debug_dir.mkdir(parents=True, exist_ok=True) (debug_dir / f"{src_path.stem}_norm.cbl").write_text(text, encoding="utf-8") (debug_dir / f"{src_path.stem}_pre.cbl").write_text( (src_path.parent / f"{src_path.stem}_pre.cbl").read_text(encoding="utf-8"), encoding="utf-8" ) except Exception: pass return pre_path, norm_path def preprocess(self, src_path: str | Path, out_dir: str | Path, copybook_dirs: list[str | Path] | None = None) -> str: """Step 1a: gixpp プリプロセス""" out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) src_path = Path(src_path) self._copybook_dirs = [Path(d) for d in copybook_dirs] if copybook_dirs else None _, norm_path = self._normalize_source(src_path) out_path = out_dir / f"{src_path.stem}_pp.cbl" cmd = [str(self.gixpp_path), "-i", str(norm_path), "-o", str(out_path), "-e"] if self._copybook_dirs: for d in self._copybook_dirs: cmd += ["-I", str(d)] logger.info(f" gixpp: {' '.join(cmd)}") r = subprocess.run(cmd, capture_output=True, timeout=30, env=self._build_env()) if r.returncode != 0: err = r.stderr.decode("utf-8", "replace")[:500] raise RuntimeError(f"gixpp failed (rc={r.returncode}): {err}") return str(out_path) def _patch_sql_identifiers(self, pp_path: Path) -> None: """Translate DB2 hyphenated identifiers to underscores inside GIXSQL SQL strings. gixsql emits the SQL text verbatim from the COBOL source (e.g. "INSERT INTO EMP-MASTER (EMP-ID, ...)"), but SQLite cannot parse bare hyphenated identifiers. Convert '-' -> '_' only inside the SQL string literals (VALUE "..." lines and & "..." continuation lines), leaving COBOL-level identifiers (cursor names, host variables, work items) untouched. """ text = pp_path.read_text(encoding="utf-8") def _fix_line(m: re.Match) -> str: return m.group(1) + m.group(2).replace('-', '_') + m.group(3) # SQL start lines: GIXSQL ... VALUE "SQL TEXT" text = re.sub( r'^(GIXSQL.*?VALUE\s+")([^"]*)(")', _fix_line, text, flags=re.MULTILINE ) # SQL continuation lines: GIXSQL & "SQL TEXT" text = re.sub( r'^(GIXSQL\s*&\s*")([^"]*)(")', _fix_line, text, flags=re.MULTILINE ) # SQLite accepts CURRENT_TIMESTAMP (no space); gixsql emits CURRENT TIMESTAMP text = re.sub(r'\bCURRENT\s+TIMESTAMP\b', 'CURRENT_TIMESTAMP', text, flags=re.IGNORECASE) pp_path.write_text(text, encoding="utf-8") def _patch_sqlcode_normalize(self, pp_path: Path) -> None: """Inject SQLITE-constraint → -803 mapping after each SQL execution. gixsql+SQLite reports duplicate-key (PRIMARY KEY / UNIQUE) violations as SQLCODE=-1555 / -2067 / -19, whereas the COBOL programs follow DB2 semantics (SQLCODE = -803). Without this mapping, `IF SQLCODE = -803` branches are unreachable under gixsql+SQLite. The injected code is program-agnostic: it only rewrites the constraint-violation codes. """ text = pp_path.read_text(encoding="utf-8") mapping = ( ' IF SQLCODE = -1555\n' ' MOVE -803 TO SQLCODE\n' ' END-IF\n' ' IF SQLCODE = -2067\n' ' MOVE -803 TO SQLCODE\n' ' END-IF\n' ' IF SQLCODE = -19\n' ' MOVE -803 TO SQLCODE\n' ' END-IF\n' ) text = re.sub( r'(GIXSQLEndSQL\s*\n?.*?END-CALL\.?)\s*\n(\s*)(IF SQLCODE\s+)', lambda m: m.group(1) + '\n' + m.group(2) + mapping + m.group(2) + m.group(3), text, flags=re.IGNORECASE | re.DOTALL ) pp_path.write_text(text, encoding="utf-8") def _patch_sqlcode_override(self, pp_path: Path) -> None: """Inject MOVE 0 TO SQLCODE after each GIXSQLEndSQL call. Workaround for gixsql DLL bug: GIXSQLExecParams/GIXSQLExec fail with 'Can't find a connection' even though the connection is valid. Setting SQLCODE=0 lets the program flow through success paths for coverage measurement. The DB state is not validated in coverage runs. """ text = pp_path.read_text(encoding="utf-8") text = re.sub( r'(GIXSQLEndSQL\s*\n?.*?END-CALL\.?)\s*\n(\s*)(IF SQLCODE\s+(?:NOT\s+)?=\s+0)', r'\1\n\2 MOVE 0 TO SQLCODE\n\2\3', text, flags=re.IGNORECASE | re.DOTALL ) pp_path.write_text(text, encoding="utf-8") def compile(self, pp_path: str | Path, exe_path: str | Path, copybook_dirs: list[str | Path] | None = None, extra_srcs: list[str | Path] | None = None) -> GixsqlBuildResult: """Step 1b: cobc 编译(gixsql 链接)""" pp_path = Path(pp_path) exe_path = Path(exe_path) exe_dir = exe_path.parent exe_dir.mkdir(parents=True, exist_ok=True) # SQL is now executed correctly (hyphen->underscore identifiers), so the # SQLCODE=0 override workaround is no longer needed. It masked real SQL # errors and caused infinite FETCH loops (SQLCODE forced to 0 prevents # "PERFORM UNTIL SQLCODE NOT = 0" from terminating at EOF=100). # self._patch_sqlcode_override(pp_path) # Translate DB2 hyphenated identifiers -> underscores in SQL strings self._patch_sql_identifiers(pp_path) # Map gixsql SQLite constraint codes (-1555/-2067/-19) -> DB2 -803 so # `IF SQLCODE = -803` branches are reachable under gixsql+SQLite. self._patch_sqlcode_normalize(pp_path) # Functions that the preprocessed COBOL actually CALLs gixsql_k = [ "-K", "GIXSQLStartSQL", "-K", "GIXSQLSetSQLParams", "-K", "GIXSQLSetResultParams", "-K", "GIXSQLExecParams", "-K", "GIXSQLExec", "-K", "GIXSQLExecSelectIntoOne", "-K", "GIXSQLEndSQL", "-K", "GIXSQLConnect", "-K", "GIXSQLCursorDeclare", "-K", "GIXSQLCursorDeclareParams", "-K", "GIXSQLCursorFetchOne", "-K", "GIXSQLCursorOpen", "-K", "GIXSQLCursorClose", ] cmd = [ self.cobc_cmd, "-x", "-L", str(self.lib_path.resolve()), *gixsql_k, "-l", "gixsql", ] if copybook_dirs: for d in copybook_dirs: cmd += ["-I", str(d)] if self.compile_flags: cmd += self.compile_flags.split() cmd += ["-o", str(exe_path)] cmd.append(str(pp_path)) if extra_srcs: for s in extra_srcs: cmd.append(str(s)) logger.info(f" cobc: {' '.join(cmd)}") try: r = subprocess.run(cmd, capture_output=True, timeout=60, env=self._build_env(), cwd=str(exe_dir)) log = (r.stdout.decode("utf-8", "replace") + "\n" + r.stderr.decode("utf-8", "replace")) if r.returncode != 0: return GixsqlBuildResult(False, log=log[:1000]) # .gcno files are generated in exe_dir (compile CWD). No need to copy. return GixsqlBuildResult(True, exe_path=str(exe_path), log=log[:500]) except subprocess.TimeoutExpired: return GixsqlBuildResult(False, log="Compile timeout (60s)") def run(self, exe_path: str | Path, work_dir: str | Path, db_path: str | Path, input_dir: str | Path | None = None, timeout: int = 30, cobol_lib_path: str | Path | None = None, env_overrides: dict[str, str] | None = None, command_args: list[str] | None = None) -> GixsqlRunResult: """Step 3: COBOL DB プログラム実行""" exe_path = Path(exe_path) work_dir = Path(work_dir) db_path = Path(db_path) work_dir.mkdir(parents=True, exist_ok=True) exe_dir = exe_path.parent exe_dir.mkdir(parents=True, exist_ok=True) lib_path = Path(self.lib_path) # Deploy correct DLLs to exe_dir so loader finds them first # Search lib_path first, then fall back to gixpp bin dir for runtime DLLs gixpp_dir = self.gixpp_path.parent dll_names = [ "libgixsql.dll", "libgixsql-sqlite.dll", "libgcc_s_dw2-1.dll", "libstdc++-6.dll", "libwinpthread-1.dll", "libiconv-2.dll", "libintl-8.dll", "zlib1.dll", ] for name in dll_names: src = lib_path / name if not src.exists(): src = gixpp_dir / name if src.exists(): dst = exe_dir / name if not dst.exists() or dst.stat().st_size != src.stat().st_size: dst.write_bytes(src.read_bytes()) # libfmt.dll search: TEMP fallback → lib_path → gixpp bin → x86/gcc subdirectory fmt_src = Path(os.environ.get("TEMP", "")) / "zan_dll" / "libfmt.dll" if not fmt_src.exists(): fmt_src = lib_path / "libfmt.dll" if not fmt_src.exists(): fmt_src = gixpp_dir / "libfmt.dll" if not fmt_src.exists(): # gixsql binary package variant: /lib/x86/gcc/libfmt.dll fmt_src = gixpp_dir.parent / "lib" / "x86" / "gcc" / "libfmt.dll" if fmt_src.exists(): (exe_dir / "libfmt.dll").write_bytes(fmt_src.read_bytes()) env = self._build_env() env["GIXSQL_DB_PATH"] = str(db_path) if cobol_lib_path: env["COB_LIBRARY_PATH"] = str(cobol_lib_path) if env_overrides: env.update(env_overrides) if input_dir: idir = Path(input_dir) if idir.exists(): for f in idir.iterdir(): if f.is_file(): dst = work_dir / f.name dst.write_bytes(f.read_bytes()) cmd = [str(exe_path)] if command_args: cmd.extend(command_args) logger.info(f" run: {' '.join(cmd)} (cwd={work_dir}, db={db_path})") try: r = subprocess.run(cmd, capture_output=True, timeout=timeout, cwd=str(work_dir), env=env) log = (r.stdout.decode("utf-8", "replace") + "\n" + r.stderr.decode("utf-8", "replace")) ok = r.returncode == 0 or r.returncode == 1 return GixsqlRunResult(ok, returncode=r.returncode, db_path=str(db_path), log=log[:1000]) except subprocess.TimeoutExpired: return GixsqlRunResult(False, log="Run timeout") def read_db_tables(self, db_path: str | Path, table_names: list[str]) -> list[GixsqlTableData]: """Step 4: SQLite DB から全テーブル読み取り""" db_path = Path(db_path) if not db_path.exists(): logger.warning(f" DB not found: {db_path}") return [] conn = sqlite3.connect(str(db_path)) conn.row_factory = sqlite3.Row results = [] for tname in table_names: try: rows = conn.execute(f"SELECT * FROM [{tname}]").fetchall() results.append(GixsqlTableData( table_name=tname, rows=[dict(r) for r in rows], )) except sqlite3.OperationalError as e: logger.warning(f" Table {tname} not found: {e}") conn.close() return results