873 lines
38 KiB
Python
873 lines
38 KiB
Python
"""Gixsql CBL Runner — gixpp + cobc pipeline for DB COBOL programs."""
|
|
|
|
from __future__ import annotations
|
|
import logging
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _strip_schema_qualifiers(text: str) -> str:
|
|
"""DB2 schema-qualified table refs (SCHEMA.TABLE) → TABLE.
|
|
|
|
SQLite has no schema objects, so gixsql-generated SQL such as
|
|
`FROM SALARYDB.EMP_MASTER` fails at OPEN time ('no such table:
|
|
SALARYDB.EMP_MASTER', SQLCODE != 0) and the program ABENDs before its
|
|
main loop. Strip the qualifier from table positions only
|
|
(after FROM/INTO/UPDATE/JOIN). Column aliases (E.EMP-ID, B.DEPT-CODE)
|
|
and host-var INTO (:DBV-X) are untouched. Program-agnostic: handles
|
|
ANY <schema>.<table>, incl. multi-part (DB2INST1.PAYROLL.TIMESHEET).
|
|
"""
|
|
def _fix(m: re.Match) -> str:
|
|
kw = m.group(1)
|
|
table = m.group(2).rsplit('.', 1)[-1]
|
|
return f"{kw} {table}"
|
|
return re.sub(
|
|
r'\b(FROM|INTO|UPDATE|JOIN)\s+([\w-]+(?:\.[\w-]+)+)',
|
|
_fix, text, flags=re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _normalize_schema_qualifiers(text: str) -> str:
|
|
"""Strip schema qualifiers inside EXEC SQL blocks (before gixpp).
|
|
|
|
Applied to the source BEFORE gixpp so the generated SQL string literals
|
|
reference plain table names. Only EXEC SQL blocks are touched; other
|
|
text (data literals, comments) is left unchanged. Mirrors
|
|
`_normalize_current_timestamp`.
|
|
"""
|
|
def _fix_block(m: re.Match) -> str:
|
|
return _strip_schema_qualifiers(m.group(0))
|
|
return re.sub(r'(?is)EXEC SQL\b.*?\bEND-EXEC', _fix_block, text)
|
|
|
|
|
|
def _normalize_current_timestamp(text: str) -> str:
|
|
"""SQLite backend: DB2 `CURRENT TIMESTAMP` → `CURRENT_TIMESTAMP` inside EXEC SQL.
|
|
|
|
Applied to the source BEFORE gixpp. gixpp wraps long SQL across continuation
|
|
lines and can split the token (`CURRENT TIMES` / `TAMP`), which defeats a
|
|
post-gixpp same-line regex; normalizing the source first makes the generated
|
|
SQL string literals concatenate to the SQLite-valid `CURRENT_TIMESTAMP`.
|
|
Only EXEC SQL blocks are touched; other text (data literals, comments) is
|
|
left unchanged.
|
|
"""
|
|
def _fix_sql(m: re.Match) -> str:
|
|
block = m.group(0)
|
|
return re.sub(r'\bCURRENT\s+TIMESTAMP\b', 'CURRENT_TIMESTAMP',
|
|
block, flags=re.IGNORECASE)
|
|
return re.sub(r'(?is)EXEC SQL\b.*?\bEND-EXEC', _fix_sql, text)
|
|
|
|
|
|
@dataclass
|
|
class GixsqlBuildResult:
|
|
success: bool
|
|
exe_path: str = ""
|
|
log: str = ""
|
|
|
|
|
|
@dataclass
|
|
class GixsqlRunResult:
|
|
success: bool
|
|
returncode: int = -1
|
|
db_path: str = ""
|
|
log: str = ""
|
|
|
|
|
|
@dataclass
|
|
class GixsqlTableData:
|
|
table_name: str = ""
|
|
rows: list[dict] = field(default_factory=list)
|
|
|
|
|
|
class GixsqlCobolRunner:
|
|
"""gixpp + cobc 管线：预处理 → 编译 → 运行 → DB读取"""
|
|
|
|
def __init__(self, gixpp_path: str | Path, lib_path: str | Path,
|
|
cobc_cmd: str = "cobc",
|
|
compile_flags: str = ""):
|
|
self.gixpp_path = Path(gixpp_path)
|
|
self.lib_path = Path(lib_path)
|
|
self.cobc_cmd = cobc_cmd
|
|
self.compile_flags = compile_flags
|
|
|
|
def _build_env(self) -> dict:
|
|
env = os.environ.copy()
|
|
env["LD_LIBRARY_PATH"] = str(self.lib_path)
|
|
if "PATH" in env:
|
|
env["PATH"] = str(self.lib_path) + ";" + env["PATH"]
|
|
else:
|
|
env["PATH"] = str(self.lib_path)
|
|
# gixsql SQLite driver: without this, autocommit=OFF + COMMIT WORK never
|
|
# persists (whole transaction rolls back on disconnect). Value must be the
|
|
# literal "ON" (libgixsql only recognises ON/OFF).
|
|
env["GIXSQL_AUTOCOMMIT"] = "ON"
|
|
return env
|
|
|
|
def _expand_copy_replacing(self, text: str, search_dirs: list[Path]) -> str:
|
|
"""Expand COPY ... REPLACING statements inline (Python-side).
|
|
gixpp's ESQL parser chokes on COPY with REPLACING pseudo-text (==...==).
|
|
"""
|
|
def _resolve_copy(m: re.Match) -> str:
|
|
name = m.group(1)
|
|
for d in search_dirs:
|
|
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
|
cp = d / f"{name}{ext}"
|
|
if cp.exists():
|
|
cpy_text = cp.read_text(encoding="utf-8-sig")
|
|
replaces_text = m.group(2)
|
|
if replaces_text:
|
|
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replaces_text)
|
|
for old_txt, new_txt in pairs:
|
|
cpy_text = cpy_text.replace(old_txt, new_txt)
|
|
return cpy_text
|
|
logger.warning(f" COPY {name} not found in {search_dirs}")
|
|
return f" * COPY {name} NOT FOUND"
|
|
|
|
text = re.sub(
|
|
r'^ {6,}COPY\s+(\w+)\s+REPLACING\s+(.+?)\.$',
|
|
_resolve_copy,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE,
|
|
)
|
|
return text
|
|
|
|
def _expand_all_copies(self, text: str, search_dirs: list[Path]) -> str:
|
|
"""Expand ALL COPY statements (including COPY SQLCA) — replaces cobc -E."""
|
|
def _resolve_copy_cb(m: re.Match) -> str:
|
|
name = m.group(1).strip().upper()
|
|
replacing_text = m.group(2)
|
|
# Try each search dir
|
|
for d in search_dirs:
|
|
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
|
cp = d / f"{name}{ext}"
|
|
if cp.exists():
|
|
cpy_text = cp.read_text(encoding="utf-8-sig")
|
|
if replacing_text:
|
|
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replacing_text)
|
|
for old_txt, new_txt in pairs:
|
|
cpy_text = cpy_text.replace(old_txt, new_txt)
|
|
return cpy_text
|
|
# If SQLCA not found in copybook dirs, provide inline definition
|
|
if name == "SQLCA":
|
|
return (
|
|
" 01 SQLCA.\n"
|
|
" 05 SQLCAID PIC X(8).\n"
|
|
" 05 SQLCABC PIC S9(9) COMP-5.\n"
|
|
" 05 SQLCODE PIC S9(9) COMP-5.\n"
|
|
" 05 SQLERRM.\n"
|
|
" 49 SQLERRML PIC S9(4) COMP-5.\n"
|
|
" 49 SQLERRMC PIC X(256).\n"
|
|
" 05 SQLERRP PIC X(8).\n"
|
|
" 05 SQLERRD PIC S9(9) COMP-5 OCCURS 6.\n"
|
|
" 05 SQLWARN.\n"
|
|
" 10 SQLWARN0 PIC X(1).\n"
|
|
" 10 SQLWARN1 PIC X(1).\n"
|
|
" 10 SQLWARN2 PIC X(1).\n"
|
|
" 10 SQLWARN3 PIC X(1).\n"
|
|
" 10 SQLWARN4 PIC X(1).\n"
|
|
" 10 SQLWARN5 PIC X(1).\n"
|
|
" 10 SQLWARN6 PIC X(1).\n"
|
|
" 10 SQLWARN7 PIC X(1).\n"
|
|
" 05 SQLEXT PIC X(8).\n"
|
|
)
|
|
logger.warning(f" COPY {name} not found in {search_dirs}")
|
|
return f" * COPY {name} NOT FOUND\n"
|
|
|
|
# Expand COPY name. and COPY name REPLACING ... .
|
|
text = re.sub(
|
|
r'^ {6,}COPY\s+(\w+(?:-\w+)*)\s*(REPLACING\s+.+?)?\.$',
|
|
_resolve_copy_cb,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE,
|
|
)
|
|
return text
|
|
|
|
def _normalize_source(self, src_path: Path) -> tuple[Path, Path]:
|
|
"""Pre-process COBOL source for gixpp: expand COPY, strip comments, fix indentation.
|
|
|
|
Returns (pre_path, norm_path) where both point to the same gixpp-ready source.
|
|
"""
|
|
text = src_path.read_text(encoding="utf-8-sig")
|
|
|
|
# 0. Strip ALL comment lines BEFORE any EXEC SQL transforms (regex may match
|
|
# 'EXEC SQL' inside Japanese comments, pulling comment text into SQL strings)
|
|
text = re.sub(r'^[ \t]{0,10}\*.*\n?', '', text, flags=re.MULTILINE)
|
|
|
|
# 1. Replace EXEC SQL INCLUDE SQLCA → COPY SQLCA
|
|
text = re.sub(r'EXEC SQL INCLUDE SQLCA END-EXEC\.', ' COPY SQLCA.', text, flags=re.IGNORECASE)
|
|
# Transform EXEC SQL CONNECT TO 'literal' → use WS variables (gixpp requires :variable not literal)
|
|
def _transform_connect(m):
|
|
inner = m.group(1)
|
|
m_lit = re.search(r"CONNECT\s+TO\s+'([^']*)'", inner, re.IGNORECASE)
|
|
if not m_lit:
|
|
return m.group(0)
|
|
conn_var = "WS-GIX-CONN"
|
|
usr_var = "WS-GIX-USR"
|
|
new_inner = re.sub(
|
|
r"CONNECT\s+TO\s+'[^']*'",
|
|
f"CONNECT TO :{conn_var} USER :{usr_var}",
|
|
inner,
|
|
flags=re.IGNORECASE | re.DOTALL
|
|
)
|
|
# Orchestrator copies seeded DB to CWD/kin and CWD/kin.db.
|
|
# gixsql native canonical form is 'sqlite://localhost/kin' (host + dot-less path),
|
|
# matching gixpp's own conversion of CONNECT TO 'data/kin.db'.
|
|
# A dotted path segment (e.g. kin.db) breaks gixsql parsing -> empty connection.
|
|
return (f"MOVE 'sqlite://localhost/kin' TO {conn_var}\n"
|
|
f" MOVE 'gix' TO {usr_var}\n"
|
|
f" EXEC SQL\n"
|
|
f" {new_inner.strip()}\n"
|
|
f" END-EXEC.")
|
|
text = re.sub(r'(?is)EXEC SQL(.*?CONNECT\s+TO.*?)END-EXEC\.', _transform_connect, text)
|
|
# Add WS-GIX-* after COPY SQLCA
|
|
text = re.sub(
|
|
r"^(\s*COPY SQLCA\.)",
|
|
r"\1\n"
|
|
r" 01 WS-GIX-VARS.\n"
|
|
r" 03 WS-GIX-CONN PIC X(256).\n"
|
|
r" 03 WS-GIX-USR PIC X(16).",
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
# 2. Expand ALL COPY statements in Python (replaces cobc -E)
|
|
if hasattr(self, '_copybook_dirs') and self._copybook_dirs:
|
|
text = self._expand_all_copies(text, self._copybook_dirs)
|
|
|
|
# 4. Collapse multiple spaces between keywords
|
|
lines = []
|
|
for line in text.splitlines(keepends=True):
|
|
line = re.sub(r'(\b\w+)\s{2,}(\b\w+\b)', lambda m: f'{m.group(1)} {m.group(2)}', line)
|
|
lines.append(line)
|
|
text = ''.join(lines)
|
|
|
|
# 5. Normalize DIVISION/SECTION headers to start at column 8 (7-space indent)
|
|
# gixpp fixed-format scanner requires headers in Area A (columns 8-11).
|
|
def _fix_header(m):
|
|
return ' ' + m.group(1).lstrip()
|
|
text = re.sub(
|
|
r'^ {6,12}((?:IDENTIFICATION|ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION\.'
|
|
r'|(?:FILE|WORKING-STORAGE|LINKAGE|CONFIGURATION|INPUT-OUTPUT)\s+SECTION\.'
|
|
r'|(?:FILE-CONTROL|I-O-CONTROL)\.'
|
|
r'|SOURCE-COMPUTER\.|OBJECT-COMPUTER\.|SPECIAL-NAMES\.'
|
|
r')',
|
|
_fix_header,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
pre_path = src_path.parent / f"{src_path.stem}_pre.cbl"
|
|
pre_path.write_text(text, encoding="utf-8")
|
|
|
|
# 6. No cobc -E — all COPY expansions done in Python above.
|
|
# Use the pre_path directly as the norm_path (gixpp input).
|
|
# 6. Fix SQL clause ordering in EXEC SQL blocks:
|
|
# gixpp expects SELECT ... INTO ... FROM ... (INTO before FROM),
|
|
# but some programs have SELECT ... FROM ... INTO ...
|
|
def _fix_sql_from_into(m):
|
|
block = m.group(0)
|
|
# Only touch SELECT ... FROM ... INTO (not INSERT/DELETE)
|
|
if re.match(r'\s*EXEC SQL\s+SELECT\b', block, re.MULTILINE | re.IGNORECASE):
|
|
# Swap FROM line and INTO line within SELECT blocks
|
|
block = re.sub(
|
|
r'^(\s+)(FROM\b.*)\n(\s+)(INTO\b.*)$',
|
|
r'\1\4\n\1\2',
|
|
block,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
return block
|
|
text = re.sub(r'(\s*EXEC SQL\n.*?\n\s*END-EXEC\.)', _fix_sql_from_into, text,
|
|
flags=re.DOTALL | re.IGNORECASE)
|
|
|
|
# 7. Add missing 01-level variables used in PROCEDURE DIVISION but
|
|
# not defined in DATA DIVISION (source program defects).
|
|
missing_vars = {
|
|
'KIN06CLD': ' 01 WS-COL-IDX PIC 9(002).\n',
|
|
}
|
|
pid = src_path.stem.upper()
|
|
if pid in missing_vars and not re.search(r'\b01\s+WS-COL-IDX\b', text):
|
|
text = re.sub(
|
|
r'^(\s*)(PROCEDURE\s+DIVISION)',
|
|
lambda m: m.group(1) + missing_vars[pid] + '\n' + m.group(1) + m.group(2),
|
|
text, count=1, flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
# 8. SQLite backend: DB2 `CURRENT TIMESTAMP` → `CURRENT_TIMESTAMP` inside
|
|
# EXEC SQL. Must run before gixpp (gixpp can split the token across
|
|
# continuation lines, defeating the post-gixpp same-line patch).
|
|
text = _normalize_current_timestamp(text)
|
|
|
|
# 9. SQLite backend: DB2 schema-qualified table refs (SCHEMA.TABLE) →
|
|
# TABLE inside EXEC SQL. gixsql would otherwise emit
|
|
# `FROM SALARYDB.EMP_MASTER`, which SQLite cannot resolve.
|
|
text = _normalize_schema_qualifiers(text)
|
|
|
|
norm_path = src_path.parent / f"{src_path.stem}_norm.cbl"
|
|
norm_path.write_text(text, encoding="utf-8")
|
|
# Save a copy in runtime for diagnosis
|
|
try:
|
|
debug_dir = Path(__file__).parent.parent / "output" / src_path.stem / "cobol" / "pre_src"
|
|
debug_dir.mkdir(parents=True, exist_ok=True)
|
|
(debug_dir / f"{src_path.stem}_norm.cbl").write_text(text, encoding="utf-8")
|
|
(debug_dir / f"{src_path.stem}_pre.cbl").write_text(
|
|
(src_path.parent / f"{src_path.stem}_pre.cbl").read_text(encoding="utf-8"),
|
|
encoding="utf-8"
|
|
)
|
|
except Exception:
|
|
pass
|
|
return pre_path, norm_path
|
|
|
|
def preprocess(self, src_path: str | Path, out_dir: str | Path,
|
|
copybook_dirs: list[str | Path] | None = None) -> str:
|
|
"""Step 1a: gixpp プリプロセス"""
|
|
out_dir = Path(out_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
src_path = Path(src_path)
|
|
|
|
self._copybook_dirs = [Path(d) for d in copybook_dirs] if copybook_dirs else None
|
|
|
|
_, norm_path = self._normalize_source(src_path)
|
|
out_path = out_dir / f"{src_path.stem}_pp.cbl"
|
|
|
|
cmd = [str(self.gixpp_path), "-i", str(norm_path), "-o", str(out_path), "-e"]
|
|
if self._copybook_dirs:
|
|
for d in self._copybook_dirs:
|
|
cmd += ["-I", str(d)]
|
|
logger.info(f" gixpp: {' '.join(cmd)}")
|
|
r = subprocess.run(cmd, capture_output=True, timeout=30, env=self._build_env())
|
|
if r.returncode != 0:
|
|
err = r.stderr.decode("utf-8", "replace")[:500]
|
|
raise RuntimeError(f"gixpp failed (rc={r.returncode}): {err}")
|
|
return str(out_path)
|
|
|
|
def _patch_sql_identifiers(self, pp_path: Path) -> None:
|
|
"""Translate DB2 hyphenated identifiers to underscores inside GIXSQL SQL strings.
|
|
|
|
gixsql emits the SQL text verbatim from the COBOL source (e.g.
|
|
"INSERT INTO EMP-MASTER (EMP-ID, ...)"), but SQLite cannot parse bare
|
|
hyphenated identifiers. Convert '-' -> '_' only inside the SQL string
|
|
literals (VALUE "..." lines and & "..." continuation lines), leaving
|
|
COBOL-level identifiers (cursor names, host variables, work items) untouched.
|
|
"""
|
|
text = pp_path.read_text(encoding="utf-8")
|
|
|
|
# gixsql emits the SQL text verbatim from the COBOL source (e.g.
|
|
# "INSERT INTO EMP-MASTER (EMP-ID, ...)"), but SQLite cannot parse bare
|
|
# hyphenated identifiers, so '-' -> '_' is needed for identifiers such as
|
|
# EMP-ID -> EMP_ID. However a hyphen surrounded by whitespace is an
|
|
# arithmetic operator (e.g. "OVT_HOURS - $1", "OVT_COUNT - 1") and MUST
|
|
# be preserved, otherwise the generated SQL is syntactically invalid
|
|
# (near "_"). Conversion is therefore restricted to hyphens that join
|
|
# two identifier characters.
|
|
#
|
|
# Each SQL string is physically split across a VALUE "..." line and
|
|
# several & "..." continuation lines, so the conversion is performed on
|
|
# the logical concatenation (length-preserving: '-' -> '_') and written
|
|
# back at the original per-line offsets.
|
|
start_re = re.compile(r'^(GIXSQL.*?VALUE\s+")(.*)(")\s*$')
|
|
cont_re = re.compile(r'^(GIXSQL\s*&\s*")(.*)(")\s*$')
|
|
lines = text.split("\n")
|
|
i = 0
|
|
while i < len(lines):
|
|
m = start_re.match(lines[i])
|
|
if not m:
|
|
i += 1
|
|
continue
|
|
group = [(m.group(1), m.group(2), m.group(3))]
|
|
j = i + 1
|
|
while j < len(lines):
|
|
c = cont_re.match(lines[j])
|
|
if not c:
|
|
break
|
|
group.append((c.group(1), c.group(2), c.group(3)))
|
|
j += 1
|
|
contents = [g[1] for g in group]
|
|
logical = "".join(contents)
|
|
if '-' in logical:
|
|
new_logical = re.sub(r'(?<=\w)-(?!\s)', '_', logical)
|
|
if new_logical != logical:
|
|
# rebuild each physical line char-by-char (offset preserved)
|
|
base = 0
|
|
for gi, (prefix, content, suffix) in enumerate(group):
|
|
seg = new_logical[base:base + len(content)]
|
|
if seg != content:
|
|
group[gi] = (prefix, seg, suffix)
|
|
base += len(content)
|
|
for gi, (prefix, content, suffix) in enumerate(group):
|
|
lines[i + gi] = prefix + content + suffix
|
|
i = j
|
|
text = "\n".join(lines)
|
|
# SQLite accepts CURRENT_TIMESTAMP (no space); gixsql emits CURRENT TIMESTAMP
|
|
text = re.sub(r'\bCURRENT\s+TIMESTAMP\b', 'CURRENT_TIMESTAMP', text, flags=re.IGNORECASE)
|
|
pp_path.write_text(text, encoding="utf-8")
|
|
|
|
def _patch_sqlcode_normalize(self, pp_path: Path) -> None:
|
|
"""Inject SQLITE-constraint → -803 mapping after each SQL execution.
|
|
|
|
gixsql+SQLite reports duplicate-key (PRIMARY KEY / UNIQUE) violations as
|
|
SQLCODE=-1555 / -2067 / -19, whereas the COBOL programs follow DB2
|
|
semantics (SQLCODE = -803). Without this mapping, `IF SQLCODE = -803`
|
|
branches are unreachable under gixsql+SQLite. The injected code is
|
|
program-agnostic: it only rewrites the constraint-violation codes.
|
|
"""
|
|
text = pp_path.read_text(encoding="utf-8")
|
|
mapping = (
|
|
' IF SQLCODE = -1555\n'
|
|
' MOVE -803 TO SQLCODE\n'
|
|
' END-IF\n'
|
|
' IF SQLCODE = -2067\n'
|
|
' MOVE -803 TO SQLCODE\n'
|
|
' END-IF\n'
|
|
' IF SQLCODE = -19\n'
|
|
' MOVE -803 TO SQLCODE\n'
|
|
' END-IF\n'
|
|
)
|
|
text = re.sub(
|
|
r'(GIXSQLEndSQL\s*\n?.*?END-CALL\.?)\s*\n(\s*)(IF SQLCODE\s+)',
|
|
lambda m: m.group(1) + '\n' + m.group(2) + mapping + m.group(2) + m.group(3),
|
|
text,
|
|
flags=re.IGNORECASE | re.DOTALL
|
|
)
|
|
pp_path.write_text(text, encoding="utf-8")
|
|
|
|
def _patch_sqlcode_override(self, pp_path: Path) -> None:
|
|
"""Inject MOVE 0 TO SQLCODE after each GIXSQLEndSQL call.
|
|
|
|
Workaround for gixsql DLL bug: GIXSQLExecParams/GIXSQLExec fail with
|
|
'Can't find a connection' even though the connection is valid.
|
|
Setting SQLCODE=0 lets the program flow through success paths for
|
|
coverage measurement. The DB state is not validated in coverage runs.
|
|
"""
|
|
text = pp_path.read_text(encoding="utf-8")
|
|
text = re.sub(
|
|
r'(GIXSQLEndSQL\s*\n?.*?END-CALL\.?)\s*\n(\s*)(IF SQLCODE\s+(?:NOT\s+)?=\s+0)',
|
|
r'\1\n\2 MOVE 0 TO SQLCODE\n\2\3',
|
|
text,
|
|
flags=re.IGNORECASE | re.DOTALL
|
|
)
|
|
pp_path.write_text(text, encoding="utf-8")
|
|
|
|
@staticmethod
|
|
def _patch_numeric_metadata(pp_path: Path) -> None:
|
|
"""Fix gixpp numeric binding metadata for implicit-decimal DISPLAY fields.
|
|
|
|
gixpp emits ``GIXSQLSetSQLParams(1, length, power, ...)`` where ``power``
|
|
is negative for implicit-decimal PICs (e.g. ``9(4)V9(1)`` → power=-1).
|
|
gixsql uses ``bytes_read = length - power``; with ``length=5, power=-1``
|
|
this reads 6 bytes, one past the actual 5-byte field, corrupting the value
|
|
with the adjacent WORKING-STORAGE byte.
|
|
|
|
Fix: reduce ``length`` so that ``length - power = actual_field_size``
|
|
(the COBOL storage bytes), keeping ``power`` unchanged so gixsql still
|
|
reads the correct number of bytes.
|
|
"""
|
|
text = pp_path.read_text(encoding="utf-8")
|
|
|
|
_CALL_RE = re.compile(
|
|
r'CALL "GIXSQLSetSQLParams" USING\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (-?\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE 0\s+'
|
|
r'(?:GIXSQL\s+)?BY REFERENCE \S+',
|
|
re.IGNORECASE | re.DOTALL
|
|
)
|
|
|
|
def _fix_numeric(m: re.Match) -> str:
|
|
type_val = int(m.group(1))
|
|
length_val = int(m.group(2))
|
|
power_val = int(m.group(3))
|
|
if type_val == 1 and power_val < 0:
|
|
new_length = length_val + power_val # power is negative
|
|
logger.debug(f" _patch_numeric_metadata: length {length_val}->{new_length}, "
|
|
f"power unchanged {power_val}")
|
|
old = m.group(0)
|
|
old_lines = old.split('\n')
|
|
new_lines = []
|
|
val_idx = 0
|
|
for line in old_lines:
|
|
if 'BY VALUE' in line:
|
|
val_idx += 1
|
|
if val_idx == 1: # type
|
|
new_lines.append(line)
|
|
elif val_idx == 2: # length — reduce to actual storage size
|
|
new_lines.append(re.sub(r'BY VALUE \d+', f'BY VALUE {new_length}', line))
|
|
else: # power, flag — keep unchanged
|
|
new_lines.append(line)
|
|
else:
|
|
new_lines.append(line)
|
|
return '\n'.join(new_lines)
|
|
return m.group(0)
|
|
|
|
text = _CALL_RE.sub(_fix_numeric, text)
|
|
pp_path.write_text(text, encoding="utf-8")
|
|
|
|
@staticmethod
|
|
def extract_power_scaling(pp_path: str | Path) -> dict[str, int]:
|
|
"""Extract power scaling info from _pp.cbl for DB post-processing.
|
|
|
|
Strategy:
|
|
1. Parse INSERT SQL from FILLER VALUE strings → get column lists.
|
|
2. Scan GIXSQLSetSQLParams in order → find which positions have
|
|
type=1, power<0 (implicit-decimal numeric).
|
|
3. Map positions to columns. Since the same column names appear
|
|
in multiple INSERTs, we just need the unique column names.
|
|
|
|
Returns mapping e.g. {"ANNUAL_LEAVE_H": -1, ...}.
|
|
"""
|
|
raw_text = Path(pp_path).read_text(encoding="utf-8")
|
|
|
|
# Step 1: Extract INSERT SQL column→position mappings from FILLER VALUEs.
|
|
_FILLER_RE = re.compile(
|
|
r'02\s+FILLER\s+PIC\s+X\(\d+\)\s+VALUE\s+"([^"]*)"'
|
|
r'((?:\s*\n\s*GIXSQL\s*&\s+"[^"]*")*)',
|
|
re.IGNORECASE
|
|
)
|
|
# Collect ALL columns that use $N params (across all INSERTs)
|
|
all_numeric_cols: set[str] = set()
|
|
max_position = 0
|
|
first_pos_map: dict[int, str] = {} # first occurrence of each position
|
|
|
|
for m in _FILLER_RE.finditer(raw_text):
|
|
first = m.group(1)
|
|
conts = m.group(2)
|
|
cont_texts = re.findall(r'"([^"]*)"', conts)
|
|
full_sql = first + ''.join(cont_texts)
|
|
ins = re.search(
|
|
r'INSERT\s+INTO\s+\w+\s*\(([^)]+)\)\s*VALUES\s*\(([^)]+)\)',
|
|
full_sql, re.IGNORECASE
|
|
)
|
|
if ins:
|
|
cols = [c.strip() for c in ins.group(1).split(',')]
|
|
vals = [v.strip() for v in ins.group(2).split(',')]
|
|
for col, val in zip(cols, vals):
|
|
pm = re.match(r'\$(\d+)', val)
|
|
if pm:
|
|
pos = int(pm.group(1))
|
|
max_position = max(max_position, pos)
|
|
if pos not in first_pos_map:
|
|
first_pos_map[pos] = col
|
|
|
|
# Step 2: Extract GIXSQLSetSQLParams calls in order.
|
|
_PARAM_RE = re.compile(
|
|
r'CALL "GIXSQLSetSQLParams" USING\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE (-?\d+)\s+'
|
|
r'(?:GIXSQL\s+)?BY VALUE 0\s+'
|
|
r'(?:GIXSQL\s+)?BY REFERENCE (\S+)',
|
|
re.IGNORECASE | re.DOTALL
|
|
)
|
|
all_params = []
|
|
for m in _PARAM_RE.finditer(raw_text):
|
|
all_params.append((int(m.group(1)), int(m.group(3)), m.group(4)))
|
|
|
|
# Step 3: Group params by their INSERT context.
|
|
# Each INSERT has its own $1..$N. The params repeat for each SQL
|
|
# statement (INSERT DAILY, DELETE, SELECT, UPDATE, INSERT MONTHLY, ...).
|
|
# We need to find which param indices map to which INSERT's $N.
|
|
#
|
|
# Key insight: for each INSERT, the $N positions are 1..N.
|
|
# The SetSQLParams before that INSERT are in the same order.
|
|
# We can identify INSERT boundaries by looking for GIXSQLExecParams
|
|
# calls that reference the SQL string (SQ0001, SQ0002, etc.).
|
|
#
|
|
# Simpler approach: since all INSERTs share the same numeric columns,
|
|
# just find ALL columns that appear at ANY $N position, and ALL
|
|
# SetSQLParams with type=1/power<0. Then match by relative position
|
|
# within each INSERT group.
|
|
#
|
|
# Even simpler: just check which columns from the DDL appear in
|
|
# ANY INSERT with a $N param, and which SetSQLParams have power<0.
|
|
# The SetSQLParams with power<0 always correspond to the numeric
|
|
# columns (DECIMAL), so we can match by counting.
|
|
|
|
# Count how many params are BEFORE the first numeric param
|
|
# and how many numeric params there are per INSERT.
|
|
# For DAILY_RECORDS: 4 string params, then 5 numeric, then 1 timestamp
|
|
# For MONTHLY_ABSENCE: 2 string params, then 5 numeric, then 1 timestamp
|
|
|
|
# Find all unique column names that have $N in INSERT SQL
|
|
numeric_cols_in_insert: list[str] = []
|
|
for pos in sorted(first_pos_map.keys()):
|
|
col = first_pos_map[pos]
|
|
# We'll filter to just the numeric ones after matching
|
|
|
|
# Match: SetSQLParams with type=1/power<0 → columns at corresponding $N
|
|
# We need to know which $N corresponds to which SetSQLParams index.
|
|
#
|
|
# Strategy: find the first INSERT's column list, then find the
|
|
# SetSQLParams group that precedes it. The numeric columns in the
|
|
# INSERT are at specific $N positions. The SetSQLParams with
|
|
# type=1/power<0 are at corresponding indices.
|
|
|
|
# Find the first INSERT DAILY_RECORDS column order
|
|
daily_cols = []
|
|
for m in _FILLER_RE.finditer(raw_text):
|
|
first = m.group(1)
|
|
conts = m.group(2)
|
|
cont_texts = re.findall(r'"([^"]*)"', conts)
|
|
full_sql = first + ''.join(cont_texts)
|
|
ins = re.search(
|
|
r'INSERT\s+INTO\s+DAILY_RECORDS\s*\(([^)]+)\)',
|
|
full_sql, re.IGNORECASE
|
|
)
|
|
if ins:
|
|
daily_cols = [c.strip() for c in ins.group(1).split(',')]
|
|
break
|
|
|
|
# Find which $N positions in DAILY_RECORDS are numeric (DECIMAL in DDL)
|
|
# by checking if the corresponding SetSQLParams has type=1/power<0.
|
|
# The $N for DAILY_RECORDS: EMP_ID=$1, TARGET_DATE=$2, ..., UPDATED_AT=$9(→CURRENT_TIMESTAMP)
|
|
# We know the SetSQLParams before the first INSERT are in order $1..$9.
|
|
# Count params before the first GIXSQLExecParams (which executes the INSERT).
|
|
|
|
# Find the first GIXSQLExecParams position
|
|
_EXEC_RE = re.compile(r'CALL "GIXSQLExecParams"', re.IGNORECASE)
|
|
first_exec = _EXEC_RE.search(raw_text)
|
|
first_exec_pos = first_exec.start() if first_exec else len(raw_text)
|
|
|
|
# Collect params that appear BEFORE the first exec
|
|
pre_exec_params = []
|
|
for ptype, ppower, pvar in all_params:
|
|
var_pattern = re.compile(rf'BY REFERENCE {re.escape(pvar)}\b', re.IGNORECASE)
|
|
var_match = var_pattern.search(raw_text)
|
|
if var_match and var_match.start() < first_exec_pos:
|
|
pre_exec_params.append((ptype, ppower, pvar))
|
|
|
|
# These are the params for the first INSERT (DAILY_RECORDS).
|
|
# Match them to the DAILY_RECORDS columns.
|
|
scaling: dict[str, int] = {}
|
|
for idx, (ptype, ppower, pvar) in enumerate(pre_exec_params):
|
|
pos = idx + 1 # 1-based
|
|
if ptype == 1 and ppower < 0:
|
|
col_name = first_pos_map.get(pos)
|
|
if col_name:
|
|
scaling[col_name] = ppower
|
|
|
|
return scaling
|
|
|
|
@staticmethod
|
|
def apply_power_scaling(db_path: str | Path,
|
|
power_map: dict[str, int]) -> int:
|
|
"""Post-process SQLite DB: divide numeric columns by 10^|power|.
|
|
|
|
gixsql stores raw digits (e.g. 402) for implicit-decimal fields
|
|
but does NOT apply the power scaling. This corrects the DB values
|
|
to match the intended COBOL decimal representation (e.g. 40.2).
|
|
|
|
Returns number of rows updated.
|
|
"""
|
|
if not power_map:
|
|
return 0
|
|
db_path = Path(db_path)
|
|
if not db_path.exists():
|
|
return 0
|
|
conn = sqlite3.connect(str(db_path))
|
|
total_updated = 0
|
|
for var_name, power in power_map.items():
|
|
col_name = var_name.replace('-', '_')
|
|
scale_factor = 10 ** abs(power)
|
|
try:
|
|
# Get table names that have this column
|
|
tables = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
).fetchall()
|
|
for (tname,) in tables:
|
|
cols = conn.execute(f"PRAGMA table_info([{tname}])").fetchall()
|
|
col_names = [c[1] for c in cols]
|
|
if col_name in col_names:
|
|
# Use CAST to force float division (SQLite integer
|
|
# division truncates: 402/10=40, need 402.0/10=40.2)
|
|
cur = conn.execute(
|
|
f"UPDATE [{tname}] SET [{col_name}] = CAST([{col_name}] AS REAL) / ?",
|
|
(scale_factor,)
|
|
)
|
|
if cur.rowcount > 0:
|
|
logger.info(
|
|
f" power_scale: {tname}.{col_name} /= {scale_factor} "
|
|
f"({cur.rowcount} rows, power={power})"
|
|
)
|
|
total_updated += cur.rowcount
|
|
except sqlite3.Error as e:
|
|
logger.warning(f" power_scale: {tname}.{col_name} failed: {e}")
|
|
conn.commit()
|
|
conn.close()
|
|
return total_updated
|
|
|
|
def compile(self, pp_path: str | Path, exe_path: str | Path,
|
|
copybook_dirs: list[str | Path] | None = None,
|
|
extra_srcs: list[str | Path] | None = None) -> GixsqlBuildResult:
|
|
"""Step 1b: cobc 编译（gixsql 链接）"""
|
|
pp_path = Path(pp_path)
|
|
exe_path = Path(exe_path)
|
|
exe_dir = exe_path.parent
|
|
exe_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# SQL is now executed correctly (hyphen->underscore identifiers), so the
|
|
# SQLCODE=0 override workaround is no longer needed. It masked real SQL
|
|
# errors and caused infinite FETCH loops (SQLCODE forced to 0 prevents
|
|
# "PERFORM UNTIL SQLCODE NOT = 0" from terminating at EOF=100).
|
|
# self._patch_sqlcode_override(pp_path)
|
|
|
|
# Translate DB2 hyphenated identifiers -> underscores in SQL strings
|
|
self._patch_sql_identifiers(pp_path)
|
|
|
|
# Map gixsql SQLite constraint codes (-1555/-2067/-19) -> DB2 -803 so
|
|
# `IF SQLCODE = -803` branches are reachable under gixsql+SQLite.
|
|
self._patch_sqlcode_normalize(pp_path)
|
|
|
|
# Fix gixpp numeric binding metadata for implicit-decimal DISPLAY fields.
|
|
# gixpp emits power=-1 for 9(4)V9(1), causing gixsql to read 6 bytes
|
|
# (length - power = 5-(-1)=6) and corrupt the value with adjacent data.
|
|
self._patch_numeric_metadata(pp_path)
|
|
|
|
# Functions that the preprocessed COBOL actually CALLs
|
|
gixsql_k = [
|
|
"-K", "GIXSQLStartSQL",
|
|
"-K", "GIXSQLSetSQLParams", "-K", "GIXSQLSetResultParams",
|
|
"-K", "GIXSQLExecParams", "-K", "GIXSQLExec",
|
|
"-K", "GIXSQLExecSelectIntoOne",
|
|
"-K", "GIXSQLEndSQL",
|
|
"-K", "GIXSQLConnect",
|
|
"-K", "GIXSQLCursorDeclare",
|
|
"-K", "GIXSQLCursorDeclareParams",
|
|
"-K", "GIXSQLCursorFetchOne",
|
|
"-K", "GIXSQLCursorOpen",
|
|
"-K", "GIXSQLCursorClose",
|
|
]
|
|
cmd = [
|
|
self.cobc_cmd, "-x",
|
|
"-L", str(self.lib_path.resolve()),
|
|
*gixsql_k,
|
|
"-l", "gixsql",
|
|
]
|
|
if copybook_dirs:
|
|
for d in copybook_dirs:
|
|
cmd += ["-I", str(d)]
|
|
if self.compile_flags:
|
|
cmd += self.compile_flags.split()
|
|
cmd += ["-o", str(exe_path)]
|
|
cmd.append(str(pp_path))
|
|
if extra_srcs:
|
|
for s in extra_srcs:
|
|
cmd.append(str(s))
|
|
|
|
logger.info(f" cobc: {' '.join(cmd)}")
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, timeout=60, env=self._build_env(),
|
|
cwd=str(exe_dir))
|
|
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
|
r.stderr.decode("utf-8", "replace"))
|
|
if r.returncode != 0:
|
|
return GixsqlBuildResult(False, log=log[:1000])
|
|
# .gcno files are generated in exe_dir (compile CWD). No need to copy.
|
|
return GixsqlBuildResult(True, exe_path=str(exe_path), log=log[:500])
|
|
except subprocess.TimeoutExpired:
|
|
return GixsqlBuildResult(False, log="Compile timeout (60s)")
|
|
|
|
def run(self, exe_path: str | Path, work_dir: str | Path,
|
|
db_path: str | Path,
|
|
input_dir: str | Path | None = None,
|
|
timeout: int = 30,
|
|
cobol_lib_path: str | Path | None = None,
|
|
env_overrides: dict[str, str] | None = None,
|
|
command_args: list[str] | None = None) -> GixsqlRunResult:
|
|
"""Step 3: COBOL DB プログラム実行"""
|
|
exe_path = Path(exe_path)
|
|
work_dir = Path(work_dir)
|
|
db_path = Path(db_path)
|
|
work_dir.mkdir(parents=True, exist_ok=True)
|
|
exe_dir = exe_path.parent
|
|
exe_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lib_path = Path(self.lib_path)
|
|
|
|
# Deploy correct DLLs to exe_dir so loader finds them first
|
|
# Search lib_path first, then fall back to gixpp bin dir for runtime DLLs
|
|
gixpp_dir = self.gixpp_path.parent
|
|
dll_names = [
|
|
"libgixsql.dll", "libgixsql-sqlite.dll",
|
|
"libgcc_s_dw2-1.dll", "libstdc++-6.dll",
|
|
"libwinpthread-1.dll", "libiconv-2.dll",
|
|
"libintl-8.dll", "zlib1.dll",
|
|
]
|
|
for name in dll_names:
|
|
src = lib_path / name
|
|
if not src.exists():
|
|
src = gixpp_dir / name
|
|
if src.exists():
|
|
dst = exe_dir / name
|
|
if not dst.exists() or dst.stat().st_size != src.stat().st_size:
|
|
dst.write_bytes(src.read_bytes())
|
|
|
|
# libfmt.dll search: TEMP fallback → lib_path → gixpp bin → x86/gcc subdirectory
|
|
fmt_src = Path(os.environ.get("TEMP", "")) / "zan_dll" / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
fmt_src = lib_path / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
fmt_src = gixpp_dir / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
# gixsql binary package variant: <gixpp-parent>/lib/x86/gcc/libfmt.dll
|
|
fmt_src = gixpp_dir.parent / "lib" / "x86" / "gcc" / "libfmt.dll"
|
|
if fmt_src.exists():
|
|
(exe_dir / "libfmt.dll").write_bytes(fmt_src.read_bytes())
|
|
|
|
env = self._build_env()
|
|
env["GIXSQL_DB_PATH"] = str(db_path)
|
|
if cobol_lib_path:
|
|
env["COB_LIBRARY_PATH"] = str(cobol_lib_path)
|
|
if env_overrides:
|
|
env.update(env_overrides)
|
|
|
|
if input_dir:
|
|
idir = Path(input_dir)
|
|
if idir.exists():
|
|
for f in idir.iterdir():
|
|
if f.is_file():
|
|
dst = work_dir / f.name
|
|
dst.write_bytes(f.read_bytes())
|
|
|
|
cmd = [str(exe_path)]
|
|
if command_args:
|
|
cmd.extend(command_args)
|
|
logger.info(f" run: {' '.join(cmd)} (cwd={work_dir}, db={db_path})")
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, timeout=timeout,
|
|
cwd=str(work_dir), env=env)
|
|
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
|
r.stderr.decode("utf-8", "replace"))
|
|
ok = r.returncode == 0 or r.returncode == 1
|
|
return GixsqlRunResult(ok, returncode=r.returncode,
|
|
db_path=str(db_path), log=log[:1000])
|
|
except subprocess.TimeoutExpired:
|
|
return GixsqlRunResult(False, log="Run timeout")
|
|
|
|
def read_db_tables(self, db_path: str | Path,
|
|
table_names: list[str]) -> list[GixsqlTableData]:
|
|
"""Step 4: SQLite DB から全テーブル読み取り"""
|
|
db_path = Path(db_path)
|
|
if not db_path.exists():
|
|
logger.warning(f" DB not found: {db_path}")
|
|
return []
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
results = []
|
|
for tname in table_names:
|
|
try:
|
|
rows = conn.execute(f"SELECT * FROM [{tname}]").fetchall()
|
|
results.append(GixsqlTableData(
|
|
table_name=tname,
|
|
rows=[dict(r) for r in rows],
|
|
))
|
|
except sqlite3.OperationalError as e:
|
|
logger.warning(f" Table {tname} not found: {e}")
|
|
conn.close()
|
|
return results
|