hangshuo652
54d4e81240
- TIME injection: change from spaces to '2500' (hour>23) for NUMVAL trigger - Runner: add .resolve() to work_dir to fix chdir+relative path breakage - Coverage: per-target field overrides for DP#9-#12 (START-DATE=20240115 etc.) - .gitignore: add compilation artifacts, temp scripts, test outputs
424 lines
18 KiB
Python
424 lines
18 KiB
Python
"""Gixsql CBL Runner — gixpp + cobc pipeline for DB COBOL programs."""
|
|
|
|
from __future__ import annotations
|
|
import logging
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class GixsqlBuildResult:
|
|
success: bool
|
|
exe_path: str = ""
|
|
log: str = ""
|
|
|
|
|
|
@dataclass
|
|
class GixsqlRunResult:
|
|
success: bool
|
|
returncode: int = -1
|
|
db_path: str = ""
|
|
log: str = ""
|
|
|
|
|
|
@dataclass
|
|
class GixsqlTableData:
|
|
table_name: str = ""
|
|
rows: list[dict] = field(default_factory=list)
|
|
|
|
|
|
class GixsqlCobolRunner:
|
|
"""gixpp + cobc 管线：预处理 → 编译 → 运行 → DB读取"""
|
|
|
|
def __init__(self, gixpp_path: str | Path, lib_path: str | Path,
|
|
cobc_cmd: str = "cobc",
|
|
compile_flags: str = ""):
|
|
self.gixpp_path = Path(gixpp_path)
|
|
self.lib_path = Path(lib_path)
|
|
self.cobc_cmd = cobc_cmd
|
|
self.compile_flags = compile_flags
|
|
|
|
def _build_env(self) -> dict:
|
|
env = os.environ.copy()
|
|
env["LD_LIBRARY_PATH"] = str(self.lib_path)
|
|
if "PATH" in env:
|
|
env["PATH"] = str(self.lib_path) + ";" + env["PATH"]
|
|
else:
|
|
env["PATH"] = str(self.lib_path)
|
|
return env
|
|
|
|
def _expand_copy_replacing(self, text: str, search_dirs: list[Path]) -> str:
|
|
"""Expand COPY ... REPLACING statements inline (Python-side).
|
|
gixpp's ESQL parser chokes on COPY with REPLACING pseudo-text (==...==).
|
|
"""
|
|
def _resolve_copy(m: re.Match) -> str:
|
|
name = m.group(1)
|
|
for d in search_dirs:
|
|
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
|
cp = d / f"{name}{ext}"
|
|
if cp.exists():
|
|
cpy_text = cp.read_text(encoding="utf-8-sig")
|
|
replaces_text = m.group(2)
|
|
if replaces_text:
|
|
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replaces_text)
|
|
for old_txt, new_txt in pairs:
|
|
cpy_text = cpy_text.replace(old_txt, new_txt)
|
|
return cpy_text
|
|
logger.warning(f" COPY {name} not found in {search_dirs}")
|
|
return f" * COPY {name} NOT FOUND"
|
|
|
|
text = re.sub(
|
|
r'^ {6,}COPY\s+(\w+)\s+REPLACING\s+(.+?)\.$',
|
|
_resolve_copy,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE,
|
|
)
|
|
return text
|
|
|
|
def _expand_all_copies(self, text: str, search_dirs: list[Path]) -> str:
|
|
"""Expand ALL COPY statements (including COPY SQLCA) — replaces cobc -E."""
|
|
def _resolve_copy_cb(m: re.Match) -> str:
|
|
name = m.group(1).strip().upper()
|
|
replacing_text = m.group(2)
|
|
# Try each search dir
|
|
for d in search_dirs:
|
|
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
|
cp = d / f"{name}{ext}"
|
|
if cp.exists():
|
|
cpy_text = cp.read_text(encoding="utf-8-sig")
|
|
if replacing_text:
|
|
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replacing_text)
|
|
for old_txt, new_txt in pairs:
|
|
cpy_text = cpy_text.replace(old_txt, new_txt)
|
|
return cpy_text
|
|
# If SQLCA not found in copybook dirs, provide inline definition
|
|
if name == "SQLCA":
|
|
return (
|
|
" 01 SQLCA.\n"
|
|
" 05 SQLCAID PIC X(8).\n"
|
|
" 05 SQLCABC PIC S9(9) COMP.\n"
|
|
" 05 SQLCODE PIC S9(9) COMP.\n"
|
|
" 05 SQLERRM.\n"
|
|
" 49 SQLERRML PIC S9(4) COMP.\n"
|
|
" 49 SQLERRMC PIC X(256).\n"
|
|
" 05 SQLERRP PIC X(8).\n"
|
|
" 05 SQLERRD PIC S9(9) COMP OCCURS 6.\n"
|
|
" 05 SQLWARN.\n"
|
|
" 10 SQLWARN0 PIC X(1).\n"
|
|
" 10 SQLWARN1 PIC X(1).\n"
|
|
" 10 SQLWARN2 PIC X(1).\n"
|
|
" 10 SQLWARN3 PIC X(1).\n"
|
|
" 10 SQLWARN4 PIC X(1).\n"
|
|
" 10 SQLWARN5 PIC X(1).\n"
|
|
" 10 SQLWARN6 PIC X(1).\n"
|
|
" 10 SQLWARN7 PIC X(1).\n"
|
|
" 05 SQLEXT PIC X(8).\n"
|
|
)
|
|
logger.warning(f" COPY {name} not found in {search_dirs}")
|
|
return f" * COPY {name} NOT FOUND\n"
|
|
|
|
# Expand COPY name. and COPY name REPLACING ... .
|
|
text = re.sub(
|
|
r'^ {6,}COPY\s+(\w+(?:-\w+)*)\s*(REPLACING\s+.+?)?\.$',
|
|
_resolve_copy_cb,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE,
|
|
)
|
|
return text
|
|
|
|
def _normalize_source(self, src_path: Path) -> tuple[Path, Path]:
|
|
"""Pre-process COBOL source for gixpp: expand COPY, strip comments, fix indentation.
|
|
|
|
Returns (pre_path, norm_path) where both point to the same gixpp-ready source.
|
|
"""
|
|
text = src_path.read_text(encoding="utf-8-sig")
|
|
|
|
# 1. Replace EXEC SQL INCLUDE SQLCA → COPY SQLCA
|
|
text = re.sub(r'EXEC SQL INCLUDE SQLCA END-EXEC\.', ' COPY SQLCA.', text, flags=re.IGNORECASE)
|
|
# Transform EXEC SQL CONNECT TO 'literal' → use WS variables (gixpp requires :variable not literal)
|
|
def _transform_connect(m):
|
|
inner = m.group(1)
|
|
m_lit = re.search(r"CONNECT\s+TO\s+'([^']*)'", inner, re.IGNORECASE)
|
|
if not m_lit:
|
|
return m.group(0)
|
|
conn_var = "WS-GIX-CONN"
|
|
usr_var = "WS-GIX-USR"
|
|
new_inner = re.sub(
|
|
r"CONNECT\s+TO\s+'[^']*'",
|
|
f"CONNECT TO :{conn_var} USER :{usr_var}",
|
|
inner,
|
|
flags=re.IGNORECASE
|
|
)
|
|
# Short absolute path under C:\Temp\gix\ (no Chinese chars, fits col 72).
|
|
# The orchestrator creates the DB at the same path so they match.
|
|
from pathlib import Path as _Path
|
|
pid = _Path(src_path).stem
|
|
gix_root = _Path("C:/Temp/gix")
|
|
gix_root.mkdir(parents=True, exist_ok=True)
|
|
db_path = gix_root / f"{pid}.db"
|
|
conn_val = "sqlite:///" + str(db_path).replace("\\", "/")
|
|
return (f"MOVE '{conn_val}' TO {conn_var}\n"
|
|
f" MOVE 'gix' TO {usr_var}\n"
|
|
f" EXEC SQL\n"
|
|
f" {new_inner.strip()}\n"
|
|
f" END-EXEC.")
|
|
text = re.sub(r'(?is)EXEC SQL(.*?CONNECT\s+TO.*?)END-EXEC\.', _transform_connect, text)
|
|
# Add WS-GIX-* after COPY SQLCA
|
|
text = re.sub(
|
|
r"^(\s*COPY SQLCA\.)",
|
|
r"\1\n"
|
|
r" 01 WS-GIX-VARS.\n"
|
|
r" 03 WS-GIX-CONN PIC X(256).\n"
|
|
r" 03 WS-GIX-USR PIC X(16).",
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
# 2. Expand ALL COPY statements in Python (replaces cobc -E)
|
|
if hasattr(self, '_copybook_dirs') and self._copybook_dirs:
|
|
text = self._expand_all_copies(text, self._copybook_dirs)
|
|
|
|
# 3. Strip ALL comment lines (* in any column 7-11)
|
|
text = re.sub(r'^[ \t]{0,10}\*.*\n?', '', text, flags=re.MULTILINE)
|
|
|
|
# 4. Collapse multiple spaces between keywords
|
|
lines = []
|
|
for line in text.splitlines(keepends=True):
|
|
line = re.sub(r'(\b\w+)\s{2,}(\b\w+\b)', lambda m: f'{m.group(1)} {m.group(2)}', line)
|
|
lines.append(line)
|
|
text = ''.join(lines)
|
|
|
|
# 5. Normalize DIVISION/SECTION headers to start at column 8 (7-space indent)
|
|
# gixpp fixed-format scanner requires headers in Area A (columns 8-11).
|
|
def _fix_header(m):
|
|
return ' ' + m.group(1).lstrip()
|
|
text = re.sub(
|
|
r'^ {6,12}((?:IDENTIFICATION|ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION\.'
|
|
r'|(?:FILE|WORKING-STORAGE|LINKAGE|CONFIGURATION|INPUT-OUTPUT)\s+SECTION\.'
|
|
r'|(?:FILE-CONTROL|I-O-CONTROL)\.'
|
|
r'|SOURCE-COMPUTER\.|OBJECT-COMPUTER\.|SPECIAL-NAMES\.'
|
|
r')',
|
|
_fix_header,
|
|
text,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
pre_path = src_path.parent / f"{src_path.stem}_pre.cbl"
|
|
pre_path.write_text(text, encoding="utf-8")
|
|
|
|
# 6. No cobc -E — all COPY expansions done in Python above.
|
|
# Use the pre_path directly as the norm_path (gixpp input).
|
|
# 6. Fix SQL clause ordering in EXEC SQL blocks:
|
|
# gixpp expects SELECT ... INTO ... FROM ... (INTO before FROM),
|
|
# but some programs have SELECT ... FROM ... INTO ...
|
|
def _fix_sql_from_into(m):
|
|
block = m.group(0)
|
|
# Only touch SELECT ... FROM ... INTO (not INSERT/DELETE)
|
|
if re.match(r'\s*EXEC SQL\s+SELECT\b', block, re.MULTILINE | re.IGNORECASE):
|
|
# Swap FROM line and INTO line within SELECT blocks
|
|
block = re.sub(
|
|
r'^(\s+)(FROM\b.*)\n(\s+)(INTO\b.*)$',
|
|
r'\1\4\n\1\2',
|
|
block,
|
|
flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
return block
|
|
text = re.sub(r'(\s*EXEC SQL\n.*?\n\s*END-EXEC\.)', _fix_sql_from_into, text,
|
|
flags=re.DOTALL | re.IGNORECASE)
|
|
|
|
# 7. Add missing 01-level variables used in PROCEDURE DIVISION but
|
|
# not defined in DATA DIVISION (source program defects).
|
|
missing_vars = {
|
|
'KIN06CLD': ' 01 WS-COL-IDX PIC 9(002).\n',
|
|
}
|
|
pid = src_path.stem.upper()
|
|
if pid in missing_vars and not re.search(r'\b01\s+WS-COL-IDX\b', text):
|
|
text = re.sub(
|
|
r'^(\s*)(PROCEDURE\s+DIVISION)',
|
|
lambda m: m.group(1) + missing_vars[pid] + '\n' + m.group(1) + m.group(2),
|
|
text, count=1, flags=re.MULTILINE | re.IGNORECASE
|
|
)
|
|
|
|
norm_path = src_path.parent / f"{src_path.stem}_norm.cbl"
|
|
norm_path.write_text(text, encoding="utf-8")
|
|
return pre_path, norm_path
|
|
|
|
def preprocess(self, src_path: str | Path, out_dir: str | Path,
|
|
copybook_dirs: list[str | Path] | None = None) -> str:
|
|
"""Step 1a: gixpp プリプロセス"""
|
|
out_dir = Path(out_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
src_path = Path(src_path)
|
|
|
|
self._copybook_dirs = [Path(d) for d in copybook_dirs] if copybook_dirs else None
|
|
|
|
_, norm_path = self._normalize_source(src_path)
|
|
out_path = out_dir / f"{src_path.stem}_pp.cbl"
|
|
|
|
cmd = [str(self.gixpp_path), "-i", str(norm_path), "-o", str(out_path), "-e"]
|
|
if self._copybook_dirs:
|
|
for d in self._copybook_dirs:
|
|
cmd += ["-I", str(d)]
|
|
logger.info(f" gixpp: {' '.join(cmd)}")
|
|
r = subprocess.run(cmd, capture_output=True, timeout=30, env=self._build_env())
|
|
if r.returncode != 0:
|
|
err = r.stderr.decode("utf-8", "replace")[:500]
|
|
raise RuntimeError(f"gixpp failed (rc={r.returncode}): {err}")
|
|
return str(out_path)
|
|
|
|
def compile(self, pp_path: str | Path, exe_path: str | Path,
|
|
copybook_dirs: list[str | Path] | None = None,
|
|
extra_srcs: list[str | Path] | None = None) -> GixsqlBuildResult:
|
|
"""Step 1b: cobc 编译（gixsql 链接）"""
|
|
pp_path = Path(pp_path)
|
|
exe_path = Path(exe_path)
|
|
exe_dir = exe_path.parent
|
|
exe_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Functions that the preprocessed COBOL actually CALLs
|
|
gixsql_k = [
|
|
"-K", "GIXSQLStartSQL",
|
|
"-K", "GIXSQLSetSQLParams", "-K", "GIXSQLSetResultParams",
|
|
"-K", "GIXSQLExecParams", "-K", "GIXSQLExec",
|
|
"-K", "GIXSQLExecSelectIntoOne",
|
|
"-K", "GIXSQLEndSQL",
|
|
"-K", "GIXSQLConnect",
|
|
"-K", "GIXSQLCursorDeclare",
|
|
"-K", "GIXSQLCursorDeclareParams",
|
|
"-K", "GIXSQLCursorFetchOne",
|
|
"-K", "GIXSQLCursorOpen",
|
|
"-K", "GIXSQLCursorClose",
|
|
]
|
|
cmd = [
|
|
self.cobc_cmd, "-x",
|
|
"-L", str(self.lib_path.resolve()),
|
|
*gixsql_k,
|
|
"-l", "gixsql",
|
|
]
|
|
if copybook_dirs:
|
|
for d in copybook_dirs:
|
|
cmd += ["-I", str(d)]
|
|
if self.compile_flags:
|
|
cmd += self.compile_flags.split()
|
|
cmd += ["-o", str(exe_path)]
|
|
cmd.append(str(pp_path))
|
|
if extra_srcs:
|
|
for s in extra_srcs:
|
|
cmd.append(str(s))
|
|
|
|
logger.info(f" cobc: {' '.join(cmd)}")
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, timeout=60, env=self._build_env(),
|
|
cwd=str(exe_dir))
|
|
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
|
r.stderr.decode("utf-8", "replace"))
|
|
if r.returncode != 0:
|
|
return GixsqlBuildResult(False, log=log[:1000])
|
|
# .gcno files are generated in exe_dir (compile CWD). No need to copy.
|
|
return GixsqlBuildResult(True, exe_path=str(exe_path), log=log[:500])
|
|
except subprocess.TimeoutExpired:
|
|
return GixsqlBuildResult(False, log="Compile timeout (60s)")
|
|
|
|
def run(self, exe_path: str | Path, work_dir: str | Path,
|
|
db_path: str | Path,
|
|
input_dir: str | Path | None = None,
|
|
timeout: int = 30,
|
|
cobol_lib_path: str | Path | None = None,
|
|
env_overrides: dict[str, str] | None = None) -> GixsqlRunResult:
|
|
"""Step 3: COBOL DB プログラム実行"""
|
|
exe_path = Path(exe_path)
|
|
work_dir = Path(work_dir)
|
|
db_path = Path(db_path)
|
|
work_dir.mkdir(parents=True, exist_ok=True)
|
|
exe_dir = exe_path.parent
|
|
exe_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
lib_path = Path(self.lib_path)
|
|
|
|
# Deploy correct DLLs to exe_dir so loader finds them first
|
|
# Search lib_path first, then fall back to gixpp bin dir for runtime DLLs
|
|
gixpp_dir = self.gixpp_path.parent
|
|
dll_names = [
|
|
"libgixsql.dll", "libgixsql-sqlite.dll",
|
|
"libgcc_s_dw2-1.dll", "libstdc++-6.dll",
|
|
"libwinpthread-1.dll", "libiconv-2.dll",
|
|
"libintl-8.dll", "zlib1.dll",
|
|
]
|
|
for name in dll_names:
|
|
src = lib_path / name
|
|
if not src.exists():
|
|
src = gixpp_dir / name
|
|
if src.exists():
|
|
dst = exe_dir / name
|
|
if not dst.exists() or dst.stat().st_size != src.stat().st_size:
|
|
dst.write_bytes(src.read_bytes())
|
|
|
|
# libfmt.dll search: TEMP fallback → lib_path → gixpp bin → x86/gcc subdirectory
|
|
fmt_src = Path(os.environ.get("TEMP", "")) / "zan_dll" / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
fmt_src = lib_path / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
fmt_src = gixpp_dir / "libfmt.dll"
|
|
if not fmt_src.exists():
|
|
# gixsql binary package variant: <gixpp-parent>/lib/x86/gcc/libfmt.dll
|
|
fmt_src = gixpp_dir.parent / "lib" / "x86" / "gcc" / "libfmt.dll"
|
|
if fmt_src.exists():
|
|
(exe_dir / "libfmt.dll").write_bytes(fmt_src.read_bytes())
|
|
|
|
env = self._build_env()
|
|
env["GIXSQL_DB_PATH"] = str(db_path)
|
|
if cobol_lib_path:
|
|
env["COB_LIBRARY_PATH"] = str(cobol_lib_path)
|
|
if env_overrides:
|
|
env.update(env_overrides)
|
|
|
|
if input_dir:
|
|
idir = Path(input_dir)
|
|
if idir.exists():
|
|
for f in idir.iterdir():
|
|
if f.is_file():
|
|
dst = work_dir / f.name
|
|
dst.write_bytes(f.read_bytes())
|
|
|
|
cmd = [str(exe_path)]
|
|
logger.info(f" run: {' '.join(cmd)} (cwd={work_dir}, db={db_path})")
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, timeout=timeout,
|
|
cwd=str(work_dir), env=env)
|
|
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
|
r.stderr.decode("utf-8", "replace"))
|
|
ok = r.returncode == 0 or r.returncode == 1
|
|
return GixsqlRunResult(ok, returncode=r.returncode,
|
|
db_path=str(db_path), log=log[:1000])
|
|
except subprocess.TimeoutExpired:
|
|
return GixsqlRunResult(False, log="Run timeout")
|
|
|
|
def read_db_tables(self, db_path: str | Path,
|
|
table_names: list[str]) -> list[GixsqlTableData]:
|
|
"""Step 4: SQLite DB から全テーブル読み取り"""
|
|
db_path = Path(db_path)
|
|
if not db_path.exists():
|
|
logger.warning(f" DB not found: {db_path}")
|
|
return []
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
results = []
|
|
for tname in table_names:
|
|
try:
|
|
rows = conn.execute(f"SELECT * FROM [{tname}]").fetchall()
|
|
results.append(GixsqlTableData(
|
|
table_name=tname,
|
|
rows=[dict(r) for r in rows],
|
|
))
|
|
except sqlite3.OperationalError as e:
|
|
logger.warning(f" Table {tname} not found: {e}")
|
|
conn.close()
|
|
return results
|