feat: DB管线补全 + 新增orchestrator_db/program_schema/to_sql + 清理临时脚本
This commit is contained in:
hangshuo652
@@ -0,0 +1,425 @@
|
||||
"""Gixsql CBL Runner — gixpp + cobc pipeline for DB COBOL programs."""
|
||||
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GixsqlBuildResult:
|
||||
success: bool
|
||||
exe_path: str = ""
|
||||
log: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class GixsqlRunResult:
|
||||
success: bool
|
||||
returncode: int = -1
|
||||
db_path: str = ""
|
||||
log: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class GixsqlTableData:
|
||||
table_name: str = ""
|
||||
rows: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
class GixsqlCobolRunner:
|
||||
"""gixpp + cobc 管线：预处理 → 编译 → 运行 → DB读取"""
|
||||
|
||||
def __init__(self, gixpp_path: str | Path, lib_path: str | Path,
|
||||
cobc_cmd: str = "cobc",
|
||||
compile_flags: str = ""):
|
||||
self.gixpp_path = Path(gixpp_path)
|
||||
self.lib_path = Path(lib_path)
|
||||
self.cobc_cmd = cobc_cmd
|
||||
self.compile_flags = compile_flags
|
||||
|
||||
def _build_env(self) -> dict:
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = str(self.lib_path)
|
||||
if "PATH" in env:
|
||||
env["PATH"] = str(self.lib_path) + ";" + env["PATH"]
|
||||
else:
|
||||
env["PATH"] = str(self.lib_path)
|
||||
return env
|
||||
|
||||
def _expand_copy_replacing(self, text: str, search_dirs: list[Path]) -> str:
|
||||
"""Expand COPY ... REPLACING statements inline (Python-side).
|
||||
gixpp's ESQL parser chokes on COPY with REPLACING pseudo-text (==...==).
|
||||
"""
|
||||
def _resolve_copy(m: re.Match) -> str:
|
||||
name = m.group(1)
|
||||
for d in search_dirs:
|
||||
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
||||
cp = d / f"{name}{ext}"
|
||||
if cp.exists():
|
||||
cpy_text = cp.read_text(encoding="utf-8-sig")
|
||||
replaces_text = m.group(2)
|
||||
if replaces_text:
|
||||
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replaces_text)
|
||||
for old_txt, new_txt in pairs:
|
||||
cpy_text = cpy_text.replace(old_txt, new_txt)
|
||||
return cpy_text
|
||||
logger.warning(f" COPY {name} not found in {search_dirs}")
|
||||
return f" * COPY {name} NOT FOUND"
|
||||
|
||||
text = re.sub(
|
||||
r'^ {6,}COPY\s+(\w+)\s+REPLACING\s+(.+?)\.$',
|
||||
_resolve_copy,
|
||||
text,
|
||||
flags=re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
return text
|
||||
|
||||
def _expand_all_copies(self, text: str, search_dirs: list[Path]) -> str:
|
||||
"""Expand ALL COPY statements (including COPY SQLCA) — replaces cobc -E."""
|
||||
def _resolve_copy_cb(m: re.Match) -> str:
|
||||
name = m.group(1).strip().upper()
|
||||
replacing_text = m.group(2)
|
||||
# Try each search dir
|
||||
for d in search_dirs:
|
||||
for ext in (".cpy", ".CPY", ".cbl", ".CBL", ""):
|
||||
cp = d / f"{name}{ext}"
|
||||
if cp.exists():
|
||||
cpy_text = cp.read_text(encoding="utf-8-sig")
|
||||
if replacing_text:
|
||||
pairs = re.findall(r'==(.*?)==\s+BY\s+==(.*?)==', replacing_text)
|
||||
for old_txt, new_txt in pairs:
|
||||
cpy_text = cpy_text.replace(old_txt, new_txt)
|
||||
return cpy_text
|
||||
# If SQLCA not found in copybook dirs, provide inline definition
|
||||
if name == "SQLCA":
|
||||
return (
|
||||
" 01 SQLCA.\n"
|
||||
" 05 SQLCAID PIC X(8).\n"
|
||||
" 05 SQLCABC PIC S9(9) COMP.\n"
|
||||
" 05 SQLCODE PIC S9(9) COMP.\n"
|
||||
" 05 SQLERRM.\n"
|
||||
" 49 SQLERRML PIC S9(4) COMP.\n"
|
||||
" 49 SQLERRMC PIC X(256).\n"
|
||||
" 05 SQLERRP PIC X(8).\n"
|
||||
" 05 SQLERRD PIC S9(9) COMP OCCURS 6.\n"
|
||||
" 05 SQLWARN.\n"
|
||||
" 10 SQLWARN0 PIC X(1).\n"
|
||||
" 10 SQLWARN1 PIC X(1).\n"
|
||||
" 10 SQLWARN2 PIC X(1).\n"
|
||||
" 10 SQLWARN3 PIC X(1).\n"
|
||||
" 10 SQLWARN4 PIC X(1).\n"
|
||||
" 10 SQLWARN5 PIC X(1).\n"
|
||||
" 10 SQLWARN6 PIC X(1).\n"
|
||||
" 10 SQLWARN7 PIC X(1).\n"
|
||||
" 05 SQLEXT PIC X(8).\n"
|
||||
)
|
||||
logger.warning(f" COPY {name} not found in {search_dirs}")
|
||||
return f" * COPY {name} NOT FOUND\n"
|
||||
|
||||
# Expand COPY name. and COPY name REPLACING ... .
|
||||
text = re.sub(
|
||||
r'^ {6,}COPY\s+(\w+(?:-\w+)*)\s*(REPLACING\s+.+?)?\.$',
|
||||
_resolve_copy_cb,
|
||||
text,
|
||||
flags=re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
return text
|
||||
|
||||
def _normalize_source(self, src_path: Path) -> tuple[Path, Path]:
|
||||
"""Pre-process COBOL source for gixpp: expand COPY, strip comments, fix indentation.
|
||||
|
||||
Returns (pre_path, norm_path) where both point to the same gixpp-ready source.
|
||||
"""
|
||||
text = src_path.read_text(encoding="utf-8-sig")
|
||||
|
||||
# 1. Replace EXEC SQL INCLUDE SQLCA → COPY SQLCA
|
||||
text = re.sub(r'EXEC SQL INCLUDE SQLCA END-EXEC\.', ' COPY SQLCA.', text, flags=re.IGNORECASE)
|
||||
# Transform EXEC SQL CONNECT TO 'literal' → use WS variables (gixpp requires :variable not literal)
|
||||
def _transform_connect(m):
|
||||
inner = m.group(1)
|
||||
m_lit = re.search(r"CONNECT\s+TO\s+'([^']*)'", inner, re.IGNORECASE)
|
||||
if not m_lit:
|
||||
return m.group(0)
|
||||
conn_var = "WS-GIX-CONN"
|
||||
usr_var = "WS-GIX-USR"
|
||||
new_inner = re.sub(
|
||||
r"CONNECT\s+TO\s+'[^']*'",
|
||||
f"CONNECT TO :{conn_var} USER :{usr_var}",
|
||||
inner,
|
||||
flags=re.IGNORECASE
|
||||
)
|
||||
# Short absolute path under C:\Temp\gix\ (no Chinese chars, fits col 72).
|
||||
# The orchestrator creates the DB at the same path so they match.
|
||||
from pathlib import Path as _Path
|
||||
pid = _Path(src_path).stem
|
||||
gix_root = _Path("C:/Temp/gix")
|
||||
gix_root.mkdir(parents=True, exist_ok=True)
|
||||
db_path = gix_root / f"{pid}.db"
|
||||
conn_val = "sqlite:///" + str(db_path).replace("\\", "/")
|
||||
return (f"MOVE '{conn_val}' TO {conn_var}\n"
|
||||
f" MOVE 'gix' TO {usr_var}\n"
|
||||
f" EXEC SQL\n"
|
||||
f" {new_inner.strip()}\n"
|
||||
f" END-EXEC.")
|
||||
text = re.sub(r'(?is)EXEC SQL(.*?CONNECT\s+TO.*?)END-EXEC\.', _transform_connect, text)
|
||||
# Add WS-GIX-* after COPY SQLCA
|
||||
text = re.sub(
|
||||
r"^(\s*COPY SQLCA\.)",
|
||||
r"\1\n"
|
||||
r" 01 WS-GIX-VARS.\n"
|
||||
r" 03 WS-GIX-CONN PIC X(256).\n"
|
||||
r" 03 WS-GIX-USR PIC X(16).",
|
||||
text,
|
||||
flags=re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
|
||||
# 2. Expand ALL COPY statements in Python (replaces cobc -E)
|
||||
if hasattr(self, '_copybook_dirs') and self._copybook_dirs:
|
||||
text = self._expand_all_copies(text, self._copybook_dirs)
|
||||
|
||||
# 3. Strip ALL comment lines (* in any column 7-11)
|
||||
text = re.sub(r'^[ \t]{0,10}\*.*\n?', '', text, flags=re.MULTILINE)
|
||||
|
||||
# 4. Collapse multiple spaces between keywords
|
||||
lines = []
|
||||
for line in text.splitlines(keepends=True):
|
||||
line = re.sub(r'(\b\w+)\s{2,}(\b\w+\b)', lambda m: f'{m.group(1)} {m.group(2)}', line)
|
||||
lines.append(line)
|
||||
text = ''.join(lines)
|
||||
|
||||
# 5. Normalize DIVISION/SECTION headers to start at column 8 (7-space indent)
|
||||
# gixpp fixed-format scanner requires headers in Area A (columns 8-11).
|
||||
def _fix_header(m):
|
||||
return ' ' + m.group(1).lstrip()
|
||||
text = re.sub(
|
||||
r'^ {6,12}((?:IDENTIFICATION|ENVIRONMENT|DATA|PROCEDURE)\s+DIVISION\.'
|
||||
r'|(?:FILE|WORKING-STORAGE|LINKAGE|CONFIGURATION|INPUT-OUTPUT)\s+SECTION\.'
|
||||
r'|(?:FILE-CONTROL|I-O-CONTROL)\.'
|
||||
r'|SOURCE-COMPUTER\.|OBJECT-COMPUTER\.|SPECIAL-NAMES\.'
|
||||
r')',
|
||||
_fix_header,
|
||||
text,
|
||||
flags=re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
|
||||
pre_path = src_path.parent / f"{src_path.stem}_pre.cbl"
|
||||
pre_path.write_text(text, encoding="utf-8")
|
||||
|
||||
# 6. No cobc -E — all COPY expansions done in Python above.
|
||||
# Use the pre_path directly as the norm_path (gixpp input).
|
||||
# 6. Fix SQL clause ordering in EXEC SQL blocks:
|
||||
# gixpp expects SELECT ... INTO ... FROM ... (INTO before FROM),
|
||||
# but some programs have SELECT ... FROM ... INTO ...
|
||||
def _fix_sql_from_into(m):
|
||||
block = m.group(0)
|
||||
# Only touch SELECT ... FROM ... INTO (not INSERT/DELETE)
|
||||
if re.match(r'\s*EXEC SQL\s+SELECT\b', block, re.MULTILINE | re.IGNORECASE):
|
||||
# Swap FROM line and INTO line within SELECT blocks
|
||||
block = re.sub(
|
||||
r'^(\s+)(FROM\b.*)\n(\s+)(INTO\b.*)$',
|
||||
r'\1\4\n\1\2',
|
||||
block,
|
||||
flags=re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
return block
|
||||
text = re.sub(r'(\s*EXEC SQL\n.*?\n\s*END-EXEC\.)', _fix_sql_from_into, text,
|
||||
flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 7. Add missing 01-level variables used in PROCEDURE DIVISION but
|
||||
# not defined in DATA DIVISION (source program defects).
|
||||
missing_vars = {
|
||||
'KIN06CLD': ' 01 WS-COL-IDX PIC 9(002).\n',
|
||||
}
|
||||
pid = src_path.stem.upper()
|
||||
if pid in missing_vars and not re.search(r'\b01\s+WS-COL-IDX\b', text):
|
||||
text = re.sub(
|
||||
r'^(\s*)(PROCEDURE\s+DIVISION)',
|
||||
lambda m: m.group(1) + missing_vars[pid] + '\n' + m.group(1) + m.group(2),
|
||||
text, count=1, flags=re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
|
||||
norm_path = src_path.parent / f"{src_path.stem}_norm.cbl"
|
||||
norm_path.write_text(text, encoding="utf-8")
|
||||
return pre_path, norm_path
|
||||
|
||||
def preprocess(self, src_path: str | Path, out_dir: str | Path,
|
||||
copybook_dirs: list[str | Path] | None = None) -> str:
|
||||
"""Step 1a: gixpp プリプロセス"""
|
||||
out_dir = Path(out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_path = Path(src_path)
|
||||
|
||||
self._copybook_dirs = [Path(d) for d in copybook_dirs] if copybook_dirs else None
|
||||
|
||||
_, norm_path = self._normalize_source(src_path)
|
||||
out_path = out_dir / f"{src_path.stem}_pp.cbl"
|
||||
|
||||
cmd = [str(self.gixpp_path), "-i", str(norm_path), "-o", str(out_path), "-e"]
|
||||
if self._copybook_dirs:
|
||||
for d in self._copybook_dirs:
|
||||
cmd += ["-I", str(d)]
|
||||
logger.info(f" gixpp: {' '.join(cmd)}")
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=30, env=self._build_env())
|
||||
if r.returncode != 0:
|
||||
err = r.stderr.decode("utf-8", "replace")[:500]
|
||||
raise RuntimeError(f"gixpp failed (rc={r.returncode}): {err}")
|
||||
return str(out_path)
|
||||
|
||||
def compile(self, pp_path: str | Path, exe_path: str | Path,
|
||||
copybook_dirs: list[str | Path] | None = None,
|
||||
extra_srcs: list[str | Path] | None = None) -> GixsqlBuildResult:
|
||||
"""Step 1b: cobc 编译（gixsql 链接）"""
|
||||
pp_path = Path(pp_path)
|
||||
exe_path = Path(exe_path)
|
||||
exe_dir = exe_path.parent
|
||||
exe_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Functions that the preprocessed COBOL actually CALLs
|
||||
gixsql_k = [
|
||||
"-K", "GIXSQLStartSQL",
|
||||
"-K", "GIXSQLSetSQLParams", "-K", "GIXSQLSetResultParams",
|
||||
"-K", "GIXSQLExecParams", "-K", "GIXSQLExec",
|
||||
"-K", "GIXSQLExecSelectIntoOne",
|
||||
"-K", "GIXSQLEndSQL",
|
||||
"-K", "GIXSQLConnect",
|
||||
]
|
||||
cmd = [
|
||||
self.cobc_cmd, "-x",
|
||||
"-L", str(self.lib_path),
|
||||
*gixsql_k,
|
||||
"-l", "gixsql",
|
||||
]
|
||||
if copybook_dirs:
|
||||
for d in copybook_dirs:
|
||||
cmd += ["-I", str(d)]
|
||||
if self.compile_flags:
|
||||
cmd += self.compile_flags.split()
|
||||
cmd += ["-o", str(exe_path)]
|
||||
cmd.append(str(pp_path))
|
||||
if extra_srcs:
|
||||
for s in extra_srcs:
|
||||
cmd.append(str(s))
|
||||
|
||||
logger.info(f" cobc: {' '.join(cmd)}")
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=60, env=self._build_env())
|
||||
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
||||
r.stderr.decode("utf-8", "replace"))
|
||||
if r.returncode != 0:
|
||||
return GixsqlBuildResult(False, log=log[:1000])
|
||||
# After successful compile, copy .gcno from CWD to exe_dir
|
||||
pp_stem = pp_path.stem # e.g. "KIN02UPD_pp"
|
||||
exe_stem = exe_path.stem # e.g. "KIN02UPD"
|
||||
copied = 0
|
||||
for gcno in Path.cwd().glob("*.gcno"):
|
||||
# Match .gcno files belonging to this compile (by subprogram name or pp_stem partial match)
|
||||
dst = exe_dir / gcno.name
|
||||
dst.write_bytes(gcno.read_bytes())
|
||||
copied += 1
|
||||
if copied:
|
||||
logger.debug(f" gcno copied: {copied} files to {exe_dir}")
|
||||
return GixsqlBuildResult(True, exe_path=str(exe_path), log=log[:500])
|
||||
except subprocess.TimeoutExpired:
|
||||
return GixsqlBuildResult(False, log="Compile timeout (60s)")
|
||||
|
||||
def run(self, exe_path: str | Path, work_dir: str | Path,
|
||||
db_path: str | Path,
|
||||
input_dir: str | Path | None = None,
|
||||
timeout: int = 30,
|
||||
cobol_lib_path: str | Path | None = None) -> GixsqlRunResult:
|
||||
"""Step 3: COBOL DB プログラム実行"""
|
||||
exe_path = Path(exe_path)
|
||||
work_dir = Path(work_dir)
|
||||
db_path = Path(db_path)
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
exe_dir = exe_path.parent
|
||||
exe_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
lib_path = Path(self.lib_path)
|
||||
|
||||
# Deploy correct DLLs to exe_dir so loader finds them first
|
||||
# Search lib_path first, then fall back to gixpp bin dir for runtime DLLs
|
||||
gixpp_dir = self.gixpp_path.parent
|
||||
dll_names = [
|
||||
"libgixsql.dll", "libgixsql-sqlite.dll",
|
||||
"libgcc_s_dw2-1.dll", "libstdc++-6.dll",
|
||||
"libwinpthread-1.dll", "libiconv-2.dll",
|
||||
"libintl-8.dll", "zlib1.dll",
|
||||
]
|
||||
for name in dll_names:
|
||||
src = lib_path / name
|
||||
if not src.exists():
|
||||
src = gixpp_dir / name
|
||||
if src.exists():
|
||||
dst = exe_dir / name
|
||||
if not dst.exists() or dst.stat().st_size != src.stat().st_size:
|
||||
dst.write_bytes(src.read_bytes())
|
||||
|
||||
# libfmt.dll search: TEMP fallback → lib_path → gixpp bin → x86/gcc subdirectory
|
||||
fmt_src = Path(os.environ.get("TEMP", "")) / "zan_dll" / "libfmt.dll"
|
||||
if not fmt_src.exists():
|
||||
fmt_src = lib_path / "libfmt.dll"
|
||||
if not fmt_src.exists():
|
||||
fmt_src = gixpp_dir / "libfmt.dll"
|
||||
if not fmt_src.exists():
|
||||
# gixsql binary package variant: <gixpp-parent>/lib/x86/gcc/libfmt.dll
|
||||
fmt_src = gixpp_dir.parent / "lib" / "x86" / "gcc" / "libfmt.dll"
|
||||
if fmt_src.exists():
|
||||
(exe_dir / "libfmt.dll").write_bytes(fmt_src.read_bytes())
|
||||
|
||||
env = self._build_env()
|
||||
env["GIXSQL_DB_PATH"] = str(db_path)
|
||||
if cobol_lib_path:
|
||||
env["COB_LIBRARY_PATH"] = str(cobol_lib_path)
|
||||
|
||||
if input_dir:
|
||||
idir = Path(input_dir)
|
||||
if idir.exists():
|
||||
for f in idir.iterdir():
|
||||
if f.is_file():
|
||||
dst = work_dir / f.name
|
||||
if not dst.exists():
|
||||
dst.write_bytes(f.read_bytes())
|
||||
|
||||
cmd = [str(exe_path)]
|
||||
logger.info(f" run: {' '.join(cmd)} (cwd={work_dir}, db={db_path})")
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=timeout,
|
||||
cwd=str(work_dir), env=env)
|
||||
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
||||
r.stderr.decode("utf-8", "replace"))
|
||||
ok = r.returncode == 0 or r.returncode == 1
|
||||
return GixsqlRunResult(ok, returncode=r.returncode,
|
||||
db_path=str(db_path), log=log[:1000])
|
||||
except subprocess.TimeoutExpired:
|
||||
return GixsqlRunResult(False, log="Run timeout")
|
||||
|
||||
def read_db_tables(self, db_path: str | Path,
|
||||
table_names: list[str]) -> list[GixsqlTableData]:
|
||||
"""Step 4: SQLite DB から全テーブル読み取り"""
|
||||
db_path = Path(db_path)
|
||||
if not db_path.exists():
|
||||
logger.warning(f" DB not found: {db_path}")
|
||||
return []
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
results = []
|
||||
for tname in table_names:
|
||||
try:
|
||||
rows = conn.execute(f"SELECT * FROM [{tname}]").fetchall()
|
||||
results.append(GixsqlTableData(
|
||||
table_name=tname,
|
||||
rows=[dict(r) for r in rows],
|
||||
))
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning(f" Table {tname} not found: {e}")
|
||||
conn.close()
|
||||
return results
|
||||
Reference in New Issue
Block a user