hangshuo652
54d4e81240
- TIME injection: change from spaces to '2500' (hour>23) for NUMVAL trigger - Runner: add .resolve() to work_dir to fix chdir+relative path breakage - Coverage: per-target field overrides for DP#9-#12 (START-DATE=20240115 etc.) - .gitignore: add compilation artifacts, temp scripts, test outputs
1317 lines
59 KiB
Python
1317 lines
59 KiB
Python
"""GixsqlOrchestrator — DB COBOL プログラムの全6Step実行"""
|
||
|
||
from __future__ import annotations
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sqlite3
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
from config import Config
|
||
from config.program_schema import ProgramSchema, load_schema, ScenarioDef
|
||
from cobol_testgen import extract_structure, generate_data
|
||
from cobol_testgen.flatfile import write_all_files, write_sysin_file
|
||
from cobol_testgen.file_io import read_output_file
|
||
from cobol_testgen.read import preprocess, resolve_copybooks, resolve_sql_includes, parse_file_control, parse_file_section, parse_data_division, extract_data_division, extract_procedure_division, scan_open_statements
|
||
from cobol_testgen.read import strip_exec_sql_from_data_div
|
||
from cobol_testgen.gcov import run_gcov
|
||
from cobol_testgen.coverage import run_coverage, generate_coverage_index
|
||
from cobol_testgen.design_mcdc import enum_paths as mcdc_enum_paths
|
||
from cobol_testgen.to_sql import collect_sql_meta, build_db_input
|
||
from cobol_testgen.core import extract_sql_assignments, classify_field_roles, _init_child_names
|
||
from cobol_testgen import expand_occurs
|
||
from cobol_testgen.design import get_term_type, generate_records
|
||
from cobol_testgen.output import output_json
|
||
from cobol_testgen.pipeline_bridge import build_branch_tree_fallback
|
||
import shutil
|
||
from data.diff_result import VerificationRun, FieldResult
|
||
from runners.gixsql_runner import GixsqlCobolRunner, GixsqlTableData
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
@dataclass
|
||
class DbPipelineResult:
|
||
"""DB 管线単体実行結果"""
|
||
program_id: str
|
||
step: int | float # pipeline step number
|
||
success: bool
|
||
message: str = ""
|
||
data: dict = field(default_factory=dict)
|
||
|
||
|
||
class GixsqlOrchestrator:
|
||
"""6Step DB 管线オーケストレーター"""
|
||
|
||
def __init__(self, config: Config, program_id: str,
|
||
cobol_src_dir: str | Path,
|
||
copybook_dirs: list[str | Path] | None = None,
|
||
work_dir: str | Path | None = None,
|
||
skip_jvm: bool = True):
|
||
self.config = config
|
||
self.program_id = program_id
|
||
self.cobol_src_dir = Path(cobol_src_dir)
|
||
self.copybook_dirs = copybook_dirs or []
|
||
self.skip_jvm = skip_jvm
|
||
v3_root = Path(__file__).parent # cobol-java-v3/
|
||
|
||
# Build artifacts in temp (ASCII-only, gixpp can't handle Chinese paths)
|
||
if work_dir is None:
|
||
temp = Path(os.environ.get("TEMP", "C:\\Temp"))
|
||
work_dir = temp / "gixsql_build" / program_id
|
||
self.work_dir = Path(work_dir)
|
||
|
||
# Runtime data under V3 (DB, flat files, CWD)
|
||
self.runtime_dir = v3_root / "runtime" / program_id
|
||
|
||
self.schema: ProgramSchema = load_schema(program_id)
|
||
|
||
self.runner = GixsqlCobolRunner(
|
||
gixpp_path=config.gixsql_path,
|
||
lib_path=config.gixsql_lib_path,
|
||
compile_flags=config.gixsql_compile_flags,
|
||
)
|
||
|
||
# Derive DB path: C:\Temp\gix\<program_id>.db (matches COBOL CONNECT TO, short enough for col 72)
|
||
self.db_path = Path("C:/Temp/gix") / f"{self.program_id}.db"
|
||
|
||
# Pipeline state
|
||
self.src_path: Optional[Path] = None
|
||
self.pp_path: Optional[Path] = None
|
||
self.exe_path: Optional[Path] = None
|
||
self.java_input_path: Optional[Path] = None
|
||
self._current_db_path: Optional[Path] = None # scenario-specific DB path
|
||
self._multi_run_gcov_data: dict[int, int] | None = None # merged multi-run gcov data
|
||
self.java_output_path: Optional[Path] = None
|
||
self.generated_records: list[dict] = []
|
||
self.generated_structure: dict | None = None
|
||
|
||
# ── Step 1: 環境整備（gixpp + compile） ──
|
||
|
||
def _copy_sources_to_workdir(self) -> tuple[Path, list[str]]:
|
||
"""Copy source + copybooks to ASCII-only workdir (gixpp can't handle Chinese paths)."""
|
||
src_dir = self.work_dir / "src"
|
||
src_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Copy main source
|
||
orig = self.cobol_src_dir / f"{self.program_id}.cbl"
|
||
ascii_src = src_dir / f"{self.program_id}.cbl"
|
||
if not ascii_src.exists():
|
||
ascii_src.write_bytes(orig.read_bytes())
|
||
self.src_path = ascii_src
|
||
|
||
# Copy copybooks
|
||
flat_cpy = []
|
||
for d in self.copybook_dirs:
|
||
pd = Path(d)
|
||
if pd.exists():
|
||
for f in pd.glob("*.cpy"):
|
||
dst = src_dir / f.name
|
||
if not dst.exists():
|
||
dst.write_bytes(f.read_bytes())
|
||
flat_cpy.append(str(dst))
|
||
|
||
# Copy SUB programs
|
||
v3_root = Path(__file__).parent
|
||
sub_dirs = [
|
||
self.cobol_src_dir,
|
||
self.cobol_src_dir.parent / "sub",
|
||
v3_root.parent / "cobol-tna-system" / "sub",
|
||
v3_root.parent / "production" / "sub",
|
||
]
|
||
for sub in self.schema.subprograms:
|
||
found = False
|
||
for sd in sub_dirs:
|
||
sp = sd / f"{sub}.cbl"
|
||
if sp.exists():
|
||
dst = src_dir / f"{sub}.cbl"
|
||
if not dst.exists():
|
||
dst.write_bytes(sp.read_bytes())
|
||
found = True
|
||
break
|
||
if not found:
|
||
logger.warning(f" SUB {sub}.cbl not found in {sub_dirs}")
|
||
|
||
return src_dir, flat_cpy
|
||
|
||
def step1_setup_environment(self) -> DbPipelineResult:
|
||
"""gixpp 前処理 → cobc コンパイル"""
|
||
try:
|
||
ascii_dir, flat_cpy = self._copy_sources_to_workdir()
|
||
src = ascii_dir / f"{self.program_id}.cbl"
|
||
|
||
pp = self.runner.preprocess(src, self.work_dir / "preprocessed",
|
||
copybook_dirs=[ascii_dir])
|
||
self.pp_path = Path(pp)
|
||
|
||
exe = self.work_dir / "bin" / f"{self.program_id}.exe"
|
||
extra_srcs = []
|
||
for sub in self.schema.subprograms:
|
||
sp = ascii_dir / f"{sub}.cbl"
|
||
if sp.exists():
|
||
extra_srcs.append(sp)
|
||
|
||
result = self.runner.compile(
|
||
pp, exe,
|
||
copybook_dirs=[ascii_dir],
|
||
extra_srcs=extra_srcs,
|
||
)
|
||
log_dir = self.runtime_dir / "logs" / "compile"
|
||
log_dir.mkdir(parents=True, exist_ok=True)
|
||
log_dir.joinpath(f"{self.program_id}.log").write_text(
|
||
result.log, encoding='utf-8')
|
||
if result.success:
|
||
self.exe_path = Path(result.exe_path)
|
||
return DbPipelineResult(
|
||
self.program_id, 1, result.success,
|
||
message=result.log[:200],
|
||
data={"exe_path": str(exe), "log": result.log[:500]},
|
||
)
|
||
except Exception as e:
|
||
return DbPipelineResult(self.program_id, 1, False, str(e))
|
||
|
||
# ── Step 2: 入力データ生成 ──
|
||
|
||
def step2_generate_inputs(self, scenario: ScenarioDef | None = None) -> DbPipelineResult:
|
||
"""テストデータ生成 + フラットファイル出力 + DB初期化
|
||
|
||
Args:
|
||
scenario: 多輪実行時のシナリオ定義。None=単輪（従来動作）。
|
||
"""
|
||
try:
|
||
src_text = self.src_path.read_text(encoding="utf-8-sig")
|
||
# Use the pre-gixpp source for Lark parsing (gixpp output contains SQLCA etc.)
|
||
parse_text = self.pp_path.read_text(encoding="utf-8") if self.pp_path and self.pp_path.exists() else src_text
|
||
|
||
# COBOL 解析 + テストデータ生成（白盒 + 機能 + 策略 統合）
|
||
cbd = [str(d) for d in self.copybook_dirs]
|
||
st = extract_structure(src_text, copybook_dirs=cbd)
|
||
self.generated_structure = st
|
||
from cobol_testgen.data_merger import generate_all_data
|
||
v3_root = Path(__file__).parent
|
||
design_doc_dir = v3_root / "詳細設計書"
|
||
if not design_doc_dir.exists():
|
||
design_doc_dir = None
|
||
# LLMClient は API key が必要な場合のみ初期化（未設定時は None → スキップ）
|
||
llm = None
|
||
if hasattr(self.config, 'llm_model') and self.config.llm_model:
|
||
from agents.llm import LLMClient
|
||
try:
|
||
llm = LLMClient(model=self.config.llm_model, timeout=self.config.llm_timeout)
|
||
except Exception:
|
||
pass
|
||
recs = generate_all_data(
|
||
program_id=self.program_id,
|
||
src_text=src_text,
|
||
st=st,
|
||
copybook_dirs=cbd,
|
||
design_doc_dir=str(design_doc_dir) if design_doc_dir else None,
|
||
llm_client=llm,
|
||
config=self.config,
|
||
)
|
||
|
||
# Post-process: link R02 cancel APPL-IDs to matching R01 insert APPL-IDs
|
||
for rec in recs:
|
||
if 'R02APPL-ID' in rec and 'R01APPL-ID' in rec:
|
||
rec['R02APPL-ID'] = rec['R01APPL-ID']
|
||
|
||
# シナリオに応じた DB パス
|
||
if scenario:
|
||
db_path = Path("C:/Temp/gix") / f"{self.program_id}_{scenario.id}.db"
|
||
self._current_db_path = db_path
|
||
else:
|
||
db_path = self.db_path
|
||
self._current_db_path = None
|
||
|
||
# DB 初期データ構築: clean stale DB first
|
||
if db_path.exists():
|
||
db_path.unlink()
|
||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||
self._init_database(db_path)
|
||
|
||
# DB 初期行投入（DELETE/UPDATE が作用する行、SELECT が返す行）
|
||
self._populate_database(db_path, src_text, recs)
|
||
# P5: inject duplicate-PK rows (scenario で制御)
|
||
if scenario is None or scenario.inject_duplicate_pk:
|
||
self._inject_sql_error_rows(db_path, recs)
|
||
|
||
# Patch R01LINE records with EMP-IDs matching the record's own EMP-ID
|
||
for i, rec in enumerate(recs):
|
||
line = rec.get('R01LINE', '')
|
||
if not line:
|
||
continue
|
||
parts = line.split(',', 1)
|
||
if len(parts) != 2:
|
||
continue
|
||
# Get EMP-ID from the record's own field (e.g., R01EMP-ID)
|
||
emp_id = rec.get('R01EMP-ID', '')
|
||
if not emp_id or emp_id == '00000000':
|
||
emp_id = rec.get('HV-EMP-ID', '')
|
||
if not emp_id or emp_id == '00000000':
|
||
emp_id = f"EMP{str(i).zfill(5)}"
|
||
# First record: empty EMP-ID to trigger R01EMP-ID = SPACE path (DP#12)
|
||
if i == 0:
|
||
rec['R01LINE'] = f"{' '*8},{parts[1]}"
|
||
rec['R01EMP-ID'] = ' ' * len(emp_id)
|
||
else:
|
||
rec['R01LINE'] = f"{emp_id.ljust(8)},{parts[1]}"
|
||
rec['R01EMP-ID'] = emp_id
|
||
# Inject duplicate EMP-IDs for last 3 records to trigger AGG UPDATE
|
||
# path (DP#19-#20). Use the LARGEST EMP-ID from the last 8 of the
|
||
# sorted unique list, so the dup ID falls in the last TARGET card
|
||
# batch (which sets TARGET-COUNT). Set R01DATE to keep YEAR_MONTH
|
||
# the same but different day avoids PK conflict in DAILY_RECORDS.
|
||
# Works for both FIXED (KIN07REC) and LINE SEQUENTIAL formats.
|
||
if len(recs) > 3:
|
||
# Collect unique EMP-IDs (non-blank, non-zero)
|
||
all_ids = set()
|
||
for r in recs:
|
||
eid = r.get('R01EMP-ID', '')
|
||
if eid and eid.strip() and eid != '00000000':
|
||
all_ids.add(eid)
|
||
sorted_ids = sorted(all_ids)
|
||
# Pick from the last chunk (matches last TARGET card batch)
|
||
# T chunk = 8 per card, so last chunk index = len % 8 or 8
|
||
n = len(sorted_ids)
|
||
last_chunk_start = n - (n % 8 or 8)
|
||
dup_eid = sorted_ids[last_chunk_start] if sorted_ids else ''
|
||
# Find a record with this EMP-ID for its DATE
|
||
src_date = ''
|
||
for r in recs:
|
||
if r.get('R01EMP-ID', '') == dup_eid:
|
||
src_date = r.get('R01DATE', '')
|
||
break
|
||
dup_date = src_date
|
||
for j in range(max(1, len(recs)-3), len(recs)):
|
||
rec = recs[j]
|
||
if not dup_eid:
|
||
continue
|
||
rec['R01EMP-ID'] = dup_eid
|
||
# FIXED format: keep same YEAR_MONTH but different day
|
||
# to avoid PK conflict in DAILY_RECORDS INSERT.
|
||
if dup_date and len(dup_date) >= 6:
|
||
dup_ym = dup_date[:6]
|
||
orig_date = rec.get('R01DATE', '')
|
||
if orig_date and len(orig_date) >= 8:
|
||
rec['R01DATE'] = dup_ym + orig_date[6:8]
|
||
else:
|
||
rec['R01DATE'] = dup_ym + '01'
|
||
# LINE SEQUENTIAL format: patch R01LINE
|
||
line = rec.get('R01LINE', '')
|
||
if line:
|
||
parts = line.split(',', 1)
|
||
if len(parts) == 2:
|
||
rec['R01LINE'] = f"{dup_eid.ljust(8)},{parts[1]}"
|
||
|
||
# ── Coverage-driven data modifications (per-scenario) ──
|
||
# Normal scenario or legacy single-run: no modifications needed.
|
||
# Collision scenario: INSERT duplicate, OVT-MONTHLY match, COMMIT threshold.
|
||
# Abnormal scenario: orphan cancel ABEND (last, to avoid polluting other branches).
|
||
|
||
apply_collision = scenario is not None and scenario.id == "collision"
|
||
apply_abnormal = scenario is not None and scenario.id == "abnormal"
|
||
|
||
if apply_collision:
|
||
# #10 T: Ensure >= 50 R01 records for COMMIT threshold (CNS-COMMIT-CNT=50)
|
||
r01_recs = [r for r in recs if 'R01APPL-ID' in r]
|
||
r01_count = len(r01_recs)
|
||
if r01_count < 50:
|
||
template = r01_recs[-1].copy() if r01_recs else {}
|
||
needed = 50 - r01_count
|
||
for i in range(needed):
|
||
nr = {}
|
||
for key, val in template.items():
|
||
if not key.startswith('R02'):
|
||
nr[key] = val
|
||
nr['R01APPL-ID'] = f"X50{str(i).zfill(5)}"
|
||
if 'R01EMP-ID' in nr:
|
||
nr['R01EMP-ID'] = str(int(str(nr.get('R01EMP-ID', '0') or '0')) + i + 10000).zfill(8)
|
||
recs.append(nr)
|
||
r01_recs = [r for r in recs if 'R01APPL-ID' in r]
|
||
logger.info(f" Coverage #10T: added {needed} R01-only records -> {len(r01_recs)} total")
|
||
|
||
# #8 T: Two R01 records with same APPL-ID -> 2nd INSERT collides -> UPDATE
|
||
if len(r01_recs) >= 4:
|
||
dup_appl_id = 'COLISN01'
|
||
for idx in (2, 3):
|
||
r01_recs[idx]['R01APPL-ID'] = dup_appl_id
|
||
if 'R02APPL-ID' in r01_recs[idx]:
|
||
r01_recs[idx]['R02APPL-ID'] = dup_appl_id
|
||
logger.info(f" Coverage #8T: set APPL-ID={dup_appl_id} on records [2]&[3] for INSERT duplicate")
|
||
|
||
# #11 T: Two R01 records with same (EMP-ID, APPL-DATE, OVT-TYPE)
|
||
if len(r01_recs) >= 2:
|
||
match_emp = r01_recs[1].get('R01EMP-ID', '00000000').strip() or '00000000'
|
||
match_date = r01_recs[1].get('R01APPL-DATE', '00000000').strip() or '00000000'
|
||
match_type = r01_recs[1].get('R01OVT-TYPE', '1').strip() or '1'
|
||
r01_recs[0]['R01EMP-ID'] = match_emp
|
||
r01_recs[0]['R01APPL-DATE'] = match_date
|
||
r01_recs[0]['R01OVT-TYPE'] = match_type
|
||
r01_recs[1]['R01EMP-ID'] = match_emp
|
||
r01_recs[1]['R01APPL-DATE'] = match_date
|
||
r01_recs[1]['R01OVT-TYPE'] = match_type
|
||
logger.info(
|
||
f" Coverage #11T: unified (EMP={match_emp} DATE={match_date}"
|
||
f" TYPE={match_type}) for R01 records [0]&[1]"
|
||
)
|
||
|
||
if apply_abnormal:
|
||
# #14 T: Last R02 record has non-existent APPL-ID -> orphan cancel ABEND
|
||
# NOTE: ABEND prevents 3000STPSOR (#19 T/F); covered by normal scenario.
|
||
r02_recs = [r for r in recs if 'R02APPL-ID' in r]
|
||
if r02_recs:
|
||
r02_recs[-1]['R02APPL-ID'] = 'ZZZZZZZZ'
|
||
logger.info(f" Coverage #14T: set last R02 APPL-ID='ZZZZZZZZ' for orphan cancel")
|
||
|
||
# 出力先ディレクトリ（シナリオ毎に分離）
|
||
run_label = f"run_{scenario.id}" if scenario else ""
|
||
output_root = self.work_dir / run_label if scenario else self.work_dir
|
||
input_dir = output_root / "main" / "input"
|
||
input_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# フラットファイル書き出し（全シナリオ同一）
|
||
flats = write_all_files(recs, src_text, input_dir,
|
||
copybook_dirs=[str(d) for d in self.copybook_dirs])
|
||
|
||
# SYSIN 設定ファイル生成（シナリオ毎に run_cfg を渡す）
|
||
run_cfg = None
|
||
if scenario:
|
||
run_cfg = {
|
||
"period": scenario.sysin.period,
|
||
"include_invalid_period": scenario.sysin.include_invalid_period,
|
||
"modes": scenario.sysin.modes,
|
||
}
|
||
sysin_path = write_sysin_file(recs, src_text, input_dir,
|
||
copybook_dirs=[str(d) for d in self.copybook_dirs],
|
||
run_cfg=run_cfg)
|
||
if sysin_path:
|
||
logger.info(f" SYSIN file written: {sysin_path}")
|
||
flats.append(("SYSIN", sysin_path, 0))
|
||
|
||
# Pre-populate MONTHLY_ABSENCE with matching EMP-ID/YEAR-MONTH for
|
||
# SELECT COUNT(*) → HV-CNT > 0 → UPDATE path (DP#27). Must run AFTER
|
||
# write_all_files (so the R01 flat file exists).
|
||
db_for_seed = self._current_db_path or self.db_path
|
||
self._seed_matching_monthly_rows(db_for_seed, recs, max_seed=20,
|
||
r01_dir=input_dir)
|
||
|
||
# ── JSON 出力（Java 検証用） ──
|
||
try:
|
||
pp = preprocess(src_text, extra_search_paths=cbd)
|
||
data_div = extract_data_division(pp)
|
||
data_fields = parse_data_division(data_div) if data_div else []
|
||
fdict = []
|
||
for f in data_fields:
|
||
fdict.append({
|
||
'name': f.name, 'level': f.level, 'pic': f.pic,
|
||
'pic_info': {
|
||
'type': f.pic_info.type if f.pic_info else 'unknown',
|
||
'digits': f.pic_info.digits if f.pic_info else 0,
|
||
'decimal': f.pic_info.decimal if f.pic_info else 0,
|
||
'length': f.pic_info.length if f.pic_info else 0,
|
||
'signed': f.pic_info.signed if f.pic_info else False,
|
||
},
|
||
'section': f.section, 'occurs': f.occurs_count,
|
||
'occurs_depending': f.occurs_depending,
|
||
'value': f.value, 'values': f.values,
|
||
'redefines': f.redefines, 'usage': f.usage,
|
||
})
|
||
fdict = expand_occurs(fdict)
|
||
proc_div = extract_procedure_division(pp)
|
||
branch_tree, assignments = build_branch_tree_fallback(proc_div, fdict)
|
||
sql_assigns = extract_sql_assignments(src_text)
|
||
for tgt, asgn_list in sql_assigns.items():
|
||
for asgn in asgn_list:
|
||
assignments.setdefault(tgt, []).append(asgn)
|
||
|
||
# fd_fields / field_to_fd
|
||
file_sec = parse_file_section(pp) or {}
|
||
fd_fields = {}
|
||
field_to_fd = {}
|
||
for fd_name, rec_names in file_sec.items():
|
||
fds = []
|
||
seen = set()
|
||
for rec in rec_names:
|
||
if rec not in seen:
|
||
fds.append(rec)
|
||
seen.add(rec)
|
||
for child in _init_child_names(rec, fdict):
|
||
if child not in seen:
|
||
fds.append(child)
|
||
seen.add(child)
|
||
fd_fields[fd_name] = fds
|
||
for child in fds:
|
||
field_to_fd[child] = fd_name
|
||
|
||
open_dir = scan_open_statements(proc_div) if proc_div else {}
|
||
|
||
# Roles + path info + termination types
|
||
roles = classify_field_roles(branch_tree, assignments, fdict,
|
||
source=src_text,
|
||
proc_text=proc_div)
|
||
branch_paths = mcdc_enum_paths(branch_tree, fdict)
|
||
path_infos = [(c, a, get_term_type(c)[1]) for c, a in branch_paths]
|
||
json_records, _, term_types = generate_records(
|
||
path_infos, fdict, assignments, file_sec=file_sec)
|
||
|
||
# DB input for JSON
|
||
data_div2, declared_columns = strip_exec_sql_from_data_div(data_div)
|
||
sql_meta = collect_sql_meta(assignments, declared_columns)
|
||
db_input = None
|
||
if sql_meta:
|
||
db_input = build_db_input(
|
||
branch_paths, fdict, assignments,
|
||
sql_meta, declared_columns, records=recs)
|
||
|
||
# Write main JSON（シナリオ毎に分離）
|
||
json_outdir = output_root / "main" / "json"
|
||
json_outdir.mkdir(parents=True, exist_ok=True)
|
||
json_path = json_outdir / f"{self.program_id}.json"
|
||
output_json(json_records, json_path, roles,
|
||
fd_fields=fd_fields, field_to_fd=field_to_fd,
|
||
open_dir=open_dir, term_types=term_types,
|
||
db_input=db_input, data_fields=fdict)
|
||
|
||
logger.info(f" JSON output: {json_path}")
|
||
flats.append(("JSON", json_path, 0))
|
||
except Exception as ej:
|
||
logger.warning(f" JSON output skipped: {ej}")
|
||
|
||
self.generated_records = recs
|
||
|
||
db_for_result = str(self._current_db_path or self.db_path)
|
||
return DbPipelineResult(
|
||
self.program_id, 2, True,
|
||
data={"records": len(recs), "flat_files": len(flats),
|
||
"db_path": db_for_result},
|
||
)
|
||
except Exception as e:
|
||
return DbPipelineResult(self.program_id, 2, False, str(e))
|
||
|
||
# ── Step 3: COBOL 実行 ──
|
||
|
||
def step3_run_cobol(self, scenario: ScenarioDef | None = None) -> DbPipelineResult:
|
||
"""COBOL DB プログラム実行（環境変数で入出力先を振り分け）
|
||
|
||
Args:
|
||
scenario: 多輪実行時のシナリオ。None=単輪。
|
||
"""
|
||
if not self.exe_path or not self.exe_path.exists():
|
||
return DbPipelineResult(self.program_id, 3, False,
|
||
"exe not found (run step1 first)")
|
||
|
||
# シナリオ毎の出力先
|
||
run_label = f"run_{scenario.id}" if scenario else ""
|
||
run_dir = self.runtime_dir / run_label if scenario else self.runtime_dir
|
||
input_dir = run_dir / "main" / "input"
|
||
output_dir = run_dir / "main" / "output"
|
||
gcov_dir = self.runtime_dir / "gcov"
|
||
input_dir.mkdir(parents=True, exist_ok=True)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
gcov_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# シナリオ毎の CWD = run_{id}/、単輪時は runtime_dir 直下
|
||
cwd = run_dir
|
||
|
||
# 入力ファイル（work_dir/run_{id}/main/input/ → runtime/run_{id}/main/input/）
|
||
gen_input_dir = self.work_dir / f"run_{scenario.id}" / "main" / "input" if scenario else self.work_dir / "main" / "input"
|
||
if gen_input_dir.exists():
|
||
for f in gen_input_dir.iterdir():
|
||
if f.is_file():
|
||
(input_dir / f.name).write_bytes(f.read_bytes())
|
||
|
||
# JSON 出力（work_dir/run_{id}/main/json/ → runtime/run_{id}/main/json/）
|
||
gen_json_dir = self.work_dir / f"run_{scenario.id}" / "main" / "json" if scenario else self.work_dir / "main" / "json"
|
||
if gen_json_dir.exists():
|
||
json_dir = run_dir / "main" / "json"
|
||
json_dir.mkdir(parents=True, exist_ok=True)
|
||
for f in gen_json_dir.iterdir():
|
||
if f.is_file() and f.suffix.lower() == '.json':
|
||
(json_dir / f.name).write_bytes(f.read_bytes())
|
||
|
||
# Scan ASSIGN TO + OPEN direction → build env overrides
|
||
assign_map = self._scan_assign_to()
|
||
env_overrides = {}
|
||
for fname, direction in assign_map.items():
|
||
if direction == "INPUT":
|
||
env_overrides[fname] = os.path.join("main", "input", fname)
|
||
else:
|
||
env_overrides[fname] = os.path.join("main", "output", fname)
|
||
|
||
# シナリオ毎の DB パス
|
||
db_path = self._current_db_path or self.db_path
|
||
# GIXSQL_DB_PATH が効かないため、デフォルト DB にシナリオ DB をコピーする
|
||
if scenario is not None and db_path != self.db_path:
|
||
if db_path.exists():
|
||
if self.db_path.exists():
|
||
self.db_path.unlink()
|
||
shutil.copy2(str(db_path), str(self.db_path))
|
||
db_path = self.db_path
|
||
|
||
# .gcda は CWD（= run_dir）に書き出されるので、実行後に gcov/run_{id}/ に移動する
|
||
|
||
# Subprogram DLLs
|
||
cobol_bin = Path(self.cobol_src_dir).parent / "bin"
|
||
|
||
result = self.runner.run(
|
||
self.exe_path, cwd,
|
||
db_path,
|
||
input_dir=None,
|
||
cobol_lib_path=str(cobol_bin) if cobol_bin.exists() else None,
|
||
env_overrides=env_overrides,
|
||
)
|
||
|
||
log_dir = self.runtime_dir / "logs"
|
||
log_dir.mkdir(parents=True, exist_ok=True)
|
||
log_dir.joinpath(f"{run_label or self.program_id}.log").write_text(
|
||
result.log, encoding='utf-8')
|
||
|
||
# .gcda を gcov/ にコピー（シナリオ毎に gcov/run_{id}/）
|
||
# GnuCOBOL は .gcno が生成された CWD (= compile CWD = exe_dir) に .gcda を書き出す。
|
||
# 複数シナリオで .gcno は共有されるため COPY で行う（MOVE 不可）。
|
||
gcda_src_dirs = [cwd] # ランタイム CWD
|
||
exe_dir_for_gcda = self.work_dir / "bin"
|
||
if exe_dir_for_gcda.exists() and exe_dir_for_gcda not in gcda_src_dirs:
|
||
gcda_src_dirs.append(exe_dir_for_gcda)
|
||
if scenario is None:
|
||
gcda_src_dirs.append(self.runtime_dir) # 従来互換
|
||
gcda_dst_dir = gcov_dir / run_label if scenario else gcov_dir
|
||
gcda_dst_dir.mkdir(parents=True, exist_ok=True)
|
||
for sd in gcda_src_dirs:
|
||
for f in sd.glob("*.gcda"):
|
||
if f.is_file() and f.stat().st_size > 0:
|
||
dst = gcda_dst_dir / f.name
|
||
if not dst.exists() or f.stat().st_mtime > dst.stat().st_mtime:
|
||
try:
|
||
shutil.copy2(str(f), str(dst))
|
||
except PermissionError:
|
||
pass
|
||
for f in sd.glob("*.gcno"):
|
||
if f.is_file() and f.stat().st_size > 0:
|
||
dst = gcda_dst_dir / f.name
|
||
try:
|
||
shutil.copy2(str(f), str(dst))
|
||
except PermissionError:
|
||
pass
|
||
|
||
return DbPipelineResult(
|
||
self.program_id, 3, result.success,
|
||
data={"returncode": result.returncode, "log": result.log[:500],
|
||
"input_dir": str(input_dir), "output_dir": str(output_dir),
|
||
"gcov_dir": str(gcov_dir)},
|
||
)
|
||
|
||
# ── マルチラン gcov マージ ──
|
||
|
||
def _merge_multi_run_gcov(self) -> dict[int, int] | None:
|
||
"""Run gcov per scenario, parse results, merge {line: count} dicts.
|
||
|
||
Returns merged gcov_data or None if no multi-run data available.
|
||
"""
|
||
from cobol_testgen.gcov import run_gcov, parse_cbl_gcov
|
||
gcov_dir = self.runtime_dir / "gcov"
|
||
run_dirs = sorted(gcov_dir.glob("run_*"))
|
||
if len(run_dirs) <= 1:
|
||
return None
|
||
|
||
# Ensure .gcno is in each run dir (copy from compile CWD if needed)
|
||
bin_gcno = self.work_dir / "bin"
|
||
if bin_gcno.exists():
|
||
for sd in run_dirs:
|
||
for f in bin_gcno.glob("*.gcno"):
|
||
dst = sd / f.name
|
||
if not dst.exists():
|
||
shutil.copy2(str(f), str(dst))
|
||
|
||
merged_data: dict[int, int] = {}
|
||
for sd in run_dirs:
|
||
data = run_gcov(f"{self.program_id}_pp", str(sd))
|
||
if data:
|
||
for line, count in data.items():
|
||
merged_data[line] = max(merged_data.get(line, 0), count)
|
||
|
||
logger.info(f" Merged gcov from {len(run_dirs)} runs ({len(merged_data)} lines)")
|
||
return merged_data
|
||
|
||
# ── カバレッジレポート（パイプライン外、オプション） ──
|
||
|
||
def generate_coverage_report(self,
|
||
output_dir: str | Path | None = None) -> DbPipelineResult:
|
||
"""COBOL 実行後：gcov データ収集 + 静的パスとマージし HTML レポート"""
|
||
try:
|
||
if not self.exe_path or not self.exe_path.exists():
|
||
# Fallback: look for exe in standard build location
|
||
fallback = self.work_dir / "bin" / f"{self.program_id}.exe"
|
||
if fallback.exists():
|
||
self.exe_path = fallback
|
||
else:
|
||
return DbPipelineResult(self.program_id, 0, False,
|
||
f"exe not found at {self.exe_path} or {fallback}")
|
||
if output_dir is None:
|
||
v3_root = Path(__file__).parent
|
||
output_dir = v3_root / "reports" / self.program_id / "coverage"
|
||
output_dir = Path(output_dir)
|
||
|
||
# 1. Use pre-merged multi-run gcov data if available (skip gcov re-run)
|
||
if self._multi_run_gcov_data is not None:
|
||
gcov_data = self._multi_run_gcov_data
|
||
# Also merge subprogram gcov data from each scenario
|
||
from cobol_testgen.gcov import run_gcov as _run_gcov
|
||
gcov_dir = self.runtime_dir / "gcov"
|
||
for sub in self.schema.subprograms:
|
||
sub_merged: dict[int, int] = {}
|
||
for sd in sorted(gcov_dir.glob("run_*")):
|
||
sub_data = _run_gcov(sub, str(sd))
|
||
if sub_data:
|
||
for line, cnt in sub_data.items():
|
||
sub_merged[line] = max(sub_merged.get(line, 0), cnt)
|
||
if sub_merged:
|
||
gcov_data.update(sub_merged)
|
||
else:
|
||
# Single-run: collect .gcno/.gcda and run gcov
|
||
gcov_dir = self.runtime_dir / "gcov"
|
||
gcov_dir.mkdir(parents=True, exist_ok=True)
|
||
v3_root = Path(__file__).parent
|
||
extra_search = list(gcov_dir.glob("run_*")) + [self.work_dir / "bin"]
|
||
for search_dir in (v3_root, self.work_dir, self.runtime_dir, gcov_dir, Path.home(), *extra_search):
|
||
for f in search_dir.glob("*.gcda"):
|
||
if f.stat().st_size > 0:
|
||
dst = gcov_dir / f.name
|
||
if not dst.exists() or f.stat().st_mtime > dst.stat().st_mtime:
|
||
try:
|
||
shutil.copy2(str(f), str(dst))
|
||
except PermissionError:
|
||
pass
|
||
for f in search_dir.glob("*.gcno"):
|
||
if f.stat().st_size > 0:
|
||
dst = gcov_dir / f.name
|
||
try:
|
||
shutil.copy2(str(f), str(dst))
|
||
except PermissionError:
|
||
pass
|
||
# Count what we have
|
||
gcno_gcda_count = 0
|
||
for ext in (".gcno", ".gcda"):
|
||
for f in gcov_dir.glob(f"*{ext}"):
|
||
if f.stat().st_size > 0 and (f.name.startswith(self.program_id) or f.name.startswith("SUB")):
|
||
gcno_gcda_count += 1
|
||
if gcno_gcda_count == 0:
|
||
for sd in (v3_root, self.work_dir, self.runtime_dir, gcov_dir):
|
||
for ext2 in (".gcno", ".gcda"):
|
||
files = list(sd.glob(f"*{ext2}"))
|
||
logger.error(f"gcov-check: {sd}\\*{ext2} -> {len(files)} files: {[f.name for f in files[:5]]}")
|
||
return DbPipelineResult(self.program_id, 0, False,
|
||
"no .gcno/.gcda found (--coverage missing?)")
|
||
|
||
# 3. Parse gcov data
|
||
gcov_data = run_gcov(f"{self.program_id}_pp", str(gcov_dir))
|
||
if not gcov_data:
|
||
gcov_data = run_gcov(self.program_id, str(gcov_dir))
|
||
for sub in self.schema.subprograms:
|
||
sd = run_gcov(sub, str(gcov_dir))
|
||
if sd:
|
||
gcov_data.update(sd)
|
||
|
||
# 4. Static branch tree from step2
|
||
st = self.generated_structure
|
||
branch_tree = st.get("branch_tree_obj") if st else None
|
||
if not branch_tree:
|
||
return DbPipelineResult(self.program_id, 0, True,
|
||
data={"gcov_lines": len(gcov_data),
|
||
"note": "no branch tree — gcov data only"})
|
||
|
||
# 5. Re-parse fields (same as generate_data)
|
||
src_text = self.src_path.read_text(encoding="utf-8-sig")
|
||
cbd = [str(d) for d in self.copybook_dirs]
|
||
pp = preprocess(src_text, extra_search_paths=cbd)
|
||
data_div = extract_data_division(pp)
|
||
data_fields = parse_data_division(data_div) if data_div else []
|
||
fdict = []
|
||
for idx, f in enumerate(data_fields):
|
||
entry = {
|
||
'name': f.name, 'level': f.level, 'pic': f.pic,
|
||
'pic_info': {
|
||
'type': f.pic_info.type if f.pic_info else 'unknown',
|
||
'digits': f.pic_info.digits if f.pic_info else 0,
|
||
'decimal': f.pic_info.decimal if f.pic_info else 0,
|
||
'length': f.pic_info.length if f.pic_info else 0,
|
||
'signed': f.pic_info.signed if f.pic_info else False,
|
||
},
|
||
'section': f.section, 'occurs': f.occurs_count,
|
||
'occurs_depending': f.occurs_depending,
|
||
'redefines': f.redefines, 'usage': f.usage,
|
||
}
|
||
if f.is_88:
|
||
entry['is_88'] = True
|
||
entry['parent'] = f.parent
|
||
fdict.append(entry)
|
||
fdict = expand_occurs(fdict)
|
||
|
||
# 6. Enumerate paths
|
||
branch_paths = mcdc_enum_paths(branch_tree, fdict)
|
||
|
||
# 7. Read preprocessed source for gcov line number matching
|
||
gcov_source = None
|
||
if self.pp_path and self.pp_path.exists():
|
||
gcov_source = self.pp_path.read_text(encoding="utf-8")
|
||
|
||
# 8. Generate merged HTML (use gcov_source for line numbers)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
prefix = str(output_dir / self.program_id)
|
||
cov_result = run_coverage(
|
||
branch_tree, branch_paths, fdict,
|
||
src_text, prefix,
|
||
index_relpath="index.html",
|
||
gcov_data=gcov_data or None,
|
||
gcov_source=gcov_source,
|
||
)
|
||
generate_coverage_index([cov_result], str(output_dir.parent))
|
||
|
||
# Clean up .gcno/.gcda from v3_root + CWD (avoid accumulation)
|
||
for clean_dir in (v3_root, Path.cwd()):
|
||
if clean_dir == gcov_dir:
|
||
continue
|
||
for ext in (".gcno", ".gcda"):
|
||
for f in clean_dir.glob(f"*{ext}"):
|
||
try:
|
||
f.unlink()
|
||
except PermissionError:
|
||
pass
|
||
|
||
total = cov_result.get("total_branches", 0)
|
||
covered = cov_result.get("covered_branches", 0)
|
||
pct = covered / total * 100 if total else 0
|
||
self._last_coverage_dict = cov_result
|
||
return DbPipelineResult(
|
||
self.program_id, 0, True,
|
||
data={
|
||
"gcov_lines": len(gcov_data),
|
||
"coverage": f"{covered}/{total} ({pct:.1f}%)",
|
||
"reports": str(output_dir),
|
||
"_cov_dict": cov_result,
|
||
},
|
||
)
|
||
except Exception as e:
|
||
logger.exception("generate_coverage_report failed")
|
||
return DbPipelineResult(self.program_id, 0, False, str(e))
|
||
|
||
# ── Step 4: DB → Java 中介データ ──
|
||
|
||
def step4_extract_intermediate(self) -> DbPipelineResult:
|
||
"""SQLite → JSON 中介データ抽出（Step 4: DB→Java中介データ）"""
|
||
db_path = self._current_db_path or self.db_path
|
||
if not db_path or not db_path.exists():
|
||
return DbPipelineResult(self.program_id, 4, False,
|
||
"db not found (run step3 first)")
|
||
|
||
try:
|
||
conn = sqlite3.connect(str(db_path))
|
||
conn.row_factory = sqlite3.Row
|
||
|
||
# Read from actual COBOL SQL tables (using sql_name or name)
|
||
output_tables = {}
|
||
for table in self.schema.db_tables:
|
||
sql_name = table.sql_name or table.name
|
||
try:
|
||
rows = conn.execute(f"SELECT * FROM [{sql_name}]").fetchall()
|
||
output_tables[table.name] = [dict(r) for r in rows]
|
||
except sqlite3.OperationalError:
|
||
output_tables[table.name] = []
|
||
|
||
conn.close()
|
||
|
||
w01_path = self.work_dir / "intermediate" / f"{self.program_id}_W01.json"
|
||
w01_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
meta = {
|
||
"program_id": self.program_id,
|
||
"tables": output_tables,
|
||
}
|
||
w01_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2))
|
||
self.java_input_path = w01_path
|
||
|
||
return DbPipelineResult(
|
||
self.program_id, 4, True,
|
||
data={"tables": len(output_tables), "w01_path": str(w01_path)},
|
||
)
|
||
except Exception as e:
|
||
return DbPipelineResult(self.program_id, 4, False, str(e))
|
||
|
||
# ── Step 5: Java 実行 ──
|
||
|
||
def step5_run_java(self, java_cmd: str = "java",
|
||
java_jar: str | Path | None = None) -> DbPipelineResult:
|
||
"""Java プログラム実行"""
|
||
if not self.java_input_path or not self.java_input_path.exists():
|
||
return DbPipelineResult(self.program_id, 5, False,
|
||
"intermediate data not found (run step4 first)")
|
||
|
||
java_out = self.work_dir / "java_output"
|
||
java_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
if java_jar:
|
||
cmd = [java_cmd, "-jar", str(java_jar),
|
||
"-i", str(self.java_input_path),
|
||
"-o", str(java_out)]
|
||
else:
|
||
cmd = [java_cmd, "-version"]
|
||
|
||
try:
|
||
r = subprocess.run(cmd, capture_output=True, timeout=60)
|
||
log = (r.stdout.decode("utf-8", "replace") + "\n" +
|
||
r.stderr.decode("utf-8", "replace"))
|
||
ok = r.returncode == 0
|
||
self.java_output_path = java_out
|
||
return DbPipelineResult(
|
||
self.program_id, 5, ok,
|
||
data={"returncode": r.returncode, "log": log[:500]},
|
||
)
|
||
except subprocess.TimeoutExpired:
|
||
return DbPipelineResult(self.program_id, 5, False, "Java timeout")
|
||
|
||
# ── Step 6: 検証 ──
|
||
|
||
def step6_verify(self) -> VerificationRun:
|
||
"""Java 出力と COBOL 期待値を比較"""
|
||
db_path = self._current_db_path or self.db_path
|
||
vr = VerificationRun(
|
||
program=self.program_id,
|
||
runner="gixsql",
|
||
gixsql_version="0.9.1",
|
||
sqlite_path=str(db_path) if db_path else "",
|
||
step_reached=6,
|
||
)
|
||
|
||
if db_path and db_path.exists():
|
||
after_tables = self.runner.read_db_tables(
|
||
db_path,
|
||
[t.name for t in self.schema.db_tables],
|
||
)
|
||
for table_data in after_tables:
|
||
vr.debug[f"table_{table_data.table_name}_rows"] = len(table_data.rows)
|
||
|
||
if self.java_output_path and self.java_output_path.exists():
|
||
java_files = list(self.java_output_path.glob("*.txt")) + \
|
||
list(self.java_output_path.glob("*.json"))
|
||
vr.debug["java_output_files"] = [str(f) for f in java_files]
|
||
vr.fields_matched = len(java_files)
|
||
|
||
vr.exit_code = 0 if vr.fields_mismatched == 0 else 1
|
||
vr.status = "PASS" if vr.exit_code == 0 else "MISMATCH"
|
||
return vr
|
||
|
||
# ── 全Step一括実行 ──
|
||
|
||
def run_all(self, skip_steps: set[int] | None = None,
|
||
generate_coverage: bool = True) -> VerificationRun:
|
||
"""Step 1 → 6 を順次実行（skip_jvm=True で Step 5/6 をスキップ）
|
||
|
||
多輪実行：schema.runs が定義されていれば各シナリオを順次実行し、最後に gcov をマージ。
|
||
schema.runs が空の場合は単輪（従来動作）。
|
||
"""
|
||
skip = set(skip_steps or [])
|
||
if self.skip_jvm:
|
||
skip.update({5, 6})
|
||
|
||
scenarios = self.schema.runs or [ScenarioDef(id="default")]
|
||
is_multi = len(scenarios) > 1 or (len(scenarios) == 1 and scenarios[0].id != "default")
|
||
|
||
# Step 1: compile once
|
||
if 1 not in skip:
|
||
logger.info(" Step 1 (compile)...")
|
||
r1 = self.step1_setup_environment()
|
||
if not r1.success:
|
||
return VerificationRun(
|
||
program=self.program_id, runner="gixsql",
|
||
status="BLOCKED", exit_code=2,
|
||
step_reached=1,
|
||
)
|
||
|
||
# Each scenario: generate inputs + run COBOL
|
||
for scenario in scenarios:
|
||
label = f" [{scenario.id}]" if is_multi else ""
|
||
logger.info(f" Step 2 (generate inputs){label}...")
|
||
r2 = self.step2_generate_inputs(scenario if is_multi else None)
|
||
if not r2.success:
|
||
return VerificationRun(
|
||
program=self.program_id, runner="gixsql",
|
||
status="BLOCKED", exit_code=2,
|
||
step_reached=2,
|
||
)
|
||
logger.info(f" Step 3 (run COBOL){label}...")
|
||
r3 = self.step3_run_cobol(scenario if is_multi else None)
|
||
if not r3.success:
|
||
return VerificationRun(
|
||
program=self.program_id, runner="gixsql",
|
||
status="BLOCKED", exit_code=2,
|
||
step_reached=3,
|
||
)
|
||
|
||
# Step 4: extract intermediate (last scenario wins for DB path)
|
||
if 4 not in skip:
|
||
logger.info(" Step 4 (extract)...")
|
||
self.step4_extract_intermediate()
|
||
|
||
if not self.skip_jvm:
|
||
steps_remaining = [5, 6]
|
||
for step_num in steps_remaining:
|
||
if step_num in skip:
|
||
continue
|
||
logger.info(f" Step {step_num}...")
|
||
if step_num == 5:
|
||
self.step5_run_java()
|
||
elif step_num == 6:
|
||
vr = self.step6_verify()
|
||
|
||
# Optional coverage report (non-blocking)
|
||
cv_flags = getattr(self.config, 'gixsql_compile_flags', '')
|
||
if '--coverage' in cv_flags and generate_coverage:
|
||
if is_multi:
|
||
merged = self._merge_multi_run_gcov()
|
||
self._multi_run_gcov_data = merged
|
||
self.generate_coverage_report()
|
||
|
||
vr = VerificationRun(
|
||
program=self.program_id, runner="gixsql",
|
||
status="PASS", exit_code=0,
|
||
step_reached=6 if not self.skip_jvm else 4,
|
||
)
|
||
return vr
|
||
|
||
# ── Internal helpers ──
|
||
|
||
def _scan_assign_to(self) -> dict[str, str]:
|
||
"""Scan COBOL source for SELECT/ASSIGN-TO + OPEN direction.
|
||
Returns {filename: direction} where direction is 'INPUT' or 'OUTPUT'.
|
||
Works for both quoted (\"KIN08R01\") and bare (KIN01R01) ASSIGN.
|
||
"""
|
||
src_text = self.src_path.read_text(encoding="utf-8-sig")
|
||
assign_map: dict[str, str] = {}
|
||
# First pass: collect all SELECT/ASSIGN-TO mappings
|
||
select_to_file: dict[str, str] = {}
|
||
for m in re.finditer(
|
||
r'SELECT\s+(\w+)\s+ASSIGN\s+TO\s+"?([^"\s.]+)',
|
||
src_text, re.IGNORECASE
|
||
):
|
||
sel_name = m.group(1)
|
||
fname = m.group(2).strip().rstrip('"')
|
||
select_to_file[sel_name] = fname
|
||
assign_map[fname] = "UNKNOWN"
|
||
|
||
# Second pass: determine direction from OPEN statements.
|
||
# COBOL allows multi-line OPEN where files listed without a direction
|
||
# keyword inherit the last stated direction:
|
||
# OPEN INPUT FILEA
|
||
# FILEB <-- inherits INPUT
|
||
# OUTPUT FILEC
|
||
# Strategy: extract OPEN body (up to terminating '.'), collapse
|
||
# whitespace, then parse direction→file pairs via splitting on
|
||
# direction keyword boundaries.
|
||
for m in re.finditer(
|
||
r'OPEN\s+(.+?)\.', src_text, re.IGNORECASE | re.DOTALL
|
||
):
|
||
full = re.sub(r'\s+', ' ', m.group(1)).strip()
|
||
# Split on direction keyword boundaries: "INPUT X Y OUTPUT Z"
|
||
# → ["INPUT X Y", "OUTPUT Z"]
|
||
tokens = re.split(r'\s+(?=(?:INPUT|OUTPUT|I-O|EXTEND)\s)', full, flags=re.IGNORECASE)
|
||
for seg in tokens:
|
||
seg = seg.strip()
|
||
if not seg:
|
||
continue
|
||
seg_m = re.match(r'(INPUT|OUTPUT|I-O|EXTEND)\s+([\w ]+)', seg, re.IGNORECASE)
|
||
if not seg_m:
|
||
continue
|
||
direction = seg_m.group(1).upper()
|
||
for fword in re.findall(r'\w+', seg_m.group(2)):
|
||
if fword in select_to_file:
|
||
fname = select_to_file[fword]
|
||
if direction in ("INPUT", "I-O"):
|
||
assign_map[fname] = "INPUT"
|
||
else:
|
||
assign_map[fname] = "OUTPUT"
|
||
|
||
return assign_map
|
||
|
||
def _init_database(self, db_path: Path):
|
||
"""Create tables from schema + COBOL EXEC SQL table definitions."""
|
||
self._create_tables(db_path)
|
||
|
||
def _create_tables(self, db_path: Path):
|
||
conn = sqlite3.connect(str(db_path))
|
||
for table in self.schema.db_tables:
|
||
col_defs = []
|
||
pk_cols = []
|
||
for col in table.columns:
|
||
col_defs.append(f"[{col.name}] {col.type}")
|
||
if col.primary_key:
|
||
pk_cols.append(f"[{col.name}]")
|
||
if pk_cols:
|
||
col_defs.append(f"PRIMARY KEY ({', '.join(pk_cols)})")
|
||
ddl = f"CREATE TABLE IF NOT EXISTS [{table.name}] (\n " + \
|
||
",\n ".join(col_defs) + "\n)"
|
||
conn.execute(ddl)
|
||
# If sql_name differs, also create the COBOL-visible SQL table name
|
||
if table.sql_name and table.sql_name != table.name:
|
||
conn.execute(ddl.replace(f"[{table.name}]", f"[{table.sql_name}]"))
|
||
conn.commit()
|
||
conn.close()
|
||
logger.info(f" DB initialized: {db_path}")
|
||
|
||
def _populate_database(self, db_path: Path, src_text: str, records: list[dict]):
|
||
"""テストデータから DB 初期行を生成し挿入する。"""
|
||
from cobol_testgen.pipeline_bridge import build_branch_tree_fallback
|
||
from cobol_testgen.read import extract_procedure_division
|
||
|
||
cbd = [str(d) for d in self.copybook_dirs]
|
||
|
||
src_resolved = resolve_copybooks(src_text, ".", extra_search_paths=cbd)
|
||
src_resolved = resolve_sql_includes(src_resolved, ".")
|
||
preprocessed = preprocess(src_resolved)
|
||
|
||
data_div = extract_data_division(preprocessed)
|
||
data_fields = parse_data_division(data_div) if data_div else []
|
||
fields_dict = []
|
||
for f in data_fields:
|
||
fields_dict.append({
|
||
'name': f.name, 'level': f.level, 'pic': f.pic,
|
||
'pic_info': {
|
||
'type': f.pic_info.type if f.pic_info else 'unknown',
|
||
'digits': f.pic_info.digits if f.pic_info else 0,
|
||
'decimal': f.pic_info.decimal if f.pic_info else 0,
|
||
'length': f.pic_info.length if f.pic_info else 0,
|
||
'signed': f.pic_info.signed if f.pic_info else False,
|
||
},
|
||
'section': f.section, 'occurs': f.occurs_count,
|
||
'occurs_depending': f.occurs_depending,
|
||
'value': f.value, 'values': f.values,
|
||
'redefines': f.redefines, 'usage': f.usage,
|
||
})
|
||
fields_dict = expand_occurs(fields_dict)
|
||
|
||
proc_div = extract_procedure_division(preprocessed)
|
||
branch_tree, assignments = build_branch_tree_fallback(proc_div, fields_dict)
|
||
|
||
# Merge SQL assignments from original source
|
||
sql_assigns = extract_sql_assignments(src_text)
|
||
for tgt, asgn_list in sql_assigns.items():
|
||
for asgn in asgn_list:
|
||
assignments.setdefault(tgt, []).append(asgn)
|
||
|
||
branch_paths = mcdc_enum_paths(branch_tree, fields_dict)
|
||
|
||
data_div2, declared_columns = strip_exec_sql_from_data_div(data_div)
|
||
sql_meta = collect_sql_meta(assignments, declared_columns)
|
||
if not sql_meta:
|
||
logger.info(" No SQL metadata found, skipping DB population")
|
||
return
|
||
|
||
db_input = build_db_input(
|
||
branch_paths, fields_dict, assignments,
|
||
sql_meta, declared_columns,
|
||
records=records,
|
||
)
|
||
if not db_input:
|
||
logger.info(" No DB input rows generated")
|
||
return
|
||
|
||
conn = sqlite3.connect(str(db_path))
|
||
for table_name, rows in db_input.items():
|
||
if not rows:
|
||
logger.info(f" Table {table_name}: 0 initial rows (will be created at runtime)")
|
||
continue
|
||
|
||
# Debug
|
||
logger.info(f" Table {table_name}: {len(rows)} rows, cols={list(rows[0].keys()) if rows else []}")
|
||
|
||
# Filter columns: only keep those that actually exist in the table
|
||
try:
|
||
pragma_cols = conn.execute(
|
||
f"PRAGMA table_info([{table_name}])"
|
||
).fetchall()
|
||
valid_cols = {r[1].upper() for r in pragma_cols}
|
||
except Exception:
|
||
valid_cols = set()
|
||
|
||
remapped_rows = []
|
||
for row in rows:
|
||
new_row = {}
|
||
for k, v in row.items():
|
||
if k.upper() in valid_cols:
|
||
new_row[k] = v
|
||
if new_row:
|
||
remapped_rows.append(new_row)
|
||
rows = remapped_rows
|
||
if not rows:
|
||
logger.info(f" Table {table_name}: all rows filtered out, skipping")
|
||
continue
|
||
|
||
col_names = list(rows[0].keys())
|
||
placeholders = ", ".join("?" for _ in col_names)
|
||
quoted_cols = ", ".join(f"[{c}]" for c in col_names)
|
||
sql = f"INSERT OR IGNORE INTO [{table_name}] ({quoted_cols}) VALUES ({placeholders})"
|
||
conn.executemany(sql, [tuple(r.get(c, "") for c in col_names) for r in rows])
|
||
logger.info(f" Table {table_name}: {len(rows)} initial rows inserted")
|
||
conn.commit()
|
||
conn.close()
|
||
logger.info(f" DB populated: {db_path}")
|
||
|
||
def _inject_sql_error_rows(self, db_path: Path, records: list[dict] | None = None):
|
||
"""Insert duplicate-PK rows to trigger SQL error handling paths in COBOL."""
|
||
conn = sqlite3.connect(str(db_path))
|
||
for table in self.schema.db_tables:
|
||
pk_cols = [c.name for c in table.columns if c.primary_key]
|
||
if not pk_cols:
|
||
continue
|
||
col_names = [c.name for c in table.columns]
|
||
try:
|
||
rows = conn.execute(f"SELECT * FROM [{table.name}] LIMIT 2").fetchall()
|
||
if len(rows) < 1:
|
||
# For empty tables, generate synthetic error rows from test record data
|
||
synthetic = self._make_synthetic_error_rows(table, records)
|
||
if synthetic:
|
||
rows = synthetic
|
||
else:
|
||
continue
|
||
quoted = ", ".join(f"[{c}]" for c in col_names)
|
||
ph = ", ".join("?" for _ in col_names)
|
||
for row in rows:
|
||
vals = tuple(str(rows[0][i]) if c in pk_cols else "X" for i, c in enumerate(col_names))
|
||
conn.execute(f"INSERT OR IGNORE INTO [{table.name}] ({quoted}) VALUES ({ph})", vals)
|
||
logger.info(f" SQL error test row injected into {table.name}")
|
||
except Exception as e:
|
||
logger.debug(f" SQL error row injection skipped: {e}")
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
def _seed_matching_monthly_rows(self, db_path: Path, records: list[dict] | None,
|
||
max_seed: int = 1,
|
||
r01_dir: Path | None = None):
|
||
"""Pre-populate MONTHLY_ABSENCE with rows matching actual R01 record data.
|
||
Reads the generated R01 flat file (200-byte fixed records, KIN07REC layout),
|
||
extracts unique (EMP_ID, YEAR_MONTH) pairs, and inserts a SUBSET of them.
|
||
This ensures some AGG entries find HV-CNT > 0 (UPDATE, DP#27) and
|
||
others find HV-CNT = 0 (INSERT, DP#28)."""
|
||
r01_path = (r01_dir or self.work_dir / "input") / "KIN08R01"
|
||
if not r01_path.exists():
|
||
logger.info(" R01 file not found, skipping MONTHLY_ABSENCE seed")
|
||
return
|
||
conn = sqlite3.connect(str(db_path))
|
||
monthly_table = None
|
||
for t in self.schema.db_tables:
|
||
if t.name == "MONTHLY_ABSENCE":
|
||
monthly_table = t
|
||
break
|
||
if not monthly_table:
|
||
conn.close()
|
||
return
|
||
col_names = [c.name for c in monthly_table.columns]
|
||
quoted = ", ".join(f"[{c}]" for c in col_names)
|
||
ph = ", ".join("?" for _ in col_names)
|
||
seen = set()
|
||
pairs = []
|
||
rows_inserted = 0
|
||
# KIN07REC layout (each record is 200 bytes):
|
||
# EMP-ID PIC 9(008) offset 0, 8 bytes
|
||
# DATE PIC 9(008) offset 8, 8 bytes
|
||
# ... remaining fields (not needed)
|
||
rec_size = 200
|
||
with open(str(r01_path), 'rb') as f:
|
||
data = f.read()
|
||
num_recs = len(data) // rec_size
|
||
for i in range(num_recs):
|
||
off = i * rec_size
|
||
emp_id = data[off:off+8].decode('ascii', errors='replace').strip()
|
||
date = data[off+8:off+16].decode('ascii', errors='replace').strip()
|
||
year_month = date[:6] if len(date) >= 6 else date
|
||
if not emp_id or not year_month or emp_id == '00000000':
|
||
continue
|
||
key = (emp_id, year_month)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
pairs.append((emp_id, year_month))
|
||
# Sort by EMP_ID for deterministic behavior, then seed only `max_seed` pairs
|
||
pairs.sort(key=lambda x: x[0])
|
||
conn = sqlite3.connect(str(db_path))
|
||
monthly_table = None
|
||
for t in self.schema.db_tables:
|
||
if t.name == "MONTHLY_ABSENCE":
|
||
monthly_table = t
|
||
break
|
||
if not monthly_table:
|
||
conn.close()
|
||
return
|
||
col_names = [c.name for c in monthly_table.columns]
|
||
quoted = ", ".join(f"[{c}]" for c in col_names)
|
||
ph = ", ".join("?" for _ in col_names)
|
||
for seed_idx, (emp_id, year_month) in enumerate(pairs):
|
||
if seed_idx >= max_seed:
|
||
break
|
||
vals = {
|
||
"EMP_ID": emp_id,
|
||
"YEAR_MONTH": year_month,
|
||
"ANNUAL_LEAVE_H": "0",
|
||
"PERSONAL_LEAVE_H": "0",
|
||
"OFFICIAL_LEAVE_H": "0",
|
||
"SICK_LEAVE_H": "0",
|
||
"UNAPPROVED_ABSENT_H": "0",
|
||
"UPDATED_AT": "2026-01-01 00:00:00",
|
||
}
|
||
row = tuple(vals.get(c, "") for c in col_names)
|
||
try:
|
||
conn.execute(f"INSERT OR IGNORE INTO [MONTHLY_ABSENCE] ({quoted}) VALUES ({ph})", row)
|
||
rows_inserted += 1
|
||
except Exception:
|
||
pass
|
||
conn.commit()
|
||
conn.close()
|
||
if rows_inserted:
|
||
logger.info(f" MONTHLY_ABSENCE: {rows_inserted}/{len(pairs)} matching rows seeded (DP#27 F + DP#28 F)")
|
||
elif pairs:
|
||
logger.info(f" MONTHLY_ABSENCE: 0 seeded — all AGG entries will INSERT (DP#28 F)")
|
||
|
||
def _make_synthetic_error_rows(self, table, records: list[dict] | None) -> list[tuple] | None:
|
||
"""Build synthetic error rows for an empty table from test record data."""
|
||
if not records or len(records) < 2:
|
||
return None
|
||
pk_cols = [c.name for c in table.columns if c.primary_key]
|
||
if not pk_cols:
|
||
return None
|
||
|
||
# Map COBOL host-variable names to table column names
|
||
# KIN08DBU DAILY_RECORDS: EMP_ID=HV-EMP-ID, TARGET_DATE=HV-TARGET-DATE
|
||
# KIN08DBU MONTHLY_ABSENCE: EMP_ID=HV-EMP-ID, YEAR_MONTH=HV-YEAR-MONTH
|
||
hv_map = {
|
||
'EMP_ID': ('HV-EMP-ID', 'R01EMP-ID', ''),
|
||
'TARGET_DATE': ('HV-TARGET-DATE', ''),
|
||
'YEAR_MONTH': ('HV-YEAR-MONTH', ''),
|
||
'TIME_IN': ('HV-TIME-IN', ''),
|
||
'TIME_OUT': ('HV-TIME-OUT', ''),
|
||
'ANNUAL_LEAVE_H': ('HV-ANNUAL-H', ''),
|
||
'PERSONAL_LEAVE_H': ('HV-PERSONAL-H', ''),
|
||
'OFFICIAL_LEAVE_H': ('HV-OFFICIAL-H', ''),
|
||
'SICK_LEAVE_H': ('HV-SICK-H', ''),
|
||
'UNAPPROVED_ABSENT_H': ('HV-ABSENT-H', ''),
|
||
}
|
||
|
||
result = []
|
||
for idx in range(min(2, len(records))):
|
||
rec = records[idx]
|
||
vals = []
|
||
for col in table.columns:
|
||
val = None
|
||
if col.name in hv_map:
|
||
for key in hv_map[col.name]:
|
||
if key and key in rec:
|
||
val = rec[key]
|
||
break
|
||
if val is None:
|
||
val = ' ' if col.name in pk_cols else ''
|
||
vals.append(str(val) if val is not None else '')
|
||
result.append(tuple(vals))
|
||
return result if result else None
|