hangshuo652/cobol-java-v3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: gixsql隐式小数DB数值修正 + LLM无Key跳过

Browse Source
This commit is contained in:
hangshuo652
2026-09-12 18:18:31 +08:00
parent bdc1584b3c
commit 7e7bc41848
4 changed files with 280 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_AI_USAGE_LOG.md
+12
View File
@@ -13,6 +13,18 @@
---
### 2026-09-10 16:00:00 - AI编码实现
- **范式步骤：** AI编码实现
- **修改摘要：** 修复gixsql DB数值差异问题：回滚旧方案（修改length导致over-read修复不完整），实施新方案（DB后处理power缩放）。从_pp.cbl的INSERT SQL和GIXSQLSetSQLParams中提取列名→power映射，COBOL执行后对SQLite DB执行`CAST(col AS REAL) / 10^|power|`修正数值。修复了3个bug：(1)列名映射错误（HV_ANNUAL_H→ANNUAL_LEAVE_H），(2)SQLite整数除法截断（需CAST AS REAL），(3)INSERT SQL多行解析（FILLER VALUE + &续行）
- **涉及文件：** `runners/gixsql_runner.py`（新增`extract_power_scaling`、`apply_power_scaling`、`_patch_numeric_metadata`），`orchestrator_db.py`（集成DB后处理调用）
- **使用模型：** deepseek/deepseek-v4-flash
### 2026-09-09 23:35:00 - AI编码实现
- **范式步骤：** AI编码实现
- **修改摘要：** 修复 KIN08DBU Java文件名映射问题：V3 DB管道的 `_prepare_db_java_inputs` 写入 `{assign}.txt` 文件，但 Java 源码常量写死 `data/KIN08S01.DAT`（带`.DAT`扩展名），导致 `FileNotFoundException`。将3个文件常量从`.DAT`改为`.TXT`：R01_FILE=`data/KIN08R01.TXT`、SYSIN_FILE=`data/KIN08S01.TXT`、W01_FILE=`data/KIN08W01.TXT`。重跑V3后 run_normal 场景 Java rc=0，COBOL 162行 vs Java 162行 DAILY_RECORDS 行数一致。
- **涉及文件：** `JavaSrc/src/Kin08DbuMain.java:39,41,43`
- **使用模型：** deepseek/deepseek-v4-flash
### 2026-09-09 21:30:00 - AI编码实现
- **范式步骤：** AI编码实现
- **修改摘要：** 修复 Java Runner 未传递 command_line 参数的 bug：在 `orchestrator_db.py` 的 `_java_run_scenario` 方法中添加 command_line 参数传递逻辑，对齐 COBOL runner 的处理方式。修改后 Java 程序能正确接收 YEARMONTH 参数。
agents/llm.py
+6 -1
View File
@@ -99,7 +99,12 @@ class LLMClient:
key = os.environ.get("LLM_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
base = os.environ.get("LLM_API_BASE", "https://api.openai.com/v1")
# 无 API Key 时直接跳过，避免超时等待
if not key:
logger.info("LLM call skipped: no API key set (LLM_API_KEY / OPENAI_API_KEY)")
return ""
for a in range(retries + 1):
try:
# 速率限制
orchestrator_db.py
+9
View File
@@ -693,6 +693,15 @@ class GixsqlOrchestrator:
command_args=command_args,
)
# Power-scaling post-process: gixsql stores raw digits for implicit-decimal
# fields (e.g. 402 for 9(4)V9(1)) but does NOT divide by 10^power.
# Fix DB values BEFORE W01 file comparison so COBOL DB matches Java.
if self.pp_path and self.pp_path.exists() and db_path and db_path.exists():
from runners.gixsql_runner import GixsqlCobolRunner
power_map = GixsqlCobolRunner.extract_power_scaling(self.pp_path)
if power_map:
GixsqlCobolRunner.apply_power_scaling(db_path, power_map)
log_dir = self.runtime_dir.parent / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
log_dir.joinpath(f"{run_label or self.program_id}.log").write_text(
runners/gixsql_runner.py
+253
View File
@@ -453,6 +453,254 @@ class GixsqlCobolRunner:
)
pp_path.write_text(text, encoding="utf-8")
@staticmethod
def _patch_numeric_metadata(pp_path: Path) -> None:
"""Fix gixpp numeric binding metadata for implicit-decimal DISPLAY fields.
gixpp emits ``GIXSQLSetSQLParams(1, length, power, ...)`` where ``power``
is negative for implicit-decimal PICs (e.g. ``9(4)V9(1)`` → power=-1).
gixsql uses ``bytes_read = length - power``; with ``length=5, power=-1``
this reads 6 bytes, one past the actual 5-byte field, corrupting the value
with the adjacent WORKING-STORAGE byte.
Fix: reduce ``length`` so that ``length - power = actual_field_size``
(the COBOL storage bytes), keeping ``power`` unchanged so gixsql still
reads the correct number of bytes.
"""
text = pp_path.read_text(encoding="utf-8")
_CALL_RE = re.compile(
r'CALL "GIXSQLSetSQLParams" USING\s+'
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE (-?\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE 0\s+'
r'(?:GIXSQL\s+)?BY REFERENCE \S+',
re.IGNORECASE | re.DOTALL
)
def _fix_numeric(m: re.Match) -> str:
type_val = int(m.group(1))
length_val = int(m.group(2))
power_val = int(m.group(3))
if type_val == 1 and power_val < 0:
new_length = length_val + power_val # power is negative
logger.debug(f" _patch_numeric_metadata: length {length_val}->{new_length}, "
f"power unchanged {power_val}")
old = m.group(0)
old_lines = old.split('\n')
new_lines = []
val_idx = 0
for line in old_lines:
if 'BY VALUE' in line:
val_idx += 1
if val_idx == 1: # type
new_lines.append(line)
elif val_idx == 2: # length — reduce to actual storage size
new_lines.append(re.sub(r'BY VALUE \d+', f'BY VALUE {new_length}', line))
else: # power, flag — keep unchanged
new_lines.append(line)
else:
new_lines.append(line)
return '\n'.join(new_lines)
return m.group(0)
text = _CALL_RE.sub(_fix_numeric, text)
pp_path.write_text(text, encoding="utf-8")
@staticmethod
def extract_power_scaling(pp_path: str | Path) -> dict[str, int]:
"""Extract power scaling info from _pp.cbl for DB post-processing.
Strategy:
1. Parse INSERT SQL from FILLER VALUE strings → get column lists.
2. Scan GIXSQLSetSQLParams in order → find which positions have
type=1, power<0 (implicit-decimal numeric).
3. Map positions to columns. Since the same column names appear
in multiple INSERTs, we just need the unique column names.
Returns mapping e.g. {"ANNUAL_LEAVE_H": -1, ...}.
"""
raw_text = Path(pp_path).read_text(encoding="utf-8")
# Step 1: Extract INSERT SQL column→position mappings from FILLER VALUEs.
_FILLER_RE = re.compile(
r'02\s+FILLER\s+PIC\s+X\(\d+\)\s+VALUE\s+"([^"]*)"'
r'((?:\s*\n\s*GIXSQL\s*&\s+"[^"]*")*)',
re.IGNORECASE
)
# Collect ALL columns that use $N params (across all INSERTs)
all_numeric_cols: set[str] = set()
max_position = 0
first_pos_map: dict[int, str] = {} # first occurrence of each position
for m in _FILLER_RE.finditer(raw_text):
first = m.group(1)
conts = m.group(2)
cont_texts = re.findall(r'"([^"]*)"', conts)
full_sql = first + ''.join(cont_texts)
ins = re.search(
r'INSERT\s+INTO\s+\w+\s*\(([^)]+)\)\s*VALUES\s*\(([^)]+)\)',
full_sql, re.IGNORECASE
)
if ins:
cols = [c.strip() for c in ins.group(1).split(',')]
vals = [v.strip() for v in ins.group(2).split(',')]
for col, val in zip(cols, vals):
pm = re.match(r'\$(\d+)', val)
if pm:
pos = int(pm.group(1))
max_position = max(max_position, pos)
if pos not in first_pos_map:
first_pos_map[pos] = col
# Step 2: Extract GIXSQLSetSQLParams calls in order.
_PARAM_RE = re.compile(
r'CALL "GIXSQLSetSQLParams" USING\s+'
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE (\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE (-?\d+)\s+'
r'(?:GIXSQL\s+)?BY VALUE 0\s+'
r'(?:GIXSQL\s+)?BY REFERENCE (\S+)',
re.IGNORECASE | re.DOTALL
)
all_params = []
for m in _PARAM_RE.finditer(raw_text):
all_params.append((int(m.group(1)), int(m.group(3)), m.group(4)))
# Step 3: Group params by their INSERT context.
# Each INSERT has its own $1..$N. The params repeat for each SQL
# statement (INSERT DAILY, DELETE, SELECT, UPDATE, INSERT MONTHLY, ...).
# We need to find which param indices map to which INSERT's $N.
#
# Key insight: for each INSERT, the $N positions are 1..N.
# The SetSQLParams before that INSERT are in the same order.
# We can identify INSERT boundaries by looking for GIXSQLExecParams
# calls that reference the SQL string (SQ0001, SQ0002, etc.).
#
# Simpler approach: since all INSERTs share the same numeric columns,
# just find ALL columns that appear at ANY $N position, and ALL
# SetSQLParams with type=1/power<0. Then match by relative position
# within each INSERT group.
#
# Even simpler: just check which columns from the DDL appear in
# ANY INSERT with a $N param, and which SetSQLParams have power<0.
# The SetSQLParams with power<0 always correspond to the numeric
# columns (DECIMAL), so we can match by counting.
# Count how many params are BEFORE the first numeric param
# and how many numeric params there are per INSERT.
# For DAILY_RECORDS: 4 string params, then 5 numeric, then 1 timestamp
# For MONTHLY_ABSENCE: 2 string params, then 5 numeric, then 1 timestamp
# Find all unique column names that have $N in INSERT SQL
numeric_cols_in_insert: list[str] = []
for pos in sorted(first_pos_map.keys()):
col = first_pos_map[pos]
# We'll filter to just the numeric ones after matching
# Match: SetSQLParams with type=1/power<0 → columns at corresponding $N
# We need to know which $N corresponds to which SetSQLParams index.
#
# Strategy: find the first INSERT's column list, then find the
# SetSQLParams group that precedes it. The numeric columns in the
# INSERT are at specific $N positions. The SetSQLParams with
# type=1/power<0 are at corresponding indices.
# Find the first INSERT DAILY_RECORDS column order
daily_cols = []
for m in _FILLER_RE.finditer(raw_text):
first = m.group(1)
conts = m.group(2)
cont_texts = re.findall(r'"([^"]*)"', conts)
full_sql = first + ''.join(cont_texts)
ins = re.search(
r'INSERT\s+INTO\s+DAILY_RECORDS\s*\(([^)]+)\)',
full_sql, re.IGNORECASE
)
if ins:
daily_cols = [c.strip() for c in ins.group(1).split(',')]
break
# Find which $N positions in DAILY_RECORDS are numeric (DECIMAL in DDL)
# by checking if the corresponding SetSQLParams has type=1/power<0.
# The $N for DAILY_RECORDS: EMP_ID=$1, TARGET_DATE=$2, ..., UPDATED_AT=$9(→CURRENT_TIMESTAMP)
# We know the SetSQLParams before the first INSERT are in order $1..$9.
# Count params before the first GIXSQLExecParams (which executes the INSERT).
# Find the first GIXSQLExecParams position
_EXEC_RE = re.compile(r'CALL "GIXSQLExecParams"', re.IGNORECASE)
first_exec = _EXEC_RE.search(raw_text)
first_exec_pos = first_exec.start() if first_exec else len(raw_text)
# Collect params that appear BEFORE the first exec
pre_exec_params = []
for ptype, ppower, pvar in all_params:
var_pattern = re.compile(rf'BY REFERENCE {re.escape(pvar)}\b', re.IGNORECASE)
var_match = var_pattern.search(raw_text)
if var_match and var_match.start() < first_exec_pos:
pre_exec_params.append((ptype, ppower, pvar))
# These are the params for the first INSERT (DAILY_RECORDS).
# Match them to the DAILY_RECORDS columns.
scaling: dict[str, int] = {}
for idx, (ptype, ppower, pvar) in enumerate(pre_exec_params):
pos = idx + 1 # 1-based
if ptype == 1 and ppower < 0:
col_name = first_pos_map.get(pos)
if col_name:
scaling[col_name] = ppower
return scaling
@staticmethod
def apply_power_scaling(db_path: str | Path,
power_map: dict[str, int]) -> int:
"""Post-process SQLite DB: divide numeric columns by 10^|power|.
gixsql stores raw digits (e.g. 402) for implicit-decimal fields
but does NOT apply the power scaling. This corrects the DB values
to match the intended COBOL decimal representation (e.g. 40.2).
Returns number of rows updated.
"""
if not power_map:
return 0
db_path = Path(db_path)
if not db_path.exists():
return 0
conn = sqlite3.connect(str(db_path))
total_updated = 0
for var_name, power in power_map.items():
col_name = var_name.replace('-', '_')
scale_factor = 10 ** abs(power)
try:
# Get table names that have this column
tables = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
for (tname,) in tables:
cols = conn.execute(f"PRAGMA table_info([{tname}])").fetchall()
col_names = [c[1] for c in cols]
if col_name in col_names:
# Use CAST to force float division (SQLite integer
# division truncates: 402/10=40, need 402.0/10=40.2)
cur = conn.execute(
f"UPDATE [{tname}] SET [{col_name}] = CAST([{col_name}] AS REAL) / ?",
(scale_factor,)
)
if cur.rowcount > 0:
logger.info(
f" power_scale: {tname}.{col_name} /= {scale_factor} "
f"({cur.rowcount} rows, power={power})"
)
total_updated += cur.rowcount
except sqlite3.Error as e:
logger.warning(f" power_scale: {tname}.{col_name} failed: {e}")
conn.commit()
conn.close()
return total_updated
def compile(self, pp_path: str | Path, exe_path: str | Path,
copybook_dirs: list[str | Path] | None = None,
extra_srcs: list[str | Path] | None = None) -> GixsqlBuildResult:
@@ -475,6 +723,11 @@ class GixsqlCobolRunner:
# `IF SQLCODE = -803` branches are reachable under gixsql+SQLite.
self._patch_sqlcode_normalize(pp_path)
# Fix gixpp numeric binding metadata for implicit-decimal DISPLAY fields.
# gixpp emits power=-1 for 9(4)V9(1), causing gixsql to read 6 bytes
# (length - power = 5-(-1)=6) and corrupt the value with adjacent data.
self._patch_numeric_metadata(pp_path)
# Functions that the preprocessed COBOL actually CALLs
gixsql_k = [
"-K", "GIXSQLStartSQL",