test: add master validation suite (Pipeline/HINA/Benchmark/QG/Retry/Report - 30/30)
This commit is contained in:
hangshuo652
@@ -0,0 +1,111 @@
|
|||||||
|
"""
|
||||||
|
Master Validation — 增强测试系统 综合验证
|
||||||
|
验证内容: Pipeline / HINA全分类 / 测试基准 / QG / Retry / Report
|
||||||
|
実行: python -X utf8 test-data/test_master_validation.py
|
||||||
|
"""
|
||||||
|
import sys, json, tempfile, shutil
|
||||||
|
from pathlib import Path
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||||
|
from data.diff_result import VerificationRun
|
||||||
|
from data.test_case import TestCase
|
||||||
|
from hina.classifier import compute_confidence
|
||||||
|
from hina.gate import check as gate_check, _compute_score
|
||||||
|
from hina.retry import RetryHandler
|
||||||
|
from report.generator import ReportGenerator
|
||||||
|
from cobol_testgen import extract_structure, generate_data
|
||||||
|
|
||||||
|
PASS, FAIL = 0, 0; LOG = []
|
||||||
|
def do(cat, name, fn):
|
||||||
|
global PASS, FAIL
|
||||||
|
try:
|
||||||
|
fn(); PASS += 1; LOG.append(f' [{cat}] {name} -> PASS')
|
||||||
|
except Exception as e:
|
||||||
|
FAIL += 1; LOG.append(f' [{cat}] {name} -> FAIL: {str(e)[:100]}')
|
||||||
|
|
||||||
|
def S():
|
||||||
|
return '\n'.join([
|
||||||
|
' IDENTIFICATION DIVISION.',
|
||||||
|
' PROGRAM-ID. T.',
|
||||||
|
' DATA DIVISION.',
|
||||||
|
' WORKING-STORAGE SECTION.',
|
||||||
|
' 01 X PIC X.',
|
||||||
|
' PROCEDURE DIVISION.',
|
||||||
|
' IF A>B MOVE 1 TO C ELSE MOVE 2 TO C.',
|
||||||
|
' GOBACK.'])
|
||||||
|
|
||||||
|
# ── Pipeline ──
|
||||||
|
do('PIPE','extract->generate', lambda: (
|
||||||
|
st:=extract_structure(S()), st['total_branches']>=2))
|
||||||
|
do('PIPE','HINA+QG', lambda: gate_check([{'x':1}],{},
|
||||||
|
{'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]})['passed'])
|
||||||
|
do('PIPE','extract+HINA+QG', lambda: (
|
||||||
|
st:=extract_structure(S()), h:=compute_confidence(S(),st),
|
||||||
|
qg:=gate_check([TestCase(id='x',fields={'a':1})],h,
|
||||||
|
{'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]}), True))
|
||||||
|
do('PIPE','report JSON HINA', lambda: (
|
||||||
|
rd:=Path(tempfile.mkdtemp()),
|
||||||
|
ReportGenerator().generate_json(VerificationRun(program='T',hina_type='DB'),rd/'r.json'),
|
||||||
|
d:=json.loads((rd/'r.json').read_text()), shutil.rmtree(rd), d['hina_type']=='DB'))
|
||||||
|
|
||||||
|
# ── HINA L1 ──
|
||||||
|
for kw, cat, conf in [
|
||||||
|
('EXEC SQL','DB操作',0.95), ('CALL\nLINKAGE','子程序调用',0.90),
|
||||||
|
('SORT ON KEY','SORT',0.95), ('MERGE ON KEY','MERGE',0.95),
|
||||||
|
('DFHCOMMAREA','online',0.95), ('SYSIN','SYSIN',0.90),
|
||||||
|
('ORGANIZATION IS','文件编成',0.99), ('ALTERNATE RECORD KEY','替代索引',0.99),
|
||||||
|
('WRITE AFTER','编辑输出',0.80)]:
|
||||||
|
do('L1', cat, lambda k=kw,c=cat,cf=conf: (
|
||||||
|
h:=compute_confidence(k,{}), h['category']==c and h['confidence']>=cf))
|
||||||
|
|
||||||
|
# ── 実プログラム ──
|
||||||
|
for fn in ['HINA001','HINA025','HINA101','HINA005','HINA007']:
|
||||||
|
do('REAL', fn, lambda f=fn: (
|
||||||
|
src:=open(f'test-data/cobol/{f}.cbl',encoding='utf-8').read(),
|
||||||
|
st:=extract_structure(src), st is not None))
|
||||||
|
|
||||||
|
# ── Benchmark ──
|
||||||
|
do('BM','COM-N001', lambda: generate_data('PROCEDURE DIVISION.GOBACK.')!=None)
|
||||||
|
do('BM','MT-N001', lambda: (
|
||||||
|
s:=open('test-data/cobol/HINA001.cbl',encoding='utf-8').read(),
|
||||||
|
extract_structure(s)['file_count']>=3))
|
||||||
|
do('BM','B-N001', lambda: extract_structure(S())['total_branches']>=2)
|
||||||
|
|
||||||
|
# ── Quality Gate ──
|
||||||
|
do('QG','pass', lambda: gate_check([{'x':1}],{},
|
||||||
|
{'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]})['passed'])
|
||||||
|
do('QG','fail', lambda: not gate_check([],{},
|
||||||
|
{'branch_rate':0.0,'paragraph_rate':0.0,'uncovered_decision_ids':[1]})['passed'])
|
||||||
|
do('QG','score', lambda: abs(_compute_score(
|
||||||
|
{'branch_rate':0.92,'paragraph_rate':1.0},{})-0.976)<0.01)
|
||||||
|
|
||||||
|
# ── Retry ──
|
||||||
|
do('RETRY','immediate', lambda: RetryHandler().run(
|
||||||
|
lambda: VerificationRun(status='PASS')).status=='PASS')
|
||||||
|
do('RETRY','fatal', lambda: RetryHandler(1,1).run(
|
||||||
|
lambda: VerificationRun(status='ERROR')).status=='FATAL')
|
||||||
|
do('RETRY','heal', lambda: (
|
||||||
|
c:=[0], h:=RetryHandler(3,1),
|
||||||
|
v:=h.run(lambda: (c.__setitem__(0,c[0]+1),
|
||||||
|
VerificationRun(status='BLOCKED',debug={'cobol_build':{'log':'not found'}}))[1]
|
||||||
|
if c[0]<=2 else VerificationRun(status='PASS')),
|
||||||
|
v.status=='PASS' and v.heal_retry>0))
|
||||||
|
|
||||||
|
# ── Report ──
|
||||||
|
do('RPT','JSON-quality', lambda: (
|
||||||
|
rd:=Path(tempfile.mkdtemp()),
|
||||||
|
ReportGenerator().generate_json(VerificationRun(program='T',quality_score=0.85),rd/'r.json'),
|
||||||
|
d:=json.loads((rd/'r.json').read_text()),shutil.rmtree(rd),d['quality_score']==0.85))
|
||||||
|
do('RPT','JSON-retry', lambda: (
|
||||||
|
rd:=Path(tempfile.mkdtemp()),
|
||||||
|
ReportGenerator().generate_json(VerificationRun(program='T',heal_retry=2),rd/'r.json'),
|
||||||
|
d:=json.loads((rd/'r.json').read_text()),shutil.rmtree(rd),d['heal_retry']==2))
|
||||||
|
do('RPT','machine-JSON', lambda: (
|
||||||
|
rd:=Path(tempfile.mkdtemp()),
|
||||||
|
ReportGenerator().generate_machine_json(VerificationRun(program='T',branch_rate=0.9),rd/'m.json'),
|
||||||
|
d:=json.loads((rd/'m.json').read_text()),shutil.rmtree(rd),d['branch_rate']==0.9))
|
||||||
|
|
||||||
|
# ── Summary ──
|
||||||
|
print(); [print(l) for l in LOG]
|
||||||
|
total = PASS+FAIL; rate = PASS/max(total,1)*100
|
||||||
|
print(f'\n═ Total: {total} | PASS: {PASS} | FAIL: {FAIL} | RATE: {rate:.1f}% ═')
|
||||||
|
sys.exit(0 if FAIL==0 else 1)
|
||||||
Reference in New Issue
Block a user