From 6ac9861c84426be63d7998184e9e871d33b25246 Mon Sep 17 00:00:00 2001 From: hangshuo652 Date: Thu, 18 Jun 2026 17:17:11 +0800 Subject: [PATCH] test: add master validation suite (Pipeline/HINA/Benchmark/QG/Retry/Report - 30/30) --- test-data/test_master_validation.py | 111 ++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 test-data/test_master_validation.py diff --git a/test-data/test_master_validation.py b/test-data/test_master_validation.py new file mode 100644 index 0000000..5c22ef6 --- /dev/null +++ b/test-data/test_master_validation.py @@ -0,0 +1,111 @@ +""" +Master Validation — 增强测试系统 综合验证 +验证内容: Pipeline / HINA全分类 / 测试基准 / QG / Retry / Report +実行: python -X utf8 test-data/test_master_validation.py +""" +import sys, json, tempfile, shutil +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent)) +from data.diff_result import VerificationRun +from data.test_case import TestCase +from hina.classifier import compute_confidence +from hina.gate import check as gate_check, _compute_score +from hina.retry import RetryHandler +from report.generator import ReportGenerator +from cobol_testgen import extract_structure, generate_data + +PASS, FAIL = 0, 0; LOG = [] +def do(cat, name, fn): + global PASS, FAIL + try: + fn(); PASS += 1; LOG.append(f' [{cat}] {name} -> PASS') + except Exception as e: + FAIL += 1; LOG.append(f' [{cat}] {name} -> FAIL: {str(e)[:100]}') + +def S(): + return '\n'.join([ + ' IDENTIFICATION DIVISION.', + ' PROGRAM-ID. T.', + ' DATA DIVISION.', + ' WORKING-STORAGE SECTION.', + ' 01 X PIC X.', + ' PROCEDURE DIVISION.', + ' IF A>B MOVE 1 TO C ELSE MOVE 2 TO C.', + ' GOBACK.']) + +# ── Pipeline ── +do('PIPE','extract->generate', lambda: ( + st:=extract_structure(S()), st['total_branches']>=2)) +do('PIPE','HINA+QG', lambda: gate_check([{'x':1}],{}, + {'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]})['passed']) +do('PIPE','extract+HINA+QG', lambda: ( + st:=extract_structure(S()), h:=compute_confidence(S(),st), + qg:=gate_check([TestCase(id='x',fields={'a':1})],h, + {'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]}), True)) +do('PIPE','report JSON HINA', lambda: ( + rd:=Path(tempfile.mkdtemp()), + ReportGenerator().generate_json(VerificationRun(program='T',hina_type='DB'),rd/'r.json'), + d:=json.loads((rd/'r.json').read_text()), shutil.rmtree(rd), d['hina_type']=='DB')) + +# ── HINA L1 ── +for kw, cat, conf in [ + ('EXEC SQL','DB操作',0.95), ('CALL\nLINKAGE','子程序调用',0.90), + ('SORT ON KEY','SORT',0.95), ('MERGE ON KEY','MERGE',0.95), + ('DFHCOMMAREA','online',0.95), ('SYSIN','SYSIN',0.90), + ('ORGANIZATION IS','文件编成',0.99), ('ALTERNATE RECORD KEY','替代索引',0.99), + ('WRITE AFTER','编辑输出',0.80)]: + do('L1', cat, lambda k=kw,c=cat,cf=conf: ( + h:=compute_confidence(k,{}), h['category']==c and h['confidence']>=cf)) + +# ── 実プログラム ── +for fn in ['HINA001','HINA025','HINA101','HINA005','HINA007']: + do('REAL', fn, lambda f=fn: ( + src:=open(f'test-data/cobol/{f}.cbl',encoding='utf-8').read(), + st:=extract_structure(src), st is not None)) + +# ── Benchmark ── +do('BM','COM-N001', lambda: generate_data('PROCEDURE DIVISION.GOBACK.')!=None) +do('BM','MT-N001', lambda: ( + s:=open('test-data/cobol/HINA001.cbl',encoding='utf-8').read(), + extract_structure(s)['file_count']>=3)) +do('BM','B-N001', lambda: extract_structure(S())['total_branches']>=2) + +# ── Quality Gate ── +do('QG','pass', lambda: gate_check([{'x':1}],{}, + {'branch_rate':1.0,'paragraph_rate':1.0,'uncovered_decision_ids':[]})['passed']) +do('QG','fail', lambda: not gate_check([],{}, + {'branch_rate':0.0,'paragraph_rate':0.0,'uncovered_decision_ids':[1]})['passed']) +do('QG','score', lambda: abs(_compute_score( + {'branch_rate':0.92,'paragraph_rate':1.0},{})-0.976)<0.01) + +# ── Retry ── +do('RETRY','immediate', lambda: RetryHandler().run( + lambda: VerificationRun(status='PASS')).status=='PASS') +do('RETRY','fatal', lambda: RetryHandler(1,1).run( + lambda: VerificationRun(status='ERROR')).status=='FATAL') +do('RETRY','heal', lambda: ( + c:=[0], h:=RetryHandler(3,1), + v:=h.run(lambda: (c.__setitem__(0,c[0]+1), + VerificationRun(status='BLOCKED',debug={'cobol_build':{'log':'not found'}}))[1] + if c[0]<=2 else VerificationRun(status='PASS')), + v.status=='PASS' and v.heal_retry>0)) + +# ── Report ── +do('RPT','JSON-quality', lambda: ( + rd:=Path(tempfile.mkdtemp()), + ReportGenerator().generate_json(VerificationRun(program='T',quality_score=0.85),rd/'r.json'), + d:=json.loads((rd/'r.json').read_text()),shutil.rmtree(rd),d['quality_score']==0.85)) +do('RPT','JSON-retry', lambda: ( + rd:=Path(tempfile.mkdtemp()), + ReportGenerator().generate_json(VerificationRun(program='T',heal_retry=2),rd/'r.json'), + d:=json.loads((rd/'r.json').read_text()),shutil.rmtree(rd),d['heal_retry']==2)) +do('RPT','machine-JSON', lambda: ( + rd:=Path(tempfile.mkdtemp()), + ReportGenerator().generate_machine_json(VerificationRun(program='T',branch_rate=0.9),rd/'m.json'), + d:=json.loads((rd/'m.json').read_text()),shutil.rmtree(rd),d['branch_rate']==0.9)) + +# ── Summary ── +print(); [print(l) for l in LOG] +total = PASS+FAIL; rate = PASS/max(total,1)*100 +print(f'\n═ Total: {total} | PASS: {PASS} | FAIL: {FAIL} | RATE: {rate:.1f}% ═') +sys.exit(0 if FAIL==0 else 1)