2026Technology-Competition/tests/measure/compare-a3.mjs

import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const OUT_DIR = join(__dirname, 'results');

const LANGS = ['js', 'css', 'java', 'sql'];
const LANG_LABEL = { js: 'JS', css: 'CSS', java: 'Java', sql: 'SQL' };

function loadBaselines() {
  const files = readdirSync(OUT_DIR).filter(f => /^manual-review-.+\.json$/.test(f)).sort();
  return files.map(f => JSON.parse(readFileSync(join(OUT_DIR, f), 'utf8')));
}

function langOf(rel) {
  const parts = rel.split('/');
  const idx = parts.indexOf('efficiency-samples');
  return parts[idx + 1];
}

function sum(arr) { return arr.reduce((s, v) => s + v, 0); }
function mean(arr) { return arr.length ? sum(arr) / arr.length : 0; }

function main() {
  const baselines = loadBaselines();
  const plugin = JSON.parse(readFileSync(join(OUT_DIR, 'a3-plugin-results.json'), 'utf8'));

  const pluginByLang = {};
  for (const row of plugin.rows) {
    pluginByLang[row.lang] = { files: row.files, diagnostics: row.diagnostics, totalMs: row.totalMs };
  }

  const perReviewer = baselines.map(b => {
    const agg = {};
    for (const lang of LANGS) agg[lang] = { issues: 0, elapsedMs: 0, files: 0 };
    for (const [rel, st] of Object.entries(b.files)) {
      const lang = langOf(rel);
      if (!agg[lang]) continue;
      agg[lang].issues += st.issues.length;
      agg[lang].elapsedMs += st.elapsedMs;
      agg[lang].files += 1;
    }
    return { reviewer: b.reviewer, level: b.level, totalElapsedMs: b.totalElapsedMs, agg };
  });

  const comparison = { generatedAt: new Date().toISOString(), reviewers: perReviewer.map(r => ({ reviewer: r.reviewer, level: r.level, totalElapsedMs: r.totalElapsedMs })), rows: [] };

  console.log('===== 每人每语言汇总 =====');
  for (const r of perReviewer) {
    console.log(`[${r.reviewer} (${r.level})] 总耗时 ${(r.totalElapsedMs / 60000).toFixed(1)} min`);
    for (const lang of LANGS) {
      const a = r.agg[lang];
      console.log(`  ${LANG_LABEL[lang]}: ${a.issues} 条, ${(a.elapsedMs / 1000).toFixed(1)}s / ${a.files} 文件`);
    }
  }

  console.log('\n===== 人工基线(3人均值) vs 插件(提效后) =====');
  console.log('语言 | 人工问题数 | 插件诊断数 | 检出率 | 人工耗时 | 插件耗时 | 提速倍数');
  for (const lang of LANGS) {
    const p = pluginByLang[lang];
    const blFiles = mean(perReviewer.map(r => r.agg[lang].files));
    const blIssues = mean(perReviewer.map(r => r.agg[lang].issues));
    const blMs = mean(perReviewer.map(r => r.agg[lang].elapsedMs));
    const recall = p.diagnostics > 0 ? blIssues / p.diagnostics : 0;
    const speedup = blMs / (p.totalMs || 1);
    comparison.rows.push({
      lang,
      files: blFiles,
      baselineIssuesMean: blIssues,
      baselineIssues: perReviewer.map(r => r.agg[lang].issues),
      pluginDiagnostics: p.diagnostics,
      recall: +recall.toFixed(3),
      baselineMsMean: +blMs.toFixed(1),
      baselineMs: perReviewer.map(r => r.agg[lang].elapsedMs),
      pluginMs: +p.totalMs.toFixed(1),
      speedup: +speedup.toFixed(1),
    });
    console.log(`${LANG_LABEL[lang]} | ${blIssues.toFixed(1)} | ${p.diagnostics} | ${(recall * 100).toFixed(0)}% | ${(blMs / 1000).toFixed(1)}s | ${(p.totalMs / 1000).toFixed(1)}s | ${speedup.toFixed(1)}x`);
  }

  const allBlMs = mean(perReviewer.map(r => r.totalElapsedMs));
  const allPluginMs = plugin.totalMs;
  const allRecall = sum(perReviewer.map(r => sum(LANGS.map(l => r.agg[l].issues)))) / perReviewer.length / plugin.totalDiagnostics;
  const allSpeedup = allBlMs / allPluginMs;
  comparison.total = {
    baselineMsMean: +allBlMs.toFixed(1),
    baselineMs: perReviewer.map(r => r.totalElapsedMs),
    pluginMs: +allPluginMs.toFixed(1),
    baselineDiagnosticsMean: +sum(perReviewer.map(r => sum(LANGS.map(l => r.agg[l].issues)))) / perReviewer.length,
    pluginDiagnostics: plugin.totalDiagnostics,
    recall: +allRecall.toFixed(3),
    speedup: +allSpeedup.toFixed(1),
  };
  console.log(`\n合计 | ${comparison.total.baselineDiagnosticsMean.toFixed(1)} | ${plugin.totalDiagnostics} | ${(allRecall * 100).toFixed(0)}% | ${(allBlMs / 60000).toFixed(1)}min | ${(allPluginMs / 1000).toFixed(1)}s | ${allSpeedup.toFixed(1)}x`);

  writeFileSync(join(OUT_DIR, 'a3-comparison.json'), JSON.stringify(comparison, null, 2));
  console.log(`\n结果已写入 ${join(OUT_DIR, 'a3-comparison.json')}`);
}

main();