提效幅度严格基准制:测试通过不构成提效证据,无基线对比数据一律C档封顶
- classifyVerifiability 新增 isPureGainDim(提效幅度/效果对比/效率提升)严格模式: 此类维度仅基准对比数据可豁免C档,修复六边形战队无量化数据仍得6/10的问题 - verif 判档增加 VERIF pipeLog 调试行(tier/capped/eff 可观测) - 补充3个单测:纯增益C封顶/基准豁免A/非纯增益B档行为不变 - AGENTS.md 同步三档机制说明
This commit is contained in:
@@ -149,7 +149,7 @@ cloneRepo → discoverFiles → countCodeStats → tryBuild → tryBrowse/trySta
|
||||
|
||||
- **排名用多次聚合**:`aggregateScores`/`aggregateEntryScores`(standard-utils.ts)——最近 N 次(默认3)`review_snapshots.score` 中位数;N=2 平均、N=1 单次。聚合前提=各快照 `standard_snapshot` 一致。`<3 次标"初评(未达聚合样本)"`,排名区分正式/初评
|
||||
- **快照分数列**:`review_snapshots.score` 存含迟交扣分的 final_score(写快照时一并存,不解析 ai_report——它只有 totalScore)。历史快照 backfill 从 totalScore best-effort
|
||||
- **可验证能力三档**(`classifyVerifiability`,效果/提效类维度,校准之前判档):A=基准证据(`entries.benchmark_json` status=done);B=测试通过或覆盖率非 null;C=数据缺位→封顶 maxScore*0.3 + note"数据缺位(未证明),非无效"。**构建成功≠效果可验证**。C 档 note 渲染到前端维度表 + PDF
|
||||
- **可验证能力三档**(`classifyVerifiability`,效果/提效类维度,校准之前判档):A=基准证据(`entries.benchmark_json` status=done);B=测试通过或覆盖率非 null;C=数据缺位→封顶 maxScore*0.3 + note"数据缺位(未证明),非无效"。**构建成功≠效果可验证**。C 档 note 渲染到前端维度表 + PDF。**纯增益维度严格制(2026-08-26)**:`提效幅度/效果对比/效率提升` 类维度(isPureGainDim)测试通过**不构成**提效证据——无基线对比数据一律 C 档封顶(实测案例:六边形战队提效幅度无量化数据 raw=6,修复后同类场景 C≤3)
|
||||
- **基准证据按 entry 落库**(`entries.benchmark_json`,非 env 变量——MAX_CONCURRENT=3 并发会串数据)
|
||||
- **确定性 L1**(`detectStructuralContradictions`):仅证据性矛盾触发(有测试/基准证据但效果≈0 → under;效果高分+实现全低 → over)。**效果维度 under 一律丢弃**(computeCalibration 内过滤)——效果维度只降不升,诚实由三档封顶负责。禁止"实现高分+无数据"当 under
|
||||
- **overall 中性边界**(`neutralizeTestEvidence`):测试 summary 含"中性"→ 标 `[中性证据]`;不含("未检测到测试框架配置"=真缺测试)→ 真实弱点。synthesizeOverall prompt 禁止把 `[中性证据]` 列为不足
|
||||
|
||||
@@ -330,6 +330,28 @@ describe('TC-VERIFY · 可验证能力三档(2026-08-19)', () => {
|
||||
expect(isEffectDim('效果与数据')).toBe(true);
|
||||
expect(isEffectDim('开发范式')).toBe(false);
|
||||
});
|
||||
|
||||
it('纯增益维度(提效幅度):测试通过不构成提效证据 → C 档封顶(2026-08-26)', () => {
|
||||
const d = { name: '提效幅度', score: 6, maxScore: 10 };
|
||||
const r = classifyVerifiability(d, { hasEffectEvidence: true });
|
||||
expect(r.tier).toBe('C');
|
||||
expect(r.capped).toBe(true);
|
||||
expect(r.effectiveScore).toBeLessThanOrEqual(3);
|
||||
expect(r.note).toContain('基线对比');
|
||||
});
|
||||
|
||||
it('纯增益维度:有基准对比证据 → A 档不封顶', () => {
|
||||
const d = { name: '提效幅度', score: 9, maxScore: 10 };
|
||||
const r = classifyVerifiability(d, { hasBenchmarkEvidence: true, hasEffectEvidence: true });
|
||||
expect(r.tier).toBe('A');
|
||||
expect(r.capped).toBe(false);
|
||||
});
|
||||
|
||||
it('非纯增益的效果维度仍可用测试证据豁免(B 档)——赛道一行为不变', () => {
|
||||
const r = classifyVerifiability(dim, { hasEffectEvidence: true });
|
||||
expect(r.tier).toBe('B');
|
||||
expect(r.capped).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('TC-CONTRAD · 确定性 L1 证据性矛盾(2026-08-19)', () => {
|
||||
|
||||
@@ -1273,6 +1273,7 @@ ${overviewFileBlock}
|
||||
};
|
||||
for (const d of dimensions) {
|
||||
const v = classifyVerifiability(d, verifEvidence);
|
||||
pipeLog(entryId, ' VERIF', `${d.name} raw=${d.score} tier=${v.tier} capped=${v.capped} eff=${v.effectiveScore}`);
|
||||
if (v.capped && d.score > v.effectiveScore) {
|
||||
d.score = v.effectiveScore;
|
||||
(d as any).verifiability = v;
|
||||
|
||||
@@ -268,14 +268,27 @@ export function isEffectDim(name: string): boolean {
|
||||
return EFFECT_EVIDENCE_KEYS.some(k => name.includes(k));
|
||||
}
|
||||
|
||||
// 纯增益对比类维度(2026-08-26):此类维度考察的是"基线 vs 改造后"的量化对比,
|
||||
// 工具自身的单元测试通过不构成提效证据——只有基准/对比数据(A档)才能豁免 C 档封顶。
|
||||
const PURE_GAIN_KEYS = ['提效幅度', '效果对比', '效率提升'];
|
||||
|
||||
export function isPureGainDim(name: string): boolean {
|
||||
return PURE_GAIN_KEYS.some(k => (name || '').includes(k));
|
||||
}
|
||||
|
||||
export function classifyVerifiability(
|
||||
dim: { name: string; score: number; maxScore: number },
|
||||
evidence: { hasBenchmarkEvidence?: boolean; hasEffectEvidence?: boolean; hasBuildEvidence?: boolean } = {}
|
||||
) {
|
||||
if (evidence.hasBenchmarkEvidence) return { tier: 'A' as const, capped: false, effectiveScore: dim.score, note: '有确定性基准证据(seed-defect benchmark)' };
|
||||
if (!isEffectDim(dim.name) || evidence.hasEffectEvidence) return { tier: 'B' as const, capped: false, effectiveScore: dim.score, note: '' };
|
||||
const strict = isPureGainDim(dim.name);
|
||||
// 非效果维度直接 B;纯增益维度即使有测试通过证据也不豁免(严格基准制)
|
||||
if (!isEffectDim(dim.name) || (evidence.hasEffectEvidence && !strict)) return { tier: 'B' as const, capped: false, effectiveScore: dim.score, note: '' };
|
||||
const cap = Math.floor(dim.maxScore * 0.3);
|
||||
return { tier: 'C' as const, capped: true, effectiveScore: Math.min(dim.score, cap), note: `数据缺位(未证明),非无效;C档封顶 ${cap}/${dim.maxScore}` };
|
||||
const note = strict
|
||||
? `提效类维度缺基线对比数据(未证明),非无效;测试通过不构成提效证据;C档封顶 ${cap}/${dim.maxScore}`
|
||||
: `数据缺位(未证明),非无效;C档封顶 ${cap}/${dim.maxScore}`;
|
||||
return { tier: 'C' as const, capped: true, effectiveScore: Math.min(dim.score, cap), note };
|
||||
}
|
||||
|
||||
// ================= 确定性 L1 证据性矛盾(2026-08-19)=================
|
||||
|
||||
Reference in New Issue
Block a user