chore: restore original directory structure (project under code-review-graph-main/)
This commit is contained in:
AuraK Developer
@@ -0,0 +1,1886 @@
|
||||
"""Quantitative code-review scoring for the unified-review workflow.
|
||||
|
||||
Implements the objectively computable Layer-2 metrics from the
|
||||
ai-code-review methodology as code, plus the git-history / graph risk
|
||||
factors used by the gstack-review workflow. The report metric set
|
||||
consists solely of the five objective heuristic metrics computed here;
|
||||
``llm_judged`` is returned empty for backward compatibility.
|
||||
|
||||
The three public entry points are:
|
||||
|
||||
* :func:`score_review` - all objective Layer-2 metrics for changed files
|
||||
* :func:`dedupe_findings` - fingerprint dedup + confidence merge
|
||||
* :func:`build_report_data` - data feed for the HTML report
|
||||
|
||||
Every metric returns a dict with ``score``/``value``, ``grade``
|
||||
(one of ``good``/``warn``/``fail``), ``thresholds`` and ``evidence`` so
|
||||
the agent can cite the numbers instead of asserting a vibe.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from .changes import (
|
||||
compute_file_churn,
|
||||
map_changes_to_nodes,
|
||||
parse_diff_ranges,
|
||||
)
|
||||
from .constants import SECURITY_KEYWORDS
|
||||
from .graph import GraphStore
|
||||
from .parser import normalize_file_path
|
||||
from collections import Counter
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thresholds (aligned with the ai-code-review Layer-2 scoring rubrics)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# good/warn/fail boundaries per metric.
|
||||
THRESHOLDS: dict[str, dict[str, float]] = {
|
||||
"redundancy_rate": {"good_max": 10.0, "warn_max": 20.0}, # % duplicate lines
|
||||
"exception_coverage": {"warn_min": 50.0, "good_min": 30.0}, # exception/normal %
|
||||
"sql_risk": {"warn_at": 1, "fail_at": 3}, # risk count
|
||||
"high_risk_density": {"warn_min": 70.0, "good_min": 90.0}, # covered % (0..100)
|
||||
"vulnerability_risk": {"warn_at": 1, "fail_at": 2}, # risk count
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Keyword tables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Exception-path markers used by the exception-coverage heuristic. These are
|
||||
# deliberately language-agnostic and conservative: a hit is counted as an
|
||||
# exception/error path, not a happy path.
|
||||
_EXCEPTION_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\btry\b"),
|
||||
re.compile(r"\bexcept\b"),
|
||||
re.compile(r"\bcatch\s*\("),
|
||||
re.compile(r"\braise\b"),
|
||||
re.compile(r"\bthrow\b"),
|
||||
re.compile(r"\?\.\.\."),
|
||||
re.compile(r"\bif\s+err\s*!=\s*nil\b"),
|
||||
re.compile(r"\bif\s+.*\berror\b"),
|
||||
re.compile(r"\.catch\s*\("),
|
||||
re.compile(r"\belse\s*\{?\s*return\s+(?:false|None|null|nil|0)\b"),
|
||||
)
|
||||
|
||||
# SQL injection / unparameterized-query markers. A hit is a warning, not a
|
||||
# proof: the agent must confirm the surrounding context before acting.
|
||||
_SQL_RISK_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"[\"'](?:SELECT|INSERT|UPDATE|DELETE)\s+.*?\+\s*[\"']", re.I | re.S),
|
||||
re.compile(r"\bf?\".*?(?:SELECT|INSERT|UPDATE|DELETE).*?(?:\$\{|\{)\w", re.I | re.S),
|
||||
re.compile(r"\bexec(?:ute)?\s*\(\s*[\"'].*?(?:SELECT|INSERT|UPDATE|DELETE)", re.I | re.S),
|
||||
re.compile(r"WHERE\s+.*?=\s*[\"']\s*\+\s*\w", re.I | re.S),
|
||||
re.compile(r"\.format\s*\(.*\).*?(?:SELECT|INSERT|UPDATE|DELETE)", re.I | re.S),
|
||||
re.compile(r"%\s*[\"']\s*%\s*\(?.*?(?:SELECT|INSERT|UPDATE|DELETE)", re.I | re.S),
|
||||
)
|
||||
|
||||
# Concurrency / transaction / data-consistency markers for high-risk density.
|
||||
_HIGH_RISK_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\b(?:async|await)\b"),
|
||||
re.compile(r"\b(?:thread|goroutine|mutex|lock|unlock|semaphore)\b"),
|
||||
re.compile(r"\b(?:transaction|commit|rollback|begin)\b", re.I),
|
||||
re.compile(r"\b(?:atomic|concurrent|race)\b", re.I),
|
||||
re.compile(r"\b(?:SELECT|INSERT|UPDATE|DELETE)\b", re.I),
|
||||
re.compile(r"\b(?:lock|with_lock|select_for_update)\b", re.I),
|
||||
re.compile(r"\b(?:cache|invalidate|evict)\b", re.I),
|
||||
)
|
||||
|
||||
# OWASP / secret markers for the vulnerability heuristic.
|
||||
_VULNERABILITY_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\b(?:password|passwd|pwd)\s*=\s*[\"'][^\"']{1,32}[\"']", re.I),
|
||||
re.compile(r"\b(?:api[_-]?key|secret|token)\s*=\s*[\"'][^\"']+[\"']", re.I),
|
||||
re.compile(r"\b(?:eval|exec)\s*\(", re.I),
|
||||
re.compile(r"\bsubprocess\s*\(.*shell\s*=\s*True", re.I | re.S),
|
||||
re.compile(r"\b(?:innerHTML|dangerouslySetInnerHTML|html\.safe|mark_safe|v-html)\b", re.I),
|
||||
re.compile(r"\b(?:<script|onerror=|javascript:)\b", re.I),
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _grade(metric: str, value: float) -> str:
|
||||
"""Classify a numeric value into good/warn/fail for one metric."""
|
||||
t = THRESHOLDS[metric]
|
||||
if metric in ("sql_risk", "vulnerability_risk"):
|
||||
# Count-based: 0 = good, warn_at count = warn, fail_at count = fail.
|
||||
if value <= 0:
|
||||
return "good"
|
||||
if value < t["fail_at"]:
|
||||
return "warn"
|
||||
return "fail"
|
||||
if metric == "redundancy_rate":
|
||||
# percentage-based, lower is better
|
||||
return "good" if value <= t["good_max"] else (
|
||||
"warn" if value <= t["warn_max"] else "fail"
|
||||
)
|
||||
# percentage-based, higher is better
|
||||
if metric == "exception_coverage":
|
||||
return "good" if value >= t["warn_min"] else (
|
||||
"warn" if value >= t["good_min"] else "fail"
|
||||
)
|
||||
return "good" if value >= t["good_min"] else (
|
||||
"warn" if value >= t["warn_min"] else "fail"
|
||||
)
|
||||
|
||||
|
||||
def _iter_source_lines(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> list[tuple[str, str, int]]:
|
||||
"""Yield (rel_path, line, line_no) for each changed source line.
|
||||
|
||||
Changed files that cannot be read are skipped silently; a missing file is
|
||||
not a scoring failure. ``changed_files`` is resolved relative to
|
||||
``repo_root`` when not absolute.
|
||||
"""
|
||||
out: list[tuple[str, str, int]] = []
|
||||
for rel in changed_files:
|
||||
raw = rel.replace("\\", "/")
|
||||
candidate = Path(rel)
|
||||
if not candidate.is_absolute():
|
||||
candidate = repo_root / raw
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
try:
|
||||
lines = candidate.read_text(
|
||||
encoding="utf-8", errors="replace",
|
||||
).splitlines()
|
||||
except OSError:
|
||||
continue
|
||||
for i, line in enumerate(lines, start=1):
|
||||
out.append((raw, line, i))
|
||||
return out
|
||||
|
||||
|
||||
def _count_matching(lines: list[tuple[str, str, int]], patterns: tuple[re.Pattern[str], ...]) -> int:
|
||||
"""Count lines matching any pattern (deduped per line)."""
|
||||
hits = 0
|
||||
for _path, line, _no in lines:
|
||||
if any(p.search(line) for p in patterns):
|
||||
hits += 1
|
||||
return hits
|
||||
|
||||
|
||||
def _normalized_signature(line: str) -> str:
|
||||
"""Normalise a source line into a reusable duplicate signature."""
|
||||
text = re.sub(r"\s+", " ", line).strip()
|
||||
text = re.sub(r"\b\d+\b", "N", text)
|
||||
text = re.sub(r"[\"'][^\"']*[\"']", '"s"', text)
|
||||
return text.lower()
|
||||
|
||||
|
||||
def _find_repeated_blocks(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Detect near-duplicate source blocks (>=3 occurrences) across the diff.
|
||||
|
||||
A coarse structural proxy for the redundancy metric: a line that appears
|
||||
with the same normalised signature in at least three places is counted as
|
||||
duplicate content. This is heuristic -- the agent confirms before acting.
|
||||
"""
|
||||
lines = _iter_source_lines(changed_files, repo_root)
|
||||
sig_count: dict[str, list[tuple[str, int]]] = {}
|
||||
for rel, line, no in lines:
|
||||
sig = _normalized_signature(line)
|
||||
if len(sig) < 24:
|
||||
continue # ignore trivially short lines
|
||||
sig_count.setdefault(sig, []).append((rel, no))
|
||||
|
||||
blocks: list[dict[str, Any]] = []
|
||||
dup_lines = 0
|
||||
for sig, occurrences in sig_count.items():
|
||||
if len(occurrences) < 3:
|
||||
continue
|
||||
dup_lines += len(occurrences)
|
||||
blocks.append({
|
||||
"signature": sig[:120],
|
||||
"occurrences": len(occurrences),
|
||||
"locations": [
|
||||
{"file": rel, "line": no}
|
||||
for rel, no in occurrences[:8]
|
||||
],
|
||||
})
|
||||
blocks.sort(key=lambda b: b["occurrences"], reverse=True)
|
||||
total = len(lines)
|
||||
rate = (dup_lines / total * 100.0) if total else 0.0
|
||||
return [blocks, rate]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_sql_risk(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan changed lines for string-interpolated / unparameterized SQL."""
|
||||
lines = _iter_source_lines(changed_files, repo_root)
|
||||
locations: list[dict[str, Any]] = []
|
||||
for rel, line, no in lines:
|
||||
if any(p.search(line) for p in _SQL_RISK_PATTERNS):
|
||||
locations.append({
|
||||
"file": rel,
|
||||
"line": no,
|
||||
"snippet": line.strip()[:200],
|
||||
})
|
||||
count = len(locations)
|
||||
return {
|
||||
"metric": "sql_risk",
|
||||
"value": count,
|
||||
"grade": _grade("sql_risk", float(count)),
|
||||
"thresholds": THRESHOLDS["sql_risk"],
|
||||
"evidence": locations[:20],
|
||||
"note": "字符串拼接 SQL 的启发式扫描。修复前请逐一确认每个位置；"
|
||||
"用 EXPLAIN 评估性能风险。",
|
||||
}
|
||||
|
||||
|
||||
def compute_exception_coverage(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Estimate exception-path coverage of changed code.
|
||||
|
||||
Ratio = exception-path lines / (normal-path lines + exception-path lines).
|
||||
The ai-code-review rubric uses ~1 exception per 2 happy paths as the
|
||||
good baseline (>=50% of normal paths have a counterpart), hence the
|
||||
thresholds on this ratio.
|
||||
"""
|
||||
lines = _iter_source_lines(changed_files, repo_root)
|
||||
total = len(lines)
|
||||
exc = _count_matching(lines, _EXCEPTION_PATTERNS)
|
||||
normal = max(0, total - exc)
|
||||
ratio = (exc / (normal + exc) * 100.0) if (normal + exc) else 0.0
|
||||
return {
|
||||
"metric": "exception_coverage",
|
||||
"value": round(ratio, 2),
|
||||
"grade": _grade("exception_coverage", ratio),
|
||||
"thresholds": THRESHOLDS["exception_coverage"],
|
||||
"evidence": {
|
||||
"total_lines": total,
|
||||
"exception_path_lines": exc,
|
||||
"normal_path_lines": normal,
|
||||
},
|
||||
"note": "异常/错误路径行的启发式占比。请人工复核边界条件与错误处理。",
|
||||
}
|
||||
|
||||
|
||||
def compute_redundancy_rate(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Estimate code redundancy in the changed files."""
|
||||
blocks, rate = _find_repeated_blocks(changed_files, repo_root)
|
||||
return {
|
||||
"metric": "redundancy_rate",
|
||||
"value": round(rate, 2),
|
||||
"grade": _grade("redundancy_rate", rate),
|
||||
"thresholds": THRESHOLDS["redundancy_rate"],
|
||||
"evidence": blocks[:20],
|
||||
"note": "重复代码块启发式占比（规范化行在 >=3 处出现）。"
|
||||
"抽取公共逻辑前请确认。",
|
||||
}
|
||||
|
||||
|
||||
def compute_high_risk_density(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Estimate high-risk scenario coverage of changed code.
|
||||
|
||||
Density = (high-risk lines with matching concurrency/transaction/security
|
||||
markers) / (high-risk-relevant lines). When no high-risk lines exist the
|
||||
metric is N/A rather than a pass/fail.
|
||||
"""
|
||||
lines = _iter_source_lines(changed_files, repo_root)
|
||||
relevant = [
|
||||
(rel, line, no) for rel, line, no in lines
|
||||
if any(p.search(line) for p in _HIGH_RISK_PATTERNS)
|
||||
]
|
||||
if not relevant:
|
||||
return {
|
||||
"metric": "high_risk_density",
|
||||
"value": None,
|
||||
"grade": "na",
|
||||
"thresholds": THRESHOLDS["high_risk_density"],
|
||||
"evidence": {},
|
||||
"note": "变更中未检出并发/事务/数据一致性模式——"
|
||||
"除非审查发现缺口，标记为 N/A。",
|
||||
}
|
||||
covered = 0
|
||||
for _rel, line, _no in relevant:
|
||||
# Every relevant line is counted as "covered by review attention";
|
||||
# the marker is a signal for the agent to verify, not a defect.
|
||||
if line.strip():
|
||||
covered += 1
|
||||
density = covered / len(relevant) * 100.0
|
||||
return {
|
||||
"metric": "high_risk_density",
|
||||
"value": round(density, 2),
|
||||
"grade": _grade("high_risk_density", density),
|
||||
"thresholds": THRESHOLDS["high_risk_density"],
|
||||
"evidence": {
|
||||
"high_risk_lines": len(relevant),
|
||||
"covered_lines": covered,
|
||||
"locations": [
|
||||
{"file": rel, "line": no, "snippet": line.strip()[:160]}
|
||||
for rel, line, no in relevant[:20]
|
||||
],
|
||||
},
|
||||
"note": "并发/事务/安全模式密度。属审查注意力信号，非正确性评分。",
|
||||
}
|
||||
|
||||
|
||||
def compute_vulnerability_heuristic(
|
||||
changed_files: list[str], repo_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan changed lines for OWASP / secret-like patterns (heuristic)."""
|
||||
lines = _iter_source_lines(changed_files, repo_root)
|
||||
locations: list[dict[str, Any]] = []
|
||||
for rel, line, no in lines:
|
||||
if any(p.search(line) for p in _VULNERABILITY_PATTERNS):
|
||||
locations.append({
|
||||
"file": rel,
|
||||
"line": no,
|
||||
"snippet": line.strip()[:200],
|
||||
})
|
||||
count = len(locations)
|
||||
return {
|
||||
"metric": "vulnerability_risk",
|
||||
"value": count,
|
||||
"grade": _grade("vulnerability_risk", float(count)),
|
||||
"thresholds": THRESHOLDS["vulnerability_risk"],
|
||||
"evidence": locations[:20],
|
||||
"note": "OWASP/密钥模式的启发式扫描。真实漏洞需依赖扫描器"
|
||||
"（npm audit、pip-audit、govulncheck）确认。",
|
||||
}
|
||||
|
||||
|
||||
def compute_risk_factors(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
changed_files: list[str],
|
||||
include_churn: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute git-history + graph risk factors for changed files.
|
||||
|
||||
Returns churn hotspots, cross-community coupling and hub dependencies --
|
||||
the structural input the agent uses to prioritise review attention.
|
||||
"""
|
||||
abs_files = [normalize_file_path(repo_root / f) for f in changed_files]
|
||||
|
||||
# git churn over the trailing window (reuses changes.compute_file_churn).
|
||||
churn: dict[str, Any] = {"enabled": include_churn, "hotspots": []}
|
||||
if include_churn:
|
||||
counts = compute_file_churn(str(repo_root))
|
||||
hotspots = [
|
||||
{"file": f, "commits": c}
|
||||
for f, c in sorted(counts.items(), key=lambda kv: kv[1], reverse=True)
|
||||
if c >= 3
|
||||
]
|
||||
churn["hotspots"] = hotspots[:20]
|
||||
|
||||
# Cross-community callers of the changed functions. A caller from a
|
||||
# different community than the changed node signals coupling that deserves
|
||||
# extra review attention.
|
||||
changed_nodes = []
|
||||
for fp in abs_files:
|
||||
changed_nodes.extend(store.get_nodes_by_file(fp))
|
||||
node_qns = {n.qualified_name for n in changed_nodes}
|
||||
|
||||
caller_qns: set[str] = set()
|
||||
incoming: dict[str, list[str]] = {}
|
||||
for qn in node_qns:
|
||||
edges = store.get_edges_by_target(qn)
|
||||
callers = [e.source_qualified for e in edges if e.kind == "CALLS"]
|
||||
incoming[qn] = callers
|
||||
caller_qns.update(callers)
|
||||
|
||||
# Batch community lookups: changed nodes and their callers.
|
||||
all_qns = list(node_qns | caller_qns)
|
||||
cid_map = store.get_community_ids_by_qualified_names(all_qns)
|
||||
|
||||
cross_community: list[dict[str, Any]] = []
|
||||
hub_dependencies: list[dict[str, Any]] = []
|
||||
for qn in node_qns:
|
||||
tgt_cid = cid_map.get(qn)
|
||||
for caller in incoming.get(qn, []):
|
||||
src_cid = cid_map.get(caller)
|
||||
if src_cid is not None and tgt_cid is not None and src_cid != tgt_cid:
|
||||
cross_community.append({
|
||||
"caller": caller,
|
||||
"callee": qn,
|
||||
"edge": "CALLS",
|
||||
})
|
||||
if len(incoming.get(qn, [])) >= 10:
|
||||
hub_dependencies.append({
|
||||
"node": qn,
|
||||
"callers": len(incoming.get(qn, [])),
|
||||
})
|
||||
|
||||
return {
|
||||
"churn": churn,
|
||||
"cross_community_edges": cross_community[:20],
|
||||
"hub_dependencies": hub_dependencies[:20],
|
||||
"note": "结构性风险因子。高变更频率 + 跨社区耦合 + 中枢依赖"
|
||||
"意味着该改动需要额外审查关注。",
|
||||
}
|
||||
|
||||
|
||||
def score_review(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
changed_files: list[str],
|
||||
include_churn: bool = True,
|
||||
progress_cb: Callable[[float, Optional[str]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute all objective Layer-2 metrics for a set of changed files.
|
||||
|
||||
Args:
|
||||
store: Open graph store (caller owns and closes it).
|
||||
repo_root: Repository root.
|
||||
changed_files: Changed file paths relative to ``repo_root``.
|
||||
include_churn: Include git-churn risk factors.
|
||||
progress_cb: Optional ``(fraction, message)`` progress callback;
|
||||
invoked once per metric (fraction = k/5).
|
||||
|
||||
Returns:
|
||||
Dict with ``metrics`` (per-metric score/grade/evidence),
|
||||
``risk_factors``, ``llm_judged`` and ``summary``.
|
||||
"""
|
||||
metric_fns = {
|
||||
"sql_risk": compute_sql_risk,
|
||||
"exception_coverage": compute_exception_coverage,
|
||||
"redundancy_rate": compute_redundancy_rate,
|
||||
"high_risk_density": compute_high_risk_density,
|
||||
"vulnerability_risk": compute_vulnerability_heuristic,
|
||||
}
|
||||
metrics: dict[str, Any] = {}
|
||||
for idx, (name, fn) in enumerate(metric_fns.items()):
|
||||
if progress_cb is not None:
|
||||
progress_cb(idx / len(metric_fns), f"computing {name}")
|
||||
metrics[name] = fn(changed_files, repo_root)
|
||||
if progress_cb is not None:
|
||||
progress_cb(1.0, "metrics done")
|
||||
|
||||
risk_factors = compute_risk_factors(
|
||||
store, repo_root, changed_files, include_churn=include_churn,
|
||||
)
|
||||
|
||||
grades = [m["grade"] for m in metrics.values() if m["grade"] != "na"]
|
||||
worst = "fail" if "fail" in grades else (
|
||||
"warn" if "warn" in grades else "good"
|
||||
)
|
||||
|
||||
summary_parts = [
|
||||
f"Scored {len(changed_files)} changed file(s) with {len(metrics)} objective metrics.",
|
||||
f"Overall objective grade: {worst.upper()}",
|
||||
]
|
||||
for name, m in metrics.items():
|
||||
if m["grade"] == "na":
|
||||
summary_parts.append(f" - {name}: N/A")
|
||||
else:
|
||||
summary_parts.append(f" - {name}: {m['grade']} (value={m['value']})")
|
||||
if risk_factors["churn"]["hotspots"]:
|
||||
summary_parts.append(
|
||||
f" - {len(risk_factors['churn']['hotspots'])} churn hotspot(s)"
|
||||
)
|
||||
if risk_factors["cross_community_edges"]:
|
||||
summary_parts.append(
|
||||
f" - {len(risk_factors['cross_community_edges'])} cross-community edge(s)"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"metrics": metrics,
|
||||
"risk_factors": risk_factors,
|
||||
"llm_judged": [],
|
||||
"objective_grade": worst,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# dedupe_findings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fingerprint(finding: dict[str, Any]) -> str:
|
||||
path = str(finding.get("path", "")).replace("\\", "/")
|
||||
line = finding.get("line")
|
||||
category = str(finding.get("category", "")).strip().lower()
|
||||
if line:
|
||||
return f"{path}:{line}:{category}"
|
||||
return f"{path}:{category}"
|
||||
|
||||
|
||||
def dedupe_findings(
|
||||
findings: list[dict[str, Any]],
|
||||
suppress_prior: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge findings by fingerprint, boost multi-source confidence.
|
||||
|
||||
Mirrors the gstack-review "collect and merge" step:
|
||||
|
||||
* same fingerprint -> keep the highest-confidence finding
|
||||
* findings confirmed by multiple sources -> confidence +1 (cap 10)
|
||||
* confidence gates: >=7 normal, 5-6 caveat, 3-4 appendix, 1-2 suppressed
|
||||
* PR quality score = max(0, 10 - (critical*2 + informational*0.5))
|
||||
|
||||
Args:
|
||||
findings: Finding dicts with at least ``path``, ``category``,
|
||||
``severity``, ``confidence`` and optional ``source``/``line``.
|
||||
suppress_prior: Findings previously skipped by the user (read from a
|
||||
prior review-log). Entries whose fingerprint matches AND whose
|
||||
file was not changed since then are dropped.
|
||||
|
||||
Returns:
|
||||
Dict with ``merged``, ``suppressed``, ``quality_score`` and
|
||||
``summary``.
|
||||
"""
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
for f in findings:
|
||||
fp = _fingerprint(f)
|
||||
groups.setdefault(fp, []).append(f)
|
||||
|
||||
merged: list[dict[str, Any]] = []
|
||||
suppressed_by_gate = 0
|
||||
for fp, items in groups.items():
|
||||
best = max(items, key=lambda x: x.get("confidence", 0))
|
||||
sources = {str(i.get("source", "")).strip() for i in items if i.get("source")}
|
||||
confirmed = len(sources) > 1
|
||||
conf = float(best.get("confidence", 0))
|
||||
if confirmed:
|
||||
conf = min(10.0, conf + 1.0)
|
||||
best["multi_source_confirmed"] = True
|
||||
best["confirming_sources"] = sorted(sources)
|
||||
best["confidence"] = conf
|
||||
|
||||
if conf < 2:
|
||||
suppressed_by_gate += 1
|
||||
continue # suppressed entirely
|
||||
if conf < 4:
|
||||
best["display"] = "appendix"
|
||||
elif conf < 7:
|
||||
best["display"] = "normal-with-caveat"
|
||||
else:
|
||||
best["display"] = "normal"
|
||||
merged.append(best)
|
||||
|
||||
# Prior-review suppression (user explicitly skipped, file unchanged).
|
||||
suppressed_prior = 0
|
||||
if suppress_prior:
|
||||
prior_fps = {_fingerprint(f) for f in suppress_prior}
|
||||
kept: list[dict[str, Any]] = []
|
||||
for f in merged:
|
||||
if _fingerprint(f) in prior_fps:
|
||||
suppressed_prior += 1
|
||||
continue
|
||||
kept.append(f)
|
||||
merged = kept
|
||||
|
||||
critical = sum(1 for f in merged if str(f.get("severity", "")).upper() in ("CRITICAL", "BLOCKER"))
|
||||
informational = sum(
|
||||
1 for f in merged
|
||||
if str(f.get("severity", "")).upper() in ("INFORMATIONAL", "MINOR", "WARN")
|
||||
)
|
||||
quality = max(0.0, 10.0 - (critical * 2.0 + informational * 0.5))
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Merged {len(findings)} raw finding(s) into {len(merged)} "
|
||||
f"unique issue(s); quality score {quality:.1f}/10."
|
||||
),
|
||||
"merged": merged,
|
||||
"suppressed_by_confidence": suppressed_by_gate,
|
||||
"suppressed_by_prior": suppressed_prior,
|
||||
"quality_score": round(quality, 2),
|
||||
"counts": {"critical": critical, "informational": informational},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report data
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _normalise_coverage(coverage: Any) -> dict[str, Any] | None:
|
||||
"""Normalise the ``coverage`` block passed into the HTML report feed.
|
||||
|
||||
If ``coverage`` is not a dict (e.g. an AI agent passed ``True`` or a
|
||||
partial subset), return ``None`` so the report renders no coverage
|
||||
section rather than crashing. When a dict is given, keep it as-is so
|
||||
every field the agent chose to transmit survives; the HTML/Markdown
|
||||
templates provide ``N/A`` fallbacks for missing counts so a partial
|
||||
transmission never shows a misleading ``0/0``.
|
||||
"""
|
||||
if not isinstance(coverage, dict) or not coverage:
|
||||
return None
|
||||
return coverage
|
||||
|
||||
|
||||
#: The five objective metrics produced by score_review_tool. Any other key
|
||||
#: an agent passes in review_data.metrics (e.g. blast_radius, objective_grade,
|
||||
#: llm_judged leftovers) is filtered out so the report only renders the
|
||||
#: canonical five rows.
|
||||
_OBJECTIVE_METRIC_KEYS: frozenset[str] = frozenset({
|
||||
"sql_risk",
|
||||
"exception_coverage",
|
||||
"redundancy_rate",
|
||||
"high_risk_density",
|
||||
"vulnerability_risk",
|
||||
})
|
||||
|
||||
|
||||
def build_report_data(
|
||||
review_data: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Normalise score_review/dedupe output into the HTML report feed.
|
||||
|
||||
``review_data`` may contain ``metrics`` (from :func:`score_review`),
|
||||
``findings`` (from :func:`dedupe_findings`) and free-form ``verdict``,
|
||||
``tier``, ``scope`` fields. The returned dict is JSON-serialisable and
|
||||
ready to be injected into ``report-template.html`` as ``{{REPORT_DATA}}``.
|
||||
"""
|
||||
# Normalise ``files``: agents sometimes pass a list instead of a
|
||||
# comma-separated string. Accept both; a list is joined so the report
|
||||
# never renders Python/JSON list syntax.
|
||||
raw_files = review_data.get("files", "")
|
||||
files_text = (
|
||||
", ".join(str(f) for f in raw_files)
|
||||
if isinstance(raw_files, list)
|
||||
else raw_files
|
||||
)
|
||||
# ``reviewed_files`` drives the collapsible <details> list. Agents may
|
||||
# pass it as an array OR as a comma-separated string; accept both (a
|
||||
# string is split so the Markdown renderer never iterates char-by-char).
|
||||
# Fall back to the ``files`` array when nothing structured was passed.
|
||||
reviewed_files = review_data.get("reviewed_files") or []
|
||||
if isinstance(reviewed_files, str):
|
||||
reviewed_files = [
|
||||
p.strip() for p in reviewed_files.split(",") if p.strip()
|
||||
]
|
||||
elif not reviewed_files and isinstance(raw_files, list):
|
||||
reviewed_files = [str(f) for f in raw_files]
|
||||
|
||||
data: dict[str, Any] = {
|
||||
"scope": review_data.get("scope", "change-level"),
|
||||
"tier": review_data.get("tier", "standard"),
|
||||
"timestamp": review_data.get("timestamp", ""),
|
||||
"files": files_text,
|
||||
"reviewed_files": reviewed_files,
|
||||
"baseline": review_data.get("baseline", "generic"),
|
||||
"verdict": review_data.get("verdict", "❌ FAIL"),
|
||||
"quality_score": review_data.get("quality_score"),
|
||||
"counts": review_data.get("counts", {}),
|
||||
"metrics": {},
|
||||
"issues": [],
|
||||
"manual_review": review_data.get("manual_review", []),
|
||||
"llm_judged": review_data.get("llm_judged", []),
|
||||
"coverage": _normalise_coverage(review_data.get("coverage")),
|
||||
"spot_check": _normalise_coverage(review_data.get("spot_check")),
|
||||
}
|
||||
|
||||
# Objective metrics only: filter out stray keys (blast_radius,
|
||||
# objective_grade, ...) so the report always renders the canonical five.
|
||||
metrics = review_data.get("metrics") or {}
|
||||
for name, m in metrics.items():
|
||||
if name not in _OBJECTIVE_METRIC_KEYS:
|
||||
continue
|
||||
if isinstance(m, dict):
|
||||
data["metrics"][name] = {
|
||||
"grade": m.get("grade"),
|
||||
"value": m.get("value"),
|
||||
"note": m.get("note", ""),
|
||||
"evidence": m.get("evidence", {}),
|
||||
}
|
||||
|
||||
findings = review_data.get("findings") or []
|
||||
for f in findings:
|
||||
data["issues"].append({
|
||||
"severity": f.get("severity", "minor"),
|
||||
"category": f.get("category", ""),
|
||||
"message": f.get("summary", f.get("message", "")),
|
||||
"location": (
|
||||
f"{f.get('path', '')}:{f.get('line', '')}"
|
||||
if f.get("line")
|
||||
else str(f.get("path", ""))
|
||||
),
|
||||
"confidence": f.get("confidence"),
|
||||
"fix": f.get("fix", ""),
|
||||
})
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown report rendering (Chinese)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Human-readable labels for the Chinese Markdown report.
|
||||
_METRIC_LABELS: dict[str, str] = {
|
||||
"sql_risk": "SQL 注入风险",
|
||||
"exception_coverage": "异常分支覆盖",
|
||||
"redundancy_rate": "代码冗余率",
|
||||
"high_risk_density": "高风险场景密度",
|
||||
"vulnerability_risk": "漏洞风险",
|
||||
}
|
||||
|
||||
_GRADE_LABELS: dict[str, str] = {
|
||||
"good": "良好",
|
||||
"warn": "警告",
|
||||
"fail": "不合格",
|
||||
"na": "不适用",
|
||||
}
|
||||
|
||||
_SEVERITY_LABELS: dict[str, str] = {
|
||||
"blocker": "🔴 阻塞",
|
||||
"critical": "🔴 严重",
|
||||
"major": "🟡 主要",
|
||||
"warn": "🟡 主要",
|
||||
"minor": "🔵 次要",
|
||||
"informational": "🔵 次要",
|
||||
}
|
||||
|
||||
|
||||
def render_markdown_report(review_data: dict[str, Any]) -> str:
|
||||
"""Render the review data as a Chinese, standalone Markdown report.
|
||||
|
||||
Mirrors the HTML report content: verdict, tier, scope, objective
|
||||
metrics, issues, manual-review items and LLM-judged metrics. The
|
||||
output is a single Markdown document with no external dependencies.
|
||||
|
||||
Args:
|
||||
review_data: Review data dict (see :func:`build_report_data`).
|
||||
|
||||
Returns:
|
||||
The Markdown report text.
|
||||
"""
|
||||
data = build_report_data(review_data)
|
||||
lines: list[str] = []
|
||||
|
||||
lines.append("# 代码审查报告\n")
|
||||
|
||||
verdict = str(data["verdict"])
|
||||
verdict_line = f"- **结论**：{verdict}"
|
||||
tier = data.get("tier") or "standard"
|
||||
scope = data.get("scope") or "change-level"
|
||||
baseline = data.get("baseline") or "generic"
|
||||
lines.append(verdict_line)
|
||||
lines.append(
|
||||
f"- **档位**：{tier} · **范围**：{scope} · **基线**：{baseline}"
|
||||
)
|
||||
if data.get("timestamp"):
|
||||
lines.append(f"- **生成时间**：{data['timestamp']}")
|
||||
if data.get("files") and not (data.get("reviewed_files") or []):
|
||||
lines.append(f"- **文件**：{data['files']}")
|
||||
qs = data.get("quality_score")
|
||||
if qs is not None:
|
||||
lines.append(f"- **PR 质量分**：{qs}/10")
|
||||
counts = data.get("counts") or {}
|
||||
if counts:
|
||||
lines.append(
|
||||
f"- **问题统计**：{counts.get('critical', 0)} 严重 · "
|
||||
f"{counts.get('informational', 0)} 次要"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Reviewed files (collapsible). Renders from the structured
|
||||
# ``reviewed_files`` array when present; otherwise the flat ``files``
|
||||
# string is used (as the meta line above). Defensive: if a raw string
|
||||
# ever slips through (build_report_data already normalises it), split it
|
||||
# so we never iterate a string char-by-char.
|
||||
reviewed = data.get("reviewed_files") or []
|
||||
if isinstance(reviewed, str):
|
||||
reviewed = [p.strip() for p in reviewed.split(",") if p.strip()]
|
||||
if reviewed:
|
||||
lines.append(f"**审查文件（{len(reviewed)} 个）**")
|
||||
lines.append("")
|
||||
lines.append("<details>")
|
||||
lines.append(f"<summary>点击展开 / 收起（{len(reviewed)} 个文件）</summary>")
|
||||
lines.append("")
|
||||
for f in reviewed:
|
||||
lines.append(f"- `{f}`")
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
# Objective metrics
|
||||
metrics = data.get("metrics") or {}
|
||||
if metrics:
|
||||
lines.append("## 客观指标\n")
|
||||
lines.append("| 指标 | 数值 | 评级 | 说明 |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for name, m in metrics.items():
|
||||
label = _METRIC_LABELS.get(name, name)
|
||||
value = m.get("value")
|
||||
value_text = str(value) if value is not None else "N/A"
|
||||
grade = m.get("grade") or "na"
|
||||
grade_text = _GRADE_LABELS.get(grade, grade)
|
||||
note = str(m.get("note", "") or "")
|
||||
note = note.replace("\n", " ")
|
||||
lines.append(f"| {label} | {value_text} | {grade_text} | {note} |")
|
||||
lines.append("")
|
||||
|
||||
# Coverage. For gate="both+line" (whole-project) the file-count and
|
||||
# high-risk rows render; for gate="line+unit" (feature reviews) the
|
||||
# engine returns coverage_pct=None so only the line/unit rows plus the
|
||||
# fail-closed status line render.
|
||||
coverage = data.get("coverage") or {}
|
||||
if coverage:
|
||||
pct = coverage.get("coverage_pct")
|
||||
hr_pct = coverage.get("high_risk_coverage_pct")
|
||||
grade = coverage.get("grade") or "na"
|
||||
grade_text = _GRADE_LABELS.get(grade, grade)
|
||||
deep_read = coverage.get("deep_read_count", "N/A")
|
||||
total = coverage.get("total_files", "N/A")
|
||||
hr_total = coverage.get("high_risk_total_files", "N/A")
|
||||
hr_deep = coverage.get("high_risk_deep_count", "N/A")
|
||||
overall_target = coverage.get("overall_target", coverage.get("target", "N/A"))
|
||||
hr_target = coverage.get("high_risk_target", coverage.get("target", "N/A"))
|
||||
reached = coverage.get("target_reached", False)
|
||||
status = "✅ 达标" if reached else "🔴 覆盖不足"
|
||||
uncovered = coverage.get("uncovered_files") or []
|
||||
silent = coverage.get("silent_files") or []
|
||||
# Line / unit coverage (gate="both+line" / "line+unit"): fail-closed.
|
||||
# A missing value means the review never ran the line-coverage gate -
|
||||
# surface it explicitly instead of silently omitting the field.
|
||||
line_pct = coverage.get("line_coverage_pct")
|
||||
unit_pct = coverage.get("unit_coverage_pct")
|
||||
line_target = coverage.get("line_target", 95.0)
|
||||
unit_target = coverage.get("unit_target", 100.0)
|
||||
line_gap_n = len(coverage.get("line_gap_files") or [])
|
||||
unit_gap_n = len(coverage.get("unit_gap_files") or [])
|
||||
missing_n = len(coverage.get("missing_data_files") or [])
|
||||
lines.append("## 覆盖度\n")
|
||||
if pct is not None:
|
||||
lines.append(
|
||||
f"- **覆盖度（全库）**：{pct}% — 已深读 {deep_read}/{total} 个源文件"
|
||||
f"（目标 {overall_target}%）"
|
||||
)
|
||||
lines.append(
|
||||
f"- **覆盖度（高风险）**：{hr_pct}% — 已深读 {hr_deep}/{hr_total} 个高风险文件"
|
||||
f"（目标 {hr_target}%）{status}"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f"- **状态**：{status}（gate=\"line+unit\"：仅行/单元覆盖，不做文件数覆盖检查）"
|
||||
)
|
||||
if line_pct is None or unit_pct is None:
|
||||
lines.append("- **行覆盖**：未执行 🔴（coverage_tool 未用 gate=\"both+line\" 或未传三件套数据）")
|
||||
else:
|
||||
line_ok = "✅" if line_pct >= line_target and line_gap_n == 0 else "🔴"
|
||||
unit_ok = "✅" if unit_pct >= unit_target and unit_gap_n == 0 else "🔴"
|
||||
lines.append(
|
||||
f"- **行覆盖**：{line_pct}% — 目标 {line_target}%（缺口 {line_gap_n} 文件）{line_ok}"
|
||||
)
|
||||
lines.append(
|
||||
f"- **单元覆盖**：{unit_pct}% — 目标 {unit_target}%（缺口 {unit_gap_n} 文件）{unit_ok}"
|
||||
)
|
||||
if missing_n:
|
||||
lines.append(
|
||||
f"- **三件套数据缺失**：{missing_n} 个文件（缺 read_ranges/语义单元，已按 fail-closed 计为缺口）"
|
||||
)
|
||||
if uncovered:
|
||||
lines.append(
|
||||
f"- **未深读文件**：{len(uncovered)} 个"
|
||||
f"（静默文件 {len(silent)} 个）"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Anti-fake spot check (three-piece suite item 3). Fail-closed: a
|
||||
# missing/incomplete spot_check renders "未执行 🔴" so reviews that
|
||||
# skipped the sampled re-read are visible instead of silently green.
|
||||
spot = data.get("spot_check")
|
||||
if spot:
|
||||
groups = spot.get("groups_sampled")
|
||||
files = spot.get("files_sampled")
|
||||
units = spot.get("units_sampled")
|
||||
fake = spot.get("fake_read_found", 0)
|
||||
rereread = spot.get("groups_rereread") or []
|
||||
if units:
|
||||
mark = "🔴 发现假读" if (fake or rereread) else "✅"
|
||||
lines.append(
|
||||
f"- **防伪抽验**：抽样 {files} 文件 / {units} 单元 / {groups} 组，"
|
||||
f"假读 {fake}{mark}"
|
||||
)
|
||||
if rereread:
|
||||
lines.append(
|
||||
f" - 因假读重读组：{', '.join(rereread)}"
|
||||
)
|
||||
else:
|
||||
lines.append("- **防伪抽验**：未执行 🔴（spot_check 已上报但单元数为 0）")
|
||||
else:
|
||||
lines.append(
|
||||
"- **防伪抽验**：未执行 🔴（主代理未回读任何语义单元；"
|
||||
"Step 5.5 应执行每组抽 2 文件 × 2-3 单元并落盘 spot_check）"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Issues
|
||||
issues = data.get("issues") or []
|
||||
lines.append(f"## 问题清单（{len(issues)}）\n")
|
||||
if not issues:
|
||||
lines.append("未发现问题。\n")
|
||||
for i, issue in enumerate(issues, start=1):
|
||||
sev = str(issue.get("severity") or "minor").lower()
|
||||
sev_label = _SEVERITY_LABELS.get(sev, sev)
|
||||
category = str(issue.get("category") or "")
|
||||
message = str(issue.get("message") or "")
|
||||
location = str(issue.get("location") or "")
|
||||
conf = issue.get("confidence")
|
||||
fix = str(issue.get("fix") or "")
|
||||
lines.append(
|
||||
f"{i}. **{sev_label}** {message}"
|
||||
f"{f'（置信度 {conf}/10）' if conf is not None else ''}"
|
||||
)
|
||||
if category:
|
||||
lines.append(f" - **类别**：{category}")
|
||||
if location:
|
||||
lines.append(f" - **位置**：`{location}`")
|
||||
if fix:
|
||||
lines.append(f" - **修复建议**：{fix}")
|
||||
lines.append("")
|
||||
|
||||
# Manual review
|
||||
manual = data.get("manual_review") or []
|
||||
if manual:
|
||||
lines.append("## 需要人工审查\n")
|
||||
for m in manual:
|
||||
lines.append(f"- {m}")
|
||||
lines.append("")
|
||||
|
||||
# LLM-judged metrics
|
||||
judged = data.get("llm_judged") or []
|
||||
if judged:
|
||||
lines.append("## 需 LLM 判断的指标\n")
|
||||
lines.append(", ".join(str(j) for j in judged))
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coverage computation (review coverage of the whole project)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Coverage targets (percentage of source files deep-read). Fixed to the
|
||||
#: single standard tier; the fast/strict tiers were removed. Per-gate
|
||||
#: targets: ``overall`` is the whole-project file-count target, ``high_risk``
|
||||
#: the signal-flagged subset target. ``gate="both"`` requires both to pass.
|
||||
COVERAGE_TARGETS: dict[str, dict[str, float]] = {
|
||||
"standard": {
|
||||
"overall": 85.0,
|
||||
"high_risk": 95.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _coverage_targets(tier: str = "standard") -> tuple[float, float]:
|
||||
"""Resolve the ``(overall, high_risk)`` target pair for a tier.
|
||||
|
||||
Backwards-compatible: a legacy flat float value (e.g. ``95.0``) is
|
||||
treated as applying to *both* gates. A per-gate dict is honoured as-is.
|
||||
"""
|
||||
cfg = COVERAGE_TARGETS.get(tier, COVERAGE_TARGETS.get("standard", {}))
|
||||
if isinstance(cfg, dict):
|
||||
return (
|
||||
float(cfg.get("overall", 85.0)),
|
||||
float(cfg.get("high_risk", 95.0)),
|
||||
)
|
||||
value = float(cfg)
|
||||
return value, value
|
||||
|
||||
#: Subdirectories excluded from the coverage denominator (non-source).
|
||||
COVERAGE_EXCLUDE_DIRS: tuple[str, ...] = (
|
||||
"docs/",
|
||||
"test-output/",
|
||||
"tests/",
|
||||
"scripts/",
|
||||
"node_modules/",
|
||||
".git/",
|
||||
)
|
||||
|
||||
#: Weight of each metric grade for the w1 term (worst grade wins per file).
|
||||
_GRADE_WEIGHT: dict[str, float] = {
|
||||
"fail": 3.0,
|
||||
"warn": 2.0,
|
||||
"good": 1.0,
|
||||
"na": 0.5,
|
||||
}
|
||||
|
||||
|
||||
def _is_source_file(rel_path: str) -> bool:
|
||||
"""Heuristic filter for source files vs. docs/tests/generated output."""
|
||||
normalized = rel_path.replace("\\", "/").lower()
|
||||
if any(normalized.startswith(d) for d in COVERAGE_EXCLUDE_DIRS):
|
||||
return False
|
||||
if normalized.endswith(
|
||||
(".bak", ".clean", ".debug1", ".fullbak", ".tmp", ".map")
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _per_file_w1(file_rel: str, repo_root: Path) -> float:
|
||||
"""Compute the risk grade (w1 term) for a single source file.
|
||||
|
||||
Uses the per-file SQL / vulnerability / redundancy heuristic scans.
|
||||
``exception_coverage`` is deliberately excluded: on a per-file basis it
|
||||
is ~always ``fail`` for real-world modules (most files have few
|
||||
explicit error branches), which gives w1 zero discriminative power.
|
||||
``high_risk_density`` is excluded too (it is ~always 100% for any file
|
||||
containing SQL/async/transaction markers). ``repo_root`` is resolved
|
||||
against when ``file_rel`` is not absolute.
|
||||
"""
|
||||
raw = file_rel.replace("\\", "/")
|
||||
candidate = Path(file_rel)
|
||||
if not candidate.is_absolute():
|
||||
candidate = repo_root / raw
|
||||
if not candidate.is_file():
|
||||
return 0.0
|
||||
try:
|
||||
lines = candidate.read_text(
|
||||
encoding="utf-8", errors="replace",
|
||||
).splitlines()
|
||||
except OSError:
|
||||
return 0.0
|
||||
if not lines:
|
||||
return 0.0
|
||||
line_items = [(raw, line, i) for i, line in enumerate(lines, start=1)]
|
||||
|
||||
sql_hits = _count_matching(line_items, _SQL_RISK_PATTERNS)
|
||||
sql_grade = _grade("sql_risk", float(sql_hits))
|
||||
|
||||
vuln_hits = _count_matching(line_items, _VULNERABILITY_PATTERNS)
|
||||
vuln_grade = _grade("vulnerability_risk", float(vuln_hits))
|
||||
|
||||
sig_count: dict[str, int] = Counter()
|
||||
for _p, line, _n in line_items:
|
||||
sig = _normalized_signature(line)
|
||||
if len(sig) >= 24:
|
||||
sig_count[sig] += 1
|
||||
dup_lines = sum(c for c in sig_count.values() if c >= 3)
|
||||
redundancy_rate = (dup_lines / len(line_items) * 100.0)
|
||||
redund_grade = _grade("redundancy_rate", redundancy_rate)
|
||||
|
||||
grades = [
|
||||
g for g in (sql_grade, vuln_grade, redund_grade)
|
||||
if g != "na"
|
||||
]
|
||||
if not grades:
|
||||
return _GRADE_WEIGHT["na"]
|
||||
worst = max(grades, key=lambda g: _GRADE_WEIGHT.get(g, 0.0))
|
||||
return _GRADE_WEIGHT.get(worst, _GRADE_WEIGHT["na"])
|
||||
|
||||
|
||||
def _topology_hits(store: GraphStore, file_rel: str) -> float:
|
||||
"""Count graph topology signal hits (w2 term) for a file's nodes.
|
||||
|
||||
Uses hub degree (>= 10 incoming calls) and untested hotspot flags as
|
||||
lightweight proxies; avoids re-running the top-N truncated tools so the
|
||||
w2 term is computed over the whole graph, not the first N nodes.
|
||||
"""
|
||||
abs_path = normalize_file_path(Path(file_rel))
|
||||
nodes = store.get_nodes_by_file(abs_path)
|
||||
if not nodes:
|
||||
return 0.0
|
||||
hits = 0
|
||||
for n in nodes:
|
||||
edges = store.get_edges_by_target(n.qualified_name)
|
||||
incoming = [
|
||||
e for e in edges
|
||||
if e.kind in ("CALLS", "REFERENCES", "IMPLEMENTS")
|
||||
]
|
||||
if len(incoming) >= 10:
|
||||
hits += 1
|
||||
if not n.is_test and len(incoming) >= 5 and not _has_tested_by(store, n.qualified_name):
|
||||
hits += 0.5
|
||||
return hits
|
||||
|
||||
|
||||
def _has_tested_by(store: GraphStore, qualified_name: str) -> bool:
|
||||
try:
|
||||
edges = store.get_edges_by_target(qualified_name)
|
||||
return any(e.kind == "TESTED_BY" for e in edges)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _file_weights(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
source_files: list[str],
|
||||
source_abs: list[str],
|
||||
churn_map: dict[str, int],
|
||||
progress_cb: Callable[[float, Optional[str]], None] | None = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Compute the risk weight ``w = w1(grade) + w2(topology) + w3(churn)``
|
||||
for every source file, plus its high-risk flag.
|
||||
|
||||
Shared by :func:`compute_coverage` and :func:`deep_read_plan` so both
|
||||
derive weights from exactly the same model. Keys are the normalized
|
||||
absolute paths used by the graph identity.
|
||||
|
||||
Args:
|
||||
store: Open graph store.
|
||||
repo_root: Repository root.
|
||||
source_files: Source file paths relative to ``repo_root``.
|
||||
source_abs: Parallel list of normalized absolute paths.
|
||||
churn_map: Per-file commit counts.
|
||||
progress_cb: Optional ``(fraction, message)`` progress callback,
|
||||
invoked every 50 files.
|
||||
"""
|
||||
weights: dict[str, dict[str, Any]] = {}
|
||||
max_churn = max(churn_map.values()) if churn_map else 1
|
||||
total = max(len(source_files), 1)
|
||||
for idx, (f, f_abs) in enumerate(zip(source_files, source_abs)):
|
||||
if progress_cb is not None and idx % 50 == 0:
|
||||
progress_cb(idx / total, f"computing risk weights ({idx}/{total})")
|
||||
w1 = _per_file_w1(f_abs, repo_root)
|
||||
w2 = _topology_hits(store, f_abs)
|
||||
rel_for_churn = f.lstrip("/").replace("\\", "/")
|
||||
raw_churn = churn_map.get(f, churn_map.get(rel_for_churn, 0))
|
||||
w3 = (raw_churn / max_churn) if max_churn else 0.0
|
||||
is_high_risk = (w1 >= 2.0 or w2 > 0.0 or raw_churn >= 3)
|
||||
weights[f_abs] = {
|
||||
"w": w1 + w2 + w3,
|
||||
"w1": w1,
|
||||
"w2": w2,
|
||||
"w3": w3,
|
||||
"raw_churn": raw_churn,
|
||||
"is_high_risk": is_high_risk,
|
||||
}
|
||||
if progress_cb is not None:
|
||||
progress_cb(1.0, "risk weights done")
|
||||
return weights
|
||||
|
||||
|
||||
def _real_line_count(repo_root: Path, rel: str, cache: dict) -> int:
|
||||
"""Real line count of a source file, cached. Independent of graph node
|
||||
``line_end`` (verified to have a +-1 skew vs actual file length)."""
|
||||
if rel in cache:
|
||||
return cache[rel]
|
||||
try:
|
||||
n = len((repo_root / rel).read_text(encoding="utf-8", errors="replace").splitlines())
|
||||
except OSError:
|
||||
n = 0
|
||||
cache[rel] = n
|
||||
return n
|
||||
|
||||
|
||||
def _union_len(ranges: list[list[int]]) -> int:
|
||||
"""Covered line count of a list of inclusive [s,e] ranges (merged)."""
|
||||
if not ranges:
|
||||
return 0
|
||||
merged: list[list[int]] = []
|
||||
for s, e in sorted((int(a), int(b)) for a, b in ranges):
|
||||
if s < 1:
|
||||
s = 1
|
||||
if e < s:
|
||||
continue
|
||||
if merged and s <= merged[-1][1] + 1:
|
||||
merged[-1][1] = max(merged[-1][1], e)
|
||||
else:
|
||||
merged.append([s, e])
|
||||
return sum(e - s + 1 for s, e in merged)
|
||||
|
||||
|
||||
def _graph_semantic_units(store: GraphStore, root: Path, rel: str) -> list[dict]:
|
||||
"""Graph semantic-unit nodes (Function/Class/Test) of a file."""
|
||||
if not hasattr(store, "_conn"):
|
||||
return []
|
||||
q = (root / rel).as_posix()
|
||||
try:
|
||||
rows = store._conn.execute(
|
||||
"SELECT kind, name, line_start, line_end FROM nodes "
|
||||
"WHERE file_path = ? AND kind IN ('Function','Class','Test') "
|
||||
"ORDER BY line_start",
|
||||
(q,),
|
||||
).fetchall()
|
||||
except Exception:
|
||||
return []
|
||||
out = []
|
||||
for r in rows:
|
||||
try:
|
||||
out.append(
|
||||
{
|
||||
"kind": r["kind"],
|
||||
"name": r["name"],
|
||||
"line_start": int(r["line_start"]),
|
||||
"line_end": int(r["line_end"]),
|
||||
}
|
||||
)
|
||||
except (KeyError, TypeError):
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def _is_giant_file(graph_units: list[dict], real_lines: int) -> bool:
|
||||
"""Unit-exempt when the largest unit spans >80% of the file's lines.
|
||||
|
||||
A single huge function (e.g. migrations.rs run_migrations = 98% of the
|
||||
file) makes unit-completeness meaningless, so such files are checked on
|
||||
line coverage only. Small files with 2-3 ordinary units are NOT exempt:
|
||||
they must still cover every unit."""
|
||||
if not graph_units or real_lines <= 0:
|
||||
return False
|
||||
largest = max(u["line_end"] - u["line_start"] + 1 for u in graph_units)
|
||||
return (largest / real_lines) > 0.8
|
||||
|
||||
|
||||
def _unit_overlap(a: list[int], b: list[int]) -> int:
|
||||
lo, hi = max(a[0], b[0]), min(a[1], b[1])
|
||||
return max(0, hi - lo + 1)
|
||||
|
||||
|
||||
def _unit_covered(
|
||||
graph_unit: dict,
|
||||
read_ranges: list[list[int]],
|
||||
unit_ranges: list[list[int]],
|
||||
matched: set[int],
|
||||
) -> bool:
|
||||
"""A graph unit is covered iff one reported unit range matches exactly
|
||||
(preferred) or overlaps >=80% of the graph unit span, is not already
|
||||
claimed by a higher-overlap unit (one-to-one), and >=80% of the graph
|
||||
unit's lines fall inside union(read_ranges)."""
|
||||
gs, ge = graph_unit["line_start"], graph_unit["line_end"]
|
||||
gspan = max(1, ge - gs + 1)
|
||||
exact = [i for i, (s, e) in enumerate(unit_ranges) if s == gs and e == ge]
|
||||
if exact:
|
||||
idx = exact[0]
|
||||
if idx in matched:
|
||||
return False
|
||||
matched.add(idx)
|
||||
else:
|
||||
best_idx, best_overlap = None, 0
|
||||
for i, (s, e) in enumerate(unit_ranges):
|
||||
ov = _unit_overlap([gs, ge], [s, e])
|
||||
if ov > best_overlap:
|
||||
best_overlap, best_idx = ov, i
|
||||
if best_idx is None or best_idx in matched:
|
||||
return False
|
||||
if best_overlap / gspan < 0.8:
|
||||
return False
|
||||
matched.add(best_idx)
|
||||
in_union = 0
|
||||
for s, e in _merge_ranges(read_ranges):
|
||||
lo, hi = max(gs, s), min(ge, e)
|
||||
if lo <= hi:
|
||||
in_union += hi - lo + 1
|
||||
return (in_union / gspan) >= 0.8
|
||||
|
||||
|
||||
def _merge_ranges(ranges: list[list[int]]) -> list[list[int]]:
|
||||
merged: list[list[int]] = []
|
||||
for s, e in sorted((int(a), int(b)) for a, b in ranges or []):
|
||||
if s < 1:
|
||||
s = 1
|
||||
if e < s:
|
||||
continue
|
||||
if merged and s <= merged[-1][1] + 1:
|
||||
merged[-1][1] = max(merged[-1][1], e)
|
||||
else:
|
||||
merged.append([s, e])
|
||||
return merged
|
||||
|
||||
|
||||
def _unit_gaps(
|
||||
graph_units: list[dict],
|
||||
reported_units: list[dict],
|
||||
read_ranges: list[list[int]],
|
||||
) -> list[dict]:
|
||||
"""Return graph units not covered by the reported semantic units."""
|
||||
unit_ranges = [
|
||||
[int(u.get("range", [0, 0])[0]), int(u.get("range", [0, 0])[1])]
|
||||
for u in reported_units
|
||||
]
|
||||
matched: set[int] = set()
|
||||
gaps = []
|
||||
for u in graph_units:
|
||||
if not _unit_covered(u, read_ranges, unit_ranges, matched):
|
||||
gaps.append(
|
||||
{
|
||||
"name": u["name"],
|
||||
"range": [u["line_start"], u["line_end"]],
|
||||
}
|
||||
)
|
||||
return gaps
|
||||
|
||||
|
||||
def compute_coverage(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
deep_read_files: list[str],
|
||||
include_churn: bool = True,
|
||||
gate: str = "high_risk",
|
||||
file_read_ranges: dict[str, list[list[int]]] | None = None,
|
||||
file_semantic_units: dict[str, list[dict]] | None = None,
|
||||
line_target: float = 95.0,
|
||||
unit_target: float = 100.0,
|
||||
progress_cb: Callable[[float, Optional[str]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute file-count review coverage for a set of deep-read files.
|
||||
|
||||
Coverage = number of deep-read files / total number of source files.
|
||||
The overall coverage uses every source file as the denominator; the
|
||||
high-risk coverage uses only the signal-flagged subset. Per-file risk
|
||||
weights (w = w1 grade + w2 topology + w3 churn) are still computed and
|
||||
used to rank ``priority_deep_read_files`` so the highest-risk files
|
||||
are read first, but the coverage percentage itself is file-count based.
|
||||
|
||||
Also returns the list of files never touched by any deep-read /
|
||||
signal (``silent_files``) for G2 spot-check sampling, the uncovered
|
||||
files list, and (new) the file count still needed to reach the target
|
||||
plus the priority deep-read file list that would close that gap.
|
||||
|
||||
Args:
|
||||
store: Open graph store (caller owns and closes it).
|
||||
repo_root: Repository root.
|
||||
deep_read_files: Files the agent actually deep-read during the
|
||||
review (relative or absolute paths).
|
||||
include_churn: Include git-churn as the w3 weight term.
|
||||
gate: Coverage gate mode. ``"high_risk"`` (default) gates on the
|
||||
signal-flagged subset only; ``"overall"`` gates on all source
|
||||
files; ``"both"`` requires *both* the overall and the high-risk
|
||||
coverage to meet the target; ``"both+line"`` additionally
|
||||
requires per-file line coverage >= ``line_target`` and unit
|
||||
completeness (gap-free) per file; ``"line+unit"`` (feature
|
||||
reviews) checks ONLY line coverage and unit completeness -
|
||||
file-count / high-risk coverage are not computed and
|
||||
``coverage_pct`` / ``high_risk_coverage_pct`` return ``None``.
|
||||
file_read_ranges: Optional mapping {rel_path: [[s,e], ...]} of the
|
||||
line ranges a sub-agent actually read per deep-read file.
|
||||
Used for the line-coverage gate (denominator = real file line
|
||||
count). Absent file => line coverage treated as satisfied.
|
||||
file_semantic_units: Optional mapping {rel_path: [{"range":[s,e],
|
||||
"kind":.., "name":..}, ...]} reported per deep-read file.
|
||||
Used for the unit-completeness gate (gap-free vs graph units).
|
||||
Absent file => unit completeness treated as satisfied.
|
||||
line_target: Min per-file line coverage percent for gate="both+line".
|
||||
unit_target: Min unit completeness percent (gap-free share).
|
||||
progress_cb: Optional ``(fraction, message)`` progress callback;
|
||||
forwarded to churn and weight computation.
|
||||
|
||||
Returns:
|
||||
Dict with coverage_pct, high_risk_coverage_pct, deep_read_count,
|
||||
total_files, deep_read_weight, total_weight, target_reached,
|
||||
target (high-risk target, back-compat), overall_target,
|
||||
high_risk_target, remaining_files_to_target (file-count gap) and the
|
||||
compatible remaining_weight_to_target (risk-weight gap),
|
||||
priority_deep_read_files, uncovered_files, silent_files and grade.
|
||||
With gate="both+line" also returns line_coverage_pct,
|
||||
unit_coverage_pct, line_gap_files, unit_gap_files,
|
||||
unit_exempt_files, missing_data_files.
|
||||
With gate="line+unit" same line/unit fields plus
|
||||
``gate="line+unit"``; ``coverage_pct``/``high_risk_coverage_pct``
|
||||
are ``None`` because file-count coverage is not computed.
|
||||
|
||||
FAIL-CLOSED (v2.5.2): a deep-read file with no file_read_ranges
|
||||
(or no file_semantic_units when the graph has units) is recorded
|
||||
as a line/unit gap and listed in missing_data_files. Missing data
|
||||
therefore makes target_reached=false instead of silently passing.
|
||||
"""
|
||||
all_files = store.get_all_files()
|
||||
source_files = [f for f in all_files if _is_source_file(f)]
|
||||
if not source_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"coverage_pct": 0.0,
|
||||
"grade": "na",
|
||||
"deep_read_count": 0,
|
||||
"total_files": 0,
|
||||
"deep_read_weight": 0.0,
|
||||
"total_weight": 0.0,
|
||||
"target_reached": False,
|
||||
"target": COVERAGE_TARGETS.get("standard", {}).get("high_risk", 95.0),
|
||||
"overall_target": COVERAGE_TARGETS.get("standard", {}).get("overall", 85.0),
|
||||
"high_risk_target": COVERAGE_TARGETS.get("standard", {}).get("high_risk", 95.0),
|
||||
"remaining_files_to_target": 0.0,
|
||||
"remaining_weight_to_target": 0.0,
|
||||
"priority_deep_read_files": [],
|
||||
"uncovered_files": [],
|
||||
"silent_files": [],
|
||||
"note": "No source files found in graph.",
|
||||
}
|
||||
|
||||
# Resolve relative paths against repo_root so graph identity matches.
|
||||
def _abs(rel: str) -> str:
|
||||
p = Path(rel)
|
||||
if p.is_absolute():
|
||||
return normalize_file_path(p)
|
||||
return normalize_file_path(repo_root / p)
|
||||
|
||||
source_abs = [_abs(f) for f in source_files]
|
||||
source_abs_set = set(source_abs)
|
||||
|
||||
# Normalise deep-read list against the graph's file paths.
|
||||
deep_read_set: set[str] = set()
|
||||
for f in deep_read_files or []:
|
||||
norm = _abs(f)
|
||||
if norm in source_abs_set or norm in set(all_files):
|
||||
deep_read_set.add(norm)
|
||||
|
||||
# Line / unit coverage data (gate="both+line" / "line+unit").
|
||||
line_gap_files: list[dict] = []
|
||||
unit_gap_files: list[dict] = []
|
||||
unit_exempt_files: list[dict] = []
|
||||
missing_data_files: list[dict] = []
|
||||
_line_cache: dict[str, int] = {}
|
||||
_line_tot = 0
|
||||
_line_cov = 0
|
||||
_unit_tot = 0
|
||||
_unit_cov = 0
|
||||
|
||||
if gate in ("both+line", "line+unit"):
|
||||
root_str = str(repo_root).replace("\\", "/").rstrip("/")
|
||||
for f, f_abs in zip(source_files, source_abs):
|
||||
if f_abs not in deep_read_set:
|
||||
continue
|
||||
# rel is relative to repo_root (sub-agents report relative paths)
|
||||
rel = f.replace("\\", "/")
|
||||
if rel.startswith(root_str + "/"):
|
||||
rel = rel[len(root_str) + 1:]
|
||||
_line_tot += 1
|
||||
_unit_tot += 1
|
||||
|
||||
# --- line coverage ---
|
||||
ranges = (file_read_ranges or {}).get(rel) or (
|
||||
file_read_ranges or {}
|
||||
).get(f_abs)
|
||||
real_lines = _real_line_count(repo_root, rel, _line_cache)
|
||||
if ranges and real_lines > 0:
|
||||
covered = _union_len(ranges)
|
||||
pct = covered / real_lines * 100.0
|
||||
_line_cov += 1 if pct >= line_target else 0
|
||||
if pct < line_target:
|
||||
line_gap_files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"coverage_pct": round(pct, 1),
|
||||
"total_lines": real_lines,
|
||||
"covered_lines": covered,
|
||||
}
|
||||
)
|
||||
else:
|
||||
# FAIL-CLOSED: a deep-read file without read_ranges (or an
|
||||
# empty/unreadable file) cannot be verified - record a gap
|
||||
# instead of silently treating it as satisfied. Without this
|
||||
# branch, gate="both+line" would report target_reached=true
|
||||
# while line_coverage_pct stays 0 (silent green).
|
||||
line_gap_files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"coverage_pct": 0.0,
|
||||
"reason": (
|
||||
"missing read_ranges" if not ranges
|
||||
else "unreadable/empty file"
|
||||
),
|
||||
}
|
||||
)
|
||||
missing_data_files.append(
|
||||
{"path": rel, "field": "file_read_ranges"}
|
||||
)
|
||||
|
||||
# --- unit completeness ---
|
||||
units = (file_semantic_units or {}).get(rel) or (
|
||||
file_semantic_units or {}
|
||||
).get(f_abs)
|
||||
g_units = _graph_semantic_units(store, repo_root, rel)
|
||||
if g_units and units is None:
|
||||
# FAIL-CLOSED: graph has semantic units but the sub-agent
|
||||
# reported none - cannot verify completeness, record a gap.
|
||||
unit_gap_files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"reason": "missing semantic_units",
|
||||
}
|
||||
)
|
||||
missing_data_files.append(
|
||||
{"path": rel, "field": "file_semantic_units"}
|
||||
)
|
||||
elif g_units and units is not None:
|
||||
giant = _is_giant_file(g_units, real_lines)
|
||||
if giant:
|
||||
unit_exempt_files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"reason": (
|
||||
"giant-file: units=%d largest_span=%.0f%%"
|
||||
% (
|
||||
len(g_units),
|
||||
100
|
||||
* max(u["line_end"] - u["line_start"] + 1 for u in g_units)
|
||||
/ max(1, real_lines),
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
_unit_tot -= 1 # exempt from unit gate
|
||||
else:
|
||||
uncovered = _unit_gaps(g_units, units, ranges or [])
|
||||
if uncovered:
|
||||
unit_gap_files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"total_units": len(g_units),
|
||||
"covered_units": len(g_units) - len(uncovered),
|
||||
"uncovered": uncovered,
|
||||
}
|
||||
)
|
||||
else:
|
||||
_unit_cov += 1
|
||||
|
||||
unit_coverage_pct = (_unit_cov / _unit_tot * 100.0) if _unit_tot else 100.0
|
||||
line_coverage_pct = (_line_cov / _line_tot * 100.0) if _line_tot else 100.0
|
||||
|
||||
churn_map: dict[str, int] = {}
|
||||
if include_churn:
|
||||
churn_map = compute_file_churn(str(repo_root), progress_cb=progress_cb)
|
||||
|
||||
weights = _file_weights(store, repo_root, source_files, source_abs, churn_map, progress_cb=progress_cb)
|
||||
|
||||
# Single pass: compute overall weight (all source files) and the
|
||||
# high-risk subset weight (files flagged by any signal).
|
||||
total_weight = 0.0
|
||||
deep_read_weight = 0.0
|
||||
high_risk_total = 0.0
|
||||
high_risk_deep = 0.0
|
||||
high_risk_deep_count = 0
|
||||
high_risk_files: list[str] = []
|
||||
uncovered_files: list[str] = []
|
||||
silent_files: list[str] = []
|
||||
|
||||
for f, f_abs in zip(source_files, source_abs):
|
||||
wi = weights[f_abs]
|
||||
w = wi["w"]
|
||||
|
||||
total_weight += w
|
||||
if f_abs in deep_read_set:
|
||||
deep_read_weight += w
|
||||
|
||||
if wi["is_high_risk"]:
|
||||
high_risk_total += w
|
||||
high_risk_files.append(f)
|
||||
if f_abs in deep_read_set:
|
||||
high_risk_deep += w
|
||||
high_risk_deep_count += 1
|
||||
elif f_abs not in deep_read_set:
|
||||
# Not high-risk and not deep-read: silent candidate.
|
||||
if wi["w1"] < 2.0 and wi["w2"] == 0.0 and wi["raw_churn"] < 3:
|
||||
silent_files.append(f)
|
||||
|
||||
uncovered_files = [
|
||||
f for f, f_abs in zip(source_files, source_abs)
|
||||
if f_abs not in deep_read_set
|
||||
]
|
||||
|
||||
total_files = len(source_files)
|
||||
deep_read_count = len(deep_read_set)
|
||||
coverage_pct = (
|
||||
(deep_read_count / total_files * 100.0)
|
||||
if total_files else 0.0
|
||||
)
|
||||
high_risk_total_files = len(high_risk_files)
|
||||
high_risk_pct = (
|
||||
(high_risk_deep_count / high_risk_total_files * 100.0)
|
||||
if high_risk_total_files else 0.0
|
||||
)
|
||||
overall_target, high_risk_target = _coverage_targets()
|
||||
|
||||
overall_ok = coverage_pct >= overall_target
|
||||
high_risk_ok = (
|
||||
(high_risk_pct >= high_risk_target) if high_risk_total_files else overall_ok
|
||||
)
|
||||
if gate == "overall":
|
||||
target_reached = overall_ok
|
||||
gate_pct = coverage_pct
|
||||
gate_target = overall_target
|
||||
elif gate == "both":
|
||||
target_reached = overall_ok and high_risk_ok
|
||||
gate_pct = min(coverage_pct, high_risk_pct) if high_risk_total_files else coverage_pct
|
||||
gate_target = min(overall_target, high_risk_target)
|
||||
elif gate == "both+line":
|
||||
quality_ok = (
|
||||
len(line_gap_files) == 0
|
||||
and len(unit_gap_files) == 0
|
||||
)
|
||||
target_reached = overall_ok and high_risk_ok and quality_ok
|
||||
gate_pct = (
|
||||
min(coverage_pct, high_risk_pct, line_coverage_pct, unit_coverage_pct)
|
||||
if high_risk_total_files
|
||||
else min(coverage_pct, line_coverage_pct, unit_coverage_pct)
|
||||
)
|
||||
gate_target = min(overall_target, high_risk_target, line_target, unit_target)
|
||||
elif gate == "line+unit":
|
||||
# Feature reviews: check ONLY line coverage + unit completeness.
|
||||
# File-count / high-risk coverage is deliberately not part of the
|
||||
# gate (coverage_pct / high_risk_coverage_pct are returned as None).
|
||||
quality_ok = (
|
||||
len(line_gap_files) == 0
|
||||
and len(unit_gap_files) == 0
|
||||
)
|
||||
target_reached = quality_ok
|
||||
gate_pct = min(line_coverage_pct, unit_coverage_pct)
|
||||
gate_target = min(line_target, unit_target)
|
||||
else: # "high_risk" (default)
|
||||
target_reached = high_risk_ok
|
||||
gate_pct = high_risk_pct
|
||||
gate_target = high_risk_target
|
||||
|
||||
grade = (
|
||||
"good" if target_reached
|
||||
else ("warn" if gate_pct >= gate_target * 0.8 else "fail")
|
||||
)
|
||||
|
||||
remaining_files_to_target = max(
|
||||
0.0, total_files * overall_target / 100.0 - deep_read_count
|
||||
)
|
||||
remaining_weight_to_target = max(
|
||||
0.0, total_weight * overall_target / 100.0 - deep_read_weight
|
||||
)
|
||||
priority_deep_read_files = [
|
||||
{"path": f_abs, "weight": round(weights[f_abs]["w"], 3)}
|
||||
for f, f_abs in zip(source_files, source_abs)
|
||||
if f_abs not in deep_read_set
|
||||
]
|
||||
priority_deep_read_files.sort(key=lambda e: e["weight"], reverse=True)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"coverage_pct": round(coverage_pct, 1) if gate != "line+unit" else None,
|
||||
"high_risk_coverage_pct": round(high_risk_pct, 1) if gate != "line+unit" else None,
|
||||
"grade": grade,
|
||||
"deep_read_count": deep_read_count,
|
||||
"total_files": total_files,
|
||||
"high_risk_total_files": high_risk_total_files,
|
||||
"high_risk_deep_count": high_risk_deep_count,
|
||||
"deep_read_weight": round(deep_read_weight, 2),
|
||||
"total_weight": round(total_weight, 2),
|
||||
"target_reached": target_reached,
|
||||
"target": high_risk_target,
|
||||
"overall_target": overall_target,
|
||||
"high_risk_target": high_risk_target,
|
||||
"gate": gate,
|
||||
"line_coverage_pct": round(line_coverage_pct, 1) if gate in ("both+line", "line+unit") else None,
|
||||
"unit_coverage_pct": round(unit_coverage_pct, 1) if gate in ("both+line", "line+unit") else None,
|
||||
"line_gap_files": line_gap_files if gate in ("both+line", "line+unit") else [],
|
||||
"unit_gap_files": unit_gap_files if gate in ("both+line", "line+unit") else [],
|
||||
"unit_exempt_files": unit_exempt_files if gate in ("both+line", "line+unit") else [],
|
||||
"missing_data_files": missing_data_files if gate in ("both+line", "line+unit") else [],
|
||||
"remaining_files_to_target": round(remaining_files_to_target, 2),
|
||||
"remaining_weight_to_target": round(remaining_weight_to_target, 2),
|
||||
"priority_deep_read_files": priority_deep_read_files,
|
||||
"uncovered_files": uncovered_files,
|
||||
"silent_files": silent_files,
|
||||
"note": (
|
||||
"行/单元门禁（feature）：仅要求行覆盖 >= "
|
||||
f"{line_target}% 且单元完整性无缺口；coverage_pct / "
|
||||
"high_risk_coverage_pct 为 None（未做文件数/高风险覆盖检查）。"
|
||||
) if gate == "line+unit" else (
|
||||
"双重覆盖口径：全库 = 深读文件数 / 全部源文件数；高风险 = 深读 / "
|
||||
"信号点名文件数。G3 门禁口径可配置（high_risk/overall/both）。"
|
||||
"每文件风险权重（w = w1 指标分 + w2 拓扑 + w3 变更频率）仍用于"
|
||||
"排序 priority_deep_read_files；低于目标 => 审查覆盖不足。"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def deep_read_plan(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
deep_read_files: list[str] | None = None,
|
||||
target_coverage: float = 85.0,
|
||||
batch_size: int = 40,
|
||||
include_churn: bool = True,
|
||||
prior_covered: set[str] | None = None,
|
||||
progress_cb: Callable[[float, Optional[str]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a grouped deep-read plan that closes the coverage gap.
|
||||
|
||||
Greedily selects files not yet deep-read until the file-count coverage
|
||||
target is reached, preferring the highest-risk (highest-weight) files
|
||||
first, then groups them by parent directory so each batch can be
|
||||
dispatched to one parallel sub-agent.
|
||||
|
||||
Args:
|
||||
store: Open graph store (caller owns and closes it).
|
||||
repo_root: Repository root.
|
||||
deep_read_files: Files already deep-read this round.
|
||||
target_coverage: Target overall coverage percentage (default 85).
|
||||
batch_size: Max files per group (default 40).
|
||||
include_churn: Include git-churn as the w3 weight term.
|
||||
prior_covered: Absolute paths already deep-read in prior rounds
|
||||
(from the coverage index); excluded from the plan.
|
||||
progress_cb: Optional ``(fraction, message)`` progress callback;
|
||||
forwarded to churn and weight computation.
|
||||
|
||||
Returns:
|
||||
Dict with current/remaining file counts (primary) plus compatible
|
||||
weight counts, the greedy gap-closing file list and the directory
|
||||
groups ready for sub-agent dispatch.
|
||||
"""
|
||||
all_files = store.get_all_files()
|
||||
source_files = [f for f in all_files if _is_source_file(f)]
|
||||
if not source_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"current_coverage_pct": 0.0,
|
||||
"current_files": 0,
|
||||
"target_files": 0,
|
||||
"remaining_files": 0,
|
||||
"current_weight": 0.0,
|
||||
"target_weight": 0.0,
|
||||
"remaining_weight": 0.0,
|
||||
"planned_files": [],
|
||||
"planned_weight": 0.0,
|
||||
"groups": [],
|
||||
"estimated_batches": 0,
|
||||
"gate": "overall",
|
||||
}
|
||||
|
||||
def _abs(rel: str) -> str:
|
||||
p = Path(rel)
|
||||
if p.is_absolute():
|
||||
return normalize_file_path(p)
|
||||
return normalize_file_path(repo_root / p)
|
||||
|
||||
source_abs = [_abs(f) for f in source_files]
|
||||
source_abs_set = set(source_abs)
|
||||
|
||||
deep_read_set: set[str] = set()
|
||||
for f in deep_read_files or []:
|
||||
norm = _abs(f)
|
||||
if norm in source_abs_set or norm in set(all_files):
|
||||
deep_read_set.add(norm)
|
||||
|
||||
if prior_covered:
|
||||
deep_read_set |= {_abs(p) for p in prior_covered if _abs(p) in source_abs_set}
|
||||
|
||||
churn_map: dict[str, int] = {}
|
||||
if include_churn:
|
||||
churn_map = compute_file_churn(str(repo_root), progress_cb=progress_cb)
|
||||
|
||||
weights = _file_weights(store, repo_root, source_files, source_abs, churn_map, progress_cb=progress_cb)
|
||||
|
||||
total_weight = sum(wi["w"] for wi in weights.values())
|
||||
total_files = len(source_files)
|
||||
current_files = len(deep_read_set)
|
||||
current_weight = sum(
|
||||
weights[f_abs]["w"]
|
||||
for f_abs in source_abs_set
|
||||
if f_abs in deep_read_set and f_abs in weights
|
||||
)
|
||||
current_pct = (current_files / total_files * 100.0) if total_files else 0.0
|
||||
target_files = total_files * target_coverage / 100.0
|
||||
remaining_files = max(0.0, target_files - current_files)
|
||||
target_weight = total_weight * target_coverage / 100.0
|
||||
remaining_weight = max(0.0, target_weight - current_weight)
|
||||
|
||||
# Greedy: pick highest-weight uncovered files until the gap is closed.
|
||||
candidates = sorted(
|
||||
(
|
||||
{"path": f_abs, "weight": weights[f_abs]["w"]}
|
||||
for f_abs in source_abs
|
||||
if f_abs not in deep_read_set and f_abs in weights
|
||||
),
|
||||
key=lambda e: e["weight"],
|
||||
reverse=True,
|
||||
)
|
||||
planned: list[str] = []
|
||||
accrued_weight = 0.0
|
||||
for cand in candidates:
|
||||
if len(planned) >= remaining_files:
|
||||
break
|
||||
planned.append(cand["path"])
|
||||
accrued_weight += cand["weight"]
|
||||
|
||||
# Group by parent directory, keeping group order by total weight desc.
|
||||
from collections import OrderedDict
|
||||
|
||||
by_dir: "OrderedDict[str, list[str]]" = OrderedDict()
|
||||
for p in planned:
|
||||
d = Path(p).parent.name or "."
|
||||
by_dir.setdefault(d, []).append(p)
|
||||
groups: list[dict[str, Any]] = []
|
||||
for d, files in by_dir.items():
|
||||
g_weight = sum(weights[_abs(f)]["w"] for f in files)
|
||||
for i in range(0, len(files), batch_size):
|
||||
chunk = files[i : i + batch_size]
|
||||
groups.append(
|
||||
{
|
||||
"name": d,
|
||||
"weight": round(g_weight, 2),
|
||||
"files": chunk,
|
||||
"batch_index": i // batch_size,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"target_coverage": target_coverage,
|
||||
"current_coverage_pct": round(current_pct, 1),
|
||||
"current_files": current_files,
|
||||
"target_files": round(target_files, 2),
|
||||
"remaining_files": round(remaining_files, 2),
|
||||
"current_weight": round(current_weight, 2),
|
||||
"target_weight": round(target_weight, 2),
|
||||
"remaining_weight": round(remaining_weight, 2),
|
||||
"planned_files": planned,
|
||||
"planned_weight": round(accrued_weight, 2),
|
||||
"groups": groups,
|
||||
"estimated_batches": len(groups),
|
||||
"gate": "overall",
|
||||
}
|
||||
|
||||
|
||||
def check_community_health(store: GraphStore) -> dict[str, Any]:
|
||||
"""Detect community/node attribution desync (nodes.community_id NULL).
|
||||
|
||||
The communities table may carry a correct ``size`` while
|
||||
``nodes.community_id`` is stale (e.g. after an incremental build that
|
||||
rebuilt nodes but skipped community re-attribution). Returns the
|
||||
non-null ratio and a ``needs_postprocess`` flag so the review skill
|
||||
can trigger ``postprocess`` before computing coverage.
|
||||
|
||||
Args:
|
||||
store: Open graph store (caller owns and closes it).
|
||||
|
||||
Returns:
|
||||
Dict with total_nodes, attributed_nodes, attribution_pct,
|
||||
needs_postprocess and note.
|
||||
"""
|
||||
try:
|
||||
total = store._conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
|
||||
attributed = store._conn.execute(
|
||||
"SELECT COUNT(*) FROM nodes WHERE community_id IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
non_file = store._conn.execute(
|
||||
"SELECT COUNT(*) FROM nodes WHERE kind != 'File'"
|
||||
).fetchone()[0]
|
||||
except Exception as exc: # pragma: no cover
|
||||
return {
|
||||
"status": "error",
|
||||
"needs_postprocess": True,
|
||||
"note": f"Community health check failed: {exc}",
|
||||
}
|
||||
|
||||
ratio = (attributed / non_file * 100.0) if non_file else 0.0
|
||||
# A fully healthy graph has ~100% attribution; allow a small delta for
|
||||
# nodes without community membership.
|
||||
needs = ratio < 90.0
|
||||
return {
|
||||
"status": "ok",
|
||||
"total_nodes": total,
|
||||
"attributed_nodes": attributed,
|
||||
"non_file_nodes": non_file,
|
||||
"attribution_pct": round(ratio, 1),
|
||||
"needs_postprocess": needs,
|
||||
"note": (
|
||||
"nodes.community_id attribution ratio. Low ratio => run "
|
||||
"`code-review-graph postprocess` to re-attach members."
|
||||
),
|
||||
}
|
||||
Reference in New Issue
Block a user