chore: enforce coverage and mutation governance

This commit is contained in:
rookiestar28
2026-03-20 12:34:09 +08:00
parent 7e0c1ee074
commit 37a3f071d5
9 changed files with 358 additions and 37 deletions
+9 -6
View File
@@ -378,30 +378,33 @@ cleanup_precommit_snapshots
# edits cannot hide until deep in the pre-push unit suite.
"$VENV_PY" scripts/check_openapi_sync.py
echo "[pre-push] 3/7 backend unit tests"
echo "[pre-push] 3/8 coverage governance check"
"$VENV_PY" scripts/verify_quality_governance.py
echo "[pre-push] 4/8 backend unit tests"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_unit" \
"$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
echo "[pre-push] 3.5/7 implementation record lint (strict)"
echo "[pre-push] 4.5/8 implementation record lint (strict)"
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
fi
echo "[pre-push] 4/7 backend real E2E lanes (R122/R123)"
echo "[pre-push] 5/8 backend real E2E lanes (R122/R123)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[pre-push] 5/7 R121 retry partition contract"
echo "[pre-push] 6/8 R121 retry partition contract"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_retry_partition" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[pre-push] 6/7 R118 adversarial gate (adaptive: smoke/extended)"
echo "[pre-push] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_adversarial" \
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
echo "[pre-push] 7/7 npm test (Playwright)"
echo "[pre-push] 8/8 npm test (Playwright)"
npm test
echo "[pre-push] PASS"
+4 -2
View File
@@ -47,6 +47,8 @@ DEFAULT_HIGH_RISK_PATTERNS = [
DEFAULT_MUTATION_ALLOWLIST_PATH = os.path.join(
os.path.dirname(__file__), "..", "tests", "mutation_survivor_allowlist.json"
)
SMOKE_MUTATION_THRESHOLD = 20.0
EXTENDED_MUTATION_THRESHOLD = 80.0
def _normalize_rel_path(path: str) -> str:
@@ -527,10 +529,10 @@ def main() -> int:
# Profile defaults
if effective_profile == "smoke":
fuzz_max_runs = 200
mutation_threshold = args.mutation_threshold or 20.0
mutation_threshold = args.mutation_threshold or SMOKE_MUTATION_THRESHOLD
else: # extended
fuzz_max_runs = 2000
mutation_threshold = args.mutation_threshold or 80.0
mutation_threshold = args.mutation_threshold or EXTENDED_MUTATION_THRESHOLD
seed = args.seed if args.seed is not None else random.randint(0, 2**31)
artifact_dir = os.path.abspath(args.artifact_dir)
+13 -10
View File
@@ -176,13 +176,13 @@ fi
echo "[tests] Node version: $(node -v)"
echo "[tests] 0/8 R120 dependency preflight"
echo "[tests] 0/9 R120 dependency preflight"
"$VENV_PY" scripts/preflight_check.py --strict
echo "[tests] 1/8 detect-secrets"
echo "[tests] 1/9 detect-secrets"
"$VENV_PY" -m pre_commit run detect-secrets --all-files
echo "[tests] 2/8 pre-commit all hooks (pass 1: autofix)"
echo "[tests] 2/9 pre-commit all hooks (pass 1: autofix)"
capture_precommit_snapshots
if "$VENV_PY" -m pre_commit run --all-files --show-diff-on-failure; then
:
@@ -195,25 +195,28 @@ if precommit_changed_repo_state; then
fi
cleanup_precommit_snapshots
echo "[tests] 3/8 backend unit tests"
echo "[tests] 3/9 coverage governance check"
"$VENV_PY" scripts/verify_quality_governance.py
echo "[tests] 4/9 backend unit tests"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
echo "[tests] 3.5/8 implementation record lint (strict)"
echo "[tests] 4.5/9 implementation record lint (strict)"
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
fi
echo "[tests] 4/8 backend real E2E lanes (R122/R123)"
echo "[tests] 5/9 backend real E2E lanes (R122/R123)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 5/8 R121 retry partition contract"
echo "[tests] 6/9 R121 retry partition contract"
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
"$VENV_PY" scripts/run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_r117_observability_redaction_e2e --enforce-skip-policy tests/skip_policy.json --max-skipped 0
@@ -221,11 +224,11 @@ echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_transport_parity --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
echo "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_adversarial" \
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
echo "[tests] 8/8 frontend E2E"
echo "[tests] 9/9 frontend E2E"
npm test
echo "[tests] PASS"
+12 -7
View File
@@ -241,21 +241,26 @@ if ($LASTEXITCODE -ne 0) {
}
Assert-PreCommitDidNotMutateRepo -BeforeWorktree $preCommitWorktreeBefore -BeforeIndex $preCommitIndexBefore
Write-Host "[tests] 3/8 backend unit tests"
Write-Host "[tests] 3/9 coverage governance check"
Invoke-Checked "coverage governance check" {
& $venvPython scripts\verify_quality_governance.py
}
Write-Host "[tests] 4/9 backend unit tests"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit"
Invoke-Checked "backend unit tests" {
& $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests\skip_policy.json
}
if ($env:OPENCLAW_IMPL_RECORD_PATH) {
Write-Host "[tests] 3.5/8 implementation record lint (strict)"
Write-Host "[tests] 4.5/9 implementation record lint (strict)"
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
Invoke-Checked "implementation record lint" {
& $venvPython scripts\lint_implementation_record.py --path $env:OPENCLAW_IMPL_RECORD_PATH --strict
}
}
Write-Host "[tests] 4/8 backend real E2E lanes (R122/R123)"
Write-Host "[tests] 5/9 backend real E2E lanes (R122/R123)"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_backend_e2e_real"
Invoke-Checked "backend real E2E lane R122" {
& $venvPython scripts\run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
@@ -264,12 +269,12 @@ Invoke-Checked "backend real E2E lane R123" {
& $venvPython scripts\run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 5/8 R121 retry partition contract"
Write-Host "[tests] 6/9 R121 retry partition contract"
Invoke-Checked "R121 retry partition contract" {
& $venvPython scripts\run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
Write-Host "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
Invoke-Checked "Slack integration gates" {
& $venvPython scripts\run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
& $venvPython scripts\run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
@@ -279,13 +284,13 @@ Invoke-Checked "Slack integration gates" {
& $venvPython scripts\run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
Write-Host "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_adversarial"
Invoke-Checked "R118 adversarial adaptive" {
& $venvPython scripts\run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp\adversarial
}
Write-Host "[tests] 8/8 frontend E2E"
Write-Host "[tests] 9/9 frontend E2E"
Invoke-Checked "frontend E2E" { npm test }
Write-Host "[tests] PASS"
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""
R156: verify coverage + mutation governance baseline configuration.
This script is intentionally stdlib-only so it can run early in local/full-test
gates before any optional tooling is installed.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
from typing import List, Optional
MIN_COVERAGE_FAIL_UNDER = 35.0
SMOKE_MUTATION_THRESHOLD = 20.0
EXTENDED_MUTATION_THRESHOLD = 80.0
def _read_text(path: Path) -> str:
return path.read_text(encoding="utf-8-sig")
def _extract_toml_section(text: str, header: str) -> Optional[str]:
pattern = re.compile(
rf"(?ms)^\[{re.escape(header)}\]\s*$\n(?P<body>.*?)(?=^\[|\Z)"
)
match = pattern.search(text)
if not match:
return None
return match.group("body")
def _extract_float_assignment(section_text: str, key: str) -> Optional[float]:
match = re.search(
rf"(?m)^\s*{re.escape(key)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$",
section_text,
)
if not match:
return None
return float(match.group(1))
def _extract_bool_assignment(section_text: str, key: str) -> Optional[bool]:
match = re.search(
rf"(?m)^\s*{re.escape(key)}\s*=\s*(true|false)\s*$", section_text
)
if not match:
return None
return match.group(1) == "true"
def _extract_python_constant(text: str, name: str) -> Optional[float]:
match = re.search(
rf"(?m)^\s*{re.escape(name)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$", text
)
if not match:
return None
return float(match.group(1))
def _require_phrase(text: str, phrase: str, failures: List[str], label: str) -> None:
if phrase not in text:
failures.append(f"{label}: missing required phrase: {phrase}")
def verify_governance(
*,
pyproject_path: Path,
adversarial_gate_path: Path,
test_sop_path: Path,
survivor_allowlist_path: Path,
) -> List[str]:
failures: List[str] = []
pyproject_text = _read_text(pyproject_path)
report_section = _extract_toml_section(pyproject_text, "tool.coverage.report")
if report_section is None:
failures.append("pyproject: missing [tool.coverage.report] section")
else:
fail_under = _extract_float_assignment(report_section, "fail_under")
if fail_under is None:
failures.append("pyproject: missing coverage fail_under")
elif fail_under < MIN_COVERAGE_FAIL_UNDER:
failures.append(
"pyproject: coverage fail_under "
f"{fail_under} below minimum baseline {MIN_COVERAGE_FAIL_UNDER}"
)
show_missing = _extract_bool_assignment(report_section, "show_missing")
if show_missing is not True:
failures.append("pyproject: coverage show_missing must be true")
skip_covered = _extract_bool_assignment(report_section, "skip_covered")
if skip_covered is not True:
failures.append("pyproject: coverage skip_covered must be true")
gate_text = _read_text(adversarial_gate_path)
smoke_threshold = _extract_python_constant(gate_text, "SMOKE_MUTATION_THRESHOLD")
if smoke_threshold != SMOKE_MUTATION_THRESHOLD:
failures.append(
"adversarial gate: smoke mutation threshold drifted "
f"(expected {SMOKE_MUTATION_THRESHOLD}, got {smoke_threshold})"
)
extended_threshold = _extract_python_constant(
gate_text, "EXTENDED_MUTATION_THRESHOLD"
)
if extended_threshold != EXTENDED_MUTATION_THRESHOLD:
failures.append(
"adversarial gate: extended mutation threshold drifted "
f"(expected {EXTENDED_MUTATION_THRESHOLD}, got {extended_threshold})"
)
test_sop_text = _read_text(test_sop_path)
_require_phrase(
test_sop_text,
"R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)",
failures,
"tests/TEST_SOP.md",
)
_require_phrase(
test_sop_text,
"global score threshold (`>= 80%` unless explicitly overridden)",
failures,
"tests/TEST_SOP.md",
)
_require_phrase(
test_sop_text,
"coverage governance check (`scripts/verify_quality_governance.py`)",
failures,
"tests/TEST_SOP.md",
)
if not survivor_allowlist_path.is_file():
failures.append(
"mutation governance: missing tests/mutation_survivor_allowlist.json"
)
else:
try:
payload = json.loads(_read_text(survivor_allowlist_path))
except json.JSONDecodeError as exc:
failures.append(f"mutation governance: invalid survivor allowlist JSON: {exc}")
else:
if not isinstance(payload, dict) or not isinstance(
payload.get("entries", []), list
):
failures.append(
"mutation governance: survivor allowlist must be an object with an entries list"
)
return failures
def main() -> int:
parser = argparse.ArgumentParser(
description="Verify repository coverage and mutation governance baselines."
)
parser.add_argument(
"--pyproject",
default="pyproject.toml",
help="Path to pyproject.toml",
)
parser.add_argument(
"--adversarial-gate",
default="scripts/run_adversarial_gate.py",
help="Path to the adversarial gate runner",
)
parser.add_argument(
"--test-sop",
default="tests/TEST_SOP.md",
help="Path to the main test SOP",
)
parser.add_argument(
"--mutation-survivor-allowlist",
default="tests/mutation_survivor_allowlist.json",
help="Path to the mutation survivor allowlist JSON",
)
args = parser.parse_args()
failures = verify_governance(
pyproject_path=Path(args.pyproject),
adversarial_gate_path=Path(args.adversarial_gate),
test_sop_path=Path(args.test_sop),
survivor_allowlist_path=Path(args.mutation_survivor_allowlist),
)
if failures:
for failure in failures:
print(f"GOVERNANCE-FAIL: {failure}")
return 1
print(
"GOVERNANCE-PASS: coverage fail_under/show_missing/skip_covered and "
"mutation thresholds are aligned."
)
return 0
if __name__ == "__main__":
raise SystemExit(main())