mirror of
https://github.com/rookiestar28/ComfyUI-OpenClaw.git
synced 2026-08-14 00:48:07 +00:00
chore: enforce coverage and mutation governance
This commit is contained in:
@@ -378,30 +378,33 @@ cleanup_precommit_snapshots
|
||||
# edits cannot hide until deep in the pre-push unit suite.
|
||||
"$VENV_PY" scripts/check_openapi_sync.py
|
||||
|
||||
echo "[pre-push] 3/7 backend unit tests"
|
||||
echo "[pre-push] 3/8 coverage governance check"
|
||||
"$VENV_PY" scripts/verify_quality_governance.py
|
||||
|
||||
echo "[pre-push] 4/8 backend unit tests"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_unit" \
|
||||
"$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
|
||||
|
||||
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
|
||||
echo "[pre-push] 3.5/7 implementation record lint (strict)"
|
||||
echo "[pre-push] 4.5/8 implementation record lint (strict)"
|
||||
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
|
||||
fi
|
||||
|
||||
echo "[pre-push] 4/7 backend real E2E lanes (R122/R123)"
|
||||
echo "[pre-push] 5/8 backend real E2E lanes (R122/R123)"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
|
||||
echo "[pre-push] 5/7 R121 retry partition contract"
|
||||
echo "[pre-push] 6/8 R121 retry partition contract"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_retry_partition" \
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
|
||||
echo "[pre-push] 6/7 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
echo "[pre-push] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_adversarial" \
|
||||
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
|
||||
|
||||
echo "[pre-push] 7/7 npm test (Playwright)"
|
||||
echo "[pre-push] 8/8 npm test (Playwright)"
|
||||
npm test
|
||||
|
||||
echo "[pre-push] PASS"
|
||||
|
||||
@@ -47,6 +47,8 @@ DEFAULT_HIGH_RISK_PATTERNS = [
|
||||
DEFAULT_MUTATION_ALLOWLIST_PATH = os.path.join(
|
||||
os.path.dirname(__file__), "..", "tests", "mutation_survivor_allowlist.json"
|
||||
)
|
||||
SMOKE_MUTATION_THRESHOLD = 20.0
|
||||
EXTENDED_MUTATION_THRESHOLD = 80.0
|
||||
|
||||
|
||||
def _normalize_rel_path(path: str) -> str:
|
||||
@@ -527,10 +529,10 @@ def main() -> int:
|
||||
# Profile defaults
|
||||
if effective_profile == "smoke":
|
||||
fuzz_max_runs = 200
|
||||
mutation_threshold = args.mutation_threshold or 20.0
|
||||
mutation_threshold = args.mutation_threshold or SMOKE_MUTATION_THRESHOLD
|
||||
else: # extended
|
||||
fuzz_max_runs = 2000
|
||||
mutation_threshold = args.mutation_threshold or 80.0
|
||||
mutation_threshold = args.mutation_threshold or EXTENDED_MUTATION_THRESHOLD
|
||||
|
||||
seed = args.seed if args.seed is not None else random.randint(0, 2**31)
|
||||
artifact_dir = os.path.abspath(args.artifact_dir)
|
||||
|
||||
@@ -176,13 +176,13 @@ fi
|
||||
|
||||
echo "[tests] Node version: $(node -v)"
|
||||
|
||||
echo "[tests] 0/8 R120 dependency preflight"
|
||||
echo "[tests] 0/9 R120 dependency preflight"
|
||||
"$VENV_PY" scripts/preflight_check.py --strict
|
||||
|
||||
echo "[tests] 1/8 detect-secrets"
|
||||
echo "[tests] 1/9 detect-secrets"
|
||||
"$VENV_PY" -m pre_commit run detect-secrets --all-files
|
||||
|
||||
echo "[tests] 2/8 pre-commit all hooks (pass 1: autofix)"
|
||||
echo "[tests] 2/9 pre-commit all hooks (pass 1: autofix)"
|
||||
capture_precommit_snapshots
|
||||
if "$VENV_PY" -m pre_commit run --all-files --show-diff-on-failure; then
|
||||
:
|
||||
@@ -195,25 +195,28 @@ if precommit_changed_repo_state; then
|
||||
fi
|
||||
cleanup_precommit_snapshots
|
||||
|
||||
echo "[tests] 3/8 backend unit tests"
|
||||
echo "[tests] 3/9 coverage governance check"
|
||||
"$VENV_PY" scripts/verify_quality_governance.py
|
||||
|
||||
echo "[tests] 4/9 backend unit tests"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
|
||||
|
||||
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
|
||||
echo "[tests] 3.5/8 implementation record lint (strict)"
|
||||
echo "[tests] 4.5/9 implementation record lint (strict)"
|
||||
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
|
||||
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
|
||||
fi
|
||||
|
||||
echo "[tests] 4/8 backend real E2E lanes (R122/R123)"
|
||||
echo "[tests] 5/9 backend real E2E lanes (R122/R123)"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
|
||||
echo "[tests] 5/8 R121 retry partition contract"
|
||||
echo "[tests] 6/9 R121 retry partition contract"
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
|
||||
echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
|
||||
echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_r117_observability_redaction_e2e --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
@@ -221,11 +224,11 @@ echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_transport_parity --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests/skip_policy.json --max-skipped 0
|
||||
|
||||
echo "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
echo "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_adversarial" \
|
||||
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
|
||||
|
||||
echo "[tests] 8/8 frontend E2E"
|
||||
echo "[tests] 9/9 frontend E2E"
|
||||
npm test
|
||||
|
||||
echo "[tests] PASS"
|
||||
|
||||
@@ -241,21 +241,26 @@ if ($LASTEXITCODE -ne 0) {
|
||||
}
|
||||
Assert-PreCommitDidNotMutateRepo -BeforeWorktree $preCommitWorktreeBefore -BeforeIndex $preCommitIndexBefore
|
||||
|
||||
Write-Host "[tests] 3/8 backend unit tests"
|
||||
Write-Host "[tests] 3/9 coverage governance check"
|
||||
Invoke-Checked "coverage governance check" {
|
||||
& $venvPython scripts\verify_quality_governance.py
|
||||
}
|
||||
|
||||
Write-Host "[tests] 4/9 backend unit tests"
|
||||
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit"
|
||||
Invoke-Checked "backend unit tests" {
|
||||
& $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests\skip_policy.json
|
||||
}
|
||||
|
||||
if ($env:OPENCLAW_IMPL_RECORD_PATH) {
|
||||
Write-Host "[tests] 3.5/8 implementation record lint (strict)"
|
||||
Write-Host "[tests] 4.5/9 implementation record lint (strict)"
|
||||
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
|
||||
Invoke-Checked "implementation record lint" {
|
||||
& $venvPython scripts\lint_implementation_record.py --path $env:OPENCLAW_IMPL_RECORD_PATH --strict
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[tests] 4/8 backend real E2E lanes (R122/R123)"
|
||||
Write-Host "[tests] 5/9 backend real E2E lanes (R122/R123)"
|
||||
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_backend_e2e_real"
|
||||
Invoke-Checked "backend real E2E lane R122" {
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
@@ -264,12 +269,12 @@ Invoke-Checked "backend real E2E lane R123" {
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
}
|
||||
|
||||
Write-Host "[tests] 5/8 R121 retry partition contract"
|
||||
Write-Host "[tests] 6/9 R121 retry partition contract"
|
||||
Invoke-Checked "R121 retry partition contract" {
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
}
|
||||
|
||||
Write-Host "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)"
|
||||
Write-Host "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
|
||||
Invoke-Checked "Slack integration gates" {
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
@@ -279,13 +284,13 @@ Invoke-Checked "Slack integration gates" {
|
||||
& $venvPython scripts\run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests\skip_policy.json --max-skipped 0
|
||||
}
|
||||
|
||||
Write-Host "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
Write-Host "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
|
||||
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_adversarial"
|
||||
Invoke-Checked "R118 adversarial adaptive" {
|
||||
& $venvPython scripts\run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp\adversarial
|
||||
}
|
||||
|
||||
Write-Host "[tests] 8/8 frontend E2E"
|
||||
Write-Host "[tests] 9/9 frontend E2E"
|
||||
Invoke-Checked "frontend E2E" { npm test }
|
||||
|
||||
Write-Host "[tests] PASS"
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
R156: verify coverage + mutation governance baseline configuration.
|
||||
|
||||
This script is intentionally stdlib-only so it can run early in local/full-test
|
||||
gates before any optional tooling is installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
MIN_COVERAGE_FAIL_UNDER = 35.0
|
||||
SMOKE_MUTATION_THRESHOLD = 20.0
|
||||
EXTENDED_MUTATION_THRESHOLD = 80.0
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8-sig")
|
||||
|
||||
|
||||
def _extract_toml_section(text: str, header: str) -> Optional[str]:
|
||||
pattern = re.compile(
|
||||
rf"(?ms)^\[{re.escape(header)}\]\s*$\n(?P<body>.*?)(?=^\[|\Z)"
|
||||
)
|
||||
match = pattern.search(text)
|
||||
if not match:
|
||||
return None
|
||||
return match.group("body")
|
||||
|
||||
|
||||
def _extract_float_assignment(section_text: str, key: str) -> Optional[float]:
|
||||
match = re.search(
|
||||
rf"(?m)^\s*{re.escape(key)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$",
|
||||
section_text,
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
def _extract_bool_assignment(section_text: str, key: str) -> Optional[bool]:
|
||||
match = re.search(
|
||||
rf"(?m)^\s*{re.escape(key)}\s*=\s*(true|false)\s*$", section_text
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1) == "true"
|
||||
|
||||
|
||||
def _extract_python_constant(text: str, name: str) -> Optional[float]:
|
||||
match = re.search(
|
||||
rf"(?m)^\s*{re.escape(name)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$", text
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
def _require_phrase(text: str, phrase: str, failures: List[str], label: str) -> None:
|
||||
if phrase not in text:
|
||||
failures.append(f"{label}: missing required phrase: {phrase}")
|
||||
|
||||
|
||||
def verify_governance(
|
||||
*,
|
||||
pyproject_path: Path,
|
||||
adversarial_gate_path: Path,
|
||||
test_sop_path: Path,
|
||||
survivor_allowlist_path: Path,
|
||||
) -> List[str]:
|
||||
failures: List[str] = []
|
||||
|
||||
pyproject_text = _read_text(pyproject_path)
|
||||
report_section = _extract_toml_section(pyproject_text, "tool.coverage.report")
|
||||
if report_section is None:
|
||||
failures.append("pyproject: missing [tool.coverage.report] section")
|
||||
else:
|
||||
fail_under = _extract_float_assignment(report_section, "fail_under")
|
||||
if fail_under is None:
|
||||
failures.append("pyproject: missing coverage fail_under")
|
||||
elif fail_under < MIN_COVERAGE_FAIL_UNDER:
|
||||
failures.append(
|
||||
"pyproject: coverage fail_under "
|
||||
f"{fail_under} below minimum baseline {MIN_COVERAGE_FAIL_UNDER}"
|
||||
)
|
||||
|
||||
show_missing = _extract_bool_assignment(report_section, "show_missing")
|
||||
if show_missing is not True:
|
||||
failures.append("pyproject: coverage show_missing must be true")
|
||||
|
||||
skip_covered = _extract_bool_assignment(report_section, "skip_covered")
|
||||
if skip_covered is not True:
|
||||
failures.append("pyproject: coverage skip_covered must be true")
|
||||
|
||||
gate_text = _read_text(adversarial_gate_path)
|
||||
smoke_threshold = _extract_python_constant(gate_text, "SMOKE_MUTATION_THRESHOLD")
|
||||
if smoke_threshold != SMOKE_MUTATION_THRESHOLD:
|
||||
failures.append(
|
||||
"adversarial gate: smoke mutation threshold drifted "
|
||||
f"(expected {SMOKE_MUTATION_THRESHOLD}, got {smoke_threshold})"
|
||||
)
|
||||
|
||||
extended_threshold = _extract_python_constant(
|
||||
gate_text, "EXTENDED_MUTATION_THRESHOLD"
|
||||
)
|
||||
if extended_threshold != EXTENDED_MUTATION_THRESHOLD:
|
||||
failures.append(
|
||||
"adversarial gate: extended mutation threshold drifted "
|
||||
f"(expected {EXTENDED_MUTATION_THRESHOLD}, got {extended_threshold})"
|
||||
)
|
||||
|
||||
test_sop_text = _read_text(test_sop_path)
|
||||
_require_phrase(
|
||||
test_sop_text,
|
||||
"R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)",
|
||||
failures,
|
||||
"tests/TEST_SOP.md",
|
||||
)
|
||||
_require_phrase(
|
||||
test_sop_text,
|
||||
"global score threshold (`>= 80%` unless explicitly overridden)",
|
||||
failures,
|
||||
"tests/TEST_SOP.md",
|
||||
)
|
||||
_require_phrase(
|
||||
test_sop_text,
|
||||
"coverage governance check (`scripts/verify_quality_governance.py`)",
|
||||
failures,
|
||||
"tests/TEST_SOP.md",
|
||||
)
|
||||
|
||||
if not survivor_allowlist_path.is_file():
|
||||
failures.append(
|
||||
"mutation governance: missing tests/mutation_survivor_allowlist.json"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
payload = json.loads(_read_text(survivor_allowlist_path))
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"mutation governance: invalid survivor allowlist JSON: {exc}")
|
||||
else:
|
||||
if not isinstance(payload, dict) or not isinstance(
|
||||
payload.get("entries", []), list
|
||||
):
|
||||
failures.append(
|
||||
"mutation governance: survivor allowlist must be an object with an entries list"
|
||||
)
|
||||
|
||||
return failures
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Verify repository coverage and mutation governance baselines."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pyproject",
|
||||
default="pyproject.toml",
|
||||
help="Path to pyproject.toml",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--adversarial-gate",
|
||||
default="scripts/run_adversarial_gate.py",
|
||||
help="Path to the adversarial gate runner",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-sop",
|
||||
default="tests/TEST_SOP.md",
|
||||
help="Path to the main test SOP",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mutation-survivor-allowlist",
|
||||
default="tests/mutation_survivor_allowlist.json",
|
||||
help="Path to the mutation survivor allowlist JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
failures = verify_governance(
|
||||
pyproject_path=Path(args.pyproject),
|
||||
adversarial_gate_path=Path(args.adversarial_gate),
|
||||
test_sop_path=Path(args.test_sop),
|
||||
survivor_allowlist_path=Path(args.mutation_survivor_allowlist),
|
||||
)
|
||||
if failures:
|
||||
for failure in failures:
|
||||
print(f"GOVERNANCE-FAIL: {failure}")
|
||||
return 1
|
||||
|
||||
print(
|
||||
"GOVERNANCE-PASS: coverage fail_under/show_missing/skip_covered and "
|
||||
"mutation thresholds are aligned."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user