From 37a3f071d5dff19698ffacfbc64e7093551e69bb Mon Sep 17 00:00:00 2001 From: rookiestar28 Date: Fri, 20 Mar 2026 12:34:09 +0800 Subject: [PATCH] chore: enforce coverage and mutation governance --- pyproject.toml | 4 + scripts/pre_push_checks.sh | 15 +- scripts/run_adversarial_gate.py | 6 +- scripts/run_full_tests_linux.sh | 23 +-- scripts/run_full_tests_windows.ps1 | 19 ++- scripts/verify_quality_governance.py | 203 ++++++++++++++++++++++++++ tests/TEST_SOP.md | 34 +++-- tests/test_r155_exception_fidelity.py | 14 +- tests/test_r156_quality_governance.py | 77 ++++++++++ 9 files changed, 358 insertions(+), 37 deletions(-) create mode 100644 scripts/verify_quality_governance.py create mode 100644 tests/test_r156_quality_governance.py diff --git a/pyproject.toml b/pyproject.toml index 948d196..849f7bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,10 @@ omit = [ ] [tool.coverage.report] +fail_under = 35.0 +show_missing = true +skip_covered = true +precision = 2 exclude_lines = [ "pragma: no cover", "def __repr__", diff --git a/scripts/pre_push_checks.sh b/scripts/pre_push_checks.sh index f9aba2b..0259218 100644 --- a/scripts/pre_push_checks.sh +++ b/scripts/pre_push_checks.sh @@ -378,30 +378,33 @@ cleanup_precommit_snapshots # edits cannot hide until deep in the pre-push unit suite. "$VENV_PY" scripts/check_openapi_sync.py -echo "[pre-push] 3/7 backend unit tests" +echo "[pre-push] 3/8 coverage governance check" +"$VENV_PY" scripts/verify_quality_governance.py + +echo "[pre-push] 4/8 backend unit tests" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_unit" \ "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then - echo "[pre-push] 3.5/7 implementation record lint (strict)" + echo "[pre-push] 4.5/8 implementation record lint (strict)" "$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict fi -echo "[pre-push] 4/7 backend real E2E lanes (R122/R123)" +echo "[pre-push] 5/8 backend real E2E lanes (R122/R123)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[pre-push] 5/7 R121 retry partition contract" +echo "[pre-push] 6/8 R121 retry partition contract" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_retry_partition" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[pre-push] 6/7 R118 adversarial gate (adaptive: smoke/extended)" +echo "[pre-push] 7/8 R118 adversarial gate (adaptive: smoke/extended)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_adversarial" \ "$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial -echo "[pre-push] 7/7 npm test (Playwright)" +echo "[pre-push] 8/8 npm test (Playwright)" npm test echo "[pre-push] PASS" diff --git a/scripts/run_adversarial_gate.py b/scripts/run_adversarial_gate.py index f9a332e..76d8b65 100644 --- a/scripts/run_adversarial_gate.py +++ b/scripts/run_adversarial_gate.py @@ -47,6 +47,8 @@ DEFAULT_HIGH_RISK_PATTERNS = [ DEFAULT_MUTATION_ALLOWLIST_PATH = os.path.join( os.path.dirname(__file__), "..", "tests", "mutation_survivor_allowlist.json" ) +SMOKE_MUTATION_THRESHOLD = 20.0 +EXTENDED_MUTATION_THRESHOLD = 80.0 def _normalize_rel_path(path: str) -> str: @@ -527,10 +529,10 @@ def main() -> int: # Profile defaults if effective_profile == "smoke": fuzz_max_runs = 200 - mutation_threshold = args.mutation_threshold or 20.0 + mutation_threshold = args.mutation_threshold or SMOKE_MUTATION_THRESHOLD else: # extended fuzz_max_runs = 2000 - mutation_threshold = args.mutation_threshold or 80.0 + mutation_threshold = args.mutation_threshold or EXTENDED_MUTATION_THRESHOLD seed = args.seed if args.seed is not None else random.randint(0, 2**31) artifact_dir = os.path.abspath(args.artifact_dir) diff --git a/scripts/run_full_tests_linux.sh b/scripts/run_full_tests_linux.sh index ac0cd06..296d9b7 100644 --- a/scripts/run_full_tests_linux.sh +++ b/scripts/run_full_tests_linux.sh @@ -176,13 +176,13 @@ fi echo "[tests] Node version: $(node -v)" -echo "[tests] 0/8 R120 dependency preflight" +echo "[tests] 0/9 R120 dependency preflight" "$VENV_PY" scripts/preflight_check.py --strict -echo "[tests] 1/8 detect-secrets" +echo "[tests] 1/9 detect-secrets" "$VENV_PY" -m pre_commit run detect-secrets --all-files -echo "[tests] 2/8 pre-commit all hooks (pass 1: autofix)" +echo "[tests] 2/9 pre-commit all hooks (pass 1: autofix)" capture_precommit_snapshots if "$VENV_PY" -m pre_commit run --all-files --show-diff-on-failure; then : @@ -195,25 +195,28 @@ if precommit_changed_repo_state; then fi cleanup_precommit_snapshots -echo "[tests] 3/8 backend unit tests" +echo "[tests] 3/9 coverage governance check" +"$VENV_PY" scripts/verify_quality_governance.py + +echo "[tests] 4/9 backend unit tests" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then - echo "[tests] 3.5/8 implementation record lint (strict)" + echo "[tests] 4.5/9 implementation record lint (strict)" # IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures. "$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict fi -echo "[tests] 4/8 backend real E2E lanes (R122/R123)" +echo "[tests] 5/9 backend real E2E lanes (R122/R123)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 5/8 R121 retry partition contract" +echo "[tests] 6/9 R121 retry partition contract" "$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)" +echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)" "$VENV_PY" scripts/run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_r117_observability_redaction_e2e --enforce-skip-policy tests/skip_policy.json --max-skipped 0 @@ -221,11 +224,11 @@ echo "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)" "$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_transport_parity --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)" +echo "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_adversarial" \ "$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial -echo "[tests] 8/8 frontend E2E" +echo "[tests] 9/9 frontend E2E" npm test echo "[tests] PASS" diff --git a/scripts/run_full_tests_windows.ps1 b/scripts/run_full_tests_windows.ps1 index 484122a..130e9a9 100644 --- a/scripts/run_full_tests_windows.ps1 +++ b/scripts/run_full_tests_windows.ps1 @@ -241,21 +241,26 @@ if ($LASTEXITCODE -ne 0) { } Assert-PreCommitDidNotMutateRepo -BeforeWorktree $preCommitWorktreeBefore -BeforeIndex $preCommitIndexBefore -Write-Host "[tests] 3/8 backend unit tests" +Write-Host "[tests] 3/9 coverage governance check" +Invoke-Checked "coverage governance check" { + & $venvPython scripts\verify_quality_governance.py +} + +Write-Host "[tests] 4/9 backend unit tests" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit" Invoke-Checked "backend unit tests" { & $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests\skip_policy.json } if ($env:OPENCLAW_IMPL_RECORD_PATH) { - Write-Host "[tests] 3.5/8 implementation record lint (strict)" + Write-Host "[tests] 4.5/9 implementation record lint (strict)" # IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures. Invoke-Checked "implementation record lint" { & $venvPython scripts\lint_implementation_record.py --path $env:OPENCLAW_IMPL_RECORD_PATH --strict } } -Write-Host "[tests] 4/8 backend real E2E lanes (R122/R123)" +Write-Host "[tests] 5/9 backend real E2E lanes (R122/R123)" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_backend_e2e_real" Invoke-Checked "backend real E2E lane R122" { & $venvPython scripts\run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 @@ -264,12 +269,12 @@ Invoke-Checked "backend real E2E lane R123" { & $venvPython scripts\run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 5/8 R121 retry partition contract" +Write-Host "[tests] 6/9 R121 retry partition contract" Invoke-Checked "R121 retry partition contract" { & $venvPython scripts\run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 6/8 Slack integration gates (R124/R125/R117/F57)" +Write-Host "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)" Invoke-Checked "Slack integration gates" { & $venvPython scripts\run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0 & $venvPython scripts\run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 @@ -279,13 +284,13 @@ Invoke-Checked "Slack integration gates" { & $venvPython scripts\run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 7/8 R118 adversarial gate (adaptive: smoke/extended)" +Write-Host "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_adversarial" Invoke-Checked "R118 adversarial adaptive" { & $venvPython scripts\run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp\adversarial } -Write-Host "[tests] 8/8 frontend E2E" +Write-Host "[tests] 9/9 frontend E2E" Invoke-Checked "frontend E2E" { npm test } Write-Host "[tests] PASS" diff --git a/scripts/verify_quality_governance.py b/scripts/verify_quality_governance.py new file mode 100644 index 0000000..4606748 --- /dev/null +++ b/scripts/verify_quality_governance.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +R156: verify coverage + mutation governance baseline configuration. + +This script is intentionally stdlib-only so it can run early in local/full-test +gates before any optional tooling is installed. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import List, Optional + +MIN_COVERAGE_FAIL_UNDER = 35.0 +SMOKE_MUTATION_THRESHOLD = 20.0 +EXTENDED_MUTATION_THRESHOLD = 80.0 + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8-sig") + + +def _extract_toml_section(text: str, header: str) -> Optional[str]: + pattern = re.compile( + rf"(?ms)^\[{re.escape(header)}\]\s*$\n(?P.*?)(?=^\[|\Z)" + ) + match = pattern.search(text) + if not match: + return None + return match.group("body") + + +def _extract_float_assignment(section_text: str, key: str) -> Optional[float]: + match = re.search( + rf"(?m)^\s*{re.escape(key)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$", + section_text, + ) + if not match: + return None + return float(match.group(1)) + + +def _extract_bool_assignment(section_text: str, key: str) -> Optional[bool]: + match = re.search( + rf"(?m)^\s*{re.escape(key)}\s*=\s*(true|false)\s*$", section_text + ) + if not match: + return None + return match.group(1) == "true" + + +def _extract_python_constant(text: str, name: str) -> Optional[float]: + match = re.search( + rf"(?m)^\s*{re.escape(name)}\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*$", text + ) + if not match: + return None + return float(match.group(1)) + + +def _require_phrase(text: str, phrase: str, failures: List[str], label: str) -> None: + if phrase not in text: + failures.append(f"{label}: missing required phrase: {phrase}") + + +def verify_governance( + *, + pyproject_path: Path, + adversarial_gate_path: Path, + test_sop_path: Path, + survivor_allowlist_path: Path, +) -> List[str]: + failures: List[str] = [] + + pyproject_text = _read_text(pyproject_path) + report_section = _extract_toml_section(pyproject_text, "tool.coverage.report") + if report_section is None: + failures.append("pyproject: missing [tool.coverage.report] section") + else: + fail_under = _extract_float_assignment(report_section, "fail_under") + if fail_under is None: + failures.append("pyproject: missing coverage fail_under") + elif fail_under < MIN_COVERAGE_FAIL_UNDER: + failures.append( + "pyproject: coverage fail_under " + f"{fail_under} below minimum baseline {MIN_COVERAGE_FAIL_UNDER}" + ) + + show_missing = _extract_bool_assignment(report_section, "show_missing") + if show_missing is not True: + failures.append("pyproject: coverage show_missing must be true") + + skip_covered = _extract_bool_assignment(report_section, "skip_covered") + if skip_covered is not True: + failures.append("pyproject: coverage skip_covered must be true") + + gate_text = _read_text(adversarial_gate_path) + smoke_threshold = _extract_python_constant(gate_text, "SMOKE_MUTATION_THRESHOLD") + if smoke_threshold != SMOKE_MUTATION_THRESHOLD: + failures.append( + "adversarial gate: smoke mutation threshold drifted " + f"(expected {SMOKE_MUTATION_THRESHOLD}, got {smoke_threshold})" + ) + + extended_threshold = _extract_python_constant( + gate_text, "EXTENDED_MUTATION_THRESHOLD" + ) + if extended_threshold != EXTENDED_MUTATION_THRESHOLD: + failures.append( + "adversarial gate: extended mutation threshold drifted " + f"(expected {EXTENDED_MUTATION_THRESHOLD}, got {extended_threshold})" + ) + + test_sop_text = _read_text(test_sop_path) + _require_phrase( + test_sop_text, + "R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)", + failures, + "tests/TEST_SOP.md", + ) + _require_phrase( + test_sop_text, + "global score threshold (`>= 80%` unless explicitly overridden)", + failures, + "tests/TEST_SOP.md", + ) + _require_phrase( + test_sop_text, + "coverage governance check (`scripts/verify_quality_governance.py`)", + failures, + "tests/TEST_SOP.md", + ) + + if not survivor_allowlist_path.is_file(): + failures.append( + "mutation governance: missing tests/mutation_survivor_allowlist.json" + ) + else: + try: + payload = json.loads(_read_text(survivor_allowlist_path)) + except json.JSONDecodeError as exc: + failures.append(f"mutation governance: invalid survivor allowlist JSON: {exc}") + else: + if not isinstance(payload, dict) or not isinstance( + payload.get("entries", []), list + ): + failures.append( + "mutation governance: survivor allowlist must be an object with an entries list" + ) + + return failures + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Verify repository coverage and mutation governance baselines." + ) + parser.add_argument( + "--pyproject", + default="pyproject.toml", + help="Path to pyproject.toml", + ) + parser.add_argument( + "--adversarial-gate", + default="scripts/run_adversarial_gate.py", + help="Path to the adversarial gate runner", + ) + parser.add_argument( + "--test-sop", + default="tests/TEST_SOP.md", + help="Path to the main test SOP", + ) + parser.add_argument( + "--mutation-survivor-allowlist", + default="tests/mutation_survivor_allowlist.json", + help="Path to the mutation survivor allowlist JSON", + ) + args = parser.parse_args() + + failures = verify_governance( + pyproject_path=Path(args.pyproject), + adversarial_gate_path=Path(args.adversarial_gate), + test_sop_path=Path(args.test_sop), + survivor_allowlist_path=Path(args.mutation_survivor_allowlist), + ) + if failures: + for failure in failures: + print(f"GOVERNANCE-FAIL: {failure}") + return 1 + + print( + "GOVERNANCE-PASS: coverage fail_under/show_missing/skip_covered and " + "mutation thresholds are aligned." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_SOP.md b/tests/TEST_SOP.md index 69da03c..e16d082 100644 --- a/tests/TEST_SOP.md +++ b/tests/TEST_SOP.md @@ -237,18 +237,20 @@ bash scripts/pre_push_checks.sh 1) `detect-secrets` 2) all `pre-commit` hooks -3) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`) -4) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`) -5) R121 retry partition contract (`tests.test_r121_retry_partition_contract`) -6) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) -7) frontend E2E (`npm test`) +3) coverage governance check (`scripts/verify_quality_governance.py`) +4) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`) +5) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`) +6) R121 retry partition contract (`tests.test_r121_retry_partition_contract`) +7) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) +8) frontend E2E (`npm test`) IMPORTANT: -- Do not remove stage (3). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later. -- Do not remove stage (4). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI. -- Do not remove stage (5) or stage (6). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI. -- Do not downgrade stage (6) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`. +- Do not remove stage (3). If governance drift is not checked locally, coverage / mutation protections can silently weaken while the main test suite still looks green. +- Do not remove stage (4). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later. +- Do not remove stage (5). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI. +- Do not remove stage (6) or stage (7). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI. +- Do not downgrade stage (7) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`. - Keep dependency bootstrap in this script aligned with `.github/workflows/ci.yml` unit-test dependencies. ## R118 Adaptive Profile + Mutation Strictness (Required) @@ -264,6 +266,20 @@ IMPORTANT: - strict zero-survivor on changed high-risk files. - Known equivalent survivors must be explicitly listed in `tests/mutation_survivor_allowlist.json`; non-allowlisted survivors fail the gate even if score threshold passes. +## Coverage Governance Baseline (Required) + +- Coverage configuration lives in `pyproject.toml` and must keep: + - `fail_under >= 35.0` + - `show_missing = true` + - `skip_covered = true` +- Governance drift check command: + +```bash +python scripts/verify_quality_governance.py +``` + +- This check must stay in the standard local/full-test flow so threshold/config drift is caught before push. + 1) Detect Secrets (baseline-based) ```bash diff --git a/tests/test_r155_exception_fidelity.py b/tests/test_r155_exception_fidelity.py index c0cc819..65e43a0 100644 --- a/tests/test_r155_exception_fidelity.py +++ b/tests/test_r155_exception_fidelity.py @@ -97,9 +97,17 @@ class TestR155ExceptionFidelity(unittest.TestCase): with ( patch("api.config.check_rate_limit", return_value=True), patch("api.config.require_admin_token", return_value=(True, None)), - patch("api.config.get_effective_config", return_value=({"provider": "openai"}, {})), - patch("services.providers.keys.get_api_key_for_provider", return_value="sk-test"), - patch("api.config.fetch_remote_model_list", side_effect=RuntimeError("boom")), + patch( + "api.config.get_effective_config", + return_value=({"provider": "openai"}, {}), + ), + patch( + "services.providers.keys.get_api_key_for_provider", + return_value="sk-test", + ), + patch( + "api.config.fetch_remote_model_list", side_effect=RuntimeError("boom") + ), ): _, tb = self._capture_runtime_error( lambda: self._run_async(llm_models_handler(request)) diff --git a/tests/test_r156_quality_governance.py b/tests/test_r156_quality_governance.py new file mode 100644 index 0000000..a5c7c1f --- /dev/null +++ b/tests/test_r156_quality_governance.py @@ -0,0 +1,77 @@ +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "verify_quality_governance.py" + + +class TestR156QualityGovernance(unittest.TestCase): + def _run_script(self, *args): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + cwd=str(ROOT), + check=False, + ) + + def test_repo_governance_baseline_passes(self): + result = self._run_script() + self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr) + self.assertIn("GOVERNANCE-PASS", result.stdout) + + def test_missing_fail_under_is_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + pyproject = tmp / "pyproject.toml" + pyproject.write_text( + textwrap.dedent( + """ + [tool.coverage.report] + show_missing = true + skip_covered = true + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + gate = tmp / "run_adversarial_gate.py" + gate.write_text( + "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", + encoding="utf-8", + ) + + sop = tmp / "TEST_SOP.md" + sop.write_text( + textwrap.dedent( + """ + R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) + global score threshold (`>= 80%` unless explicitly overridden) + coverage governance check (`scripts/verify_quality_governance.py`) + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + allowlist = tmp / "mutation_survivor_allowlist.json" + allowlist.write_text('{"entries":[]}\n', encoding="utf-8") + + result = self._run_script( + "--pyproject", + str(pyproject), + "--adversarial-gate", + str(gate), + "--test-sop", + str(sop), + "--mutation-survivor-allowlist", + str(allowlist), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("missing coverage fail_under", result.stdout)