diff --git a/docs/release/ci_regression_policy.md b/docs/release/ci_regression_policy.md index 737d7e8..39cbc00 100644 --- a/docs/release/ci_regression_policy.md +++ b/docs/release/ci_regression_policy.md @@ -12,6 +12,7 @@ All pull requests must pass the repository SOP gate before merge. | Backend dependency audit | `pip-audit -r requirements.txt` | Audit declared Python project dependencies without scanning unrelated CI runner/toolchain packages | | GitHub CodeQL analysis | `.github/workflows/codeql.yml` | Run repository-native static security analysis for Python, JavaScript/TypeScript, and GitHub Actions on push, pull request, and weekly schedule | | Coverage governance | `python scripts/verify_quality_governance.py` | Fail closed on coverage-policy, mutation-threshold, SOP-guidance, and survivor-allowlist drift | +| Test debt governance | `python scripts/verify_test_debt_governance.py` | Fail closed on stale or under-documented skip-policy / mutation allowlist debt entries | | Backend unit tests | `python scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json` | Validate backend behavior and skip governance | | Adversarial gate | `python scripts/run_adversarial_gate.py --profile auto --seed 42` | Enforce adaptive fuzz/mutation verification with smoke=>extended escalation on high-risk diffs | | Frontend E2E | `npm test` | Validate UI and frontend/backend integration | @@ -56,6 +57,10 @@ If a change intentionally modifies contract behavior: - `fail_under` must match the current stage floor declared in `tests/coverage_governance_policy.json`; do not ratchet the floor by editing `pyproject.toml` alone. - Coverage hotspot review should use: - `python scripts/report_coverage_governance.py --coverage-json ` +- Test debt governance remains fail-closed: + - no-skip modules in `tests/skip_policy.json` must keep explicit metadata (`reason` + `review_after`) and point at live test modules + - mutation survivor allowlist entries must carry `review_after` dates and point at live repo files + - review dates in the past are governance debt, not advisory comments - Mutation governance remains adaptive: - smoke profile threshold: `20.0%` - extended profile threshold: `80.0%` diff --git a/scripts/pre_push_checks.sh b/scripts/pre_push_checks.sh index 31907d1..55eb35a 100644 --- a/scripts/pre_push_checks.sh +++ b/scripts/pre_push_checks.sh @@ -383,33 +383,36 @@ cleanup_precommit_snapshots # edits cannot hide until deep in the pre-push unit suite. "$VENV_PY" scripts/check_openapi_sync.py -echo "[pre-push] 3/8 coverage governance check" +echo "[pre-push] 3/9 coverage governance check" "$VENV_PY" scripts/verify_quality_governance.py -echo "[pre-push] 4/8 backend unit tests" +echo "[pre-push] 4/9 test debt governance check" +"$VENV_PY" scripts/verify_test_debt_governance.py + +echo "[pre-push] 5/9 backend unit tests" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_unit" \ "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then - echo "[pre-push] 4.5/8 implementation record lint (strict)" + echo "[pre-push] 5.5/9 implementation record lint (strict)" "$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict fi -echo "[pre-push] 5/8 backend real E2E lanes (R122/R123)" +echo "[pre-push] 6/9 backend real E2E lanes (R122/R123)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[pre-push] 6/8 R121 retry partition contract" +echo "[pre-push] 7/9 R121 retry partition contract" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_retry_partition" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[pre-push] 7/8 R118 adversarial gate (adaptive: smoke/extended)" +echo "[pre-push] 8/9 R118 adversarial gate (adaptive: smoke/extended)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_adversarial" \ "$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial -echo "[pre-push] 8/8 npm test (Playwright)" +echo "[pre-push] 9/9 npm test (Playwright)" npm test echo "[pre-push] PASS" diff --git a/scripts/quality_governance_common.py b/scripts/quality_governance_common.py index 7d087e1..d0616b0 100644 --- a/scripts/quality_governance_common.py +++ b/scripts/quality_governance_common.py @@ -7,7 +7,6 @@ from fnmatch import fnmatch from pathlib import Path from typing import Any, Iterable - REQUIRED_HOTSPOT_FAMILIES = ( "safe_io", "security_boundary", @@ -42,8 +41,10 @@ def _validate_hotspot_family( seen_ids.add(family_id) paths = family.get("paths") - if not isinstance(paths, list) or not paths or not all( - isinstance(path, str) and path.strip() for path in paths + if ( + not isinstance(paths, list) + or not paths + or not all(isinstance(path, str) and path.strip() for path in paths) ): failures.append( f"coverage policy: hotspot family {family_id} must define a non-empty paths list" @@ -93,7 +94,9 @@ def load_and_validate_policy(path: Path) -> tuple[dict[str, Any] | None, list[st f"coverage policy: stage {stage_id} missing numeric min_fail_under" ) continue - stages.append(CoverageStage(stage_id=stage_id, min_fail_under=float(min_fail_under))) + stages.append( + CoverageStage(stage_id=stage_id, min_fail_under=float(min_fail_under)) + ) for previous, current in zip(stages, stages[1:]): if current.min_fail_under <= previous.min_fail_under: @@ -243,7 +246,9 @@ def summarize_coverage( summary = files[file_path].get("summary", {}) covered_lines += int(summary.get("covered_lines", 0)) num_statements += int(summary.get("num_statements", 0)) - percent = round((covered_lines / num_statements) * 100, 2) if num_statements else 0.0 + percent = ( + round((covered_lines / num_statements) * 100, 2) if num_statements else 0.0 + ) hotspot_summary[family_id] = { "matched_files": matched_files, "missing_paths": missing_paths, @@ -263,7 +268,9 @@ def summarize_coverage( "current_stage_fail_under": current_stage_threshold(policy), "next_stage": next_policy_stage["id"] if next_policy_stage else None, "next_stage_fail_under": ( - float(next_policy_stage["min_fail_under"]) if next_policy_stage else None + float(next_policy_stage["min_fail_under"]) + if next_policy_stage + else None ), }, "overall": { diff --git a/scripts/report_coverage_governance.py b/scripts/report_coverage_governance.py index bcba6b6..5760aff 100644 --- a/scripts/report_coverage_governance.py +++ b/scripts/report_coverage_governance.py @@ -10,7 +10,11 @@ import json import sys from pathlib import Path -from quality_governance_common import load_and_validate_policy, read_json, summarize_coverage +from quality_governance_common import ( + load_and_validate_policy, + read_json, + summarize_coverage, +) def _render_text(summary: dict[str, object]) -> str: diff --git a/scripts/run_full_tests_linux.sh b/scripts/run_full_tests_linux.sh index 3531881..99a7a3c 100644 --- a/scripts/run_full_tests_linux.sh +++ b/scripts/run_full_tests_linux.sh @@ -210,28 +210,31 @@ if precommit_changed_repo_state; then fi cleanup_precommit_snapshots -echo "[tests] 3/9 coverage governance check" +echo "[tests] 3/10 coverage governance check" "$VENV_PY" scripts/verify_quality_governance.py -echo "[tests] 4/9 backend unit tests" +echo "[tests] 4/10 test debt governance check" +"$VENV_PY" scripts/verify_test_debt_governance.py + +echo "[tests] 5/10 backend unit tests" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then - echo "[tests] 4.5/9 implementation record lint (strict)" + echo "[tests] 5.5/10 implementation record lint (strict)" # IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures. "$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict fi -echo "[tests] 5/9 backend real E2E lanes (R122/R123)" +echo "[tests] 6/10 backend real E2E lanes (R122/R123)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \ "$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 6/9 R121 retry partition contract" +echo "[tests] 7/10 R121 retry partition contract" "$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)" +echo "[tests] 8/10 Slack integration gates (R124/R125/R117/F57)" "$VENV_PY" scripts/run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_r117_observability_redaction_e2e --enforce-skip-policy tests/skip_policy.json --max-skipped 0 @@ -239,11 +242,11 @@ echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)" "$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_transport_parity --enforce-skip-policy tests/skip_policy.json --max-skipped 0 "$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests/skip_policy.json --max-skipped 0 -echo "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)" +echo "[tests] 9/10 R118 adversarial gate (adaptive: smoke/extended)" MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_adversarial" \ "$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial -echo "[tests] 9/9 frontend E2E" +echo "[tests] 10/10 frontend E2E" # IMPORTANT: full-gate acceptance must provision Playwright browsers itself; do # not assume a warmed local browser cache when running on fresh WSL/Linux hosts. OPENCLAW_PLAYWRIGHT_INSTALL=1 OPENCLAW_PLAYWRIGHT_BROWSERS=chromium npm test diff --git a/scripts/run_full_tests_windows.ps1 b/scripts/run_full_tests_windows.ps1 index cd76b74..ec75a4f 100644 --- a/scripts/run_full_tests_windows.ps1 +++ b/scripts/run_full_tests_windows.ps1 @@ -266,26 +266,31 @@ if ($LASTEXITCODE -ne 0) { } Assert-PreCommitDidNotMutateRepo -BeforeWorktree $preCommitWorktreeBefore -BeforeIndex $preCommitIndexBefore -Write-Host "[tests] 3/9 coverage governance check" +Write-Host "[tests] 3/10 coverage governance check" Invoke-Checked "coverage governance check" { & $venvPython scripts\verify_quality_governance.py } -Write-Host "[tests] 4/9 backend unit tests" +Write-Host "[tests] 4/10 test debt governance check" +Invoke-Checked "test debt governance check" { + & $venvPython scripts\verify_test_debt_governance.py +} + +Write-Host "[tests] 5/10 backend unit tests" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit" Invoke-Checked "backend unit tests" { & $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests\skip_policy.json } if ($env:OPENCLAW_IMPL_RECORD_PATH) { - Write-Host "[tests] 4.5/9 implementation record lint (strict)" + Write-Host "[tests] 5.5/10 implementation record lint (strict)" # IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures. Invoke-Checked "implementation record lint" { & $venvPython scripts\lint_implementation_record.py --path $env:OPENCLAW_IMPL_RECORD_PATH --strict } } -Write-Host "[tests] 5/9 backend real E2E lanes (R122/R123)" +Write-Host "[tests] 6/10 backend real E2E lanes (R122/R123)" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_backend_e2e_real" Invoke-Checked "backend real E2E lane R122" { & $venvPython scripts\run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 @@ -294,12 +299,12 @@ Invoke-Checked "backend real E2E lane R123" { & $venvPython scripts\run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 6/9 R121 retry partition contract" +Write-Host "[tests] 7/10 R121 retry partition contract" Invoke-Checked "R121 retry partition contract" { & $venvPython scripts\run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)" +Write-Host "[tests] 8/10 Slack integration gates (R124/R125/R117/F57)" Invoke-Checked "Slack integration gates" { & $venvPython scripts\run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0 & $venvPython scripts\run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0 @@ -309,13 +314,13 @@ Invoke-Checked "Slack integration gates" { & $venvPython scripts\run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests\skip_policy.json --max-skipped 0 } -Write-Host "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)" +Write-Host "[tests] 9/10 R118 adversarial gate (adaptive: smoke/extended)" $env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_adversarial" Invoke-Checked "R118 adversarial adaptive" { & $venvPython scripts\run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp\adversarial } -Write-Host "[tests] 9/9 frontend E2E" +Write-Host "[tests] 10/10 frontend E2E" $env:OPENCLAW_PLAYWRIGHT_INSTALL = "1" $env:OPENCLAW_PLAYWRIGHT_BROWSERS = "chromium" # IMPORTANT: full-gate acceptance must provision Playwright browsers itself; do diff --git a/scripts/verify_test_debt_governance.py b/scripts/verify_test_debt_governance.py new file mode 100644 index 0000000..c483502 --- /dev/null +++ b/scripts/verify_test_debt_governance.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +R171: validate skip-policy and mutation-survivor debt metadata. + +This script is intentionally stdlib-only so it can run early in local/full-test +gates before optional dependencies are installed. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import date +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +def _read_json_object(path: Path) -> Dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(payload, dict): + raise ValueError(f"{path}: expected a JSON object") + return payload + + +def _normalize_repo_rel_path(path: str) -> str: + return PurePosixPath(path.replace("\\", "/")).as_posix().lstrip("./") + + +def _parse_review_after(value: Any, *, label: str, failures: List[str]) -> None: + if not isinstance(value, str) or not value.strip(): + failures.append(f"{label}: missing review_after") + return + try: + review_after = date.fromisoformat(value) + except ValueError: + failures.append(f"{label}: invalid review_after '{value}'") + return + if review_after < date.today(): + failures.append(f"{label}: review_after {value} is in the past") + + +def _validate_reason(value: Any, *, label: str, failures: List[str]) -> None: + if not isinstance(value, str) or not value.strip(): + failures.append(f"{label}: missing non-empty reason") + + +def _resolve_test_module_path(repo_root: Path, module_name: str) -> Path: + return repo_root / Path(module_name.replace(".", "/")).with_suffix(".py") + + +def _validate_skip_policy(repo_root: Path, path: Path) -> List[str]: + failures: List[str] = [] + payload = _read_json_object(path) + + max_skipped = payload.get("max_skipped") + if not isinstance(max_skipped, int) or max_skipped < 0: + failures.append("skip policy: max_skipped must be a non-negative integer") + + modules = payload.get("no_skip_modules", []) + if not isinstance(modules, list) or any( + not isinstance(item, str) or not item.strip() for item in modules + ): + failures.append("skip policy: no_skip_modules must be a list of non-empty strings") + return failures + + seen = set() + duplicates = set() + normalized_modules: List[str] = [] + for module in modules: + normalized = module.strip() + normalized_modules.append(normalized) + if normalized in seen: + duplicates.add(normalized) + seen.add(normalized) + module_path = _resolve_test_module_path(repo_root, normalized) + if not module_path.is_file(): + failures.append( + f"skip policy: module path does not exist for {normalized} -> {module_path.relative_to(repo_root)}" + ) + if duplicates: + failures.append( + "skip policy: duplicate no-skip modules: " + ", ".join(sorted(duplicates)) + ) + + metadata = payload.get("no_skip_module_metadata") + if not isinstance(metadata, dict): + failures.append( + "skip policy: no_skip_module_metadata must be an object keyed by module name" + ) + return failures + + metadata_keys = {str(key).strip() for key in metadata.keys()} + missing_metadata = [module for module in normalized_modules if module not in metadata_keys] + extra_metadata = sorted( + key for key in metadata_keys if key and key not in set(normalized_modules) + ) + if missing_metadata: + failures.append( + "skip policy: missing metadata for no-skip modules: " + + ", ".join(sorted(missing_metadata)) + ) + if extra_metadata: + failures.append( + "skip policy: stale metadata without matching no-skip module: " + + ", ".join(extra_metadata) + ) + + for module_name in normalized_modules: + raw_meta = metadata.get(module_name) + label = f"skip policy metadata[{module_name}]" + if not isinstance(raw_meta, dict): + failures.append(f"{label}: metadata entry must be an object") + continue + _validate_reason(raw_meta.get("reason"), label=label, failures=failures) + _parse_review_after( + raw_meta.get("review_after"), label=label, failures=failures + ) + return failures + + +def _validate_mutation_allowlist(repo_root: Path, path: Path) -> List[str]: + failures: List[str] = [] + payload = _read_json_object(path) + entries = payload.get("entries", []) + if not isinstance(entries, list): + return ["mutation allowlist: entries must be a list"] + + seen: set[Tuple[str, int]] = set() + duplicates: set[Tuple[str, int]] = set() + for index, raw_entry in enumerate(entries): + label = f"mutation allowlist entry[{index}]" + if not isinstance(raw_entry, dict): + failures.append(f"{label}: entry must be an object") + continue + file_path = _normalize_repo_rel_path(str(raw_entry.get("file", ""))) + mutation_index = raw_entry.get("mutation_index") + if not file_path: + failures.append(f"{label}: missing file") + elif not (repo_root / Path(file_path)).is_file(): + failures.append(f"{label}: file does not exist in repo: {file_path}") + if not isinstance(mutation_index, int) or mutation_index < 0: + failures.append(f"{label}: mutation_index must be a non-negative integer") + else: + key = (file_path, mutation_index) + if key in seen: + duplicates.add(key) + seen.add(key) + _validate_reason(raw_entry.get("reason"), label=label, failures=failures) + _parse_review_after( + raw_entry.get("review_after"), label=label, failures=failures + ) + + if duplicates: + failures.append( + "mutation allowlist: duplicate (file, mutation_index) entries: " + + ", ".join(f"{file}@{mutation_index}" for file, mutation_index in sorted(duplicates)) + ) + return failures + + +def verify_test_debt_governance( + *, + repo_root: Path, + skip_policy_path: Path, + mutation_allowlist_path: Path, +) -> List[str]: + failures: List[str] = [] + if not skip_policy_path.is_file(): + failures.append(f"missing skip policy: {skip_policy_path}") + else: + try: + failures.extend(_validate_skip_policy(repo_root, skip_policy_path)) + except Exception as exc: + failures.append(f"skip policy: failed to validate {skip_policy_path}: {exc}") + + if not mutation_allowlist_path.is_file(): + failures.append(f"missing mutation survivor allowlist: {mutation_allowlist_path}") + else: + try: + failures.extend( + _validate_mutation_allowlist(repo_root, mutation_allowlist_path) + ) + except Exception as exc: + failures.append( + f"mutation allowlist: failed to validate {mutation_allowlist_path}: {exc}" + ) + return failures + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Validate skip-policy and mutation-survivor debt metadata." + ) + parser.add_argument( + "--repo-root", + default=".", + help="Repository root used to resolve test modules and file paths.", + ) + parser.add_argument( + "--skip-policy", + default="tests/skip_policy.json", + help="Path to tests/skip_policy.json", + ) + parser.add_argument( + "--mutation-survivor-allowlist", + default="tests/mutation_survivor_allowlist.json", + help="Path to tests/mutation_survivor_allowlist.json", + ) + args = parser.parse_args() + + repo_root = Path(args.repo_root).resolve() + failures = verify_test_debt_governance( + repo_root=repo_root, + skip_policy_path=Path(args.skip_policy), + mutation_allowlist_path=Path(args.mutation_survivor_allowlist), + ) + if failures: + for failure in failures: + print(f"TEST-DEBT-GOVERNANCE-FAIL: {failure}") + return 1 + + print( + "TEST-DEBT-GOVERNANCE-PASS: skip-policy and mutation-survivor debt metadata are current." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_SOP.md b/tests/TEST_SOP.md index fe78e31..4071507 100644 --- a/tests/TEST_SOP.md +++ b/tests/TEST_SOP.md @@ -242,24 +242,26 @@ Then every `git push` will run: bash scripts/pre_push_checks.sh ``` -`scripts/pre_push_checks.sh` is the CI-parity guard and must include all 7 stages: +`scripts/pre_push_checks.sh` is the CI-parity guard and must include all 9 stages: 1) `detect-secrets` 2) all `pre-commit` hooks 3) coverage governance check (`scripts/verify_quality_governance.py`) -4) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`) -5) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`) -6) R121 retry partition contract (`tests.test_r121_retry_partition_contract`) -7) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) -8) frontend E2E (`npm test`) +4) test debt governance check (`scripts/verify_test_debt_governance.py`) +5) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`) +6) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`) +7) R121 retry partition contract (`tests.test_r121_retry_partition_contract`) +8) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) +9) frontend E2E (`npm test`) IMPORTANT: - Do not remove stage (3). If governance drift is not checked locally, coverage / mutation protections can silently weaken while the main test suite still looks green. -- Do not remove stage (4). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later. -- Do not remove stage (5). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI. -- Do not remove stage (6) or stage (7). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI. -- Do not downgrade stage (7) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`. +- Do not remove stage (4). If stale skip-policy or mutation allowlist debt is not checked locally, CI can silently accumulate unreviewed governance exceptions. +- Do not remove stage (5). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later. +- Do not remove stage (6). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI. +- Do not remove stage (7) or stage (8). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI. +- Do not downgrade stage (8) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`. - Keep dependency bootstrap in this script aligned with `.github/workflows/ci.yml` unit-test dependencies. ## R118 Adaptive Profile + Mutation Strictness (Required) @@ -300,6 +302,23 @@ python scripts/verify_quality_governance.py - This check must stay in the standard local/full-test flow so threshold/config drift is caught before push. +## Test Debt Governance Baseline (Required) + +- Governance drift check command: + +```bash +python scripts/verify_test_debt_governance.py +``` + +- No-skip modules in `tests/skip_policy.json` must keep explicit metadata: + - `reason` + - `review_after` +- Mutation survivor allowlist entries in `tests/mutation_survivor_allowlist.json` must keep: + - `reason` + - `review_after` +- Both governance files must point only at live repo paths; stale module/file references are gate failures. +- `review_after` dates in the past are treated as actionable governance debt and must be refreshed or removed before acceptance. + 1) Detect Secrets (baseline-based) ```bash diff --git a/tests/mutation_survivor_allowlist.json b/tests/mutation_survivor_allowlist.json index 6a8b24d..4d6c007 100644 --- a/tests/mutation_survivor_allowlist.json +++ b/tests/mutation_survivor_allowlist.json @@ -4,12 +4,14 @@ { "file": "services/access_control.py", "mutation_index": 9, - "reason": "Equivalent guard: compare_digest still fails when client token is empty." + "reason": "Equivalent guard: compare_digest still fails when client token is empty.", + "review_after": "2026-10-31" }, { "file": "services/access_control.py", "mutation_index": 11, - "reason": "Equivalent guard: compare_digest still fails when client token is empty." + "reason": "Equivalent guard: compare_digest still fails when client token is empty.", + "review_after": "2026-10-31" } ] } diff --git a/tests/quality_governance_test_utils.py b/tests/quality_governance_test_utils.py new file mode 100644 index 0000000..9f3c9d0 --- /dev/null +++ b/tests/quality_governance_test_utils.py @@ -0,0 +1,160 @@ +import json +import textwrap +from pathlib import Path +from typing import Any, Dict, Iterable, Optional + + +DEFAULT_REQUIRED_HOTSPOT_FAMILIES = [ + "safe_io", + "security_boundary", + "connector_config", + "config_bootstrap", +] + + +def sample_policy_payload( + *, + current_stage: str = "baseline-35", + stages: Optional[Iterable[Dict[str, Any]]] = None, + required_hotspot_families: Optional[Iterable[str]] = None, + hotspot_families: Optional[Iterable[Dict[str, Any]]] = None, + exceptions: Optional[Iterable[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + required_families = list( + required_hotspot_families or DEFAULT_REQUIRED_HOTSPOT_FAMILIES + ) + return { + "schema_version": 1, + "current_stage": current_stage, + "stages": list( + stages + or [ + { + "id": "baseline-35", + "min_fail_under": 35.0, + "promotion_requires": [ + "coverage summary reviewed", + "no unresolved hotspot exceptions", + ], + "rollback_triggers": [ + "coverage regression", + "critical hotspot slip", + ], + }, + { + "id": "ratchet-45", + "min_fail_under": 45.0, + "promotion_requires": ["two consecutive clean reviews"], + "rollback_triggers": ["new unresolved exceptions"], + }, + ] + ), + "required_hotspot_families": required_families, + "hotspot_families": list( + hotspot_families + or [ + {"id": "safe_io", "paths": ["services/safe_io.py"]}, + {"id": "security_boundary", "paths": ["services/security_gate.py"]}, + {"id": "connector_config", "paths": ["connector/config.py"]}, + { + "id": "config_bootstrap", + "paths": ["config.py", "services/runtime_config.py"], + }, + ] + ), + "exceptions": list(exceptions or []), + } + + +def sample_policy_json(**kwargs: Any) -> str: + return json.dumps(sample_policy_payload(**kwargs), indent=2) + "\n" + + +def sample_sop_text(*, include_test_debt_phrase: bool = False) -> str: + lines = [ + "R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)", + "global score threshold (`>= 80%` unless explicitly overridden)", + "coverage governance check (`scripts/verify_quality_governance.py`)", + "staged coverage ratchet policy (`tests/coverage_governance_policy.json`)", + ] + if include_test_debt_phrase: + lines.append( + "test debt governance check (`scripts/verify_test_debt_governance.py`)" + ) + return "\n".join(lines) + "\n" + + +def sample_release_policy_text(*, include_test_debt_phrase: bool = False) -> str: + lines = [ + "staged coverage ratchet policy (`tests/coverage_governance_policy.json`)", + "`fail_under` must match the current stage floor declared in `tests/coverage_governance_policy.json`", + ] + if include_test_debt_phrase: + lines.append( + "test debt governance check (`scripts/verify_test_debt_governance.py`)" + ) + return "\n".join(lines) + "\n" + + +def write_governance_baseline_fixture( + tmp: Path, + *, + fail_under: Optional[float] = 35.0, + coverage_policy_payload: Optional[Dict[str, Any]] = None, + mutation_allowlist_payload: Optional[Dict[str, Any]] = None, + include_test_debt_phrase: bool = False, +) -> Dict[str, Path]: + pyproject = tmp / "pyproject.toml" + if fail_under is None: + report_lines = [ + "[tool.coverage.report]", + "show_missing = true", + "skip_covered = true", + ] + else: + report_lines = [ + "[tool.coverage.report]", + f"fail_under = {fail_under}", + "show_missing = true", + "skip_covered = true", + ] + pyproject.write_text("\n".join(report_lines) + "\n", encoding="utf-8") + + adversarial_gate = tmp / "run_adversarial_gate.py" + adversarial_gate.write_text( + "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", + encoding="utf-8", + ) + + test_sop = tmp / "TEST_SOP.md" + test_sop.write_text( + sample_sop_text(include_test_debt_phrase=include_test_debt_phrase), + encoding="utf-8", + ) + + survivor_allowlist = tmp / "mutation_survivor_allowlist.json" + survivor_allowlist.write_text( + json.dumps(mutation_allowlist_payload or {"entries": []}, indent=2) + "\n", + encoding="utf-8", + ) + + coverage_policy = tmp / "coverage_governance_policy.json" + coverage_policy.write_text( + json.dumps(coverage_policy_payload or sample_policy_payload(), indent=2) + "\n", + encoding="utf-8", + ) + + release_policy_doc = tmp / "ci_regression_policy.md" + release_policy_doc.write_text( + sample_release_policy_text(include_test_debt_phrase=include_test_debt_phrase), + encoding="utf-8", + ) + + return { + "pyproject": pyproject, + "adversarial_gate": adversarial_gate, + "test_sop": test_sop, + "survivor_allowlist": survivor_allowlist, + "coverage_policy": coverage_policy, + "release_policy_doc": release_policy_doc, + } diff --git a/tests/skip_policy.json b/tests/skip_policy.json index 5534303..8638fa6 100644 --- a/tests/skip_policy.json +++ b/tests/skip_policy.json @@ -19,5 +19,79 @@ "tests.test_connector_installation_registry", "tests.test_connector_callback_contract", "tests.test_s70_ssrf_pinning_regression" - ] + ], + "no_skip_module_metadata": { + "tests.test_s58_bridge_token_lifecycle": { + "reason": "Bridge token lifecycle coverage is a hard regression seam for token expiry and revoke behavior.", + "review_after": "2026-10-31" + }, + "tests.test_s59_webhook_mapping_clamp": { + "reason": "Webhook mapping clamp must stay no-skip because route/mapping drift can silently reopen unsafe ingress paths.", + "review_after": "2026-10-31" + }, + "tests.test_s60_mae_route_segmentation": { + "reason": "Public MAE route segmentation is a published hard-guarantee suite and cannot degrade to skip-only coverage.", + "review_after": "2026-10-31" + }, + "tests.test_s60_routes_startup_gate": { + "reason": "Startup gate coverage must stay active to catch route registration drift before public deployment.", + "review_after": "2026-10-31" + }, + "tests.test_s61_registry_signature": { + "reason": "Registry signature enforcement is part of the release-safety boundary for extension metadata.", + "review_after": "2026-10-31" + }, + "tests.security.test_endpoint_drift": { + "reason": "Endpoint drift detection guards the published endpoint manifest and must remain non-skippable.", + "review_after": "2026-10-31" + }, + "tests.test_r122_real_backend_lane": { + "reason": "Real backend lane is the low-mock parity seam for LLM/backend integration and cannot silently skip.", + "review_after": "2026-10-31" + }, + "tests.test_r123_real_backend_model_list_lane": { + "reason": "Model-list real backend lane protects loopback SSRF and provider-model discovery parity.", + "review_after": "2026-10-31" + }, + "tests.test_r121_retry_partition_contract": { + "reason": "Retry partition contract is an explicit robustness gate for failure-domain isolation.", + "review_after": "2026-10-31" + }, + "tests.test_r124_slack_ingress_contract": { + "reason": "Slack ingress contract must stay enforced because connector auth and callback parsing are production paths.", + "review_after": "2026-10-31" + }, + "tests.test_r125_slack_real_backend_lane": { + "reason": "Slack real-backend lane protects connector-to-backend parity across approval and run flows.", + "review_after": "2026-10-31" + }, + "tests.test_r117_observability_redaction_e2e": { + "reason": "Observability redaction E2E covers production-facing data minimization and cannot be treated as optional.", + "review_after": "2026-10-31" + }, + "tests.test_r117_observability_redaction_endpoints": { + "reason": "Endpoint-level redaction assertions are part of the audit/privacy boundary and must remain live.", + "review_after": "2026-10-31" + }, + "tests.test_f57_slack_transport_parity": { + "reason": "Slack transport parity catches protocol drift between connector routing and HTTP contract handling.", + "review_after": "2026-10-31" + }, + "tests.test_f57_slack_socket_mode_startup": { + "reason": "Slack socket mode startup coverage protects bootstrap compatibility for a supported deployment mode.", + "review_after": "2026-10-31" + }, + "tests.test_connector_installation_registry": { + "reason": "Connector installation registry governs multi-platform connector ownership and must stay no-skip.", + "review_after": "2026-10-31" + }, + "tests.test_connector_callback_contract": { + "reason": "Connector callback contract protects external ingress normalization and approval wiring.", + "review_after": "2026-10-31" + }, + "tests.test_s70_ssrf_pinning_regression": { + "reason": "SSRF pinning regression is a security-critical boundary test and must never degrade to skip coverage.", + "review_after": "2026-10-31" + } + } } diff --git a/tests/test_r156_quality_governance.py b/tests/test_r156_quality_governance.py index 834d2dc..9c8a809 100644 --- a/tests/test_r156_quality_governance.py +++ b/tests/test_r156_quality_governance.py @@ -2,63 +2,18 @@ import json import subprocess import sys import tempfile -import textwrap import unittest from pathlib import Path +from tests.quality_governance_test_utils import ( + sample_policy_payload, + write_governance_baseline_fixture, +) + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "verify_quality_governance.py" -def _sample_policy_json(): - return ( - json.dumps( - { - "schema_version": 1, - "current_stage": "baseline-35", - "stages": [ - { - "id": "baseline-35", - "min_fail_under": 35.0, - "promotion_requires": [ - "coverage summary reviewed", - "no unresolved hotspot exceptions", - ], - "rollback_triggers": [ - "coverage regression", - "critical hotspot slip", - ], - }, - { - "id": "ratchet-45", - "min_fail_under": 45.0, - "promotion_requires": ["two consecutive clean reviews"], - "rollback_triggers": ["new unresolved exceptions"], - }, - ], - "required_hotspot_families": [ - "safe_io", - "security_boundary", - "connector_config", - "config_bootstrap", - ], - "hotspot_families": [ - {"id": "safe_io", "paths": ["services/safe_io.py"]}, - {"id": "security_boundary", "paths": ["services/security_gate.py"]}, - {"id": "connector_config", "paths": ["connector/config.py"]}, - { - "id": "config_bootstrap", - "paths": ["config.py", "services/runtime_config.py"], - }, - ], - "exceptions": [], - }, - indent=2, - ) - + "\n" - ) - - class TestR156QualityGovernance(unittest.TestCase): def _run_script(self, *args): return subprocess.run( @@ -77,52 +32,19 @@ class TestR156QualityGovernance(unittest.TestCase): def test_missing_coverage_policy_is_rejected(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) - pyproject = tmp / "pyproject.toml" - pyproject.write_text( - textwrap.dedent( - """ - [tool.coverage.report] - fail_under = 35.0 - show_missing = true - skip_covered = true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - gate = tmp / "run_adversarial_gate.py" - gate.write_text( - "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", - encoding="utf-8", - ) - - sop = tmp / "TEST_SOP.md" - sop.write_text( - textwrap.dedent( - """ - R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) - global score threshold (`>= 80%` unless explicitly overridden) - coverage governance check (`scripts/verify_quality_governance.py`) - staged coverage ratchet policy (`tests/coverage_governance_policy.json`) - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - allowlist = tmp / "mutation_survivor_allowlist.json" - allowlist.write_text('{"entries":[]}\n', encoding="utf-8") + fixture = write_governance_baseline_fixture(tmp) result = self._run_script( "--pyproject", - str(pyproject), + str(fixture["pyproject"]), "--adversarial-gate", - str(gate), + str(fixture["adversarial_gate"]), "--test-sop", - str(sop), + str(fixture["test_sop"]), "--mutation-survivor-allowlist", - str(allowlist), + str(fixture["survivor_allowlist"]), + "--release-policy-doc", + str(fixture["release_policy_doc"]), "--coverage-policy", str(tmp / "missing_policy.json"), ) @@ -132,76 +54,33 @@ class TestR156QualityGovernance(unittest.TestCase): def test_non_monotonic_policy_thresholds_are_rejected(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) - pyproject = tmp / "pyproject.toml" - pyproject.write_text( - textwrap.dedent( - """ - [tool.coverage.report] - fail_under = 35.0 - show_missing = true - skip_covered = true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - gate = tmp / "run_adversarial_gate.py" - gate.write_text( - "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", - encoding="utf-8", - ) - - sop = tmp / "TEST_SOP.md" - sop.write_text( - textwrap.dedent( - """ - R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) - global score threshold (`>= 80%` unless explicitly overridden) - coverage governance check (`scripts/verify_quality_governance.py`) - staged coverage ratchet policy (`tests/coverage_governance_policy.json`) - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - allowlist = tmp / "mutation_survivor_allowlist.json" - allowlist.write_text('{"entries":[]}\n', encoding="utf-8") - - policy = tmp / "coverage_governance_policy.json" - policy.write_text( - json.dumps( - { - "schema_version": 1, - "current_stage": "baseline-35", - "stages": [ - {"id": "baseline-35", "min_fail_under": 35.0}, - {"id": "ratchet-30", "min_fail_under": 30.0}, - ], - "required_hotspot_families": ["safe_io"], - "hotspot_families": [ - {"id": "safe_io", "paths": ["services/safe_io.py"]} - ], - "exceptions": [], - }, - indent=2, - ) - + "\n", - encoding="utf-8", + fixture = write_governance_baseline_fixture( + tmp, + coverage_policy_payload=sample_policy_payload( + stages=[ + {"id": "baseline-35", "min_fail_under": 35.0}, + {"id": "ratchet-30", "min_fail_under": 30.0}, + ], + required_hotspot_families=["safe_io"], + hotspot_families=[ + {"id": "safe_io", "paths": ["services/safe_io.py"]} + ], + ), ) result = self._run_script( "--pyproject", - str(pyproject), + str(fixture["pyproject"]), "--adversarial-gate", - str(gate), + str(fixture["adversarial_gate"]), "--test-sop", - str(sop), + str(fixture["test_sop"]), "--mutation-survivor-allowlist", - str(allowlist), + str(fixture["survivor_allowlist"]), + "--release-policy-doc", + str(fixture["release_policy_doc"]), "--coverage-policy", - str(policy), + str(fixture["coverage_policy"]), ) self.assertNotEqual(result.returncode, 0) self.assertIn("coverage stages must increase strictly", result.stdout) @@ -209,76 +88,29 @@ class TestR156QualityGovernance(unittest.TestCase): def test_missing_required_hotspot_family_is_rejected(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) - pyproject = tmp / "pyproject.toml" - pyproject.write_text( - textwrap.dedent( - """ - [tool.coverage.report] - fail_under = 35.0 - show_missing = true - skip_covered = true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - gate = tmp / "run_adversarial_gate.py" - gate.write_text( - "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", - encoding="utf-8", - ) - - sop = tmp / "TEST_SOP.md" - sop.write_text( - textwrap.dedent( - """ - R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) - global score threshold (`>= 80%` unless explicitly overridden) - coverage governance check (`scripts/verify_quality_governance.py`) - staged coverage ratchet policy (`tests/coverage_governance_policy.json`) - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - allowlist = tmp / "mutation_survivor_allowlist.json" - allowlist.write_text('{"entries":[]}\n', encoding="utf-8") - - policy = tmp / "coverage_governance_policy.json" - policy.write_text( - json.dumps( - { - "schema_version": 1, - "current_stage": "baseline-35", - "stages": [ - {"id": "baseline-35", "min_fail_under": 35.0}, - {"id": "ratchet-45", "min_fail_under": 45.0}, - ], - "required_hotspot_families": ["safe_io", "connector_config"], - "hotspot_families": [ - {"id": "safe_io", "paths": ["services/safe_io.py"]} - ], - "exceptions": [], - }, - indent=2, - ) - + "\n", - encoding="utf-8", + fixture = write_governance_baseline_fixture( + tmp, + coverage_policy_payload=sample_policy_payload( + required_hotspot_families=["safe_io", "connector_config"], + hotspot_families=[ + {"id": "safe_io", "paths": ["services/safe_io.py"]} + ], + ), ) result = self._run_script( "--pyproject", - str(pyproject), + str(fixture["pyproject"]), "--adversarial-gate", - str(gate), + str(fixture["adversarial_gate"]), "--test-sop", - str(sop), + str(fixture["test_sop"]), "--mutation-survivor-allowlist", - str(allowlist), + str(fixture["survivor_allowlist"]), + "--release-policy-doc", + str(fixture["release_policy_doc"]), "--coverage-policy", - str(policy), + str(fixture["coverage_policy"]), ) self.assertNotEqual(result.returncode, 0) self.assertIn("missing required hotspot families", result.stdout) @@ -286,56 +118,21 @@ class TestR156QualityGovernance(unittest.TestCase): def test_missing_fail_under_is_rejected(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) - pyproject = tmp / "pyproject.toml" - pyproject.write_text( - textwrap.dedent( - """ - [tool.coverage.report] - show_missing = true - skip_covered = true - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - gate = tmp / "run_adversarial_gate.py" - gate.write_text( - "SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n", - encoding="utf-8", - ) - - sop = tmp / "TEST_SOP.md" - sop.write_text( - textwrap.dedent( - """ - R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`) - global score threshold (`>= 80%` unless explicitly overridden) - coverage governance check (`scripts/verify_quality_governance.py`) - staged coverage ratchet policy (`tests/coverage_governance_policy.json`) - """ - ).strip() - + "\n", - encoding="utf-8", - ) - - allowlist = tmp / "mutation_survivor_allowlist.json" - allowlist.write_text('{"entries":[]}\n', encoding="utf-8") - - policy = tmp / "coverage_governance_policy.json" - policy.write_text(_sample_policy_json(), encoding="utf-8") + fixture = write_governance_baseline_fixture(tmp, fail_under=None) result = self._run_script( "--pyproject", - str(pyproject), + str(fixture["pyproject"]), "--adversarial-gate", - str(gate), + str(fixture["adversarial_gate"]), "--test-sop", - str(sop), + str(fixture["test_sop"]), "--mutation-survivor-allowlist", - str(allowlist), + str(fixture["survivor_allowlist"]), + "--release-policy-doc", + str(fixture["release_policy_doc"]), "--coverage-policy", - str(policy), + str(fixture["coverage_policy"]), ) self.assertNotEqual(result.returncode, 0) self.assertIn("missing coverage fail_under", result.stdout) diff --git a/tests/test_r171_test_debt_governance.py b/tests/test_r171_test_debt_governance.py new file mode 100644 index 0000000..4cae170 --- /dev/null +++ b/tests/test_r171_test_debt_governance.py @@ -0,0 +1,247 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "verify_test_debt_governance.py" + + +def _write_repo_fixture(tmp: Path) -> dict[str, Path]: + tests_dir = tmp / "tests" + services_dir = tmp / "services" + tests_dir.mkdir() + services_dir.mkdir() + + guarded_test = tests_dir / "test_guarded_lane.py" + guarded_test.write_text("import unittest\n", encoding="utf-8") + access_control = services_dir / "access_control.py" + access_control.write_text("TOKEN = 'ok'\n", encoding="utf-8") + + skip_policy = tests_dir / "skip_policy.json" + skip_policy.write_text( + json.dumps( + { + "max_skipped": 1, + "no_skip_modules": ["tests.test_guarded_lane"], + "no_skip_module_metadata": { + "tests.test_guarded_lane": { + "reason": "Guarded lane must stay no-skip in CI parity.", + "review_after": "2026-10-31", + } + }, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + mutation_allowlist = tests_dir / "mutation_survivor_allowlist.json" + mutation_allowlist.write_text( + json.dumps( + { + "version": 1, + "entries": [ + { + "file": "services/access_control.py", + "mutation_index": 9, + "reason": "Equivalent compare_digest branch with empty token.", + "review_after": "2026-10-31", + } + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return { + "repo_root": tmp, + "skip_policy": skip_policy, + "mutation_allowlist": mutation_allowlist, + } + + +class TestR171TestDebtGovernance(unittest.TestCase): + def _run_script(self, *args: str): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + cwd=str(ROOT), + check=False, + ) + + def test_repo_governance_baseline_passes(self): + result = self._run_script() + self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr) + self.assertIn("TEST-DEBT-GOVERNANCE-PASS", result.stdout) + + def test_duplicate_no_skip_modules_are_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + fixture = _write_repo_fixture(Path(tmpdir)) + fixture["skip_policy"].write_text( + json.dumps( + { + "max_skipped": 1, + "no_skip_modules": [ + "tests.test_guarded_lane", + "tests.test_guarded_lane", + ], + "no_skip_module_metadata": { + "tests.test_guarded_lane": { + "reason": "Guarded lane must stay no-skip in CI parity.", + "review_after": "2026-10-31", + } + }, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + result = self._run_script( + "--repo-root", + str(fixture["repo_root"]), + "--skip-policy", + str(fixture["skip_policy"]), + "--mutation-survivor-allowlist", + str(fixture["mutation_allowlist"]), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("duplicate no-skip modules", result.stdout) + + def test_missing_no_skip_module_metadata_is_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + fixture = _write_repo_fixture(Path(tmpdir)) + fixture["skip_policy"].write_text( + json.dumps( + { + "max_skipped": 1, + "no_skip_modules": ["tests.test_guarded_lane"], + "no_skip_module_metadata": {}, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + result = self._run_script( + "--repo-root", + str(fixture["repo_root"]), + "--skip-policy", + str(fixture["skip_policy"]), + "--mutation-survivor-allowlist", + str(fixture["mutation_allowlist"]), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("missing metadata for no-skip modules", result.stdout) + + def test_duplicate_mutation_allowlist_entries_are_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + fixture = _write_repo_fixture(Path(tmpdir)) + fixture["mutation_allowlist"].write_text( + json.dumps( + { + "version": 1, + "entries": [ + { + "file": "services/access_control.py", + "mutation_index": 9, + "reason": "Equivalent compare_digest branch with empty token.", + "review_after": "2026-10-31", + }, + { + "file": "services/access_control.py", + "mutation_index": 9, + "reason": "Duplicate entry for regression coverage.", + "review_after": "2026-10-31", + }, + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + result = self._run_script( + "--repo-root", + str(fixture["repo_root"]), + "--skip-policy", + str(fixture["skip_policy"]), + "--mutation-survivor-allowlist", + str(fixture["mutation_allowlist"]), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("duplicate (file, mutation_index) entries", result.stdout) + + def test_stale_mutation_allowlist_file_is_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + fixture = _write_repo_fixture(Path(tmpdir)) + fixture["mutation_allowlist"].write_text( + json.dumps( + { + "version": 1, + "entries": [ + { + "file": "services/missing.py", + "mutation_index": 9, + "reason": "Stale path should fail closed.", + "review_after": "2026-10-31", + } + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + result = self._run_script( + "--repo-root", + str(fixture["repo_root"]), + "--skip-policy", + str(fixture["skip_policy"]), + "--mutation-survivor-allowlist", + str(fixture["mutation_allowlist"]), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("file does not exist in repo", result.stdout) + + def test_past_review_after_is_rejected(self): + with tempfile.TemporaryDirectory() as tmpdir: + fixture = _write_repo_fixture(Path(tmpdir)) + fixture["mutation_allowlist"].write_text( + json.dumps( + { + "version": 1, + "entries": [ + { + "file": "services/access_control.py", + "mutation_index": 9, + "reason": "Expired reviews must not linger.", + "review_after": "2025-01-01", + } + ], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + result = self._run_script( + "--repo-root", + str(fixture["repo_root"]), + "--skip-policy", + str(fixture["skip_policy"]), + "--mutation-survivor-allowlist", + str(fixture["mutation_allowlist"]), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("review_after 2025-01-01 is in the past", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_r174_quality_governance_report.py b/tests/test_r174_quality_governance_report.py index b2aa8b3..d127ea0 100644 --- a/tests/test_r174_quality_governance_report.py +++ b/tests/test_r174_quality_governance_report.py @@ -5,6 +5,8 @@ import tempfile import unittest from pathlib import Path +from tests.quality_governance_test_utils import sample_policy_payload + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "report_coverage_governance.py" @@ -27,27 +29,20 @@ class TestR174QualityGovernanceReport(unittest.TestCase): policy.write_text( json.dumps( - { - "schema_version": 1, - "current_stage": "baseline-35", - "stages": [ - {"id": "baseline-35", "min_fail_under": 35.0}, - {"id": "ratchet-45", "min_fail_under": 45.0}, - ], - "required_hotspot_families": [ - "safe_io", - "security_boundary", - "connector_config", - "config_bootstrap", - ], - "hotspot_families": [ + sample_policy_payload( + hotspot_families=[ {"id": "safe_io", "paths": ["services/safe_io.py"]}, - {"id": "connector_config", "paths": ["connector/config.py"]}, - {"id": "security_boundary", "paths": ["services/security_gate.py"]}, + { + "id": "connector_config", + "paths": ["connector/config.py"], + }, + { + "id": "security_boundary", + "paths": ["services/security_gate.py"], + }, {"id": "config_bootstrap", "paths": ["config.py"]}, - ], - "exceptions": [], - }, + ] + ), indent=2, ) + "\n", @@ -103,8 +98,12 @@ class TestR174QualityGovernanceReport(unittest.TestCase): self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr) payload = json.loads(result.stdout) self.assertEqual(payload["overall"]["percent_covered"], 64.35) - self.assertEqual(payload["hotspot_families"]["safe_io"]["percent_covered"], 80.0) - self.assertEqual(payload["hotspot_families"]["connector_config"]["percent_covered"], 60.0) + self.assertEqual( + payload["hotspot_families"]["safe_io"]["percent_covered"], 80.0 + ) + self.assertEqual( + payload["hotspot_families"]["connector_config"]["percent_covered"], 60.0 + ) def test_missing_hotspot_files_are_reported_deterministically(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -114,27 +113,20 @@ class TestR174QualityGovernanceReport(unittest.TestCase): policy.write_text( json.dumps( - { - "schema_version": 1, - "current_stage": "baseline-35", - "stages": [ - {"id": "baseline-35", "min_fail_under": 35.0}, - {"id": "ratchet-45", "min_fail_under": 45.0}, - ], - "required_hotspot_families": [ - "safe_io", - "security_boundary", - "connector_config", - "config_bootstrap", - ], - "hotspot_families": [ + sample_policy_payload( + hotspot_families=[ {"id": "safe_io", "paths": ["services/safe_io.py"]}, - {"id": "security_boundary", "paths": ["services/security_gate.py"]}, - {"id": "connector_config", "paths": ["connector/config.py"]}, + { + "id": "security_boundary", + "paths": ["services/security_gate.py"], + }, + { + "id": "connector_config", + "paths": ["connector/config.py"], + }, {"id": "config_bootstrap", "paths": ["config.py"]}, - ], - "exceptions": [], - }, + ] + ), indent=2, ) + "\n",