admin/ComfyUI-OpenClaw
1
0
Fork 0
mirror of https://github.com/rookiestar28/ComfyUI-OpenClaw.git synced 2026-08-14 00:48:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(governance): validate test debt metadata

Browse Source
This commit is contained in:
rookiestar28
2026-04-20 05:38:56 +08:00
parent 723215ac48
commit 71199efa07
14 changed files with 887 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/release/ci_regression_policy.md
+5
View File
@@ -12,6 +12,7 @@ All pull requests must pass the repository SOP gate before merge.
| Backend dependency audit | `pip-audit -r requirements.txt` | Audit declared Python project dependencies without scanning unrelated CI runner/toolchain packages |
| GitHub CodeQL analysis | `.github/workflows/codeql.yml` | Run repository-native static security analysis for Python, JavaScript/TypeScript, and GitHub Actions on push, pull request, and weekly schedule |
| Coverage governance | `python scripts/verify_quality_governance.py` | Fail closed on coverage-policy, mutation-threshold, SOP-guidance, and survivor-allowlist drift |
| Test debt governance | `python scripts/verify_test_debt_governance.py` | Fail closed on stale or under-documented skip-policy / mutation allowlist debt entries |
| Backend unit tests | `python scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json` | Validate backend behavior and skip governance |
| Adversarial gate | `python scripts/run_adversarial_gate.py --profile auto --seed 42` | Enforce adaptive fuzz/mutation verification with smoke=>extended escalation on high-risk diffs |
| Frontend E2E | `npm test` | Validate UI and frontend/backend integration |
@@ -56,6 +57,10 @@ If a change intentionally modifies contract behavior:
- `fail_under` must match the current stage floor declared in `tests/coverage_governance_policy.json`; do not ratchet the floor by editing `pyproject.toml` alone.
- Coverage hotspot review should use:
- `python scripts/report_coverage_governance.py --coverage-json <path-to-coverage.json>`
- Test debt governance remains fail-closed:
- no-skip modules in `tests/skip_policy.json` must keep explicit metadata (`reason` + `review_after`) and point at live test modules
- mutation survivor allowlist entries must carry `review_after` dates and point at live repo files
- review dates in the past are governance debt, not advisory comments
- Mutation governance remains adaptive:
- smoke profile threshold: `20.0%`
- extended profile threshold: `80.0%`
scripts/pre_push_checks.sh
+10 -7
View File
@@ -383,33 +383,36 @@ cleanup_precommit_snapshots
# edits cannot hide until deep in the pre-push unit suite.
"$VENV_PY" scripts/check_openapi_sync.py
echo "[pre-push] 3/8 coverage governance check"
echo "[pre-push] 3/9 coverage governance check"
"$VENV_PY" scripts/verify_quality_governance.py
echo "[pre-push] 4/8 backend unit tests"
echo "[pre-push] 4/9 test debt governance check"
"$VENV_PY" scripts/verify_test_debt_governance.py
echo "[pre-push] 5/9 backend unit tests"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_unit" \
"$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
echo "[pre-push] 4.5/8 implementation record lint (strict)"
echo "[pre-push] 5.5/9 implementation record lint (strict)"
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
fi
echo "[pre-push] 5/8 backend real E2E lanes (R122/R123)"
echo "[pre-push] 6/9 backend real E2E lanes (R122/R123)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[pre-push] 6/8 R121 retry partition contract"
echo "[pre-push] 7/9 R121 retry partition contract"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_retry_partition" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[pre-push] 7/8 R118 adversarial gate (adaptive: smoke/extended)"
echo "[pre-push] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_pre_push_adversarial" \
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
echo "[pre-push] 8/8 npm test (Playwright)"
echo "[pre-push] 9/9 npm test (Playwright)"
npm test
echo "[pre-push] PASS"
scripts/quality_governance_common.py
+13 -6
View File
@@ -7,7 +7,6 @@ from fnmatch import fnmatch
from pathlib import Path
from typing import Any, Iterable
REQUIRED_HOTSPOT_FAMILIES = (
"safe_io",
"security_boundary",
@@ -42,8 +41,10 @@ def _validate_hotspot_family(
seen_ids.add(family_id)
paths = family.get("paths")
if not isinstance(paths, list) or not paths or not all(
isinstance(path, str) and path.strip() for path in paths
if (
not isinstance(paths, list)
or not paths
or not all(isinstance(path, str) and path.strip() for path in paths)
):
failures.append(
f"coverage policy: hotspot family {family_id} must define a non-empty paths list"
@@ -93,7 +94,9 @@ def load_and_validate_policy(path: Path) -> tuple[dict[str, Any] | None, list[st
f"coverage policy: stage {stage_id} missing numeric min_fail_under"
)
continue
stages.append(CoverageStage(stage_id=stage_id, min_fail_under=float(min_fail_under)))
stages.append(
CoverageStage(stage_id=stage_id, min_fail_under=float(min_fail_under))
)
for previous, current in zip(stages, stages[1:]):
if current.min_fail_under <= previous.min_fail_under:
@@ -243,7 +246,9 @@ def summarize_coverage(
summary = files[file_path].get("summary", {})
covered_lines += int(summary.get("covered_lines", 0))
num_statements += int(summary.get("num_statements", 0))
percent = round((covered_lines / num_statements) * 100, 2) if num_statements else 0.0
percent = (
round((covered_lines / num_statements) * 100, 2) if num_statements else 0.0
)
hotspot_summary[family_id] = {
"matched_files": matched_files,
"missing_paths": missing_paths,
@@ -263,7 +268,9 @@ def summarize_coverage(
"current_stage_fail_under": current_stage_threshold(policy),
"next_stage": next_policy_stage["id"] if next_policy_stage else None,
"next_stage_fail_under": (
float(next_policy_stage["min_fail_under"]) if next_policy_stage else None
float(next_policy_stage["min_fail_under"])
if next_policy_stage
else None
),
},
"overall": {
scripts/report_coverage_governance.py
+5 -1
View File
@@ -10,7 +10,11 @@ import json
import sys
from pathlib import Path
from quality_governance_common import load_and_validate_policy, read_json, summarize_coverage
from quality_governance_common import (
load_and_validate_policy,
read_json,
summarize_coverage,
)
def _render_text(summary: dict[str, object]) -> str:
scripts/run_full_tests_linux.sh
+11 -8
View File
@@ -210,28 +210,31 @@ if precommit_changed_repo_state; then
fi
cleanup_precommit_snapshots
echo "[tests] 3/9 coverage governance check"
echo "[tests] 3/10 coverage governance check"
"$VENV_PY" scripts/verify_quality_governance.py
echo "[tests] 4/9 backend unit tests"
echo "[tests] 4/10 test debt governance check"
"$VENV_PY" scripts/verify_test_debt_governance.py
echo "[tests] 5/10 backend unit tests"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json
if [ -n "${OPENCLAW_IMPL_RECORD_PATH:-}" ]; then
echo "[tests] 4.5/9 implementation record lint (strict)"
echo "[tests] 5.5/10 implementation record lint (strict)"
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
"$VENV_PY" scripts/lint_implementation_record.py --path "$OPENCLAW_IMPL_RECORD_PATH" --strict
fi
echo "[tests] 5/9 backend real E2E lanes (R122/R123)"
echo "[tests] 6/10 backend real E2E lanes (R122/R123)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_backend_e2e_real" \
"$VENV_PY" scripts/run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 6/9 R121 retry partition contract"
echo "[tests] 7/10 R121 retry partition contract"
"$VENV_PY" scripts/run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
echo "[tests] 8/10 Slack integration gates (R124/R125/R117/F57)"
"$VENV_PY" scripts/run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_r117_observability_redaction_e2e --enforce-skip-policy tests/skip_policy.json --max-skipped 0
@@ -239,11 +242,11 @@ echo "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_transport_parity --enforce-skip-policy tests/skip_policy.json --max-skipped 0
"$VENV_PY" scripts/run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests/skip_policy.json --max-skipped 0
echo "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
echo "[tests] 9/10 R118 adversarial gate (adaptive: smoke/extended)"
MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_adversarial" \
"$VENV_PY" scripts/run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp/adversarial
echo "[tests] 9/9 frontend E2E"
echo "[tests] 10/10 frontend E2E"
# IMPORTANT: full-gate acceptance must provision Playwright browsers itself; do
# not assume a warmed local browser cache when running on fresh WSL/Linux hosts.
OPENCLAW_PLAYWRIGHT_INSTALL=1 OPENCLAW_PLAYWRIGHT_BROWSERS=chromium npm test
scripts/run_full_tests_windows.ps1
+13 -8
View File
@@ -266,26 +266,31 @@ if ($LASTEXITCODE -ne 0) {
}
Assert-PreCommitDidNotMutateRepo -BeforeWorktree $preCommitWorktreeBefore -BeforeIndex $preCommitIndexBefore
Write-Host "[tests] 3/9 coverage governance check"
Write-Host "[tests] 3/10 coverage governance check"
Invoke-Checked "coverage governance check" {
& $venvPython scripts\verify_quality_governance.py
}
Write-Host "[tests] 4/9 backend unit tests"
Write-Host "[tests] 4/10 test debt governance check"
Invoke-Checked "test debt governance check" {
& $venvPython scripts\verify_test_debt_governance.py
}
Write-Host "[tests] 5/10 backend unit tests"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit"
Invoke-Checked "backend unit tests" {
& $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py" --enforce-skip-policy tests\skip_policy.json
}
if ($env:OPENCLAW_IMPL_RECORD_PATH) {
Write-Host "[tests] 4.5/9 implementation record lint (strict)"
Write-Host "[tests] 5.5/10 implementation record lint (strict)"
# IMPORTANT: strict mode is opt-in via OPENCLAW_IMPL_RECORD_PATH to avoid retroactive legacy record failures.
Invoke-Checked "implementation record lint" {
& $venvPython scripts\lint_implementation_record.py --path $env:OPENCLAW_IMPL_RECORD_PATH --strict
}
}
Write-Host "[tests] 5/9 backend real E2E lanes (R122/R123)"
Write-Host "[tests] 6/10 backend real E2E lanes (R122/R123)"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_backend_e2e_real"
Invoke-Checked "backend real E2E lane R122" {
& $venvPython scripts\run_unittests.py --module tests.test_r122_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
@@ -294,12 +299,12 @@ Invoke-Checked "backend real E2E lane R123" {
& $venvPython scripts\run_unittests.py --module tests.test_r123_real_backend_model_list_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 6/9 R121 retry partition contract"
Write-Host "[tests] 7/10 R121 retry partition contract"
Invoke-Checked "R121 retry partition contract" {
& $venvPython scripts\run_unittests.py --module tests.test_r121_retry_partition_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 7/9 Slack integration gates (R124/R125/R117/F57)"
Write-Host "[tests] 8/10 Slack integration gates (R124/R125/R117/F57)"
Invoke-Checked "Slack integration gates" {
& $venvPython scripts\run_unittests.py --module tests.test_r124_slack_ingress_contract --enforce-skip-policy tests\skip_policy.json --max-skipped 0
& $venvPython scripts\run_unittests.py --module tests.test_r125_slack_real_backend_lane --enforce-skip-policy tests\skip_policy.json --max-skipped 0
@@ -309,13 +314,13 @@ Invoke-Checked "Slack integration gates" {
& $venvPython scripts\run_unittests.py --module tests.test_f57_slack_socket_mode_startup --enforce-skip-policy tests\skip_policy.json --max-skipped 0
}
Write-Host "[tests] 8/9 R118 adversarial gate (adaptive: smoke/extended)"
Write-Host "[tests] 9/10 R118 adversarial gate (adaptive: smoke/extended)"
$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_adversarial"
Invoke-Checked "R118 adversarial adaptive" {
& $venvPython scripts\run_adversarial_gate.py --profile auto --seed 42 --artifact-dir .tmp\adversarial
}
Write-Host "[tests] 9/9 frontend E2E"
Write-Host "[tests] 10/10 frontend E2E"
$env:OPENCLAW_PLAYWRIGHT_INSTALL = "1"
$env:OPENCLAW_PLAYWRIGHT_BROWSERS = "chromium"
# IMPORTANT: full-gate acceptance must provision Playwright browsers itself; do
scripts/verify_test_debt_governance.py
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""
R171: validate skip-policy and mutation-survivor debt metadata.
This script is intentionally stdlib-only so it can run early in local/full-test
gates before optional dependencies are installed.
"""
from __future__ import annotations
import argparse
import json
from datetime import date
from pathlib import Path, PurePosixPath
from typing import Any, Dict, Iterable, List, Optional, Tuple
def _read_json_object(path: Path) -> Dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(payload, dict):
raise ValueError(f"{path}: expected a JSON object")
return payload
def _normalize_repo_rel_path(path: str) -> str:
return PurePosixPath(path.replace("\\", "/")).as_posix().lstrip("./")
def _parse_review_after(value: Any, *, label: str, failures: List[str]) -> None:
if not isinstance(value, str) or not value.strip():
failures.append(f"{label}: missing review_after")
return
try:
review_after = date.fromisoformat(value)
except ValueError:
failures.append(f"{label}: invalid review_after '{value}'")
return
if review_after < date.today():
failures.append(f"{label}: review_after {value} is in the past")
def _validate_reason(value: Any, *, label: str, failures: List[str]) -> None:
if not isinstance(value, str) or not value.strip():
failures.append(f"{label}: missing non-empty reason")
def _resolve_test_module_path(repo_root: Path, module_name: str) -> Path:
return repo_root / Path(module_name.replace(".", "/")).with_suffix(".py")
def _validate_skip_policy(repo_root: Path, path: Path) -> List[str]:
failures: List[str] = []
payload = _read_json_object(path)
max_skipped = payload.get("max_skipped")
if not isinstance(max_skipped, int) or max_skipped < 0:
failures.append("skip policy: max_skipped must be a non-negative integer")
modules = payload.get("no_skip_modules", [])
if not isinstance(modules, list) or any(
not isinstance(item, str) or not item.strip() for item in modules
):
failures.append("skip policy: no_skip_modules must be a list of non-empty strings")
return failures
seen = set()
duplicates = set()
normalized_modules: List[str] = []
for module in modules:
normalized = module.strip()
normalized_modules.append(normalized)
if normalized in seen:
duplicates.add(normalized)
seen.add(normalized)
module_path = _resolve_test_module_path(repo_root, normalized)
if not module_path.is_file():
failures.append(
f"skip policy: module path does not exist for {normalized} -> {module_path.relative_to(repo_root)}"
)
if duplicates:
failures.append(
"skip policy: duplicate no-skip modules: " + ", ".join(sorted(duplicates))
)
metadata = payload.get("no_skip_module_metadata")
if not isinstance(metadata, dict):
failures.append(
"skip policy: no_skip_module_metadata must be an object keyed by module name"
)
return failures
metadata_keys = {str(key).strip() for key in metadata.keys()}
missing_metadata = [module for module in normalized_modules if module not in metadata_keys]
extra_metadata = sorted(
key for key in metadata_keys if key and key not in set(normalized_modules)
)
if missing_metadata:
failures.append(
"skip policy: missing metadata for no-skip modules: "
+ ", ".join(sorted(missing_metadata))
)
if extra_metadata:
failures.append(
"skip policy: stale metadata without matching no-skip module: "
+ ", ".join(extra_metadata)
)
for module_name in normalized_modules:
raw_meta = metadata.get(module_name)
label = f"skip policy metadata[{module_name}]"
if not isinstance(raw_meta, dict):
failures.append(f"{label}: metadata entry must be an object")
continue
_validate_reason(raw_meta.get("reason"), label=label, failures=failures)
_parse_review_after(
raw_meta.get("review_after"), label=label, failures=failures
)
return failures
def _validate_mutation_allowlist(repo_root: Path, path: Path) -> List[str]:
failures: List[str] = []
payload = _read_json_object(path)
entries = payload.get("entries", [])
if not isinstance(entries, list):
return ["mutation allowlist: entries must be a list"]
seen: set[Tuple[str, int]] = set()
duplicates: set[Tuple[str, int]] = set()
for index, raw_entry in enumerate(entries):
label = f"mutation allowlist entry[{index}]"
if not isinstance(raw_entry, dict):
failures.append(f"{label}: entry must be an object")
continue
file_path = _normalize_repo_rel_path(str(raw_entry.get("file", "")))
mutation_index = raw_entry.get("mutation_index")
if not file_path:
failures.append(f"{label}: missing file")
elif not (repo_root / Path(file_path)).is_file():
failures.append(f"{label}: file does not exist in repo: {file_path}")
if not isinstance(mutation_index, int) or mutation_index < 0:
failures.append(f"{label}: mutation_index must be a non-negative integer")
else:
key = (file_path, mutation_index)
if key in seen:
duplicates.add(key)
seen.add(key)
_validate_reason(raw_entry.get("reason"), label=label, failures=failures)
_parse_review_after(
raw_entry.get("review_after"), label=label, failures=failures
)
if duplicates:
failures.append(
"mutation allowlist: duplicate (file, mutation_index) entries: "
+ ", ".join(f"{file}@{mutation_index}" for file, mutation_index in sorted(duplicates))
)
return failures
def verify_test_debt_governance(
*,
repo_root: Path,
skip_policy_path: Path,
mutation_allowlist_path: Path,
) -> List[str]:
failures: List[str] = []
if not skip_policy_path.is_file():
failures.append(f"missing skip policy: {skip_policy_path}")
else:
try:
failures.extend(_validate_skip_policy(repo_root, skip_policy_path))
except Exception as exc:
failures.append(f"skip policy: failed to validate {skip_policy_path}: {exc}")
if not mutation_allowlist_path.is_file():
failures.append(f"missing mutation survivor allowlist: {mutation_allowlist_path}")
else:
try:
failures.extend(
_validate_mutation_allowlist(repo_root, mutation_allowlist_path)
)
except Exception as exc:
failures.append(
f"mutation allowlist: failed to validate {mutation_allowlist_path}: {exc}"
)
return failures
def main() -> int:
parser = argparse.ArgumentParser(
description="Validate skip-policy and mutation-survivor debt metadata."
)
parser.add_argument(
"--repo-root",
default=".",
help="Repository root used to resolve test modules and file paths.",
)
parser.add_argument(
"--skip-policy",
default="tests/skip_policy.json",
help="Path to tests/skip_policy.json",
)
parser.add_argument(
"--mutation-survivor-allowlist",
default="tests/mutation_survivor_allowlist.json",
help="Path to tests/mutation_survivor_allowlist.json",
)
args = parser.parse_args()
repo_root = Path(args.repo_root).resolve()
failures = verify_test_debt_governance(
repo_root=repo_root,
skip_policy_path=Path(args.skip_policy),
mutation_allowlist_path=Path(args.mutation_survivor_allowlist),
)
if failures:
for failure in failures:
print(f"TEST-DEBT-GOVERNANCE-FAIL: {failure}")
return 1
print(
"TEST-DEBT-GOVERNANCE-PASS: skip-policy and mutation-survivor debt metadata are current."
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
tests/TEST_SOP.md
+29 -10
View File
@@ -242,24 +242,26 @@ Then every `git push` will run:
bash scripts/pre_push_checks.sh
```
`scripts/pre_push_checks.sh` is the CI-parity guard and must include all 7 stages:
`scripts/pre_push_checks.sh` is the CI-parity guard and must include all 9 stages:
1) `detect-secrets`
2) all `pre-commit` hooks
3) coverage governance check (`scripts/verify_quality_governance.py`)
4) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`)
5) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`)
6) R121 retry partition contract (`tests.test_r121_retry_partition_contract`)
7) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
8) frontend E2E (`npm test`)
4) test debt governance check (`scripts/verify_test_debt_governance.py`)
5) backend unit tests (`scripts/run_unittests.py --pattern "test_*.py" --enforce-skip-policy tests/skip_policy.json`)
6) backend real E2E lanes (`tests.test_r122_real_backend_lane` + `tests.test_r123_real_backend_model_list_lane`)
7) R121 retry partition contract (`tests.test_r121_retry_partition_contract`)
8) R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
9) frontend E2E (`npm test`)
IMPORTANT:
- Do not remove stage (3). If governance drift is not checked locally, coverage / mutation protections can silently weaken while the main test suite still looks green.
- Do not remove stage (4). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later.
- Do not remove stage (5). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI.
- Do not remove stage (6) or stage (7). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI.
- Do not downgrade stage (7) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`.
- Do not remove stage (4). If stale skip-policy or mutation allowlist debt is not checked locally, CI can silently accumulate unreviewed governance exceptions.
- Do not remove stage (5). If pre-push skips backend unit tests, local pushes can pass while GitHub CI fails later.
- Do not remove stage (6). If pre-push skips real-backend lanes, model-list/webhook wiring regressions can bypass local checks and fail later in CI.
- Do not remove stage (7) or stage (8). If pre-push skips retry partition or adversarial gates, verification hardening regressions can bypass local checks and fail later in CI.
- Do not downgrade stage (8) back to fixed smoke profile. Adaptive mode is required so high-risk diffs auto-escalate to `extended`.
- Keep dependency bootstrap in this script aligned with `.github/workflows/ci.yml` unit-test dependencies.
## R118 Adaptive Profile + Mutation Strictness (Required)
@@ -300,6 +302,23 @@ python scripts/verify_quality_governance.py
- This check must stay in the standard local/full-test flow so threshold/config drift is caught before push.
## Test Debt Governance Baseline (Required)
- Governance drift check command:
```bash
python scripts/verify_test_debt_governance.py
```
- No-skip modules in `tests/skip_policy.json` must keep explicit metadata:
- `reason`
- `review_after`
- Mutation survivor allowlist entries in `tests/mutation_survivor_allowlist.json` must keep:
- `reason`
- `review_after`
- Both governance files must point only at live repo paths; stale module/file references are gate failures.
- `review_after` dates in the past are treated as actionable governance debt and must be refreshed or removed before acceptance.
1) Detect Secrets (baseline-based)
```bash
tests/mutation_survivor_allowlist.json
+4 -2
View File
@@ -4,12 +4,14 @@
{
"file": "services/access_control.py",
"mutation_index": 9,
"reason": "Equivalent guard: compare_digest still fails when client token is empty."
"reason": "Equivalent guard: compare_digest still fails when client token is empty.",
"review_after": "2026-10-31"
},
{
"file": "services/access_control.py",
"mutation_index": 11,
"reason": "Equivalent guard: compare_digest still fails when client token is empty."
"reason": "Equivalent guard: compare_digest still fails when client token is empty.",
"review_after": "2026-10-31"
}
]
}
tests/quality_governance_test_utils.py
+160
View File
@@ -0,0 +1,160 @@
import json
import textwrap
from pathlib import Path
from typing import Any, Dict, Iterable, Optional
DEFAULT_REQUIRED_HOTSPOT_FAMILIES = [
"safe_io",
"security_boundary",
"connector_config",
"config_bootstrap",
]
def sample_policy_payload(
*,
current_stage: str = "baseline-35",
stages: Optional[Iterable[Dict[str, Any]]] = None,
required_hotspot_families: Optional[Iterable[str]] = None,
hotspot_families: Optional[Iterable[Dict[str, Any]]] = None,
exceptions: Optional[Iterable[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
required_families = list(
required_hotspot_families or DEFAULT_REQUIRED_HOTSPOT_FAMILIES
)
return {
"schema_version": 1,
"current_stage": current_stage,
"stages": list(
stages
or [
{
"id": "baseline-35",
"min_fail_under": 35.0,
"promotion_requires": [
"coverage summary reviewed",
"no unresolved hotspot exceptions",
],
"rollback_triggers": [
"coverage regression",
"critical hotspot slip",
],
},
{
"id": "ratchet-45",
"min_fail_under": 45.0,
"promotion_requires": ["two consecutive clean reviews"],
"rollback_triggers": ["new unresolved exceptions"],
},
]
),
"required_hotspot_families": required_families,
"hotspot_families": list(
hotspot_families
or [
{"id": "safe_io", "paths": ["services/safe_io.py"]},
{"id": "security_boundary", "paths": ["services/security_gate.py"]},
{"id": "connector_config", "paths": ["connector/config.py"]},
{
"id": "config_bootstrap",
"paths": ["config.py", "services/runtime_config.py"],
},
]
),
"exceptions": list(exceptions or []),
}
def sample_policy_json(**kwargs: Any) -> str:
return json.dumps(sample_policy_payload(**kwargs), indent=2) + "\n"
def sample_sop_text(*, include_test_debt_phrase: bool = False) -> str:
lines = [
"R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)",
"global score threshold (`>= 80%` unless explicitly overridden)",
"coverage governance check (`scripts/verify_quality_governance.py`)",
"staged coverage ratchet policy (`tests/coverage_governance_policy.json`)",
]
if include_test_debt_phrase:
lines.append(
"test debt governance check (`scripts/verify_test_debt_governance.py`)"
)
return "\n".join(lines) + "\n"
def sample_release_policy_text(*, include_test_debt_phrase: bool = False) -> str:
lines = [
"staged coverage ratchet policy (`tests/coverage_governance_policy.json`)",
"`fail_under` must match the current stage floor declared in `tests/coverage_governance_policy.json`",
]
if include_test_debt_phrase:
lines.append(
"test debt governance check (`scripts/verify_test_debt_governance.py`)"
)
return "\n".join(lines) + "\n"
def write_governance_baseline_fixture(
tmp: Path,
*,
fail_under: Optional[float] = 35.0,
coverage_policy_payload: Optional[Dict[str, Any]] = None,
mutation_allowlist_payload: Optional[Dict[str, Any]] = None,
include_test_debt_phrase: bool = False,
) -> Dict[str, Path]:
pyproject = tmp / "pyproject.toml"
if fail_under is None:
report_lines = [
"[tool.coverage.report]",
"show_missing = true",
"skip_covered = true",
]
else:
report_lines = [
"[tool.coverage.report]",
f"fail_under = {fail_under}",
"show_missing = true",
"skip_covered = true",
]
pyproject.write_text("\n".join(report_lines) + "\n", encoding="utf-8")
adversarial_gate = tmp / "run_adversarial_gate.py"
adversarial_gate.write_text(
"SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n",
encoding="utf-8",
)
test_sop = tmp / "TEST_SOP.md"
test_sop.write_text(
sample_sop_text(include_test_debt_phrase=include_test_debt_phrase),
encoding="utf-8",
)
survivor_allowlist = tmp / "mutation_survivor_allowlist.json"
survivor_allowlist.write_text(
json.dumps(mutation_allowlist_payload or {"entries": []}, indent=2) + "\n",
encoding="utf-8",
)
coverage_policy = tmp / "coverage_governance_policy.json"
coverage_policy.write_text(
json.dumps(coverage_policy_payload or sample_policy_payload(), indent=2) + "\n",
encoding="utf-8",
)
release_policy_doc = tmp / "ci_regression_policy.md"
release_policy_doc.write_text(
sample_release_policy_text(include_test_debt_phrase=include_test_debt_phrase),
encoding="utf-8",
)
return {
"pyproject": pyproject,
"adversarial_gate": adversarial_gate,
"test_sop": test_sop,
"survivor_allowlist": survivor_allowlist,
"coverage_policy": coverage_policy,
"release_policy_doc": release_policy_doc,
}
tests/skip_policy.json
+75 -1
View File
@@ -19,5 +19,79 @@
"tests.test_connector_installation_registry",
"tests.test_connector_callback_contract",
"tests.test_s70_ssrf_pinning_regression"
]
],
"no_skip_module_metadata": {
"tests.test_s58_bridge_token_lifecycle": {
"reason": "Bridge token lifecycle coverage is a hard regression seam for token expiry and revoke behavior.",
"review_after": "2026-10-31"
},
"tests.test_s59_webhook_mapping_clamp": {
"reason": "Webhook mapping clamp must stay no-skip because route/mapping drift can silently reopen unsafe ingress paths.",
"review_after": "2026-10-31"
},
"tests.test_s60_mae_route_segmentation": {
"reason": "Public MAE route segmentation is a published hard-guarantee suite and cannot degrade to skip-only coverage.",
"review_after": "2026-10-31"
},
"tests.test_s60_routes_startup_gate": {
"reason": "Startup gate coverage must stay active to catch route registration drift before public deployment.",
"review_after": "2026-10-31"
},
"tests.test_s61_registry_signature": {
"reason": "Registry signature enforcement is part of the release-safety boundary for extension metadata.",
"review_after": "2026-10-31"
},
"tests.security.test_endpoint_drift": {
"reason": "Endpoint drift detection guards the published endpoint manifest and must remain non-skippable.",
"review_after": "2026-10-31"
},
"tests.test_r122_real_backend_lane": {
"reason": "Real backend lane is the low-mock parity seam for LLM/backend integration and cannot silently skip.",
"review_after": "2026-10-31"
},
"tests.test_r123_real_backend_model_list_lane": {
"reason": "Model-list real backend lane protects loopback SSRF and provider-model discovery parity.",
"review_after": "2026-10-31"
},
"tests.test_r121_retry_partition_contract": {
"reason": "Retry partition contract is an explicit robustness gate for failure-domain isolation.",
"review_after": "2026-10-31"
},
"tests.test_r124_slack_ingress_contract": {
"reason": "Slack ingress contract must stay enforced because connector auth and callback parsing are production paths.",
"review_after": "2026-10-31"
},
"tests.test_r125_slack_real_backend_lane": {
"reason": "Slack real-backend lane protects connector-to-backend parity across approval and run flows.",
"review_after": "2026-10-31"
},
"tests.test_r117_observability_redaction_e2e": {
"reason": "Observability redaction E2E covers production-facing data minimization and cannot be treated as optional.",
"review_after": "2026-10-31"
},
"tests.test_r117_observability_redaction_endpoints": {
"reason": "Endpoint-level redaction assertions are part of the audit/privacy boundary and must remain live.",
"review_after": "2026-10-31"
},
"tests.test_f57_slack_transport_parity": {
"reason": "Slack transport parity catches protocol drift between connector routing and HTTP contract handling.",
"review_after": "2026-10-31"
},
"tests.test_f57_slack_socket_mode_startup": {
"reason": "Slack socket mode startup coverage protects bootstrap compatibility for a supported deployment mode.",
"review_after": "2026-10-31"
},
"tests.test_connector_installation_registry": {
"reason": "Connector installation registry governs multi-platform connector ownership and must stay no-skip.",
"review_after": "2026-10-31"
},
"tests.test_connector_callback_contract": {
"reason": "Connector callback contract protects external ingress normalization and approval wiring.",
"review_after": "2026-10-31"
},
"tests.test_s70_ssrf_pinning_regression": {
"reason": "SSRF pinning regression is a security-critical boundary test and must never degrade to skip coverage.",
"review_after": "2026-10-31"
}
}
}
tests/test_r156_quality_governance.py
+54 -257
View File
@@ -2,63 +2,18 @@ import json
import subprocess
import sys
import tempfile
import textwrap
import unittest
from pathlib import Path
from tests.quality_governance_test_utils import (
sample_policy_payload,
write_governance_baseline_fixture,
)
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "verify_quality_governance.py"
def _sample_policy_json():
return (
json.dumps(
{
"schema_version": 1,
"current_stage": "baseline-35",
"stages": [
{
"id": "baseline-35",
"min_fail_under": 35.0,
"promotion_requires": [
"coverage summary reviewed",
"no unresolved hotspot exceptions",
],
"rollback_triggers": [
"coverage regression",
"critical hotspot slip",
],
},
{
"id": "ratchet-45",
"min_fail_under": 45.0,
"promotion_requires": ["two consecutive clean reviews"],
"rollback_triggers": ["new unresolved exceptions"],
},
],
"required_hotspot_families": [
"safe_io",
"security_boundary",
"connector_config",
"config_bootstrap",
],
"hotspot_families": [
{"id": "safe_io", "paths": ["services/safe_io.py"]},
{"id": "security_boundary", "paths": ["services/security_gate.py"]},
{"id": "connector_config", "paths": ["connector/config.py"]},
{
"id": "config_bootstrap",
"paths": ["config.py", "services/runtime_config.py"],
},
],
"exceptions": [],
},
indent=2,
)
+ "\n"
)
class TestR156QualityGovernance(unittest.TestCase):
def _run_script(self, *args):
return subprocess.run(
@@ -77,52 +32,19 @@ class TestR156QualityGovernance(unittest.TestCase):
def test_missing_coverage_policy_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pyproject = tmp / "pyproject.toml"
pyproject.write_text(
textwrap.dedent(
"""
[tool.coverage.report]
fail_under = 35.0
show_missing = true
skip_covered = true
"""
).strip()
+ "\n",
encoding="utf-8",
)
gate = tmp / "run_adversarial_gate.py"
gate.write_text(
"SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n",
encoding="utf-8",
)
sop = tmp / "TEST_SOP.md"
sop.write_text(
textwrap.dedent(
"""
R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
global score threshold (`>= 80%` unless explicitly overridden)
coverage governance check (`scripts/verify_quality_governance.py`)
staged coverage ratchet policy (`tests/coverage_governance_policy.json`)
"""
).strip()
+ "\n",
encoding="utf-8",
)
allowlist = tmp / "mutation_survivor_allowlist.json"
allowlist.write_text('{"entries":[]}\n', encoding="utf-8")
fixture = write_governance_baseline_fixture(tmp)
result = self._run_script(
"--pyproject",
str(pyproject),
str(fixture["pyproject"]),
"--adversarial-gate",
str(gate),
str(fixture["adversarial_gate"]),
"--test-sop",
str(sop),
str(fixture["test_sop"]),
"--mutation-survivor-allowlist",
str(allowlist),
str(fixture["survivor_allowlist"]),
"--release-policy-doc",
str(fixture["release_policy_doc"]),
"--coverage-policy",
str(tmp / "missing_policy.json"),
)
@@ -132,76 +54,33 @@ class TestR156QualityGovernance(unittest.TestCase):
def test_non_monotonic_policy_thresholds_are_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pyproject = tmp / "pyproject.toml"
pyproject.write_text(
textwrap.dedent(
"""
[tool.coverage.report]
fail_under = 35.0
show_missing = true
skip_covered = true
"""
).strip()
+ "\n",
encoding="utf-8",
)
gate = tmp / "run_adversarial_gate.py"
gate.write_text(
"SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n",
encoding="utf-8",
)
sop = tmp / "TEST_SOP.md"
sop.write_text(
textwrap.dedent(
"""
R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
global score threshold (`>= 80%` unless explicitly overridden)
coverage governance check (`scripts/verify_quality_governance.py`)
staged coverage ratchet policy (`tests/coverage_governance_policy.json`)
"""
).strip()
+ "\n",
encoding="utf-8",
)
allowlist = tmp / "mutation_survivor_allowlist.json"
allowlist.write_text('{"entries":[]}\n', encoding="utf-8")
policy = tmp / "coverage_governance_policy.json"
policy.write_text(
json.dumps(
{
"schema_version": 1,
"current_stage": "baseline-35",
"stages": [
{"id": "baseline-35", "min_fail_under": 35.0},
{"id": "ratchet-30", "min_fail_under": 30.0},
],
"required_hotspot_families": ["safe_io"],
"hotspot_families": [
{"id": "safe_io", "paths": ["services/safe_io.py"]}
],
"exceptions": [],
},
indent=2,
)
+ "\n",
encoding="utf-8",
fixture = write_governance_baseline_fixture(
tmp,
coverage_policy_payload=sample_policy_payload(
stages=[
{"id": "baseline-35", "min_fail_under": 35.0},
{"id": "ratchet-30", "min_fail_under": 30.0},
],
required_hotspot_families=["safe_io"],
hotspot_families=[
{"id": "safe_io", "paths": ["services/safe_io.py"]}
],
),
)
result = self._run_script(
"--pyproject",
str(pyproject),
str(fixture["pyproject"]),
"--adversarial-gate",
str(gate),
str(fixture["adversarial_gate"]),
"--test-sop",
str(sop),
str(fixture["test_sop"]),
"--mutation-survivor-allowlist",
str(allowlist),
str(fixture["survivor_allowlist"]),
"--release-policy-doc",
str(fixture["release_policy_doc"]),
"--coverage-policy",
str(policy),
str(fixture["coverage_policy"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("coverage stages must increase strictly", result.stdout)
@@ -209,76 +88,29 @@ class TestR156QualityGovernance(unittest.TestCase):
def test_missing_required_hotspot_family_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pyproject = tmp / "pyproject.toml"
pyproject.write_text(
textwrap.dedent(
"""
[tool.coverage.report]
fail_under = 35.0
show_missing = true
skip_covered = true
"""
).strip()
+ "\n",
encoding="utf-8",
)
gate = tmp / "run_adversarial_gate.py"
gate.write_text(
"SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n",
encoding="utf-8",
)
sop = tmp / "TEST_SOP.md"
sop.write_text(
textwrap.dedent(
"""
R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
global score threshold (`>= 80%` unless explicitly overridden)
coverage governance check (`scripts/verify_quality_governance.py`)
staged coverage ratchet policy (`tests/coverage_governance_policy.json`)
"""
).strip()
+ "\n",
encoding="utf-8",
)
allowlist = tmp / "mutation_survivor_allowlist.json"
allowlist.write_text('{"entries":[]}\n', encoding="utf-8")
policy = tmp / "coverage_governance_policy.json"
policy.write_text(
json.dumps(
{
"schema_version": 1,
"current_stage": "baseline-35",
"stages": [
{"id": "baseline-35", "min_fail_under": 35.0},
{"id": "ratchet-45", "min_fail_under": 45.0},
],
"required_hotspot_families": ["safe_io", "connector_config"],
"hotspot_families": [
{"id": "safe_io", "paths": ["services/safe_io.py"]}
],
"exceptions": [],
},
indent=2,
)
+ "\n",
encoding="utf-8",
fixture = write_governance_baseline_fixture(
tmp,
coverage_policy_payload=sample_policy_payload(
required_hotspot_families=["safe_io", "connector_config"],
hotspot_families=[
{"id": "safe_io", "paths": ["services/safe_io.py"]}
],
),
)
result = self._run_script(
"--pyproject",
str(pyproject),
str(fixture["pyproject"]),
"--adversarial-gate",
str(gate),
str(fixture["adversarial_gate"]),
"--test-sop",
str(sop),
str(fixture["test_sop"]),
"--mutation-survivor-allowlist",
str(allowlist),
str(fixture["survivor_allowlist"]),
"--release-policy-doc",
str(fixture["release_policy_doc"]),
"--coverage-policy",
str(policy),
str(fixture["coverage_policy"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("missing required hotspot families", result.stdout)
@@ -286,56 +118,21 @@ class TestR156QualityGovernance(unittest.TestCase):
def test_missing_fail_under_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pyproject = tmp / "pyproject.toml"
pyproject.write_text(
textwrap.dedent(
"""
[tool.coverage.report]
show_missing = true
skip_covered = true
"""
).strip()
+ "\n",
encoding="utf-8",
)
gate = tmp / "run_adversarial_gate.py"
gate.write_text(
"SMOKE_MUTATION_THRESHOLD = 20.0\nEXTENDED_MUTATION_THRESHOLD = 80.0\n",
encoding="utf-8",
)
sop = tmp / "TEST_SOP.md"
sop.write_text(
textwrap.dedent(
"""
R118 adversarial adaptive gate (`scripts/run_adversarial_gate.py --profile auto --seed 42`)
global score threshold (`>= 80%` unless explicitly overridden)
coverage governance check (`scripts/verify_quality_governance.py`)
staged coverage ratchet policy (`tests/coverage_governance_policy.json`)
"""
).strip()
+ "\n",
encoding="utf-8",
)
allowlist = tmp / "mutation_survivor_allowlist.json"
allowlist.write_text('{"entries":[]}\n', encoding="utf-8")
policy = tmp / "coverage_governance_policy.json"
policy.write_text(_sample_policy_json(), encoding="utf-8")
fixture = write_governance_baseline_fixture(tmp, fail_under=None)
result = self._run_script(
"--pyproject",
str(pyproject),
str(fixture["pyproject"]),
"--adversarial-gate",
str(gate),
str(fixture["adversarial_gate"]),
"--test-sop",
str(sop),
str(fixture["test_sop"]),
"--mutation-survivor-allowlist",
str(allowlist),
str(fixture["survivor_allowlist"]),
"--release-policy-doc",
str(fixture["release_policy_doc"]),
"--coverage-policy",
str(policy),
str(fixture["coverage_policy"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("missing coverage fail_under", result.stdout)
tests/test_r171_test_debt_governance.py
+247
View File
@@ -0,0 +1,247 @@
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "verify_test_debt_governance.py"
def _write_repo_fixture(tmp: Path) -> dict[str, Path]:
tests_dir = tmp / "tests"
services_dir = tmp / "services"
tests_dir.mkdir()
services_dir.mkdir()
guarded_test = tests_dir / "test_guarded_lane.py"
guarded_test.write_text("import unittest\n", encoding="utf-8")
access_control = services_dir / "access_control.py"
access_control.write_text("TOKEN = 'ok'\n", encoding="utf-8")
skip_policy = tests_dir / "skip_policy.json"
skip_policy.write_text(
json.dumps(
{
"max_skipped": 1,
"no_skip_modules": ["tests.test_guarded_lane"],
"no_skip_module_metadata": {
"tests.test_guarded_lane": {
"reason": "Guarded lane must stay no-skip in CI parity.",
"review_after": "2026-10-31",
}
},
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
mutation_allowlist = tests_dir / "mutation_survivor_allowlist.json"
mutation_allowlist.write_text(
json.dumps(
{
"version": 1,
"entries": [
{
"file": "services/access_control.py",
"mutation_index": 9,
"reason": "Equivalent compare_digest branch with empty token.",
"review_after": "2026-10-31",
}
],
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
return {
"repo_root": tmp,
"skip_policy": skip_policy,
"mutation_allowlist": mutation_allowlist,
}
class TestR171TestDebtGovernance(unittest.TestCase):
def _run_script(self, *args: str):
return subprocess.run(
[sys.executable, str(SCRIPT), *args],
capture_output=True,
text=True,
cwd=str(ROOT),
check=False,
)
def test_repo_governance_baseline_passes(self):
result = self._run_script()
self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
self.assertIn("TEST-DEBT-GOVERNANCE-PASS", result.stdout)
def test_duplicate_no_skip_modules_are_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
fixture = _write_repo_fixture(Path(tmpdir))
fixture["skip_policy"].write_text(
json.dumps(
{
"max_skipped": 1,
"no_skip_modules": [
"tests.test_guarded_lane",
"tests.test_guarded_lane",
],
"no_skip_module_metadata": {
"tests.test_guarded_lane": {
"reason": "Guarded lane must stay no-skip in CI parity.",
"review_after": "2026-10-31",
}
},
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
result = self._run_script(
"--repo-root",
str(fixture["repo_root"]),
"--skip-policy",
str(fixture["skip_policy"]),
"--mutation-survivor-allowlist",
str(fixture["mutation_allowlist"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("duplicate no-skip modules", result.stdout)
def test_missing_no_skip_module_metadata_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
fixture = _write_repo_fixture(Path(tmpdir))
fixture["skip_policy"].write_text(
json.dumps(
{
"max_skipped": 1,
"no_skip_modules": ["tests.test_guarded_lane"],
"no_skip_module_metadata": {},
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
result = self._run_script(
"--repo-root",
str(fixture["repo_root"]),
"--skip-policy",
str(fixture["skip_policy"]),
"--mutation-survivor-allowlist",
str(fixture["mutation_allowlist"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("missing metadata for no-skip modules", result.stdout)
def test_duplicate_mutation_allowlist_entries_are_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
fixture = _write_repo_fixture(Path(tmpdir))
fixture["mutation_allowlist"].write_text(
json.dumps(
{
"version": 1,
"entries": [
{
"file": "services/access_control.py",
"mutation_index": 9,
"reason": "Equivalent compare_digest branch with empty token.",
"review_after": "2026-10-31",
},
{
"file": "services/access_control.py",
"mutation_index": 9,
"reason": "Duplicate entry for regression coverage.",
"review_after": "2026-10-31",
},
],
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
result = self._run_script(
"--repo-root",
str(fixture["repo_root"]),
"--skip-policy",
str(fixture["skip_policy"]),
"--mutation-survivor-allowlist",
str(fixture["mutation_allowlist"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("duplicate (file, mutation_index) entries", result.stdout)
def test_stale_mutation_allowlist_file_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
fixture = _write_repo_fixture(Path(tmpdir))
fixture["mutation_allowlist"].write_text(
json.dumps(
{
"version": 1,
"entries": [
{
"file": "services/missing.py",
"mutation_index": 9,
"reason": "Stale path should fail closed.",
"review_after": "2026-10-31",
}
],
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
result = self._run_script(
"--repo-root",
str(fixture["repo_root"]),
"--skip-policy",
str(fixture["skip_policy"]),
"--mutation-survivor-allowlist",
str(fixture["mutation_allowlist"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("file does not exist in repo", result.stdout)
def test_past_review_after_is_rejected(self):
with tempfile.TemporaryDirectory() as tmpdir:
fixture = _write_repo_fixture(Path(tmpdir))
fixture["mutation_allowlist"].write_text(
json.dumps(
{
"version": 1,
"entries": [
{
"file": "services/access_control.py",
"mutation_index": 9,
"reason": "Expired reviews must not linger.",
"review_after": "2025-01-01",
}
],
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
result = self._run_script(
"--repo-root",
str(fixture["repo_root"]),
"--skip-policy",
str(fixture["skip_policy"]),
"--mutation-survivor-allowlist",
str(fixture["mutation_allowlist"]),
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("review_after 2025-01-01 is in the past", result.stdout)
if __name__ == "__main__":
unittest.main()
tests/test_r174_quality_governance_report.py
+32 -40
View File
@@ -5,6 +5,8 @@ import tempfile
import unittest
from pathlib import Path
from tests.quality_governance_test_utils import sample_policy_payload
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "report_coverage_governance.py"
@@ -27,27 +29,20 @@ class TestR174QualityGovernanceReport(unittest.TestCase):
policy.write_text(
json.dumps(
{
"schema_version": 1,
"current_stage": "baseline-35",
"stages": [
{"id": "baseline-35", "min_fail_under": 35.0},
{"id": "ratchet-45", "min_fail_under": 45.0},
],
"required_hotspot_families": [
"safe_io",
"security_boundary",
"connector_config",
"config_bootstrap",
],
"hotspot_families": [
sample_policy_payload(
hotspot_families=[
{"id": "safe_io", "paths": ["services/safe_io.py"]},
{"id": "connector_config", "paths": ["connector/config.py"]},
{"id": "security_boundary", "paths": ["services/security_gate.py"]},
{
"id": "connector_config",
"paths": ["connector/config.py"],
},
{
"id": "security_boundary",
"paths": ["services/security_gate.py"],
},
{"id": "config_bootstrap", "paths": ["config.py"]},
],
"exceptions": [],
},
]
),
indent=2,
)
+ "\n",
@@ -103,8 +98,12 @@ class TestR174QualityGovernanceReport(unittest.TestCase):
self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
payload = json.loads(result.stdout)
self.assertEqual(payload["overall"]["percent_covered"], 64.35)
self.assertEqual(payload["hotspot_families"]["safe_io"]["percent_covered"], 80.0)
self.assertEqual(payload["hotspot_families"]["connector_config"]["percent_covered"], 60.0)
self.assertEqual(
payload["hotspot_families"]["safe_io"]["percent_covered"], 80.0
)
self.assertEqual(
payload["hotspot_families"]["connector_config"]["percent_covered"], 60.0
)
def test_missing_hotspot_files_are_reported_deterministically(self):
with tempfile.TemporaryDirectory() as tmpdir:
@@ -114,27 +113,20 @@ class TestR174QualityGovernanceReport(unittest.TestCase):
policy.write_text(
json.dumps(
{
"schema_version": 1,
"current_stage": "baseline-35",
"stages": [
{"id": "baseline-35", "min_fail_under": 35.0},
{"id": "ratchet-45", "min_fail_under": 45.0},
],
"required_hotspot_families": [
"safe_io",
"security_boundary",
"connector_config",
"config_bootstrap",
],
"hotspot_families": [
sample_policy_payload(
hotspot_families=[
{"id": "safe_io", "paths": ["services/safe_io.py"]},
{"id": "security_boundary", "paths": ["services/security_gate.py"]},
{"id": "connector_config", "paths": ["connector/config.py"]},
{
"id": "security_boundary",
"paths": ["services/security_gate.py"],
},
{
"id": "connector_config",
"paths": ["connector/config.py"],
},
{"id": "config_bootstrap", "paths": ["config.py"]},
],
"exceptions": [],
},
]
),
indent=2,
)
+ "\n",