test(sop): enforce project .venv in full test runners with auto-bootstrap for pre-commit and aiohttp

.gitattributes

@@ -1,6 +1,9 @@
 # Normalize line endings for cross-platform consistency (pre-commit safe on Windows).
 * text=auto eol=lf
 
+# POSIX shell scripts must stay LF.
+*.sh text eol=lf
+
 # Windows scripts should remain CRLF.
 *.bat text eol=crlf
 *.cmd text eol=crlf

scripts/pre_push_checks.sh

@@ -40,35 +40,92 @@ require_cmd() {
   fi
 }
 
-ensure_single_pre_commit_source() {
-  # CRITICAL (Windows): mixed pre-commit installations (e.g. Roaming Python + conda)
-  # can spawn competing process trees and repeatedly lock hook executables.
-  # Fail fast here so users fix PATH/source before running expensive checks.
-  local paths=()
-  local line
-  while IFS= read -r line; do
-    paths+=("$line")
-  done < <(type -a pre-commit 2>/dev/null | sed -n 's/^pre-commit is //p' | awk '!seen[$0]++')
-
-  [ "${#paths[@]}" -eq 0 ] && return 0
-
+resolve_venv_python() {
   case "$UNAME_S" in
     MINGW*|MSYS*|CYGWIN*)
-      if [ "${#paths[@]}" -gt 1 ]; then
-        echo "[pre-push] ERROR: multiple pre-commit executables detected on PATH." >&2
-        for line in "${paths[@]}"; do
-          echo "[pre-push]   - $line" >&2
-        done
-        echo "[pre-push] Fix: keep a single source (recommended: conda env), uninstall the others, then retry." >&2
-        echo "[pre-push] Example: py -3.12 -m pip uninstall pre-commit" >&2
-        exit 1
-      fi
+      echo "$ROOT_DIR/.venv/Scripts/python.exe"
+      ;;
+    *)
+      echo "$ROOT_DIR/.venv/bin/python"
       ;;
   esac
 }
 
-require_cmd pre-commit
-ensure_single_pre_commit_source
+is_venv_python_healthy() {
+  local venv_py="$1"
+  # CRITICAL: on Git Bash/Windows, `test -x` is unreliable for `.exe`.
+  # Use existence + actual interpreter execution probe instead.
+  [ -f "$venv_py" ] || return 1
+  "$venv_py" -c "import sys; print(sys.executable)" >/dev/null 2>&1
+}
+
+bootstrap_venv() {
+  local venv_py
+  venv_py="$(resolve_venv_python)"
+  if is_venv_python_healthy "$venv_py"; then
+    echo "$venv_py"
+    return 0
+  fi
+
+  if [ -e "$venv_py" ]; then
+    echo "[pre-push] WARN: existing .venv is invalid; recreating with a Windows-native Python." >&2
+    rm -rf "$ROOT_DIR/.venv"
+  fi
+
+  echo "[pre-push] INFO: creating project .venv ..." >&2
+  case "$UNAME_S" in
+    MINGW*|MSYS*|CYGWIN*)
+      # CRITICAL: on Git Bash, `python3` may resolve to MSYS `/usr/bin/python`,
+      # which creates a broken Windows venv (`No Python at "/usr/bin\python.exe"`).
+      # Always prefer Windows-native launchers/interpreters.
+      if command -v py.exe >/dev/null 2>&1; then
+        py.exe -3 -m venv "$ROOT_DIR/.venv"
+      elif [ -x "/c/Windows/py.exe" ]; then
+        /c/Windows/py.exe -3 -m venv "$ROOT_DIR/.venv"
+      elif command -v python.exe >/dev/null 2>&1; then
+        python.exe -m venv "$ROOT_DIR/.venv"
+      elif command -v py >/dev/null 2>&1; then
+        py -3 -m venv "$ROOT_DIR/.venv"
+      else
+        echo "[pre-push] ERROR: no Windows Python launcher found (py.exe/python.exe)." >&2
+        exit 1
+      fi
+      ;;
+    *)
+      if command -v python3 >/dev/null 2>&1; then
+        python3 -m venv "$ROOT_DIR/.venv"
+      elif command -v python >/dev/null 2>&1; then
+        python -m venv "$ROOT_DIR/.venv"
+      else
+        echo "[pre-push] ERROR: no bootstrap Python found (python3/python)." >&2
+        exit 1
+      fi
+      ;;
+  esac
+
+  if ! is_venv_python_healthy "$venv_py"; then
+    echo "[pre-push] ERROR: failed to initialize project .venv." >&2
+    exit 1
+  fi
+  echo "$venv_py"
+}
+
+pre_commit_cmd() {
+  "$VENV_PY" -m pre_commit "$@"
+}
+
+# CRITICAL: pre-push must always run pre-commit from project .venv.
+# Do not switch this back to global `pre-commit` command lookup.
+VENV_PY="$(bootstrap_venv)"
+if ! "$VENV_PY" -m pre_commit --version >/dev/null 2>&1; then
+  echo "[pre-push] INFO: installing pre-commit into project .venv ..." >&2
+  "$VENV_PY" -m pip install -U pip pre-commit
+fi
+if ! "$VENV_PY" -c "import black" >/dev/null 2>&1; then
+  # Keep black in the same interpreter used by local black-single hook.
+  echo "[pre-push] INFO: installing black into project .venv ..." >&2
+  "$VENV_PY" -m pip install black==24.1.1
+fi
 require_cmd npm
 
 run_pre_commit_safe() {
@@ -93,7 +150,7 @@ run_pre_commit_safe() {
     fi
   }
 
-  if pre-commit "$@" 2>&1 | tee "$tmp_log"; then
+  if pre_commit_cmd "$@" 2>&1 | tee "$tmp_log"; then
     rm -f "$tmp_log"
     rm -f "$lower_log"
     return 0
@@ -103,12 +160,12 @@ run_pre_commit_safe() {
 
   if grep -q "invalidmanifesterror" "$lower_log"; then
     echo "[pre-push] WARN: pre-commit cache manifest is corrupted; running clean + cache reset + single retry." >&2
-    if ! pre-commit clean; then
+    if ! pre_commit_cmd clean; then
       echo "[pre-push] WARN: 'pre-commit clean' failed; trying manual cache reset." >&2
       reset_cache
     fi
     reset_cache
-    pre-commit "$@"
+    pre_commit_cmd "$@"
     rm -f "$tmp_log"
     rm -f "$lower_log"
     return 0
@@ -123,7 +180,7 @@ run_pre_commit_safe() {
       rm -rf "$BLACK_CACHE_DIR" 2>/dev/null || true
       mkdir -p "$BLACK_CACHE_DIR"
     fi
-    pre-commit "$@"
+    pre_commit_cmd "$@"
     rm -f "$tmp_log"
     rm -f "$lower_log"
     return 0

scripts/precommit_black_single.sh

@@ -6,16 +6,42 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 export BLACK_CACHE_DIR="${BLACK_CACHE_DIR:-$ROOT_DIR/.tmp/black-cache}"
 mkdir -p "$BLACK_CACHE_DIR"
 
-if command -v python >/dev/null 2>&1; then
-  # CRITICAL cross-platform guard:
-  # WSL sometimes has only `python3`, while Windows shells usually expose `python`.
-  # Keep this fallback chain to avoid false hook failures like "Executable `python` not found".
-  exec python -B scripts/precommit_black_single.py "$@"
+# CRITICAL: Always prefer project-local .venv interpreter for Black.
+# Without this, Windows can accidentally pick global Python (e.g. C:\Program Files\Python312)
+# where `black` is not installed, causing flaky pre-commit failures.
+can_use_python() {
+  local candidate="$1"
+  [ -f "$candidate" ] || return 1
+  "$candidate" -c "import sys; print(sys.executable)" >/dev/null 2>&1
+}
+
+has_project_venv=false
+[ -d "$ROOT_DIR/.venv" ] && has_project_venv=true
+
+if can_use_python "$ROOT_DIR/.venv/Scripts/python.exe"; then
+  PY_CMD="$ROOT_DIR/.venv/Scripts/python.exe"
+elif can_use_python "$ROOT_DIR/.venv/bin/python"; then
+  PY_CMD="$ROOT_DIR/.venv/bin/python"
+elif [ "$has_project_venv" = true ]; then
+  # CRITICAL: if .venv exists but is broken, fail fast instead of silently
+  # falling back to a random global Python (which reintroduces flakiness).
+  echo "ERROR: project .venv exists but Python is unusable. Recreate .venv and retry." >&2
+  exit 1
+elif command -v python >/dev/null 2>&1; then
+  # Fallback chain only for environments that intentionally do not use .venv.
+  PY_CMD="$(command -v python)"
+elif command -v python3 >/dev/null 2>&1; then
+  PY_CMD="$(command -v python3)"
+else
+  echo "ERROR: python interpreter not found (need python or python3 in PATH)." >&2
+  exit 127
 fi
 
-if command -v python3 >/dev/null 2>&1; then
-  exec python3 -B scripts/precommit_black_single.py "$@"
+# Ensure black exists in the selected interpreter.
+# This self-heals first-run environments and prevents recurring "No module named black".
+if ! "$PY_CMD" -c "import black" >/dev/null 2>&1; then
+  echo "[black-single] INFO: installing black==24.1.1 into selected Python env ..." >&2
+  "$PY_CMD" -m pip install black==24.1.1 >/dev/null
 fi
 
-echo "ERROR: python interpreter not found (need python or python3 in PATH)." >&2
-exit 127
+exec "$PY_CMD" -B scripts/precommit_black_single.py "$@"

scripts/run_full_tests_linux.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -o errtrace
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+trap 'echo "[tests] ERROR at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR
+
+echo "[tests] repo: $ROOT_DIR"
+
+# Cache isolation for pre-commit + black
+export PRE_COMMIT_HOME="${PRE_COMMIT_HOME:-$ROOT_DIR/.tmp/pre-commit}"
+export BLACK_CACHE_DIR="${BLACK_CACHE_DIR:-$ROOT_DIR/.tmp/black-cache}"
+mkdir -p "$PRE_COMMIT_HOME" "$BLACK_CACHE_DIR"
+
+require_cmd() {
+  local cmd="$1"
+  if ! command -v "$cmd" >/dev/null 2>&1; then
+    echo "[tests] ERROR: missing command: $cmd" >&2
+    exit 1
+  fi
+}
+
+require_cmd node
+require_cmd npm
+
+# Always use project-local venv to avoid global interpreter / tool drift.
+VENV_PY="$ROOT_DIR/.venv/bin/python"
+if [ ! -x "$VENV_PY" ]; then
+  echo "[tests] Creating project venv at $ROOT_DIR/.venv ..."
+  if command -v python3 >/dev/null 2>&1; then
+    python3 -m venv "$ROOT_DIR/.venv"
+  elif command -v python >/dev/null 2>&1; then
+    python -m venv "$ROOT_DIR/.venv"
+  else
+    echo "[tests] ERROR: no bootstrap Python found (need python3 or python)" >&2
+    exit 1
+  fi
+fi
+
+if ! "$VENV_PY" -m pre_commit --version >/dev/null 2>&1; then
+  echo "[tests] Installing pre-commit into project venv ..."
+  "$VENV_PY" -m pip install -U pip pre-commit
+fi
+
+if ! "$VENV_PY" -c "import aiohttp" >/dev/null 2>&1; then
+  echo "[tests] Installing aiohttp into project venv ..."
+  "$VENV_PY" -m pip install aiohttp
+fi
+
+NODE_MAJOR="$(node -p "process.versions.node.split('.')[0]")"
+if [ "$NODE_MAJOR" -lt 18 ]; then
+  # Best-effort: try to use nvm if available
+  if [ -n "${NVM_DIR:-}" ] && [ -s "${NVM_DIR}/nvm.sh" ]; then
+    # shellcheck disable=SC1090
+    . "${NVM_DIR}/nvm.sh"
+  elif [ -s "${HOME}/.nvm/nvm.sh" ]; then
+    # shellcheck disable=SC1091
+    . "${HOME}/.nvm/nvm.sh"
+  fi
+  if command -v nvm >/dev/null 2>&1; then
+    nvm use 18 >/dev/null 2>&1 || true
+  fi
+  NODE_MAJOR="$(node -p "process.versions.node.split('.')[0]")"
+fi
+
+if [ "$NODE_MAJOR" -lt 18 ]; then
+  echo "[tests] ERROR: Node >=18 required, current=$(node -v)" >&2
+  echo "[tests] Hint: source ~/.nvm/nvm.sh && nvm use 18" >&2
+  exit 1
+fi
+
+echo "[tests] Node version: $(node -v)"
+
+echo "[tests] 1/4 detect-secrets"
+"$VENV_PY" -m pre_commit run detect-secrets --all-files
+
+echo "[tests] 2/4 pre-commit all hooks"
+"$VENV_PY" -m pre_commit run --all-files --show-diff-on-failure
+
+echo "[tests] 3/4 backend unit tests"
+MOLTBOT_STATE_DIR="$ROOT_DIR/moltbot_state/_local_unit" "$VENV_PY" scripts/run_unittests.py --start-dir tests --pattern "test_*.py"
+
+echo "[tests] 4/4 frontend E2E"
+npm test
+
+echo "[tests] PASS"

scripts/run_full_tests_windows.ps1

@@ -0,0 +1,87 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$root = Split-Path -Parent $MyInvocation.MyCommand.Path | Split-Path -Parent
+Set-Location $root
+
+Write-Host "[tests] repo: $root"
+
+# Cache isolation for pre-commit + black
+$env:PRE_COMMIT_HOME = if ($env:PRE_COMMIT_HOME) { $env:PRE_COMMIT_HOME } else { "$root\.tmp\pre-commit-win" }
+$env:BLACK_CACHE_DIR = if ($env:BLACK_CACHE_DIR) { $env:BLACK_CACHE_DIR } else { "$root\.tmp\black-cache" }
+New-Item -ItemType Directory -Force $env:PRE_COMMIT_HOME | Out-Null
+New-Item -ItemType Directory -Force $env:BLACK_CACHE_DIR | Out-Null
+
+function Require-Cmd($cmd) {
+  if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) {
+    throw "[tests] ERROR: missing command: $cmd"
+  }
+}
+
+Require-Cmd node
+Require-Cmd npm
+
+# Prefer project-local virtualenv to avoid global PATH / cache conflicts on Windows.
+$venvPython = Join-Path $root ".venv\Scripts\python.exe"
+if (-not (Test-Path $venvPython)) {
+  Write-Host "[tests] Creating project venv at $root\.venv ..."
+  if (Get-Command py -ErrorAction SilentlyContinue) {
+    & py -3 -m venv .venv
+  } elseif (Get-Command python -ErrorAction SilentlyContinue) {
+    & python -m venv .venv
+  } else {
+    throw "[tests] ERROR: no bootstrap Python found (need py or python)"
+  }
+}
+
+$hasPreCommit = $true
+try {
+  & $venvPython -m pre_commit --version | Out-Null
+} catch {
+  $hasPreCommit = $false
+}
+if (-not $hasPreCommit) {
+  Write-Host "[tests] Installing pre-commit into project venv ..."
+  & $venvPython -m pip install -U pip pre-commit
+}
+
+$hasAiohttp = $true
+try {
+  & $venvPython -c "import aiohttp" | Out-Null
+} catch {
+  $hasAiohttp = $false
+}
+if (-not $hasAiohttp) {
+  Write-Host "[tests] Installing aiohttp into project venv ..."
+  & $venvPython -m pip install aiohttp
+}
+
+# Ensure Node >= 18
+$nodeMajor = [int]((& node -p "process.versions.node.split('.')[0]").Trim())
+if ($nodeMajor -lt 18) {
+  Write-Host "[tests] WARN: Node < 18 detected. Trying nvm use 18..."
+  if (Get-Command nvm -ErrorAction SilentlyContinue) {
+    nvm use 18 | Out-Null
+    $nodeMajor = [int]((& node -p "process.versions.node.split('.')[0]").Trim())
+  }
+}
+if ($nodeMajor -lt 18) {
+  throw "[tests] ERROR: Node >=18 required, current=$(node -v)"
+}
+
+Write-Host "[tests] Node version: $(node -v)"
+
+Write-Host "[tests] 1/4 detect-secrets"
+& $venvPython -m pre_commit run detect-secrets --all-files
+
+Write-Host "[tests] 2/4 pre-commit all hooks"
+& $venvPython -m pre_commit run --all-files --show-diff-on-failure
+
+Write-Host "[tests] 3/4 backend unit tests"
+$env:MOLTBOT_STATE_DIR = "$root\moltbot_state\_local_unit"
+& $venvPython scripts\run_unittests.py --start-dir tests --pattern "test_*.py"
+
+Write-Host "[tests] 4/4 frontend E2E"
+npm test
+
+Write-Host "[tests] PASS"

tests/TEST_SOP.md

@@ -100,8 +100,35 @@ Rules:
 - Do not run multiple pre-commit commands in parallel on Windows.
 - Do not mark tests as passed if hooks were interrupted by lock errors.
 
+### Windows PATH and Process Reality Checks
+
+Use these checks before assuming the hook runner is broken:
+
+1) `where pre-commit` can be empty in PowerShell even when module execution works.
+   - Prefer:
+     - `python -m pre_commit --version`
+     - `Get-Command pre-commit -All`
+2) If multiple Python installations exist, always run:
+   - `python -m pre_commit ...`
+   instead of relying on bare `pre-commit` resolution.
+3) If process cleanup looks inconsistent, inspect actual command lines:
+   - `Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'pre-commit|detect-secrets|black' } | Select-Object ProcessId,ParentProcessId,Name,CommandLine`
+4) `taskkill` may report "no running instance" when the PID already exited between scans.
+   - Re-run the `Get-CimInstance` query above before deciding a process is still stuck.
+
 ## Required Pre-Push Workflow (Must Run)
 
+### Optional: One-Command Full Test Scripts (Fastest)
+
+Use these if you want a single command that runs **all required steps** (detect-secrets, pre-commit, unit tests, E2E). These scripts also handle the most common environment issues (Windows cache locks, Black cache, Node 18).
+Both scripts enforce a project-local `.venv` and will bootstrap missing test tooling (`pre-commit`, and `aiohttp` where needed for imports).
+If `.venv` exists but is invalid for the current OS (for example created in WSL then reused in Windows), rerun via the script so it can recreate the environment.
+
+- Linux/WSL:
+  - `bash scripts/run_full_tests_linux.sh`
+- Windows (PowerShell):
+  - `powershell -File scripts/run_full_tests_windows.ps1`
+
 ### Optional automation (recommended)
 
 Enable the repository-managed Git pre-push hook once: