feat(startup): model bootstrap lifecycle outcomes

This commit is contained in:
rookiestar28
2026-07-31 05:40:58 +08:00
parent 5babb01a56
commit 8c175f47ab
8 changed files with 1808 additions and 293 deletions
+11 -1
View File
@@ -53,7 +53,17 @@ def _bootstrap_openclaw_routes() -> None:
from .services.route_bootstrap import register_routes_once from .services.route_bootstrap import register_routes_once
else: else:
from services.route_bootstrap import register_routes_once from services.route_bootstrap import register_routes_once
except Exception: except Exception as exc:
try:
if __package__:
from .services.startup_lifecycle import mark_bootstrap_import_failed
else:
from services.startup_lifecycle import mark_bootstrap_import_failed
mark_bootstrap_import_failed(exc)
except Exception:
# IMPORTANT: diagnostics must not mask the original compatibility fallback.
pass
return return
register_routes_once() register_routes_once()
+17 -1
View File
@@ -137,7 +137,23 @@ async def health_response(request: Any, deps: RouteHandlerDependencies) -> Any:
from services.startup_lifecycle import get_startup_diagnostics from services.startup_lifecycle import get_startup_diagnostics
startup_diagnostics = get_startup_diagnostics() startup_diagnostics = get_startup_diagnostics()
except Exception: except Exception:
startup_diagnostics = {"state": "unknown", "ready": False, "warmups": {}} # SECURITY: keep the public fallback deterministic and content-free even when
# startup diagnostics cannot be imported.
startup_diagnostics = {
"schema_version": 1,
"phase": "package_import",
"state": "fatal",
"reason_code": "bootstrap_import_failed",
"ready": False,
"degraded": False,
"fatal": True,
"attempt": 0,
"max_attempts": 0,
"elapsed_ms": 0,
"phase_elapsed_ms": 0,
"ready_elapsed_ms": None,
"warmups": [],
}
job_stats = {} job_stats = {}
try: try:
+307 -81
View File
@@ -11,8 +11,17 @@ import os
import sys import sys
import threading import threading
import time import time
from collections.abc import Callable
_routes_registered = False _routes_registered = False
_registration_condition = threading.Condition(threading.RLock())
_registration_inflight = False
_registration_started = False
_registration_error: Exception | None = None
_registration_retry_thread: threading.Thread | None = None
_registration_generation = 0
_REGISTRATION_MAX_ATTEMPTS = 10
_REGISTRATION_INITIAL_DELAY_SEC = 2.0
def _resolve_optional_warmup_timeout_sec() -> float: def _resolve_optional_warmup_timeout_sec() -> float:
@@ -41,30 +50,62 @@ def _build_optional_startup_warmups():
def _mark_startup_ready_and_start_warmups() -> None: def _mark_startup_ready_and_start_warmups() -> None:
try: from .startup_lifecycle import mark_startup_ready, start_optional_warmups
from .startup_lifecycle import mark_startup_ready, start_optional_warmups
mark_startup_ready("routes") # Required readiness is part of successful route registration and must not be
# hidden behind the optional warmup boundary.
mark_startup_ready("routes")
try:
start_optional_warmups(_build_optional_startup_warmups()) start_optional_warmups(_build_optional_startup_warmups())
except Exception: except Exception as exc:
# IMPORTANT: optional warmup diagnostics must not undo successful route startup. # IMPORTANT: optional warmup diagnostics must not undo successful route startup.
logging.getLogger("ComfyUI-OpenClaw").exception( logging.getLogger("ComfyUI-OpenClaw").error(
"R188: failed to start optional startup warmups" "Optional startup warmups could not be started (error_type=%s)",
type(exc).__name__,
) )
def _mark_startup_fatal(phase: str, exc: BaseException) -> None: def _mark_startup_fatal(
phase: str,
exc: BaseException,
*,
reason_code=None,
) -> None:
try: try:
from .startup_lifecycle import mark_startup_fatal from .startup_lifecycle import mark_startup_fatal
mark_startup_fatal(phase, exc) mark_startup_fatal(phase, exc, reason_code=reason_code)
except Exception: except Exception as diagnostics_exc:
# IMPORTANT: preserve the original bootstrap exception even if diagnostics fail. # IMPORTANT: preserve the original bootstrap exception even if diagnostics fail.
logging.getLogger("ComfyUI-OpenClaw").exception( logging.getLogger("ComfyUI-OpenClaw").error(
"R188: failed to record fatal startup state" "Startup diagnostics update failed (error_type=%s)",
type(diagnostics_exc).__name__,
) )
def _mark_required_initialization_started() -> None:
from .startup_lifecycle import mark_required_initialization_started
mark_required_initialization_started()
def _mark_host_waiting(*, attempt: int, max_attempts: int) -> None:
from .startup_lifecycle import mark_host_waiting
mark_host_waiting(attempt=attempt, max_attempts=max_attempts)
def _mark_route_registration_started(
*, attempt: int = 0, max_attempts: int = 0
) -> None:
from .startup_lifecycle import mark_route_registration_started
mark_route_registration_started(
attempt=attempt,
max_attempts=max_attempts,
)
def _load_plugin_shutdown_registrars(): def _load_plugin_shutdown_registrars():
"""Load optional startup registrars behind one patchable compatibility seam.""" """Load optional startup registrars behind one patchable compatibility seam."""
@@ -125,8 +166,7 @@ def _initialize_registries_and_security_gate() -> None:
# CRITICAL: pass db_path as keyword (first positional arg is backend object). # CRITICAL: pass db_path as keyword (first positional arg is backend object).
IdempotencyStore().configure_durable(db_path=db_path, strict_mode=True) IdempotencyStore().configure_durable(db_path=db_path, strict_mode=True)
logging.getLogger("ComfyUI-OpenClaw").info( logging.getLogger("ComfyUI-OpenClaw").info(
"IdempotencyStore durable backend configured at: %s (strict_mode=True)", "IdempotencyStore durable backend configured (strict_mode=True)"
db_path,
) )
if config.bridge_enabled: if config.bridge_enabled:
@@ -145,9 +185,10 @@ def _initialize_registries_and_security_gate() -> None:
from .security_gate import enforce_startup_gate from .security_gate import enforce_startup_gate
enforce_startup_gate() enforce_startup_gate()
except Exception as e: except Exception as exc:
logging.getLogger("ComfyUI-OpenClaw").error( logging.getLogger("ComfyUI-OpenClaw").error(
f"Failed to initialize registries: {e}" "Required registry initialization failed (error_type=%s)",
type(exc).__name__,
) )
# CRITICAL: keep bootstrap fail-closed; swallowing startup gate errors # CRITICAL: keep bootstrap fail-closed; swallowing startup gate errors
# silently degrades security posture and can expose partial registration. # silently degrades security posture and can expose partial registration.
@@ -249,7 +290,6 @@ def _do_full_registration(server) -> None:
require_admin_token_fn=require_admin_token, require_admin_token_fn=require_admin_token,
submit_fn=unified_submit_fn, submit_fn=unified_submit_fn,
) )
_mark_startup_ready_and_start_warmups()
_BRIDGE_ROUTE_SPECS = ( _BRIDGE_ROUTE_SPECS = (
@@ -284,85 +324,271 @@ def _register_bridge_routes(router, bridge_handlers) -> None:
raise raise
def _start_registration_retry_loop() -> None: def _resolve_prompt_server():
"""R25: Retry route registration while PromptServer is warming up.""" ps_mod = sys.modules.get("server")
prompt_server = getattr(ps_mod, "PromptServer", None) if ps_mod else None
return getattr(prompt_server, "instance", None) if prompt_server else None
def _retry_worker():
global _routes_registered
attempts = 0
max_attempts = 10
delay = 2.0
logger = logging.getLogger("ComfyUI-OpenClaw")
while not _routes_registered and attempts < max_attempts: def reset_route_bootstrap_for_tests() -> None:
"""Invalidate background ownership and reset the route bootstrap seam."""
global _routes_registered
global _registration_error
global _registration_generation
global _registration_inflight
global _registration_retry_thread
global _registration_started
with _registration_condition:
_registration_generation += 1
_routes_registered = False
_registration_inflight = False
_registration_started = False
_registration_error = None
_registration_retry_thread = None
_registration_condition.notify_all()
def _store_registration_success(*, generation: int | None = None) -> bool:
global _routes_registered
global _registration_error
global _registration_inflight
global _registration_started
with _registration_condition:
if generation is not None and generation != _registration_generation:
return False
_routes_registered = True
_registration_inflight = False
_registration_started = True
_registration_error = None
_registration_condition.notify_all()
return True
def _store_registration_failure(
exc: Exception,
*,
generation: int | None = None,
) -> bool:
global _registration_error
global _registration_inflight
global _registration_started
with _registration_condition:
if generation is not None and generation != _registration_generation:
return False
_registration_inflight = False
_registration_started = True
_registration_error = exc
_registration_condition.notify_all()
return True
def _run_registration_retry_loop(
*,
max_attempts: int = _REGISTRATION_MAX_ATTEMPTS,
initial_delay: float = _REGISTRATION_INITIAL_DELAY_SEC,
sleep_fn: Callable[[float], None] = time.sleep,
generation: int | None = None,
) -> None:
"""Run the sole bounded host-wait owner with explicit terminal outcomes."""
global _registration_retry_thread
logger = logging.getLogger("ComfyUI-OpenClaw")
with _registration_condition:
owner_generation = (
_registration_generation if generation is None else generation
)
delay = max(0.0, float(initial_delay))
try:
for attempt in range(1, max_attempts + 1):
with _registration_condition:
if owner_generation != _registration_generation:
return
try: try:
ps_mod = sys.modules.get("server") server = _resolve_prompt_server()
PromptServer = getattr(ps_mod, "PromptServer", None) if ps_mod else None except Exception as exc:
if PromptServer and getattr(PromptServer, "instance", None) is not None: _mark_startup_fatal("route_registration", exc)
_do_full_registration(PromptServer.instance) _store_registration_failure(exc, generation=owner_generation)
_routes_registered = True logger.error(
logger.info( "PromptServer resolution failed (attempt=%s, error_type=%s)",
"Routes registered successfully on attempt %s", attempts + 1 attempt,
type(exc).__name__,
)
return
if server is not None:
try:
_mark_route_registration_started(
attempt=attempt,
max_attempts=max_attempts,
)
_do_full_registration(server)
_mark_startup_ready_and_start_warmups()
except Exception as exc:
_mark_startup_fatal("route_registration", exc)
_store_registration_failure(exc, generation=owner_generation)
logger.error(
"Route registration failed " "(attempt=%s, error_type=%s)",
attempt,
type(exc).__name__,
) )
return return
logger.debug( _store_registration_success(generation=owner_generation)
"PromptServer.instance not ready (attempt %s)", attempts + 1 logger.info(
"Routes registered successfully (attempt=%s)",
attempt,
) )
except Exception: return
logger.exception("Error registering routes (attempt %s)", attempts + 1)
time.sleep(delay) _mark_host_waiting(attempt=attempt, max_attempts=max_attempts)
delay = min(delay * 1.5, 30) logger.debug("PromptServer not ready (attempt=%s)", attempt)
attempts += 1 if attempt < max_attempts:
sleep_fn(delay)
delay = min(delay * 1.5, 30.0)
if not _routes_registered: failure = RuntimeError("route registration retry exhausted")
_mark_startup_fatal( _mark_startup_fatal(
"route_registration_retry", "host_wait",
RuntimeError( failure,
f"Failed to register routes after {max_attempts} attempts" reason_code="retry_exhausted",
), )
) _store_registration_failure(failure, generation=owner_generation)
logger.error( logger.error(
"Failed to register routes after %s attempts. API endpoints unavailable.", "Route registration retry exhausted (attempts=%s)",
max_attempts, max_attempts,
) )
finally:
with _registration_condition:
if owner_generation == _registration_generation:
current = threading.current_thread()
if _registration_retry_thread is current:
_registration_retry_thread = None
_registration_condition.notify_all()
t = threading.Thread(target=_retry_worker, name="openclaw-route-retry", daemon=True)
t.start() def _start_registration_retry_loop() -> None:
"""Start at most one background host-wait owner."""
global _registration_retry_thread
with _registration_condition:
existing = _registration_retry_thread
if existing is not None and existing.is_alive():
return
generation = _registration_generation
thread = threading.Thread(
target=_run_registration_retry_loop,
kwargs={"generation": generation},
name="openclaw-route-retry",
daemon=True,
)
_registration_retry_thread = thread
thread.start()
def register_routes_once() -> None: def register_routes_once() -> None:
global _routes_registered """Initialize and register routes through one process-wide bootstrap owner."""
if _routes_registered:
return global _registration_inflight
global _registration_started
logger = logging.getLogger("ComfyUI-OpenClaw")
# CRITICAL: one condition owns initialization, registration, retry creation, and
# terminal error replay. Independent flags reintroduce duplicate side effects.
with _registration_condition:
if _routes_registered:
return
if _registration_error is not None:
raise _registration_error
if _registration_started:
while _registration_inflight:
_registration_condition.wait()
if _registration_error is not None:
raise _registration_error
return
_registration_started = True
_registration_inflight = True
generation = _registration_generation
try: try:
_register_plugins_and_shutdown_hooks() _mark_required_initialization_started()
_initialize_registries_and_security_gate() try:
except Exception as exc: _register_plugins_and_shutdown_hooks()
_mark_startup_fatal("required_startup", exc) _initialize_registries_and_security_gate()
raise except Exception as exc:
_mark_startup_fatal("required_initialization", exc)
try: _store_registration_failure(exc, generation=generation)
ps_mod = sys.modules.get("server") logger.error(
PromptServer = getattr(ps_mod, "PromptServer", None) if ps_mod else None "Required startup initialization failed (error_type=%s)",
type(exc).__name__,
if PromptServer and getattr(PromptServer, "instance", None) is not None:
_do_full_registration(PromptServer.instance)
_routes_registered = True
logging.getLogger("ComfyUI-OpenClaw").info(
"Routes registered successfully on initial attempt."
) )
else: raise
logging.getLogger("ComfyUI-OpenClaw").info(
"PromptServer not ready, starting background registration retry loop..." try:
) server = _resolve_prompt_server()
_start_registration_retry_loop() except Exception as exc:
except Exception:
_exc_type, exc, _tb = sys.exc_info()
if exc is not None:
_mark_startup_fatal("route_registration", exc) _mark_startup_fatal("route_registration", exc)
logging.getLogger("ComfyUI-OpenClaw").exception("Route registration failed") _store_registration_failure(exc, generation=generation)
# CRITICAL: initial registration failures must fail closed. The retry loop is logger.error(
# only for PromptServer warm-up, not for hiding broken route/bootstrap state. "Initial PromptServer resolution failed (error_type=%s)",
type(exc).__name__,
)
raise
if server is None:
_mark_host_waiting(
attempt=0,
max_attempts=_REGISTRATION_MAX_ATTEMPTS,
)
try:
_start_registration_retry_loop()
except Exception as exc:
_mark_startup_fatal(
"host_wait",
exc,
reason_code="retry_exhausted",
)
_store_registration_failure(exc, generation=generation)
logger.error(
"Route registration retry owner failed to start " "(error_type=%s)",
type(exc).__name__,
)
raise
logger.info(
"PromptServer not ready; route registration retry owner started"
)
return
_mark_route_registration_started()
try:
_do_full_registration(server)
_mark_startup_ready_and_start_warmups()
except Exception as exc:
_mark_startup_fatal("route_registration", exc)
_store_registration_failure(exc, generation=generation)
logger.error(
"Initial route registration failed (error_type=%s)",
type(exc).__name__,
)
# CRITICAL: only host availability is retryable. Broken route
# registration remains fail-closed and replays the same error.
raise
_store_registration_success(generation=generation)
logger.info("Routes registered successfully on initial attempt")
except BaseException:
with _registration_condition:
if (
generation == _registration_generation
and _registration_error is None
and not _routes_registered
):
_registration_started = False
_registration_inflight = False
_registration_condition.notify_all()
raise raise
finally:
with _registration_condition:
if generation == _registration_generation:
_registration_inflight = False
_registration_condition.notify_all()
+633 -149
View File
@@ -1,214 +1,698 @@
""" """Typed startup lifecycle outcomes and redacted public diagnostics.
Startup lifecycle diagnostics and optional warmup boundaries.
Required startup work still fails closed in callers. This module only tracks Required startup work still fails closed in callers. This module owns only the
readiness and runs optional warmups without delaying route availability. phase/result state machine and optional post-ready warmup observations; it does
not own ComfyUI's application lifecycle.
""" """
from __future__ import annotations from __future__ import annotations
import logging import logging
import math
import re
import threading import threading
import time import time
from typing import Any, Callable, Dict, Iterable, Optional from collections.abc import Callable, Iterable
from dataclasses import dataclass
from enum import Enum
from typing import Any
logger = logging.getLogger("ComfyUI-OpenClaw.services.startup_lifecycle") logger = logging.getLogger("ComfyUI-OpenClaw.services.startup_lifecycle")
STARTUP_STARTING = "starting" SCHEMA_VERSION = 1
STARTUP_READY = "ready" MAX_DIAGNOSTIC_MS = 86_400_000
STARTUP_DEGRADED_WARMUP = "degraded-warmup" MAX_WARMUPS = 16
STARTUP_FATAL = "fatal-startup" STARTUP_DIAGNOSTIC_KEYS = (
"schema_version",
"phase",
"state",
"reason_code",
"ready",
"degraded",
"fatal",
"attempt",
"max_attempts",
"elapsed_ms",
"phase_elapsed_ms",
"ready_elapsed_ms",
"warmups",
)
WARMUP_PENDING = "pending"
WARMUP_RUNNING = "running" class StartupPhase(str, Enum):
WARMUP_SUCCEEDED = "succeeded" PACKAGE_IMPORT = "package_import"
WARMUP_FAILED = "failed" REQUIRED_INITIALIZATION = "required_initialization"
WARMUP_TIMED_OUT = "timed_out" HOST_WAIT = "host_wait"
ROUTE_REGISTRATION = "route_registration"
COMPLETE = "complete"
OPTIONAL_WARMUP = "optional_warmup"
class StartupState(str, Enum):
STARTING = "starting"
INITIALIZING = "initializing"
WAITING_FOR_HOST = "waiting_for_host"
REGISTERING_ROUTES = "registering_routes"
READY = "ready"
DEGRADED = "degraded"
FATAL = "fatal"
class StartupReason(str, Enum):
BOOTSTRAP_STARTED = "bootstrap_started"
BOOTSTRAP_IMPORT_FAILED = "bootstrap_import_failed"
REQUIRED_INITIALIZATION_STARTED = "required_initialization_started"
REQUIRED_INITIALIZATION_FAILED = "required_initialization_failed"
HOST_NOT_READY = "host_not_ready"
ROUTE_REGISTRATION_STARTED = "route_registration_started"
ROUTE_REGISTRATION_SUCCEEDED = "route_registration_succeeded"
ROUTE_REGISTRATION_FAILED = "route_registration_failed"
RETRY_EXHAUSTED = "retry_exhausted"
WARMUP_STARTED = "warmup_started"
WARMUP_SUCCEEDED = "warmup_succeeded"
WARMUP_FAILED = "warmup_failed"
WARMUP_TIMED_OUT = "warmup_timed_out"
class WarmupState(str, Enum):
PENDING = "pending"
RUNNING = "running"
SUCCEEDED = "succeeded"
FAILED = "failed"
TIMED_OUT = "timed_out"
# Compatibility constants retained for existing internal imports.
STARTUP_STARTING = StartupState.STARTING.value
STARTUP_READY = StartupState.READY.value
STARTUP_DEGRADED_WARMUP = StartupState.DEGRADED.value
STARTUP_FATAL = StartupState.FATAL.value
WARMUP_PENDING = WarmupState.PENDING.value
WARMUP_RUNNING = WarmupState.RUNNING.value
WARMUP_SUCCEEDED = WarmupState.SUCCEEDED.value
WARMUP_FAILED = WarmupState.FAILED.value
WARMUP_TIMED_OUT = WarmupState.TIMED_OUT.value
WarmupSpec = tuple[str, Callable[[], Any], float] WarmupSpec = tuple[str, Callable[[], Any], float]
_LOCK = threading.RLock()
_STARTED_AT = time.time() class StartupTransitionError(RuntimeError):
_READY = False """Stable transition failure that never embeds caller/source content."""
_READY_PHASE: Optional[str] = None
_READY_AT: Optional[float] = None def __init__(self, code: str):
_FATAL: Optional[Dict[str, Any]] = None self.code = code
_WARMUPS: Dict[str, Dict[str, Any]] = {} super().__init__(code)
@dataclass(frozen=True)
class WarmupOutcome:
name: str
state: WarmupState
reason_code: StartupReason
timeout_ms: int
duration_ms: int
def to_diagnostics(self) -> dict[str, Any]:
return {
"name": self.name,
"state": self.state.value,
"reason_code": self.reason_code.value,
"timeout_ms": self.timeout_ms,
"duration_ms": self.duration_ms,
}
@dataclass(frozen=True)
class StartupOutcome:
schema_version: int
phase: StartupPhase
state: StartupState
reason_code: StartupReason
ready: bool
degraded: bool
fatal: bool
attempt: int
max_attempts: int
elapsed_ms: int
phase_elapsed_ms: int
ready_elapsed_ms: int | None
warmups: tuple[WarmupOutcome, ...]
def to_diagnostics(self) -> dict[str, Any]:
"""Return a fresh, ordered, JSON-safe public projection."""
return {
"schema_version": self.schema_version,
"phase": self.phase.value,
"state": self.state.value,
"reason_code": self.reason_code.value,
"ready": self.ready,
"degraded": self.degraded,
"fatal": self.fatal,
"attempt": self.attempt,
"max_attempts": self.max_attempts,
"elapsed_ms": self.elapsed_ms,
"phase_elapsed_ms": self.phase_elapsed_ms,
"ready_elapsed_ms": self.ready_elapsed_ms,
"warmups": [warmup.to_diagnostics() for warmup in self.warmups],
}
@dataclass
class _WarmupRecord:
name: str
state: WarmupState
reason_code: StartupReason
timeout_sec: float
started_at: float | None = None
completed_at: float | None = None
def _bounded_ms(seconds: float) -> int:
if not math.isfinite(seconds) or seconds <= 0:
return 0
return min(round(seconds * 1000.0), MAX_DIAGNOSTIC_MS)
_WARMUP_NAME_RE = re.compile(r"[^a-zA-Z0-9_.-]+")
def _safe_warmup_name(value: Any) -> str:
normalized = _WARMUP_NAME_RE.sub("_", str(value or "warmup")).strip("._-")
return (normalized or "warmup")[:64]
class StartupLifecycle:
"""Single lock-protected owner of startup phase and warmup outcomes."""
def __init__(self, *, monotonic_fn: Callable[[], float] = time.monotonic):
self._clock = monotonic_fn
self._lock = threading.RLock()
self._generation = 0
self._reset_locked()
def _reset_locked(self) -> None:
now = self._clock()
self._started_at = now
self._phase_started_at = now
self._ready_at: float | None = None
self._phase = StartupPhase.PACKAGE_IMPORT
self._state = StartupState.STARTING
self._reason_code = StartupReason.BOOTSTRAP_STARTED
self._attempt = 0
self._max_attempts = 0
self._warmups: dict[str, _WarmupRecord] = {}
def reset(self) -> None:
with self._lock:
self._generation += 1
self._reset_locked()
def _require_nonterminal(self) -> None:
if self._state is StartupState.FATAL:
raise StartupTransitionError("TERMINAL_STATE")
def _set(
self,
*,
phase: StartupPhase,
state: StartupState,
reason_code: StartupReason,
attempt: int | None = None,
max_attempts: int | None = None,
) -> None:
now = self._clock()
if phase is not self._phase:
self._phase_started_at = now
self._phase = phase
self._state = state
self._reason_code = reason_code
if attempt is not None:
self._attempt = attempt
if max_attempts is not None:
self._max_attempts = max_attempts
if state is StartupState.READY and self._ready_at is None:
self._ready_at = now
def mark_required_initialization_started(self) -> None:
with self._lock:
self._require_nonterminal()
if self._state is not StartupState.STARTING:
raise StartupTransitionError("INVALID_TRANSITION")
self._set(
phase=StartupPhase.REQUIRED_INITIALIZATION,
state=StartupState.INITIALIZING,
reason_code=StartupReason.REQUIRED_INITIALIZATION_STARTED,
)
def mark_host_waiting(self, *, attempt: int, max_attempts: int) -> None:
with self._lock:
self._require_nonterminal()
if max_attempts <= 0 or attempt < 0 or attempt > max_attempts:
raise StartupTransitionError("ATTEMPT_OUT_OF_RANGE")
if self._state is StartupState.INITIALIZING:
if attempt != 0:
raise StartupTransitionError("ATTEMPT_OUT_OF_RANGE")
elif self._state is StartupState.WAITING_FOR_HOST:
if attempt <= self._attempt:
raise StartupTransitionError("ATTEMPT_NOT_INCREASING")
if max_attempts != self._max_attempts:
raise StartupTransitionError("ATTEMPT_BOUND_CHANGED")
else:
raise StartupTransitionError("INVALID_TRANSITION")
self._set(
phase=StartupPhase.HOST_WAIT,
state=StartupState.WAITING_FOR_HOST,
reason_code=StartupReason.HOST_NOT_READY,
attempt=attempt,
max_attempts=max_attempts,
)
def mark_route_registration_started(
self, *, attempt: int = 0, max_attempts: int = 0
) -> None:
with self._lock:
self._require_nonterminal()
if self._state not in {
StartupState.INITIALIZING,
StartupState.WAITING_FOR_HOST,
}:
raise StartupTransitionError("INVALID_TRANSITION")
if attempt < 0 or max_attempts < 0 or attempt > max_attempts:
raise StartupTransitionError("ATTEMPT_OUT_OF_RANGE")
if self._state is StartupState.INITIALIZING:
if attempt != 0 or max_attempts != 0:
raise StartupTransitionError("ATTEMPT_OUT_OF_RANGE")
else:
if max_attempts != self._max_attempts:
raise StartupTransitionError("ATTEMPT_BOUND_CHANGED")
if attempt <= self._attempt:
raise StartupTransitionError("ATTEMPT_NOT_INCREASING")
self._set(
phase=StartupPhase.ROUTE_REGISTRATION,
state=StartupState.REGISTERING_ROUTES,
reason_code=StartupReason.ROUTE_REGISTRATION_STARTED,
attempt=attempt,
max_attempts=max_attempts,
)
def mark_ready(self) -> None:
with self._lock:
self._require_nonterminal()
if self._state is not StartupState.REGISTERING_ROUTES:
raise StartupTransitionError("INVALID_TRANSITION")
self._set(
phase=StartupPhase.COMPLETE,
state=StartupState.READY,
reason_code=StartupReason.ROUTE_REGISTRATION_SUCCEEDED,
)
def mark_fatal(
self,
*,
phase: StartupPhase,
reason_code: StartupReason,
) -> None:
with self._lock:
self._require_nonterminal()
allowed = {
StartupReason.BOOTSTRAP_IMPORT_FAILED,
StartupReason.REQUIRED_INITIALIZATION_FAILED,
StartupReason.ROUTE_REGISTRATION_FAILED,
StartupReason.RETRY_EXHAUSTED,
}
if reason_code not in allowed:
raise StartupTransitionError("INVALID_FATAL_REASON")
expected_phase = {
StartupReason.BOOTSTRAP_IMPORT_FAILED: StartupPhase.PACKAGE_IMPORT,
StartupReason.REQUIRED_INITIALIZATION_FAILED: (
StartupPhase.REQUIRED_INITIALIZATION
),
StartupReason.ROUTE_REGISTRATION_FAILED: (
StartupPhase.ROUTE_REGISTRATION
),
StartupReason.RETRY_EXHAUSTED: StartupPhase.HOST_WAIT,
}[reason_code]
if phase is not expected_phase:
raise StartupTransitionError("FATAL_PHASE_MISMATCH")
allowed_states = {
StartupReason.BOOTSTRAP_IMPORT_FAILED: {
StartupState.STARTING,
},
StartupReason.REQUIRED_INITIALIZATION_FAILED: {
StartupState.INITIALIZING,
},
StartupReason.ROUTE_REGISTRATION_FAILED: {
StartupState.INITIALIZING,
StartupState.WAITING_FOR_HOST,
StartupState.REGISTERING_ROUTES,
},
StartupReason.RETRY_EXHAUSTED: {
StartupState.WAITING_FOR_HOST,
},
}[reason_code]
if self._state not in allowed_states:
raise StartupTransitionError("INVALID_FATAL_TRANSITION")
self._set(
phase=phase,
state=StartupState.FATAL,
reason_code=reason_code,
)
def mark_retry_exhausted(self) -> None:
with self._lock:
self._require_nonterminal()
if self._state is not StartupState.WAITING_FOR_HOST:
raise StartupTransitionError("INVALID_TRANSITION")
self._set(
phase=StartupPhase.HOST_WAIT,
state=StartupState.FATAL,
reason_code=StartupReason.RETRY_EXHAUSTED,
)
def begin_warmup(self, name: str, timeout_sec: float) -> tuple[bool, int, str]:
safe_name = _safe_warmup_name(name)
timeout_sec = max(0.01, min(float(timeout_sec or 5.0), 60.0))
with self._lock:
self._require_nonterminal()
if self._state not in {StartupState.READY, StartupState.DEGRADED}:
raise StartupTransitionError("WARMUP_BEFORE_READY")
existing = self._warmups.get(safe_name)
if existing is not None:
return False, self._generation, safe_name
if len(self._warmups) >= MAX_WARMUPS:
raise StartupTransitionError("WARMUP_LIMIT_EXCEEDED")
self._warmups[safe_name] = _WarmupRecord(
name=safe_name,
state=WarmupState.PENDING,
reason_code=StartupReason.WARMUP_STARTED,
timeout_sec=timeout_sec,
)
return True, self._generation, safe_name
def mark_warmup_running(self, name: str, generation: int) -> None:
with self._lock:
if generation != self._generation:
return
record = self._warmups.get(name)
if record is None:
return
record.state = WarmupState.RUNNING
record.reason_code = StartupReason.WARMUP_STARTED
record.started_at = self._clock()
if self._state is StartupState.READY:
self._set(
phase=StartupPhase.OPTIONAL_WARMUP,
state=StartupState.READY,
reason_code=StartupReason.WARMUP_STARTED,
)
def finish_warmup(
self,
name: str,
generation: int,
*,
state: WarmupState,
) -> None:
reason_by_state = {
WarmupState.SUCCEEDED: StartupReason.WARMUP_SUCCEEDED,
WarmupState.FAILED: StartupReason.WARMUP_FAILED,
WarmupState.TIMED_OUT: StartupReason.WARMUP_TIMED_OUT,
}
reason = reason_by_state.get(state)
if reason is None:
raise StartupTransitionError("INVALID_WARMUP_RESULT")
with self._lock:
if generation != self._generation:
return
record = self._warmups.get(name)
if record is None:
return
if record.state in {
WarmupState.SUCCEEDED,
WarmupState.FAILED,
WarmupState.TIMED_OUT,
}:
return
record.state = state
record.reason_code = reason
record.completed_at = self._clock()
if state in {WarmupState.FAILED, WarmupState.TIMED_OUT}:
self._set(
phase=StartupPhase.OPTIONAL_WARMUP,
state=StartupState.DEGRADED,
reason_code=reason,
)
elif self._state is StartupState.READY:
self._set(
phase=StartupPhase.OPTIONAL_WARMUP,
state=StartupState.READY,
reason_code=reason,
)
def snapshot(self) -> StartupOutcome:
with self._lock:
now = self._clock()
warmups = []
for name in sorted(self._warmups):
record = self._warmups[name]
started_at = record.started_at
completed_at = record.completed_at
if started_at is None:
duration = 0.0
else:
duration = (
completed_at if completed_at is not None else now
) - started_at
warmups.append(
WarmupOutcome(
name=record.name,
state=record.state,
reason_code=record.reason_code,
timeout_ms=_bounded_ms(record.timeout_sec),
duration_ms=_bounded_ms(duration),
)
)
ready = self._state in {StartupState.READY, StartupState.DEGRADED}
return StartupOutcome(
schema_version=SCHEMA_VERSION,
phase=self._phase,
state=self._state,
reason_code=self._reason_code,
ready=ready,
degraded=self._state is StartupState.DEGRADED,
fatal=self._state is StartupState.FATAL,
attempt=self._attempt,
max_attempts=self._max_attempts,
elapsed_ms=_bounded_ms(now - self._started_at),
phase_elapsed_ms=_bounded_ms(now - self._phase_started_at),
ready_elapsed_ms=(
_bounded_ms(now - self._ready_at)
if self._ready_at is not None
else None
),
warmups=tuple(warmups),
)
_LIFECYCLE = StartupLifecycle()
def get_startup_outcome() -> StartupOutcome:
return _LIFECYCLE.snapshot()
def get_startup_diagnostics() -> dict[str, Any]:
return get_startup_outcome().to_diagnostics()
def mark_required_initialization_started() -> None:
_LIFECYCLE.mark_required_initialization_started()
def mark_host_waiting(*, attempt: int, max_attempts: int) -> None:
_LIFECYCLE.mark_host_waiting(attempt=attempt, max_attempts=max_attempts)
def mark_route_registration_started(*, attempt: int = 0, max_attempts: int = 0) -> None:
_LIFECYCLE.mark_route_registration_started(
attempt=attempt,
max_attempts=max_attempts,
)
def mark_startup_ready(phase: str = "routes") -> None: def mark_startup_ready(phase: str = "routes") -> None:
"""Mark required startup work as ready.""" """Compatibility facade that reaches the required ready transition."""
global _READY, _READY_AT, _READY_PHASE
with _LOCK: _ = phase
if _FATAL is not None: outcome = _LIFECYCLE.snapshot()
return if outcome.state is StartupState.STARTING:
_READY = True _LIFECYCLE.mark_required_initialization_started()
_READY_PHASE = str(phase or "routes") _LIFECYCLE.mark_route_registration_started()
_READY_AT = time.time() _LIFECYCLE.mark_ready()
return
if outcome.state is StartupState.REGISTERING_ROUTES:
_LIFECYCLE.mark_ready()
return
if outcome.state in {StartupState.READY, StartupState.DEGRADED}:
return
raise StartupTransitionError("INVALID_TRANSITION")
def mark_startup_fatal(phase: str, exc: BaseException) -> None: def mark_startup_fatal(
"""Record a fatal required-startup failure.""" phase: str,
global _FATAL, _READY exc: BaseException | None = None,
with _LOCK: *,
_READY = False reason_code: StartupReason | str | None = None,
_FATAL = { ) -> None:
"phase": str(phase or "startup"), """Record a stable fatal classification without retaining ``exc``."""
"error_type": type(exc).__name__,
"error": str(exc)[:500], _ = exc
"ts": time.time(), phase_map = {
"package_import": StartupPhase.PACKAGE_IMPORT,
"required_startup": StartupPhase.REQUIRED_INITIALIZATION,
"required_initialization": StartupPhase.REQUIRED_INITIALIZATION,
"route_registration": StartupPhase.ROUTE_REGISTRATION,
"route_registration_retry": StartupPhase.HOST_WAIT,
"host_wait": StartupPhase.HOST_WAIT,
}
resolved_phase = phase_map.get(str(phase), StartupPhase.REQUIRED_INITIALIZATION)
if reason_code is None:
default_reasons = {
StartupPhase.PACKAGE_IMPORT: StartupReason.BOOTSTRAP_IMPORT_FAILED,
StartupPhase.REQUIRED_INITIALIZATION: (
StartupReason.REQUIRED_INITIALIZATION_FAILED
),
StartupPhase.ROUTE_REGISTRATION: (StartupReason.ROUTE_REGISTRATION_FAILED),
StartupPhase.HOST_WAIT: StartupReason.RETRY_EXHAUSTED,
} }
resolved_reason = default_reasons[resolved_phase]
else:
resolved_reason = (
reason_code
if isinstance(reason_code, StartupReason)
else StartupReason(str(reason_code))
)
_LIFECYCLE.mark_fatal(
phase=resolved_phase,
reason_code=resolved_reason,
)
def mark_bootstrap_import_failed(exc: BaseException) -> None:
mark_startup_fatal(
"package_import",
exc,
reason_code=StartupReason.BOOTSTRAP_IMPORT_FAILED,
)
def mark_retry_exhausted() -> None:
_LIFECYCLE.mark_retry_exhausted()
def start_optional_warmups(specs: Iterable[WarmupSpec]) -> None: def start_optional_warmups(specs: Iterable[WarmupSpec]) -> None:
"""Start optional warmups in background monitor threads.""" for name, fn, timeout_sec in tuple(specs or ()):
for name, fn, timeout_sec in list(specs or []):
_start_optional_warmup(str(name), fn, float(timeout_sec)) _start_optional_warmup(str(name), fn, float(timeout_sec))
def get_startup_diagnostics() -> Dict[str, Any]:
"""Return a bounded diagnostic snapshot for health/operator views."""
with _LOCK:
warmups = {name: dict(record) for name, record in _WARMUPS.items()}
fatal = dict(_FATAL) if _FATAL else None
ready = bool(_READY and fatal is None)
degraded = any(
record.get("state") in {WARMUP_FAILED, WARMUP_TIMED_OUT}
for record in warmups.values()
)
if fatal:
state = STARTUP_FATAL
elif ready and degraded:
state = STARTUP_DEGRADED_WARMUP
elif ready:
state = STARTUP_READY
else:
state = STARTUP_STARTING
return {
"state": state,
"ready": ready,
"ready_phase": _READY_PHASE,
"started_at": _STARTED_AT,
"ready_at": _READY_AT,
"fatal": fatal,
"warmups": warmups,
}
def reset_startup_lifecycle_for_tests() -> None: def reset_startup_lifecycle_for_tests() -> None:
"""Reset in-memory lifecycle state for tests.""" _LIFECYCLE.reset()
global _READY, _READY_AT, _READY_PHASE, _FATAL, _STARTED_AT
with _LOCK:
_STARTED_AT = time.time()
_READY = False
_READY_PHASE = None
_READY_AT = None
_FATAL = None
_WARMUPS.clear()
def _start_optional_warmup( def _start_optional_warmup(
name: str, fn: Callable[[], Any], timeout_sec: float name: str, fn: Callable[[], Any], timeout_sec: float
) -> None: ) -> None:
timeout_sec = max(0.01, min(float(timeout_sec or 5.0), 60.0)) should_start, generation, safe_name = _LIFECYCLE.begin_warmup(name, timeout_sec)
with _LOCK: if not should_start:
existing = _WARMUPS.get(name) return
if existing and existing.get("state") in {WARMUP_RUNNING, WARMUP_SUCCEEDED}:
return
_WARMUPS[name] = {
"state": WARMUP_PENDING,
"timeout_sec": timeout_sec,
"started_at": None,
"completed_at": None,
"duration_sec": None,
"error_type": None,
"error": None,
}
monitor = threading.Thread( monitor = threading.Thread(
target=_warmup_monitor, target=_warmup_monitor,
args=(name, fn, timeout_sec), args=(safe_name, generation, fn, max(0.01, min(timeout_sec, 60.0))),
name=f"openclaw-warmup-monitor-{name}", name=f"openclaw-warmup-monitor-{safe_name}",
daemon=True, daemon=True,
) )
monitor.start() try:
monitor.start()
except Exception:
_LIFECYCLE.finish_warmup(
safe_name,
generation,
state=WarmupState.FAILED,
)
raise
def _warmup_monitor(name: str, fn: Callable[[], Any], timeout_sec: float) -> None: def _warmup_monitor(
started_at = time.time() name: str,
generation: int,
fn: Callable[[], Any],
timeout_sec: float,
) -> None:
done = threading.Event() done = threading.Event()
result: Dict[str, Any] = {} result: dict[str, bool] = {}
def _worker() -> None: def _worker() -> None:
try: try:
result["value"] = fn() fn()
result["ok"] = True result["ok"] = True
except Exception as exc: # pragma: no cover - defensive outer guard except Exception:
# SECURITY: never retain or log arbitrary exception content.
result["ok"] = False result["ok"] = False
result["exc"] = exc
finally: finally:
done.set() done.set()
with _LOCK: _LIFECYCLE.mark_warmup_running(name, generation)
if name in _WARMUPS:
_WARMUPS[name]["state"] = WARMUP_RUNNING
_WARMUPS[name]["started_at"] = started_at
worker = threading.Thread( worker = threading.Thread(
target=_worker, target=_worker,
name=f"openclaw-warmup-{name}", name=f"openclaw-warmup-{name}",
daemon=True, daemon=True,
) )
worker.start() try:
worker.start()
if not done.wait(timeout=timeout_sec): except Exception as exc:
_finish_warmup( _LIFECYCLE.finish_warmup(
name, name,
WARMUP_TIMED_OUT, generation,
started_at, state=WarmupState.FAILED,
error_type="TimeoutError",
error=f"optional warmup exceeded {timeout_sec:.2f}s",
) )
logger.warning( logger.warning(
"R188: optional startup warmup timed out: %s (%.2fs)", "Optional startup warmup worker could not start "
"(component=%s, error_type=%s)",
name, name,
timeout_sec, type(exc).__name__,
)
return
if not done.wait(timeout=timeout_sec):
_LIFECYCLE.finish_warmup(
name,
generation,
state=WarmupState.TIMED_OUT,
)
logger.warning(
"Optional startup warmup timed out (component=%s, reason_code=%s)",
name,
StartupReason.WARMUP_TIMED_OUT.value,
) )
return return
exc = result.get("exc")
if result.get("ok"): if result.get("ok"):
_finish_warmup(name, WARMUP_SUCCEEDED, started_at) _LIFECYCLE.finish_warmup(
logger.info("R188: optional startup warmup completed: %s", name) name,
generation,
state=WarmupState.SUCCEEDED,
)
logger.info("Optional startup warmup completed (component=%s)", name)
return return
_finish_warmup( _LIFECYCLE.finish_warmup(
name, name,
WARMUP_FAILED, generation,
started_at, state=WarmupState.FAILED,
error_type=type(exc).__name__ if exc else "Exception", )
error=str(exc)[:500] if exc else "unknown warmup failure", logger.warning(
"Optional startup warmup failed (component=%s, reason_code=%s)",
name,
StartupReason.WARMUP_FAILED.value,
) )
logger.warning("R188: optional startup warmup failed: %s: %s", name, exc)
def _finish_warmup(
name: str,
state: str,
started_at: float,
*,
error_type: Optional[str] = None,
error: Optional[str] = None,
) -> None:
completed_at = time.time()
with _LOCK:
record = _WARMUPS.setdefault(name, {})
record.update(
{
"state": state,
"completed_at": completed_at,
"duration_sec": max(0.0, completed_at - started_at),
"error_type": error_type,
"error": error,
}
)
+7 -7
View File
@@ -126,19 +126,19 @@
"review_after": "2027-01-11" "review_after": "2027-01-11"
}, },
{ {
"scope": "_start_registration_retry_loop._retry_worker", "scope": "_run_registration_retry_loop",
"expected_count": 1, "expected_count": 2,
"classification": "allowed_boundary_guard", "classification": "allowed_boundary_guard",
"reason": "Background route warmup retries after transient registration failures.", "reason": "The bounded retry owner classifies PromptServer resolution and route registration failures without exposing exception content.",
"regression_owner": "tests/test_route_registration.py", "regression_owner": "tests/test_bootstrap_lifecycle_outcome.py",
"review_after": "2027-01-11" "review_after": "2027-01-11"
}, },
{ {
"scope": "register_routes_once", "scope": "register_routes_once",
"expected_count": 2, "expected_count": 5,
"classification": "allowed_boundary_guard", "classification": "allowed_boundary_guard",
"reason": "Required startup failures are recorded and re-raised instead of hidden.", "reason": "Required, host-resolution, retry-owner-start, and route failures are classified and replayed fail-closed, while BaseException cleanup re-raises cancellation unchanged.",
"regression_owner": "tests/test_r180_exception_boundary_governance.py", "regression_owner": "tests/test_bootstrap_lifecycle_outcome.py",
"review_after": "2027-01-11" "review_after": "2027-01-11"
} }
] ]
-42
View File
@@ -1750,13 +1750,6 @@
"message": "Unused \"type: ignore\" comment", "message": "Unused \"type: ignore\" comment",
"count": 1 "count": 1
}, },
{
"tool": "mypy",
"path": "services/route_bootstrap.py",
"code": "misc",
"message": "Assignment to variable \"exc\" outside except: block",
"count": 1
},
{ {
"tool": "mypy", "tool": "mypy",
"path": "services/runtime_config.py", "path": "services/runtime_config.py",
@@ -8386,13 +8379,6 @@
"message": "Variable `BridgeHandlers` in function should be lowercase", "message": "Variable `BridgeHandlers` in function should be lowercase",
"count": 1 "count": 1
}, },
{
"tool": "ruff",
"path": "services/route_bootstrap.py",
"code": "N806",
"message": "Variable `PromptServer` in function should be lowercase",
"count": 2
},
{ {
"tool": "ruff", "tool": "ruff",
"path": "services/runtime_config.py", "path": "services/runtime_config.py",
@@ -9667,34 +9653,6 @@
"message": "Import from `collections.abc` instead: `Mapping`", "message": "Import from `collections.abc` instead: `Mapping`",
"count": 1 "count": 1
}, },
{
"tool": "ruff",
"path": "services/startup_lifecycle.py",
"code": "UP006",
"message": "Use `dict` instead of `Dict` for type annotation",
"count": 5
},
{
"tool": "ruff",
"path": "services/startup_lifecycle.py",
"code": "UP035",
"message": "Import from `collections.abc` instead: `Callable`, `Iterable`",
"count": 1
},
{
"tool": "ruff",
"path": "services/startup_lifecycle.py",
"code": "UP035",
"message": "`typing.Dict` is deprecated, use `dict` instead",
"count": 1
},
{
"tool": "ruff",
"path": "services/startup_lifecycle.py",
"code": "UP045",
"message": "Use `X | None` for type annotations",
"count": 5
},
{ {
"tool": "ruff", "tool": "ruff",
"path": "services/startup_profile_gate.py", "path": "services/startup_profile_gate.py",
+804
View File
@@ -0,0 +1,804 @@
import asyncio
import json
import runpy
import sys
import threading
import time
import unittest
from dataclasses import FrozenInstanceError
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from services import route_bootstrap
from services.startup_lifecycle import (
STARTUP_DIAGNOSTIC_KEYS,
MAX_DIAGNOSTIC_MS,
MAX_WARMUPS,
StartupLifecycle,
StartupPhase,
StartupReason,
StartupState,
StartupTransitionError,
WarmupState,
get_startup_diagnostics,
get_startup_outcome,
mark_bootstrap_import_failed,
reset_startup_lifecycle_for_tests,
)
class _Clock:
def __init__(self, value: float = 10.0):
self.value = value
def __call__(self) -> float:
return self.value
def advance(self, seconds: float) -> None:
self.value += seconds
class TestStartupOutcomeContract(unittest.TestCase):
def test_initial_snapshot_is_frozen_versioned_and_deterministic(self):
clock = _Clock()
lifecycle = StartupLifecycle(monotonic_fn=clock)
outcome = lifecycle.snapshot()
first = outcome.to_diagnostics()
second = outcome.to_diagnostics()
self.assertEqual(tuple(first), STARTUP_DIAGNOSTIC_KEYS)
self.assertEqual(first, second)
self.assertEqual(first["schema_version"], 1)
self.assertEqual(first["phase"], "package_import")
self.assertEqual(first["state"], "starting")
self.assertEqual(first["reason_code"], "bootstrap_started")
self.assertFalse(first["ready"])
self.assertFalse(first["degraded"])
self.assertFalse(first["fatal"])
self.assertEqual(first["warmups"], [])
self.assertIsInstance(outcome.warmups, tuple)
with self.assertRaises(FrozenInstanceError):
outcome.ready = True
def test_legal_transition_path_has_bounded_monotonic_timing(self):
clock = _Clock()
lifecycle = StartupLifecycle(monotonic_fn=clock)
lifecycle.mark_required_initialization_started()
clock.advance(0.125)
lifecycle.mark_host_waiting(attempt=0, max_attempts=3)
clock.advance(0.125)
lifecycle.mark_host_waiting(attempt=1, max_attempts=3)
lifecycle.mark_route_registration_started(attempt=2, max_attempts=3)
clock.advance(200_000)
lifecycle.mark_ready()
diagnostics = lifecycle.snapshot().to_diagnostics()
self.assertEqual(diagnostics["phase"], "complete")
self.assertEqual(diagnostics["state"], "ready")
self.assertEqual(
diagnostics["reason_code"], "route_registration_succeeded"
)
self.assertTrue(diagnostics["ready"])
self.assertFalse(diagnostics["fatal"])
self.assertEqual(diagnostics["attempt"], 2)
self.assertEqual(diagnostics["max_attempts"], 3)
for field in ("elapsed_ms", "phase_elapsed_ms", "ready_elapsed_ms"):
self.assertIsInstance(diagnostics[field], int)
self.assertGreaterEqual(diagnostics[field], 0)
self.assertLessEqual(diagnostics[field], MAX_DIAGNOSTIC_MS)
def test_invalid_and_post_fatal_transitions_do_not_mutate_state(self):
lifecycle = StartupLifecycle()
before = lifecycle.snapshot()
with self.assertRaises(StartupTransitionError) as invalid:
lifecycle.mark_ready()
self.assertEqual(invalid.exception.code, "INVALID_TRANSITION")
self.assertEqual(lifecycle.snapshot(), before)
lifecycle.mark_fatal(
phase=StartupPhase.PACKAGE_IMPORT,
reason_code=StartupReason.BOOTSTRAP_IMPORT_FAILED,
)
fatal = lifecycle.snapshot()
with self.assertRaises(StartupTransitionError) as terminal:
lifecycle.mark_required_initialization_started()
self.assertEqual(terminal.exception.code, "TERMINAL_STATE")
self.assertEqual(lifecycle.snapshot(), fatal)
def test_retry_attempt_must_increase_and_stay_within_bound(self):
lifecycle = StartupLifecycle()
lifecycle.mark_required_initialization_started()
lifecycle.mark_host_waiting(attempt=0, max_attempts=2)
for attempt, code in ((0, "ATTEMPT_NOT_INCREASING"), (3, "ATTEMPT_OUT_OF_RANGE")):
with self.subTest(attempt=attempt):
before = lifecycle.snapshot()
with self.assertRaises(StartupTransitionError) as ctx:
lifecycle.mark_host_waiting(attempt=attempt, max_attempts=2)
self.assertEqual(ctx.exception.code, code)
self.assertEqual(lifecycle.snapshot(), before)
with self.assertRaises(StartupTransitionError) as route_regression:
lifecycle.mark_route_registration_started(
attempt=0,
max_attempts=2,
)
self.assertEqual(
route_regression.exception.code,
"ATTEMPT_NOT_INCREASING",
)
def test_fatal_reason_must_match_its_phase(self):
lifecycle = StartupLifecycle()
before = lifecycle.snapshot()
with self.assertRaises(StartupTransitionError) as mismatch:
lifecycle.mark_fatal(
phase=StartupPhase.PACKAGE_IMPORT,
reason_code=StartupReason.ROUTE_REGISTRATION_FAILED,
)
self.assertEqual(mismatch.exception.code, "FATAL_PHASE_MISMATCH")
self.assertEqual(lifecycle.snapshot(), before)
with self.assertRaises(StartupTransitionError) as source:
lifecycle.mark_fatal(
phase=StartupPhase.ROUTE_REGISTRATION,
reason_code=StartupReason.ROUTE_REGISTRATION_FAILED,
)
self.assertEqual(source.exception.code, "INVALID_FATAL_TRANSITION")
self.assertEqual(lifecycle.snapshot(), before)
def test_exception_payload_is_never_retained_or_serialized(self):
marker = "PRIVATE_R231_SECRET C:/private/token.txt"
reset_startup_lifecycle_for_tests()
mark_bootstrap_import_failed(RuntimeError(marker))
outcome = get_startup_outcome()
rendered = json.dumps(get_startup_diagnostics(), sort_keys=True)
self.assertEqual(outcome.state, StartupState.FATAL)
self.assertEqual(outcome.reason_code, StartupReason.BOOTSTRAP_IMPORT_FAILED)
self.assertNotIn(marker, repr(outcome))
self.assertNotIn(marker, rendered)
self.assertNotIn("RuntimeError", rendered)
self.assertNotIn("started_at", rendered)
self.assertNotIn("traceback", rendered.lower())
class TestStartupWarmupProjection(unittest.TestCase):
def setUp(self):
reset_startup_lifecycle_for_tests()
from services.startup_lifecycle import mark_startup_ready
mark_startup_ready("routes")
def tearDown(self):
reset_startup_lifecycle_for_tests()
def _wait_for_warmup(self, name: str, state: str) -> dict:
deadline = time.monotonic() + 1.0
while time.monotonic() < deadline:
diagnostics = get_startup_diagnostics()
warmup = next(
(
item
for item in diagnostics["warmups"]
if item["name"] == name
),
None,
)
if warmup and warmup["state"] == state:
return diagnostics
time.sleep(0.005)
self.fail(f"warmup {name} did not reach {state}")
def test_failure_and_timeout_degrade_without_leaking_exception_content(self):
from services.startup_lifecycle import start_optional_warmups
release = threading.Event()
marker = "PRIVATE_WARMUP_FAILURE C:/private/model"
def fail():
raise RuntimeError(marker)
def block():
release.wait(timeout=1)
with self.assertLogs("ComfyUI-OpenClaw", level="WARNING") as captured:
start_optional_warmups(
[
("z_failure", fail, 0.5),
("a_timeout", block, 0.01),
]
)
self._wait_for_warmup("z_failure", "failed")
diagnostics = self._wait_for_warmup("a_timeout", "timed_out")
release.set()
time.sleep(0.02)
self.assertTrue(diagnostics["ready"])
self.assertTrue(diagnostics["degraded"])
self.assertFalse(diagnostics["fatal"])
self.assertEqual(diagnostics["state"], "degraded")
self.assertEqual(
[item["name"] for item in diagnostics["warmups"]],
["a_timeout", "z_failure"],
)
rendered = json.dumps(diagnostics, sort_keys=True)
self.assertNotIn(marker, rendered)
self.assertNotIn("RuntimeError", rendered)
self.assertNotIn("error", rendered)
self.assertNotIn(marker, "\n".join(captured.output))
final = get_startup_diagnostics()
timed_out = next(
item for item in final["warmups"] if item["name"] == "a_timeout"
)
self.assertEqual(timed_out["state"], "timed_out")
def test_terminal_warmup_cannot_restart_or_be_overwritten(self):
lifecycle = StartupLifecycle()
lifecycle.mark_required_initialization_started()
lifecycle.mark_route_registration_started()
lifecycle.mark_ready()
should_start, generation, name = lifecycle.begin_warmup("provider", 0.01)
self.assertTrue(should_start)
lifecycle.mark_warmup_running(name, generation)
lifecycle.finish_warmup(
name,
generation,
state=WarmupState.TIMED_OUT,
)
should_restart, _, _ = lifecycle.begin_warmup("provider", 0.01)
lifecycle.finish_warmup(
name,
generation,
state=WarmupState.SUCCEEDED,
)
self.assertFalse(should_restart)
outcome = lifecycle.snapshot()
self.assertEqual(outcome.state, StartupState.DEGRADED)
self.assertEqual(outcome.warmups[0].state, WarmupState.TIMED_OUT)
def test_warmup_projection_has_a_non_mutating_cardinality_bound(self):
lifecycle = StartupLifecycle()
lifecycle.mark_required_initialization_started()
lifecycle.mark_route_registration_started()
lifecycle.mark_ready()
for index in range(MAX_WARMUPS):
should_start, _, _ = lifecycle.begin_warmup(
f"provider_{index}",
0.01,
)
self.assertTrue(should_start)
before = lifecycle.snapshot()
with self.assertRaises(StartupTransitionError) as limit:
lifecycle.begin_warmup("one_too_many", 0.01)
self.assertEqual(limit.exception.code, "WARMUP_LIMIT_EXCEEDED")
self.assertEqual(lifecycle.snapshot(), before)
self.assertEqual(len(before.warmups), MAX_WARMUPS)
def test_monitor_thread_start_failure_degrades_and_reraises_content_free(self):
from services.startup_lifecycle import start_optional_warmups
marker = "PRIVATE_MONITOR_START C:/private/monitor"
failure = RuntimeError(marker)
with patch(
"services.startup_lifecycle.threading.Thread"
) as thread_factory:
thread_factory.return_value.start.side_effect = failure
with self.assertRaises(RuntimeError) as ctx:
start_optional_warmups(
[("monitor_provider", lambda: None, 0.01)]
)
self.assertIs(ctx.exception, failure)
diagnostics = get_startup_diagnostics()
self.assertTrue(diagnostics["degraded"])
warmup = next(
item
for item in diagnostics["warmups"]
if item["name"] == "monitor_provider"
)
self.assertEqual(warmup["state"], "failed")
self.assertNotIn(marker, json.dumps(diagnostics))
def test_worker_thread_start_failure_degrades_without_escaping_monitor(self):
from services.startup_lifecycle import _LIFECYCLE, _warmup_monitor
marker = "PRIVATE_WORKER_START C:/private/worker"
failure = RuntimeError(marker)
_, generation, name = _LIFECYCLE.begin_warmup(
"worker_provider",
0.01,
)
with (
patch(
"services.startup_lifecycle.threading.Thread"
) as thread_factory,
self.assertLogs("ComfyUI-OpenClaw", level="WARNING") as captured,
):
thread_factory.return_value.start.side_effect = failure
_warmup_monitor(name, generation, lambda: None, 0.01)
diagnostics = get_startup_diagnostics()
self.assertTrue(diagnostics["degraded"])
warmup = next(
item
for item in diagnostics["warmups"]
if item["name"] == "worker_provider"
)
self.assertEqual(warmup["state"], "failed")
self.assertNotIn(marker, json.dumps(diagnostics))
self.assertNotIn(marker, "\n".join(captured.output))
class TestRouteBootstrapOutcomeIntegration(unittest.TestCase):
def setUp(self):
route_bootstrap.reset_route_bootstrap_for_tests()
reset_startup_lifecycle_for_tests()
def tearDown(self):
route_bootstrap.reset_route_bootstrap_for_tests()
reset_startup_lifecycle_for_tests()
@staticmethod
def _host_module(instance):
return SimpleNamespace(PromptServer=SimpleNamespace(instance=instance))
def test_duplicate_concurrent_calls_share_one_sync_owner_and_retry_owner(self):
entered = threading.Event()
release = threading.Event()
errors = []
def required_init():
entered.set()
release.wait(timeout=1)
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks") as optional,
patch.object(
route_bootstrap,
"_initialize_registries_and_security_gate",
side_effect=required_init,
) as required,
patch.object(route_bootstrap, "_start_registration_retry_loop") as retry,
patch.dict(sys.modules, {"server": self._host_module(None)}),
):
first = threading.Thread(
target=self._call_registration,
args=(errors,),
)
second = threading.Thread(
target=self._call_registration,
args=(errors,),
)
first.start()
self.assertTrue(entered.wait(timeout=1))
second.start()
release.set()
first.join(timeout=1)
second.join(timeout=1)
self.assertFalse(first.is_alive())
self.assertFalse(second.is_alive())
self.assertEqual(errors, [])
self.assertEqual(optional.call_count, 1)
self.assertEqual(required.call_count, 1)
self.assertEqual(retry.call_count, 1)
diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "waiting_for_host")
self.assertFalse(diagnostics["ready"])
@staticmethod
def _call_registration(errors):
try:
route_bootstrap.register_routes_once()
except BaseException as exc: # test capture; production must not swallow
errors.append(exc)
def test_required_failure_is_shared_fail_closed_and_content_free(self):
marker = "PRIVATE_REQUIRED_FAILURE C:/private/config"
failure = RuntimeError(marker)
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object(
route_bootstrap,
"_initialize_registries_and_security_gate",
side_effect=failure,
) as required,
patch.dict(sys.modules, {"server": self._host_module(None)}),
):
with self.assertLogs("ComfyUI-OpenClaw", level="ERROR") as captured:
observed = []
for _ in range(2):
with self.assertRaises(RuntimeError) as ctx:
route_bootstrap.register_routes_once()
observed.append(ctx.exception)
self.assertEqual(required.call_count, 1)
self.assertIs(observed[0], failure)
self.assertIs(observed[1], failure)
diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "fatal")
self.assertEqual(
diagnostics["reason_code"], "required_initialization_failed"
)
self.assertNotIn(marker, json.dumps(diagnostics))
self.assertNotIn(marker, "\n".join(captured.output))
def test_initial_registration_failure_is_fatal_and_reraised(self):
marker = "PRIVATE_ROUTE_FAILURE C:/private/route"
failure = RuntimeError(marker)
server = SimpleNamespace(app=object())
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object(route_bootstrap, "_initialize_registries_and_security_gate"),
patch.object(
route_bootstrap,
"_do_full_registration",
side_effect=failure,
),
patch.dict(sys.modules, {"server": self._host_module(server)}),
):
with self.assertLogs("ComfyUI-OpenClaw", level="ERROR") as captured:
with self.assertRaises(RuntimeError) as ctx:
route_bootstrap.register_routes_once()
self.assertIs(ctx.exception, failure)
diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "fatal")
self.assertEqual(diagnostics["reason_code"], "route_registration_failed")
self.assertNotIn(marker, json.dumps(diagnostics))
self.assertNotIn(marker, "\n".join(captured.output))
def test_initial_host_resolution_failure_is_terminal_and_replayed(self):
marker = "PRIVATE_HOST_RESOLUTION C:/private/host"
failure = RuntimeError(marker)
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object(route_bootstrap, "_initialize_registries_and_security_gate"),
patch.object(
route_bootstrap,
"_resolve_prompt_server",
side_effect=failure,
) as resolve,
):
observed = []
for _ in range(2):
with self.assertRaises(RuntimeError) as ctx:
route_bootstrap.register_routes_once()
observed.append(ctx.exception)
self.assertEqual(resolve.call_count, 1)
self.assertIs(observed[0], failure)
self.assertIs(observed[1], failure)
diagnostics = get_startup_diagnostics()
self.assertTrue(diagnostics["fatal"])
self.assertEqual(diagnostics["reason_code"], "route_registration_failed")
self.assertNotIn(marker, json.dumps(diagnostics))
def test_retry_owner_start_failure_is_terminal_and_replayed(self):
marker = "PRIVATE_THREAD_START C:/private/thread"
failure = RuntimeError(marker)
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object(route_bootstrap, "_initialize_registries_and_security_gate"),
patch.object(route_bootstrap, "_resolve_prompt_server", return_value=None),
patch.object(
route_bootstrap,
"_start_registration_retry_loop",
side_effect=failure,
) as start_retry,
):
observed = []
for _ in range(2):
with self.assertRaises(RuntimeError) as ctx:
route_bootstrap.register_routes_once()
observed.append(ctx.exception)
self.assertEqual(start_retry.call_count, 1)
self.assertIs(observed[0], failure)
self.assertIs(observed[1], failure)
diagnostics = get_startup_diagnostics()
self.assertTrue(diagnostics["fatal"])
self.assertEqual(diagnostics["reason_code"], "retry_exhausted")
self.assertNotIn(marker, json.dumps(diagnostics))
def test_retry_success_and_exhaustion_have_distinct_outcomes(self):
from services.startup_lifecycle import (
mark_host_waiting,
mark_required_initialization_started,
)
server = SimpleNamespace(app=object())
sequence = iter([None, server])
mark_required_initialization_started()
mark_host_waiting(attempt=0, max_attempts=3)
with (
patch.object(
route_bootstrap,
"_resolve_prompt_server",
side_effect=lambda: next(sequence),
),
patch.object(route_bootstrap, "_do_full_registration") as register,
patch.object(
route_bootstrap,
"_build_optional_startup_warmups",
return_value=[],
),
):
route_bootstrap._run_registration_retry_loop(
max_attempts=3,
initial_delay=0,
sleep_fn=lambda _delay: None,
)
register.assert_called_once_with(server)
diagnostics = get_startup_diagnostics()
self.assertTrue(diagnostics["ready"])
self.assertEqual(
diagnostics["reason_code"], "route_registration_succeeded"
)
reset_startup_lifecycle_for_tests()
route_bootstrap.reset_route_bootstrap_for_tests()
mark_required_initialization_started()
mark_host_waiting(attempt=0, max_attempts=2)
with patch.object(
route_bootstrap,
"_resolve_prompt_server",
return_value=None,
):
route_bootstrap._run_registration_retry_loop(
max_attempts=2,
initial_delay=0,
sleep_fn=lambda _delay: None,
)
diagnostics = get_startup_diagnostics()
self.assertFalse(diagnostics["ready"])
self.assertTrue(diagnostics["fatal"])
self.assertEqual(diagnostics["reason_code"], "retry_exhausted")
self.assertEqual(diagnostics["attempt"], 2)
def test_base_exception_is_not_swallowed_or_converted_to_fatal(self):
signal = KeyboardInterrupt()
with (
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object(
route_bootstrap,
"_initialize_registries_and_security_gate",
side_effect=signal,
),
):
with self.assertRaises(KeyboardInterrupt) as ctx:
route_bootstrap.register_routes_once()
self.assertIs(ctx.exception, signal)
diagnostics = get_startup_diagnostics()
self.assertFalse(diagnostics["fatal"])
self.assertEqual(diagnostics["state"], "initializing")
class TestPublicHealthLifecycleProjection(unittest.TestCase):
def setUp(self):
reset_startup_lifecycle_for_tests()
def tearDown(self):
reset_startup_lifecycle_for_tests()
@staticmethod
def _health_payload(*, diagnostics_side_effect=None):
from api.route_handlers import health_response
web = SimpleNamespace(json_response=lambda data, **_kwargs: data)
deps = SimpleNamespace(
web=web,
pack_start_time=time.time(),
pack_name="openclaw",
pack_version="test",
metrics=SimpleNamespace(
get_snapshot=lambda: {
"errors_captured": 0,
"logs_processed": 0,
}
),
get_executor_diagnostics=lambda: {},
check_dependency=lambda _name: True,
)
client = MagicMock()
client.get_provider_summary.return_value = {
"provider": "openai",
"model": "test",
"key_configured": False,
}
diagnostics_patch = patch(
"services.startup_lifecycle.get_startup_diagnostics",
side_effect=diagnostics_side_effect,
)
if diagnostics_side_effect is None:
diagnostics_patch = patch(
"services.startup_lifecycle.get_startup_diagnostics",
wraps=get_startup_diagnostics,
)
with (
patch("services.llm_client.LLMClient", return_value=client),
patch("services.providers.keys.requires_api_key", return_value=True),
patch(
"services.job_events.get_job_event_store",
return_value=SimpleNamespace(stats=lambda: {}),
),
patch(
"services.capabilities._get_control_plane_info",
return_value={},
),
patch(
"services.runtime_profile.get_runtime_profile",
return_value="minimal",
),
diagnostics_patch,
):
return asyncio.run(health_response(SimpleNamespace(), deps))
def test_health_uses_exact_schema_and_content_free_fallback(self):
normal = self._health_payload()
self.assertEqual(
tuple(normal["startup"]),
STARTUP_DIAGNOSTIC_KEYS,
)
marker = "PRIVATE_HEALTH_FAILURE C:/private/health"
fallback = self._health_payload(
diagnostics_side_effect=RuntimeError(marker)
)
self.assertEqual(
tuple(fallback["startup"]),
STARTUP_DIAGNOSTIC_KEYS,
)
self.assertEqual(fallback["startup"]["schema_version"], 1)
self.assertEqual(fallback["startup"]["state"], "fatal")
self.assertEqual(
fallback["startup"]["reason_code"],
"bootstrap_import_failed",
)
self.assertNotIn(marker, json.dumps(fallback["startup"]))
def test_health_metadata_remains_public_and_all_aliases_share_one_handler(self):
from api.route_registrars import build_core_route_specs
from api.routes import health_handler
from services.endpoint_manifest import AuthTier, get_metadata
metadata = get_metadata(health_handler)
self.assertIsNotNone(metadata)
self.assertEqual(metadata.auth_tier, AuthTier.PUBLIC)
handlers = {"health_handler": health_handler}
sentinel = MagicMock()
handlers.update(
{
key: sentinel
for key in (
"remote_admin_page_handler",
"logs_tail_handler",
"jobs_handler",
"trace_handler",
"webhook_handler",
"webhook_submit_handler",
"webhook_validate_handler",
"capabilities_handler",
"config_get_handler",
"config_put_handler",
"llm_test_handler",
"llm_chat_handler",
"llm_models_handler",
"templates_list_handler",
"preflight_handler",
"inventory_handler",
"pnginfo_handler",
"list_checkpoints_handler",
"create_checkpoint_handler",
"get_checkpoint_handler",
"delete_checkpoint_handler",
"rewrite_recipes_list_handler",
"rewrite_recipe_create_handler",
"rewrite_recipe_get_handler",
"rewrite_recipe_update_handler",
"rewrite_recipe_delete_handler",
"rewrite_recipe_dry_run_handler",
"rewrite_recipe_apply_handler",
"model_search_handler",
"model_download_create_handler",
"model_download_list_handler",
"model_download_get_handler",
"model_download_cancel_handler",
"model_import_handler",
"model_installations_list_handler",
"secrets_status_handler",
"secrets_put_handler",
"events_stream_handler",
"events_poll_handler",
"secrets_delete_handler",
"security_doctor_handler",
"tools_list_handler",
"tools_run_handler",
"create_sweep_handler",
"create_compare_handler",
"list_experiments_handler",
"get_experiment_handler",
"update_experiment_handler",
"select_apply_winner_handler",
)
}
)
for prefix in ("/openclaw", "/api/openclaw", "/moltbot", "/api/moltbot"):
specs = build_core_route_specs(prefix, handlers)
health = next(spec for spec in specs if spec.path == f"{prefix}/health")
self.assertIs(health.handler, health_handler)
class TestPackageBootstrapLifecycleProjection(unittest.TestCase):
def setUp(self):
reset_startup_lifecycle_for_tests()
def tearDown(self):
reset_startup_lifecycle_for_tests()
@staticmethod
def _run_entrypoint_with_import_signal(signal):
original_import = __import__
def guarded_import(name, *args, **kwargs):
if name == "services.route_bootstrap":
raise signal
return original_import(name, *args, **kwargs)
root = Path(__file__).resolve().parents[1]
with patch("builtins.__import__", side_effect=guarded_import):
return runpy.run_path(
str(root / "__init__.py"),
run_name="openclaw_bootstrap_lifecycle_probe",
)
def test_package_import_failure_is_stably_classified_without_payload(self):
marker = "PRIVATE_IMPORT_FAILURE C:/private/import"
self._run_entrypoint_with_import_signal(ImportError(marker))
diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "fatal")
self.assertEqual(
diagnostics["reason_code"],
"bootstrap_import_failed",
)
self.assertNotIn(marker, json.dumps(diagnostics))
def test_package_import_base_exception_is_reraised_unchanged(self):
signal = KeyboardInterrupt()
with self.assertRaises(KeyboardInterrupt) as ctx:
self._run_entrypoint_with_import_signal(signal)
self.assertIs(ctx.exception, signal)
self.assertFalse(get_startup_diagnostics()["fatal"])
if __name__ == "__main__":
unittest.main()
+29 -12
View File
@@ -39,28 +39,39 @@ class TestStartupLifecycleDiagnostics(unittest.TestCase):
diagnostics = get_startup_diagnostics() diagnostics = get_startup_diagnostics()
while time.monotonic() < deadline: while time.monotonic() < deadline:
diagnostics = get_startup_diagnostics() diagnostics = get_startup_diagnostics()
if diagnostics["warmups"]["slow_provider"]["state"] == "timed_out": warmup = next(
(
item
for item in diagnostics["warmups"]
if item["name"] == "slow_provider"
),
None,
)
if warmup and warmup["state"] == "timed_out":
break break
time.sleep(0.01) time.sleep(0.01)
release.set() release.set()
self.assertEqual(diagnostics["state"], "degraded-warmup") self.assertEqual(diagnostics["state"], "degraded")
self.assertEqual(diagnostics["ready"], True) self.assertEqual(diagnostics["ready"], True)
self.assertEqual(diagnostics["warmups"]["slow_provider"]["state"], "timed_out") self.assertEqual(warmup["state"], "timed_out")
def test_fatal_startup_state_is_distinct_from_warmup_degradation(self): def test_fatal_startup_state_is_distinct_from_warmup_degradation(self):
from services.startup_lifecycle import ( from services.startup_lifecycle import (
get_startup_diagnostics, get_startup_diagnostics,
mark_required_initialization_started,
mark_startup_fatal, mark_startup_fatal,
) )
mark_startup_fatal("security_gate", RuntimeError("blocked")) mark_required_initialization_started()
mark_startup_fatal("required_initialization", RuntimeError("blocked"))
diagnostics = get_startup_diagnostics() diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "fatal-startup") self.assertEqual(diagnostics["state"], "fatal")
self.assertFalse(diagnostics["ready"]) self.assertFalse(diagnostics["ready"])
self.assertEqual(diagnostics["fatal"]["phase"], "security_gate") self.assertTrue(diagnostics["fatal"])
self.assertIn("RuntimeError", diagnostics["fatal"]["error_type"]) self.assertEqual(diagnostics["phase"], "required_initialization")
self.assertEqual(diagnostics["reason_code"], "required_initialization_failed")
class _DummyRoutes: class _DummyRoutes:
@@ -117,13 +128,17 @@ class _DummyBridgeHandlers:
class TestRouteBootstrapWarmupBoundary(unittest.TestCase): class TestRouteBootstrapWarmupBoundary(unittest.TestCase):
def setUp(self): def setUp(self):
from services import route_bootstrap
from services.startup_lifecycle import reset_startup_lifecycle_for_tests from services.startup_lifecycle import reset_startup_lifecycle_for_tests
route_bootstrap.reset_route_bootstrap_for_tests()
reset_startup_lifecycle_for_tests() reset_startup_lifecycle_for_tests()
def tearDown(self): def tearDown(self):
from services import route_bootstrap
from services.startup_lifecycle import reset_startup_lifecycle_for_tests from services.startup_lifecycle import reset_startup_lifecycle_for_tests
route_bootstrap.reset_route_bootstrap_for_tests()
reset_startup_lifecycle_for_tests() reset_startup_lifecycle_for_tests()
def test_full_registration_marks_ready_before_optional_warmup_finishes(self): def test_full_registration_marks_ready_before_optional_warmup_finishes(self):
@@ -169,6 +184,7 @@ class TestRouteBootstrapWarmupBoundary(unittest.TestCase):
get_runner.return_value = MagicMock() get_runner.return_value = MagicMock()
started_at = time.monotonic() started_at = time.monotonic()
route_bootstrap._do_full_registration(server) route_bootstrap._do_full_registration(server)
route_bootstrap._mark_startup_ready_and_start_warmups()
elapsed = time.monotonic() - started_at elapsed = time.monotonic() - started_at
diagnostics = get_startup_diagnostics() diagnostics = get_startup_diagnostics()
@@ -179,16 +195,16 @@ class TestRouteBootstrapWarmupBoundary(unittest.TestCase):
self.assertTrue(app.triggers) self.assertTrue(app.triggers)
self.assertTrue(app.approvals) self.assertTrue(app.approvals)
self.assertTrue(diagnostics["ready"]) self.assertTrue(diagnostics["ready"])
self.assertIn( warmup = next(
diagnostics["warmups"]["slow_provider"]["state"], {"running", "succeeded"} item for item in diagnostics["warmups"] if item["name"] == "slow_provider"
) )
self.assertIn(warmup["state"], {"running", "succeeded"})
def test_register_routes_once_marks_fatal_when_required_startup_fails(self): def test_register_routes_once_marks_fatal_when_required_startup_fails(self):
from services import route_bootstrap from services import route_bootstrap
from services.startup_lifecycle import get_startup_diagnostics from services.startup_lifecycle import get_startup_diagnostics
with ( with (
patch.object(route_bootstrap, "_routes_registered", False),
patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"), patch.object(route_bootstrap, "_register_plugins_and_shutdown_hooks"),
patch.object( patch.object(
route_bootstrap, route_bootstrap,
@@ -200,9 +216,10 @@ class TestRouteBootstrapWarmupBoundary(unittest.TestCase):
route_bootstrap.register_routes_once() route_bootstrap.register_routes_once()
diagnostics = get_startup_diagnostics() diagnostics = get_startup_diagnostics()
self.assertEqual(diagnostics["state"], "fatal-startup") self.assertEqual(diagnostics["state"], "fatal")
self.assertFalse(diagnostics["ready"]) self.assertFalse(diagnostics["ready"])
self.assertEqual(diagnostics["fatal"]["phase"], "required_startup") self.assertTrue(diagnostics["fatal"])
self.assertEqual(diagnostics["reason_code"], "required_initialization_failed")
if __name__ == "__main__": if __name__ == "__main__":