hybrid: de-hardcode user-specific paths in scripts, defaults, docs

- Replace hardcoded /matx/u/aspark/... paths with
  OPENJARVIS_HYBRID_EXPERIMENTS_DIR env var defaulting to
  ~/.openjarvis/experiments/hybrid across runner, sweep orchestrator,
  report builder, backfill, and the SWE-bench dataset regression test.
- run_sweep.py REPO_ROOT now inferred from __file__; VENV_PY +
  LOG_DIR overridable via env.
- archon.py ARCHON_SRC env var no longer falls back to a personal clone;
  docstring points at the upstream repo.
- README / docstring scrub: remove personal github link, personal install
  paths, '(Andrew's lane)' comment, /matx/u/aspark/CLAUDE.md spec ref.
- Add minion_logs/ and oj-debug.*.json to .gitignore.

.gitignore

@@ -125,3 +125,8 @@ learning.db
 **/learning/benchmarks/
 **/teacher_traces/
 *.session.json
+
+# Local dev artifacts (hybrid worker logs + cli debug dumps)
+minion_logs/
+*.oj-debug.json
+oj-debug.*.json

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "OpenJarvis"
-version = "1.0.2"
+version = "0.1.1"
 description = "OpenJarvis — modular AI assistant backend with composable intelligence primitives"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -29,8 +29,6 @@ dependencies = [
     "ddgs>=9.11.4",
     "httpx>=0.27",
     "openai>=1.30",
-    "posthog>=3.0",
-    "nvidia-ml-py>=12.560.30",
     "python-telegram-bot>=22.6",
     "rich>=13",
     "tomli>=2.0; python_version < '3.11'",
@@ -80,19 +78,13 @@ server = [
     "python-multipart>=0.0.9",
 ]
 openhands = ["openhands-sdk>=1.0; python_version >= '3.12'"]
-gpu-metrics = ["nvidia-ml-py>=12.560.30"]
+gpu-metrics = ["pynvml>=12.0"]
 energy-amd = ["amdsmi>=6.1"]
 energy-apple = ["zeus-ml[apple]"]
-energy-all = ["nvidia-ml-py>=12.560.30", "amdsmi>=6.1", "zeus-ml[apple]"]
+energy-all = ["pynvml>=12.0", "amdsmi>=6.1", "zeus-ml[apple]"]
 orchestrator-training = ["torch>=2.0", "transformers>=4.40"]
 learning-dspy = ["dspy>=2.6"]
 learning-gepa = ["gepa>=0.1"]
-# ACE (Agentic Context Engineering) is supported via
-# ``openjarvis.learning.agents.ace_optimizer`` but ACE upstream isn't on
-# PyPI and isn't structured as an installable Python package as of
-# v1.0.1, so there's no ``learning-ace`` extra. To use ACE, follow the
-# manual setup in docs/learning/ace.md (clone the upstream repo, add
-# its ``src/`` to PYTHONPATH).
 channel-telegram = ["python-telegram-bot>=21.0"]
 channel-discord = ["discord.py>=2.3"]
 channel-slack = ["slack-sdk>=3.27"]
@@ -175,6 +167,7 @@ markers = [
     "nvidia: requires NVIDIA GPU",
     "slow: long-running test",
     "live_external: requires HERMES_AGENT_PATH and OPENCLAW_PATH; spawns real foreign-framework subprocesses",
+    "modal: requires Modal token + network; runs real swebench harness on Modal",
 ]
 
 [tool.ruff]
@@ -190,10 +183,6 @@ select = ["E", "F", "I", "W"]
 # hybrid/ is research code with long prompt strings and paradigm-specific
 # config dicts — same relaxation as evals research code above.
 "src/openjarvis/agents/hybrid/*.py" = ["E501"]
-# research_loop.py carries the multi-paragraph planner system prompt as
-# inline string literals; line-length wrapping would harm readability of
-# the prompt itself.
-"src/openjarvis/agents/research_loop.py" = ["E501"]
 
 [dependency-groups]
 dev = [

scripts/ablation/backfill_tool_calls.py

@@ -0,0 +1,284 @@
+"""Backfill `tool_calls_total` into existing hybrid n=100 summaries.
+
+The `tool_calls_total` field was added to the canonical row-write path in
+commit `7f432453` (2026-05-18). Earlier n=100 cells don't have it in
+either `results.jsonl` rows or `summary.json`. Rerunning is too expensive,
+but per-task log files at
+``experiments/hybrid/runs/<cell>/logs/<task_id>.json`` preserve enough
+state to derive the count.
+
+Per-paradigm derivation (matches the in-process counter installed by
+commit `7f432453`):
+
+* ``baseline_cloud`` (cells named ``cloud-only-*``):
+  - SWE: ``metadata.turns`` (== bash invocations + final patch turn).
+  - GAIA one-shot: 0 (no tools).
+  - GAIA opus47 with web_search: ``traces.n_web_searches`` if present,
+    else 0 (older cells pre-date the wiring).
+* ``advisors``:
+  - SWE: ``metadata.turns - 1`` (subtract the advisor critique pass).
+  - GAIA: ``metadata.turns - 2`` is NOT meaningful here — old cells did
+    no web_search and the executor passes were one-shot, so 0.
+* ``minions``:
+  - GAIA: ``traces.prefetch.n_searches`` (only the prefetch hits tools;
+    supervisor↔worker protocol is text-only).
+  - SWE: ``metadata.turns - 1`` (subtract supervisor turn; worker runs
+    the bash agent loop).
+* ``skillorchestra`` (old single-file version):
+  - GAIA: 0 (one-shot routed call).
+  - SWE: count ``skillorch_(local|cloud)_turn`` events in the log
+    (router's ``anthropic`` event is the routing call, not a tool turn).
+
+The script writes the derived total into ``summary.json`` under the same
+``tool_calls_total`` key and preserves all other keys. Rows that were
+already populated get re-confirmed (idempotent — no overwrite unless
+``--force``). Cells whose logs are missing (no `logs/` dir) get skipped
+with a warning.
+
+Usage::
+
+    .venv/bin/python scripts/ablation/backfill_tool_calls.py
+    .venv/bin/python scripts/ablation/backfill_tool_calls.py --dry-run
+    .venv/bin/python scripts/ablation/backfill_tool_calls.py --force
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+HYBRID_RUNS = Path(
+    os.environ.get(
+        "HYBRID_RUNS_DIR",
+        Path(
+            os.environ.get(
+                "OPENJARVIS_HYBRID_EXPERIMENTS_DIR",
+                Path.home() / ".openjarvis" / "experiments" / "hybrid",
+            )
+        )
+        / "runs",
+    )
+)
+
+
+def _load_log(p: Path) -> Optional[Dict[str, Any]]:
+    try:
+        return json.loads(p.read_text())
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def _count_event_kind(log: Dict[str, Any], kind_suffix: str) -> int:
+    """Count events whose ``kind`` ends with ``kind_suffix``."""
+    n = 0
+    for ev in log.get("events", []) or []:
+        if isinstance(ev, dict) and str(ev.get("kind", "")).endswith(kind_suffix):
+            n += 1
+    return n
+
+
+def _derive_one(cell_name: str, log: Dict[str, Any]) -> Optional[int]:
+    """Derive tool_calls for a single task log. ``None`` if undetermined."""
+    md = log.get("metadata", {}) or {}
+    tr = (md.get("traces") or {}) if isinstance(md, dict) else {}
+    turns = int(md.get("turns") or 0)
+
+    if cell_name.startswith("cloud-only-"):
+        # baseline_cloud
+        if tr.get("mode") in ("anthropic_agent_loop",):
+            return int(tr.get("n_web_searches") or 0)
+        if tr.get("mode") == "one_shot":
+            return 0
+        if tr.get("backbone") == "cloud":
+            # SWE branch: turns == LLM turns; matches the post-7f432453
+            # convention (`tool_calls = int(out["turns"])`).
+            return turns
+        # GAIA one-shot fall-through
+        if tr.get("is_swe") is False:
+            return 0
+        return turns if turns > 0 else 0
+
+    if cell_name.startswith("advisors-"):
+        if tr.get("swe_mode"):
+            # initial_out["turns"] + 1 (advisor) + final_out["turns"] = turns
+            # post-7f432453 stores tool_calls = initial+final = turns - 1.
+            return max(0, turns - 1)
+        # GAIA: newer cells (post commit 24da67d2) ran web_search and
+        # logged ``n_web_searches`` in traces. Older cells did neither.
+        if tr.get("web_search_enabled"):
+            return int(tr.get("n_web_searches") or 0)
+        return 0
+
+    if cell_name.startswith("minions-"):
+        if tr.get("swe_mode"):
+            # 1 (supervisor) + worker bash-agent turns = turns.
+            return max(0, turns - 1)
+        # GAIA: prefetch n_searches is the only tool surface.
+        prefetch = tr.get("prefetch") or {}
+        return int(prefetch.get("n_searches") or 0)
+
+    if cell_name.startswith("skillorchestra-"):
+        # Old single-file version routes via `chosen_agent`; SWE goes
+        # through run_swe_agent_loop with trace_prefix=skillorch_(local|cloud).
+        if "chosen_agent" in tr:
+            # Count `*_turn` events specific to the SWE loop. The
+            # ``skillorchestra_route`` and bare ``anthropic`` events are
+            # the routing call, NOT a tool turn.
+            n_local = _count_event_kind(log, "skillorch_local_turn")
+            n_cloud = _count_event_kind(log, "skillorch_cloud_turn")
+            n = n_local + n_cloud
+            if n > 0:
+                return n
+            # GAIA path: one-shot routed call, no tools.
+            return 0
+
+    return None
+
+
+def _derive_cell(cell_dir: Path) -> Tuple[int, int, List[str], str]:
+    """Sum tool_calls across all per-task evidence in ``cell_dir/``.
+
+    Prefers the canonical ``results.jsonl`` ``tool_calls`` field
+    (post-7f432453 wiring writes it directly). Falls back to per-task
+    log derivation for pre-wiring cells.
+
+    Returns ``(total, n_tasks_counted, warnings, source)`` where ``source``
+    is ``"results.jsonl"``, ``"logs"``, or ``"none"``.
+    """
+    cell = cell_dir.name
+    warnings: List[str] = []
+
+    # Source 1: results.jsonl with tool_calls field (canonical).
+    res_p = cell_dir / "results.jsonl"
+    rows: List[Dict[str, Any]] = []
+    if res_p.exists():
+        try:
+            rows = [json.loads(line) for line in res_p.read_text().splitlines() if line.strip()]
+        except Exception as e:  # noqa: BLE001
+            warnings.append(f"{cell}: failed to parse results.jsonl ({e})")
+            rows = []
+        if rows and all("tool_calls" in r for r in rows):
+            return sum(int(r.get("tool_calls") or 0) for r in rows), len(rows), warnings, "results.jsonl"
+
+    # Source 2: per-task log files (pre-wiring fallback).
+    logs_dir = cell_dir / "logs"
+    have_logs = logs_dir.is_dir() and any(logs_dir.glob("*.json"))
+    if have_logs:
+        total = 0
+        n = 0
+        for log_path in sorted(logs_dir.glob("*.json")):
+            log = _load_log(log_path)
+            if log is None:
+                warnings.append(f"{cell}: failed to parse {log_path.name}")
+                continue
+            v = _derive_one(cell, log)
+            if v is None:
+                warnings.append(
+                    f"{cell}: undetermined for {log_path.stem} "
+                    f"(traces.keys={sorted((log.get('metadata') or {}).get('traces', {}).keys())})"
+                )
+                continue
+            total += v
+            n += 1
+        if n > 0:
+            return total, n, warnings, "logs"
+
+    # Source 3: legacy results.jsonl `traces.n_searches` (pre-mini-SWE-agent
+    # cells stored the GAIA web_search count under this key, before the
+    # tool_calls schema landed). Only safe for cells that actually used
+    # web_search (advisors/cloud GAIA cells); for SWE the field is
+    # spurious. Heuristic: name contains "gaia" => use it.
+    if rows and "gaia" in cell.lower() and any(
+        "n_searches" in (r.get("traces") or {}) for r in rows
+    ):
+        total = sum(int((r.get("traces") or {}).get("n_searches") or 0) for r in rows)
+        return total, len(rows), warnings, "results.jsonl(legacy n_searches)"
+
+    return 0, 0, warnings + [f"{cell}: no derivable evidence"], "none"
+
+
+def _load_summary(p: Path) -> Optional[Dict[str, Any]]:
+    try:
+        return json.loads(p.read_text())
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def _save_summary(p: Path, d: Dict[str, Any]) -> None:
+    tmp = p.with_suffix(p.suffix + ".tmp")
+    tmp.write_text(json.dumps(d, indent=2))
+    tmp.replace(p)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dry-run", action="store_true", help="Don't write summaries.")
+    ap.add_argument("--force", action="store_true",
+                    help="Overwrite existing tool_calls_total too.")
+    ap.add_argument("--glob", default="*-n100",
+                    help="Cell directory glob (default: *-n100).")
+    ap.add_argument("--runs-dir", default=str(HYBRID_RUNS),
+                    help="hybrid runs root dir.")
+    args = ap.parse_args()
+
+    root = Path(args.runs_dir)
+    rows: List[Tuple[str, Any, Any, str, int]] = []
+    all_warnings: List[str] = []
+
+    for cell_dir in sorted(root.glob(args.glob)):
+        if not cell_dir.is_dir():
+            continue
+        summ_p = cell_dir / "summary.json"
+        if not summ_p.exists():
+            continue
+        summ = _load_summary(summ_p)
+        if summ is None:
+            all_warnings.append(f"{cell_dir.name}: failed to parse summary.json")
+            continue
+        before = summ.get("tool_calls_total", None)
+        derived, n_tasks, warnings, source = _derive_cell(cell_dir)
+        all_warnings.extend(warnings)
+
+        action = ""
+        if before is None:
+            if n_tasks == 0:
+                action = "skip (no derivable evidence)"
+            else:
+                action = f"backfill -> {derived} (from {n_tasks} {source})"
+                if not args.dry_run:
+                    summ["tool_calls_total"] = int(derived)
+                    _save_summary(summ_p, summ)
+        elif args.force and isinstance(before, int) and before != derived and n_tasks > 0:
+            action = f"force-update {before} -> {derived} (from {source})"
+            if not args.dry_run:
+                summ["tool_calls_total"] = int(derived)
+                _save_summary(summ_p, summ)
+        elif n_tasks > 0 and isinstance(before, int) and before != derived:
+            action = f"mismatch (have {before}, {source} sum {derived}) — keep, use --force to overwrite"
+        else:
+            action = f"already has {before}"
+
+        rows.append((cell_dir.name, before, derived, action, n_tasks))
+
+    # Print table
+    col1 = max(len(r[0]) for r in rows) if rows else 10
+    print(f"{'cell'.ljust(col1)}  {'before':>8}  {'logs_n':>6}  {'derived':>8}  action")
+    print("-" * (col1 + 40))
+    for name, before, derived, action, n_tasks in rows:
+        b = "—" if before is None else str(before)
+        print(f"{name.ljust(col1)}  {b:>8}  {n_tasks:>6}  {derived:>8}  {action}")
+
+    if all_warnings:
+        print(f"\n{len(all_warnings)} warnings:")
+        for w in all_warnings[:50]:
+            print(f"  ! {w}")
+        if len(all_warnings) > 50:
+            print(f"  ... and {len(all_warnings) - 50} more")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ablation/make_report.py

@@ -0,0 +1,561 @@
+#!/usr/bin/env python3
+"""Build ``$OPENJARVIS_HYBRID_EXPERIMENTS_DIR/docs/index.html`` plus a set
+of PNG pareto plots in the same dir. The HTML embeds the PNGs via
+``<img src="...">`` so the browser never has to render anything itself —
+just shows static images.
+
+Re-running is idempotent: regenerates index.html + all PNGs from whatever
+cells have summary.json at run-time.
+"""
+from __future__ import annotations
+
+import html as html_lib
+import json
+import os
+import sys
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from statistics import median
+from typing import Callable, Optional
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+HYBRID_ROOT = Path(
+    os.environ.get(
+        "OPENJARVIS_HYBRID_EXPERIMENTS_DIR",
+        Path.home() / ".openjarvis" / "experiments" / "hybrid",
+    )
+)
+RUNS_DIR = HYBRID_ROOT / "runs"
+DOCS_DIR = HYBRID_ROOT / "docs"
+OUT_HTML = DOCS_DIR / "index.html"
+RAW_MD = DOCS_DIR / "results-table.md"
+PLOTS_DIR = DOCS_DIR / "plots-n100"
+PLOTS_DIR.mkdir(exist_ok=True)
+
+CLOUD_TOKENS = {
+    "opus47": "Opus 4.7",
+    "haiku45": "Haiku 4.5",
+    "gpt5mini": "GPT-5 mini",
+    "gpt5": "GPT-5.5",
+    "gemini25pro": "Gemini 3.1 Pro",
+    "gemini25flash": "Gemini 3.1 Flash",
+}
+CLOUD_TOKEN_ORDER = [
+    "gpt5mini", "gemini25flash", "gemini25pro", "gpt5", "opus47", "haiku45",
+]
+BENCH_LABELS = {"gaia": "GAIA", "swe": "SWE-bench"}
+
+PARADIGM_COLOR = {
+    "skillorchestra": "#3b82f6",
+    "cloud-only":     "#9ca3af",
+    "minions":        "#10b981",
+    "advisors":       "#f59e0b",
+}
+PARADIGM_ORDER = ["cloud-only", "skillorchestra", "minions", "advisors"]
+
+
+def parse_cell_name(name: str) -> Optional[dict]:
+    if not name.endswith("-n100"):
+        return None
+    parts = name[:-len("-n100")].split("-")
+    if not parts:
+        return None
+    paradigm = parts[0]
+    rest = parts[1:]
+    if paradigm == "cloud":
+        if rest and rest[0] == "only":
+            rest = rest[1:]
+        paradigm = "cloud-only"
+        local = None
+    elif paradigm in ("skillorchestra", "minions", "advisors"):
+        if not rest:
+            return None
+        local = rest[0]
+        rest = rest[1:]
+    else:
+        return None
+    if len(rest) < 2:
+        return None
+    cloud_token = None
+    for tok in CLOUD_TOKEN_ORDER:
+        if tok in rest:
+            cloud_token = tok
+            break
+    if cloud_token is None:
+        return None
+    bench = rest[-1]
+    if bench not in BENCH_LABELS:
+        return None
+    return {"paradigm": paradigm, "local": local, "cloud": cloud_token, "bench": bench}
+
+
+@dataclass
+class Cell:
+    name: str
+    paradigm: str
+    local: Optional[str]
+    cloud: str
+    bench: str
+    accuracy: float
+    cost_usd: float
+    tokens_local_total: int
+    tokens_cloud_total: int
+    n_done: int
+    latency_med_s: Optional[float]
+    tool_calls_mean: Optional[float]
+    web_searches_mean: Optional[float]
+
+
+def load_cells() -> list[Cell]:
+    out = []
+    for d in sorted(RUNS_DIR.iterdir()):
+        if not (d.is_dir() and d.name.endswith("-n100")):
+            continue
+        sj = d / "summary.json"
+        if not sj.exists():
+            continue
+        parsed = parse_cell_name(d.name)
+        if parsed is None:
+            continue
+        try:
+            s = json.loads(sj.read_text())
+        except Exception:
+            continue
+        # latency median
+        rj = d / "results.jsonl"
+        lat_med = None
+        if rj.exists():
+            lats = []
+            for line in rj.read_text().splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    row = json.loads(line)
+                except Exception:
+                    continue
+                v = row.get("latency_s")
+                if isinstance(v, (int, float)):
+                    lats.append(float(v))
+            if lats:
+                lat_med = median(lats)
+        # tool calls mean — count any flavor of tool use per task:
+        #   * SWE / mini-swe-agent fires events with kind="<something>_bash" (one per bash exec)
+        #   * GAIA / agentic flows record tool calls inline on anthropic/openai events
+        #     as `tool_calls: [...]` (anthropic server_tool_use, openai function calls)
+        #     and Anthropic-side web search uses `n_web_searches`
+        #   * minions does a hidden pre-fetch step recorded under metadata.traces.prefetch.n_searches
+        # Sum all of them so the metric isn't silently zero on GAIA.
+        #
+        # `web_searches_mean` is the same idea but ONLY counts Anthropic
+        # server-side web_search invocations — useful to see how often
+        # GAIA cells actually leveraged the new opt-in web_search tool.
+        tc_mean = None
+        ws_mean = None
+        logs = d / "logs"
+        if logs.is_dir():
+            counts = []
+            ws_counts = []
+            for log_p in logs.iterdir():
+                if not log_p.name.endswith(".json"):
+                    continue
+                try:
+                    lg = json.loads(log_p.read_text())
+                except Exception:
+                    continue
+                ev = lg.get("events") or []
+                bash_c = sum(1 for e in ev if isinstance(e, dict)
+                             and isinstance(e.get("kind"), str)
+                             and "_bash" in e["kind"])
+                tc_c = sum(len(e.get("tool_calls") or [])
+                           for e in ev if isinstance(e, dict))
+                ws_c = sum(int(e.get("n_web_searches") or 0)
+                           for e in ev if isinstance(e, dict))
+                # n_web_searches is Anthropic's server-side web_search count; it's
+                # already represented as one tool_call per search above, so don't
+                # double-count — only fall back to it if tool_calls list is empty.
+                if tc_c == 0:
+                    tc_c = ws_c
+                # prefetch (minions) — recorded out-of-band in metadata.traces.prefetch
+                meta = lg.get("metadata") or {}
+                pf = ((meta.get("traces") or {}).get("prefetch") or {}) if isinstance(meta, dict) else {}
+                pf_c = int(pf.get("n_searches") or 0) if isinstance(pf, dict) else 0
+                counts.append(bash_c + tc_c + pf_c)
+                ws_counts.append(ws_c + pf_c)
+            if counts:
+                tc_mean = sum(counts) / len(counts)
+            if ws_counts:
+                ws_mean = sum(ws_counts) / len(ws_counts)
+        out.append(Cell(
+            name=d.name,
+            paradigm=parsed["paradigm"],
+            local=parsed["local"],
+            cloud=parsed["cloud"],
+            bench=parsed["bench"],
+            accuracy=float(s.get("accuracy") or 0.0),
+            cost_usd=float(s.get("cost_usd_total") or 0.0),
+            tokens_local_total=int(s.get("tokens_local_total") or 0),
+            tokens_cloud_total=int(s.get("tokens_cloud_total") or 0),
+            n_done=int(s.get("n_done") or 0),
+            latency_med_s=lat_med,
+            tool_calls_mean=tc_mean,
+            web_searches_mean=ws_mean,
+        ))
+    return out
+
+
+@dataclass
+class Axis:
+    key: str
+    title: str
+    description: str
+    filter_fn: Callable[[Cell], bool]
+
+
+def axis_definitions() -> list[Axis]:
+    return [
+        Axis("all-cells", "0. All cells overview",
+             "Every cell on one plot. Color = paradigm. Pareto frontier at a glance.",
+             lambda c: True),
+        Axis("cloud-anthropic", "1. Cloud-size within Anthropic",
+             "Local = Qwen-27B; vary Anthropic cloud (Opus 4.7 vs Haiku 4.5). All paradigms overlaid.",
+             lambda c: c.cloud in ("opus47", "haiku45")),
+        Axis("cloud-openai", "2. Cloud-size within OpenAI",
+             "Local = Qwen-27B; vary OpenAI cloud (GPT-5.5 vs GPT-5 mini). All paradigms overlaid.",
+             lambda c: c.cloud in ("gpt5", "gpt5mini")),
+        Axis("cloud-google", "3. Cloud-size within Google",
+             "Local = Qwen-27B; vary Google cloud (Gemini 3.1 Pro vs Flash). All paradigms overlaid.",
+             lambda c: c.cloud in ("gemini25pro", "gemini25flash")),
+        Axis("cloud-family-frontier", "4. Cloud-family — frontier tier",
+             "Compare frontier clouds across vendors (Opus 4.7, GPT-5.5, Gemini 3.1 Pro).",
+             lambda c: c.cloud in ("opus47", "gpt5", "gemini25pro")),
+        Axis("cloud-family-mini", "5. Cloud-family — mini/flash tier",
+             "Compare cost-floor clouds across vendors (Haiku 4.5, GPT-5 mini, Gemini 3.1 Flash).",
+             lambda c: c.cloud in ("haiku45", "gpt5mini", "gemini25flash")),
+        Axis("paradigm-skillorch", "6. Skillorchestra only",
+             "Just the skillorchestra cells. See how its router behaves across cloud choices.",
+             lambda c: c.paradigm == "skillorchestra"),
+        Axis("paradigm-cloud-only", "7. Cloud-only baseline",
+             "Baseline cloud-only runs across all 6 clouds — no local model in the loop.",
+             lambda c: c.paradigm == "cloud-only"),
+    ]
+
+
+METRIC_SPECS = [
+    ("cost_usd",           "Cost (USD)",            True),
+    ("latency_med_s",      "Latency median (s)",    True),
+    ("tokens_cloud_total", "Tokens cloud (total)",  True),
+]
+
+
+def render_axis_png(axis: Axis, cells: list[Cell]) -> Optional[Path]:
+    pts = [c for c in cells if axis.filter_fn(c)]
+    if not pts:
+        return None
+    n_metrics = len(METRIC_SPECS)
+    fig, axes = plt.subplots(
+        n_metrics, 2,
+        figsize=(15, 4.2 * n_metrics),
+        squeeze=False,
+    )
+    fig.suptitle(axis.title, fontsize=16, fontweight="bold", y=0.998)
+    used_paradigms = sorted({c.paradigm for c in pts}, key=lambda p: PARADIGM_ORDER.index(p) if p in PARADIGM_ORDER else 99)
+
+    for row_idx, (mkey, mlabel, logx) in enumerate(METRIC_SPECS):
+        for col_idx, bench in enumerate(("gaia", "swe")):
+            ax = axes[row_idx][col_idx]
+            bench_pts = [c for c in pts if c.bench == bench]
+            for p in used_paradigms:
+                p_pts = [c for c in bench_pts if c.paradigm == p]
+                xs, ys, labels = [], [], []
+                for c in p_pts:
+                    v = getattr(c, mkey)
+                    if v is None:
+                        continue
+                    if logx and v <= 0:
+                        continue
+                    xs.append(v); ys.append(c.accuracy); labels.append(CLOUD_TOKENS.get(c.cloud, c.cloud))
+                if xs:
+                    ax.scatter(xs, ys, c=PARADIGM_COLOR.get(p, "#000"),
+                               s=120, alpha=0.85, edgecolors="white", linewidths=1.5,
+                               label=p if (row_idx == 0 and col_idx == 0) else None)
+                    for x, y, lbl in zip(xs, ys, labels):
+                        ax.annotate(lbl, (x, y), xytext=(6, 6), textcoords="offset points",
+                                    fontsize=9, color="#1f2937", alpha=0.9)
+            if logx:
+                ax.set_xscale("log")
+            # Generous y-axis: 0 → max(observed)+0.15, floored at 0.7 so small numbers don't look cramped
+            ax.set_ylim(0, max(0.75, max((c.accuracy for c in bench_pts), default=0.5) + 0.15))
+            ax.set_xlabel(mlabel, fontsize=10)
+            ax.set_ylabel("accuracy" if col_idx == 0 else "")
+            ax.set_title(f"{mlabel} — {BENCH_LABELS[bench]}", fontsize=11, fontweight="bold")
+            ax.grid(True, color="#e5e7eb", linewidth=0.5)
+            ax.set_axisbelow(True)
+            for spine in ax.spines.values():
+                spine.set_color("#d1d5db")
+
+    # legend on top
+    handles, labels = axes[0][0].get_legend_handles_labels()
+    if handles:
+        fig.legend(handles, labels, loc="upper right", bbox_to_anchor=(0.99, 0.99),
+                   ncol=len(used_paradigms), frameon=True, facecolor="white",
+                   edgecolor="#d1d5db", fontsize=10)
+    plt.subplots_adjust(left=0.07, right=0.97, top=0.96, bottom=0.04,
+                        hspace=0.55, wspace=0.18)
+    out = PLOTS_DIR / f"{axis.key}.png"
+    fig.savefig(out, dpi=110, bbox_inches="tight")
+    plt.close(fig)
+    return out
+
+
+# ---------- Markdown → HTML (small) ----------
+
+def md_to_html(md: str) -> str:
+    lines = md.splitlines()
+    out: list[str] = []
+    in_table = False
+    table_rows: list[list[str]] = []
+
+    def flush_table():
+        nonlocal table_rows
+        if not table_rows:
+            return
+        head = table_rows[0]
+        body = table_rows[2:] if len(table_rows) > 2 else []
+        out.append("<table class='md'><thead><tr>" +
+                   "".join(f"<th>{html_lib.escape(h.strip())}</th>" for h in head) +
+                   "</tr></thead><tbody>")
+        for row in body:
+            out.append("<tr>" + "".join(
+                f"<td>{html_lib.escape(cell.strip())}</td>" for cell in row) + "</tr>")
+        out.append("</tbody></table>")
+        table_rows = []
+
+    def inline(s: str) -> str:
+        s = html_lib.escape(s)
+        # bold
+        import re as _re
+        s = _re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", s)
+        s = _re.sub(r"`(.+?)`", r"<code>\1</code>", s)
+        return s
+
+    for raw in lines:
+        line = raw.rstrip()
+        if line.startswith("|") and "|" in line[1:]:
+            if not in_table:
+                in_table = True
+                table_rows = []
+            cells = [c for c in line.strip().strip("|").split("|")]
+            table_rows.append(cells)
+            continue
+        if in_table:
+            flush_table()
+            in_table = False
+        if line.startswith("### "):
+            out.append(f"<h3>{inline(line[4:])}</h3>")
+        elif line.startswith("## "):
+            out.append(f"<h2>{inline(line[3:])}</h2>")
+        elif line.startswith("# "):
+            out.append(f"<h1>{inline(line[2:])}</h1>")
+        elif line.startswith("> "):
+            out.append(f"<blockquote>{inline(line[2:])}</blockquote>")
+        elif line.startswith("- "):
+            out.append(f"<li>{inline(line[2:])}</li>")
+        elif line.strip() == "---":
+            out.append("<hr>")
+        elif line.strip() == "":
+            out.append("")
+        else:
+            out.append(f"<p>{inline(line)}</p>")
+    if in_table:
+        flush_table()
+    return "\n".join(out)
+
+
+# ---------- HTML output ----------
+
+CSS = """
+body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+       background: #fafbfc; color: #1a1a1a; max-width: 1280px; margin: 0 auto;
+       padding: 24px 16px 60px; line-height: 1.55; }
+h1 { border-bottom: 2px solid #1a1a1a; padding-bottom: 8px; }
+h2 { margin-top: 40px; border-bottom: 1px solid #d1d5db; padding-bottom: 6px; }
+.card { background: white; border: 1px solid #e5e7eb; border-radius: 8px;
+        box-shadow: 0 1px 3px rgba(0,0,0,0.05); padding: 16px 20px; margin: 18px 0; }
+.hero { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+        gap: 12px; margin: 16px 0 24px; }
+.hero .item { background: white; border: 1px solid #e5e7eb; border-left: 3px solid #3b82f6;
+              border-radius: 6px; padding: 12px 14px; }
+.hero .item.cost { border-left-color: #10b981; }
+.hero .label { font-size: 0.78em; color: #6b7280; text-transform: uppercase; letter-spacing: 0.5px; }
+.hero .value { font-size: 1.15em; font-weight: 600; margin-top: 4px; }
+.hero .sub { font-size: 0.85em; color: #4b5563; margin-top: 2px; }
+table.summary { width: 100%; border-collapse: collapse; font-size: 0.85em; margin-top: 12px; }
+table.summary th, table.summary td { border: 1px solid #e5e7eb; padding: 6px 8px; text-align: left; }
+table.summary th { background: #f3f4f6; cursor: pointer; user-select: none; }
+table.summary tr:nth-child(even) { background: #fafafa; }
+table.summary .best { background: #dcfce7 !important; }
+section { margin-top: 28px; }
+section img { max-width: 100%; height: auto; display: block; margin: 8px 0;
+              border: 1px solid #e5e7eb; border-radius: 6px; background: white; }
+.legend { font-size: 0.85em; color: #6b7280; }
+.legend .sw { display: inline-block; width: 11px; height: 11px; border-radius: 50%;
+              vertical-align: middle; margin-right: 4px; }
+.desc { color: #4b5563; }
+table.md { width: 100%; border-collapse: collapse; font-size: 0.85em; margin: 12px 0; }
+table.md th, table.md td { border: 1px solid #e5e7eb; padding: 4px 8px; text-align: left; }
+table.md th { background: #f3f4f6; }
+table.md tr:nth-child(even) { background: #fafafa; }
+code { background: #f3f4f6; padding: 1px 4px; border-radius: 3px; font-size: 0.92em; }
+hr { border: none; border-top: 1px solid #d1d5db; margin: 20px 0; }
+blockquote { border-left: 3px solid #d1d5db; padding-left: 12px; color: #4b5563; margin: 8px 0; }
+"""
+
+SORT_JS = """
+document.querySelectorAll('table.summary th').forEach((th, idx) => {
+  th.addEventListener('click', () => {
+    const tbl = th.closest('table');
+    const tbody = tbl.querySelector('tbody');
+    const rows = Array.from(tbody.querySelectorAll('tr'));
+    const asc = th.dataset.dir !== 'asc';
+    rows.sort((a, b) => {
+      const av = a.cells[idx].dataset.sort ?? a.cells[idx].innerText;
+      const bv = b.cells[idx].dataset.sort ?? b.cells[idx].innerText;
+      const af = parseFloat(av), bf = parseFloat(bv);
+      const an = isNaN(af) ? av : af, bn = isNaN(bf) ? bv : bf;
+      return (an < bn ? -1 : an > bn ? 1 : 0) * (asc ? 1 : -1);
+    });
+    rows.forEach(r => tbody.appendChild(r));
+    tbl.querySelectorAll('th').forEach(t => delete t.dataset.dir);
+    th.dataset.dir = asc ? 'asc' : 'desc';
+  });
+});
+"""
+
+
+def build_html(cells: list[Cell], plots: list[tuple[Axis, Path]]) -> str:
+    ts = datetime.now().strftime("%Y-%m-%d %H:%M")
+    # headline cards
+    by_bench = {"gaia": [c for c in cells if c.bench == "gaia"],
+                "swe":  [c for c in cells if c.bench == "swe"]}
+    def best_acc(lst):
+        return max(lst, key=lambda c: c.accuracy) if lst else None
+    def best_acc_per_dollar(lst):
+        cand = [c for c in lst if c.cost_usd > 0]
+        return max(cand, key=lambda c: c.accuracy / c.cost_usd) if cand else None
+
+    hero = ['<div class="hero">']
+    for bench_key, label in (("gaia", "GAIA"), ("swe", "SWE-bench")):
+        b = best_acc(by_bench[bench_key])
+        if b:
+            hero.append(f"""<div class="item"><div class="label">Best acc — {label}</div>
+                <div class="value">{b.accuracy:.3f} · {b.paradigm} × {CLOUD_TOKENS.get(b.cloud, b.cloud)}</div>
+                <div class="sub">cost: ${b.cost_usd:.2f}</div></div>""")
+    for bench_key, label in (("gaia", "GAIA"), ("swe", "SWE-bench")):
+        b = best_acc_per_dollar(by_bench[bench_key])
+        if b:
+            ratio = b.accuracy / b.cost_usd
+            hero.append(f"""<div class="item cost"><div class="label">Best acc / $ — {label}</div>
+                <div class="value">{b.accuracy:.3f} for ${b.cost_usd:.2f}</div>
+                <div class="sub">{b.paradigm} × {CLOUD_TOKENS.get(b.cloud, b.cloud)} ({ratio:.3f} acc/$)</div></div>""")
+    hero.append("</div>")
+
+    # summary table
+    legend_html = '<div class="legend">'
+    for p in PARADIGM_ORDER:
+        col = PARADIGM_COLOR.get(p, "#000")
+        legend_html += f'<span class="sw" style="background:{col}"></span>{p}&nbsp;&nbsp;'
+    legend_html += '</div>'
+
+    rows_html = []
+    sorted_cells = sorted(cells, key=lambda c: (c.bench, -c.accuracy))
+    # mark best per bench
+    best_per_bench = {b: best_acc(by_bench[b]) for b in by_bench}
+    for c in sorted_cells:
+        is_best = best_per_bench.get(c.bench) is c
+        cls = "best" if is_best else ""
+        rows_html.append(f"""<tr class="{cls}">
+            <td>{c.paradigm}</td>
+            <td>{c.local or '—'}</td>
+            <td>{CLOUD_TOKENS.get(c.cloud, c.cloud)}</td>
+            <td>{BENCH_LABELS[c.bench]}</td>
+            <td data-sort="{c.accuracy}">{c.accuracy:.3f}</td>
+            <td data-sort="{c.cost_usd}">${c.cost_usd:.2f}</td>
+            <td data-sort="{c.latency_med_s or 0}">{('%.1f' % c.latency_med_s + 's') if c.latency_med_s else '—'}</td>
+            <td data-sort="{c.tool_calls_mean if c.tool_calls_mean is not None else 0}">{('%.1f' % c.tool_calls_mean) if c.tool_calls_mean is not None else '—'}</td>
+            <td data-sort="{c.web_searches_mean if c.web_searches_mean is not None else 0}">{('%.2f' % c.web_searches_mean) if c.web_searches_mean is not None else '—'}</td>
+            <td data-sort="{c.tokens_local_total}">{c.tokens_local_total:,}</td>
+            <td data-sort="{c.tokens_cloud_total}">{c.tokens_cloud_total:,}</td>
+        </tr>""")
+    summary_table = f"""<table class="summary"><thead><tr>
+      <th>paradigm</th><th>local</th><th>cloud</th><th>bench</th>
+      <th>accuracy ↕</th><th>cost ↕</th><th>latency_med ↕</th><th>tool_calls ↕</th>
+      <th>web_searches ↕</th>
+      <th>tokens_local ↕</th><th>tokens_cloud ↕</th>
+    </tr></thead><tbody>{''.join(rows_html)}</tbody></table>"""
+
+    # axis sections (embed PNGs)
+    axis_sections = []
+    for axis, png_path in plots:
+        rel = f"plots-n100/{png_path.name}"
+        axis_sections.append(f"""<section>
+            <h2>{html_lib.escape(axis.title)}</h2>
+            <p class="desc">{html_lib.escape(axis.description)}</p>
+            <img src="{rel}" alt="{axis.title}">
+        </section>""")
+
+    md_html = ""
+    if RAW_MD.exists():
+        md_html = md_to_html(RAW_MD.read_text())
+
+    return f"""<!doctype html>
+<html><head><meta charset="utf-8">
+<title>OpenJarvis Hybrid n=100 Ablation</title>
+<style>{CSS}</style>
+</head><body>
+<h1>OpenJarvis Hybrid n=100 Ablation</h1>
+<p class="desc">Comparing local-cloud paradigms across 6 cloud models on GAIA + SWE-bench-Verified · {len(cells)} cells · generated {ts}</p>
+
+<div class="card">
+<h2 style="border:0; margin-top:0">Headline findings</h2>
+{''.join(hero)}
+</div>
+
+<div class="card">
+<h2 style="border:0; margin-top:0">All cells (sortable)</h2>
+{legend_html}
+{summary_table}
+</div>
+
+{''.join(axis_sections)}
+
+<div class="card">
+<h2 style="border:0; margin-top:0">results-table.md (full)</h2>
+{md_html}
+</div>
+
+<script>{SORT_JS}</script>
+</body></html>
+"""
+
+
+def main():
+    cells = load_cells()
+    print(f"loaded {len(cells)} cells")
+    plots = []
+    for axis in axis_definitions():
+        png = render_axis_png(axis, cells)
+        if png is not None:
+            plots.append((axis, png))
+            print(f"  rendered {png.name}")
+    html = build_html(cells, plots)
+    OUT_HTML.write_text(html)
+    print(f"wrote {OUT_HTML} ({len(html):,} bytes)")
+
+
+if __name__ == "__main__":
+    main()

scripts/ablation/rescore_gaia.py

@@ -0,0 +1,350 @@
+"""Re-grade GAIA cells whose answers scored 0 under the old regex scorer.
+
+The old ``runner._score_gaia`` only credited answers that emitted a literal
+``FINAL ANSWER:`` line and exact-string-matched it. A verbose answer that
+states the right answer in prose ("...therefore the answer is 4") silently
+scored 0. Opus emits the marker ~92% of the time; GPT-5-mini / Haiku almost
+never do, so their GAIA cells were badly undercounted.
+
+The agents already produced correct answers in ``<cell>/results.jsonl`` —
+only the grading step was broken. This script re-scores every non-error row
+with the proper ``GAIAScorer`` (normalized exact-match + LLM-judge fallback)
+and rewrites the row's ``score``. Error rows are left untouched (they really
+did fail). ``summary.json`` accuracy is recomputed.
+
+Usage:
+    source .env   # OpenAI key for the judge
+    .venv/bin/python scripts/ablation/rescore_gaia.py --all-gaia
+    .venv/bin/python scripts/ablation/rescore_gaia.py \\
+        --cells minions-qwen27b-haiku45-gaia-n100,minions-qwen27b-gpt5-gaia-n100
+
+Idempotent: each cell writes ``_rescored_gaia_ids.txt`` listing task_ids
+already rescored; reruns skip those. A one-time ``results.jsonl.bak-gaia*``
+backup is made before the first rewrite.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from threading import Lock
+from typing import Any, Dict, List, Optional, Tuple
+
+# Make `openjarvis` importable when running this script directly.
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(_REPO_ROOT / "src"))
+
+from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend  # noqa: E402
+from openjarvis.evals.core.types import EvalRecord  # noqa: E402
+from openjarvis.evals.datasets.gaia import GAIADataset  # noqa: E402
+from openjarvis.evals.scorers.gaia_exact import GAIAScorer  # noqa: E402
+
+HYBRID_DIR = Path(os.path.expanduser("~/.openjarvis/experiments/hybrid"))
+RUNS_DIR = HYBRID_DIR / "runs"
+DOCS_TABLE = HYBRID_DIR / "docs" / "results-table.md"
+
+MAX_WORKERS = 8
+RETRY_ATTEMPTS = 4
+JUDGE_MODEL = os.environ.get("OPENJARVIS_GAIA_JUDGE_MODEL", "gpt-5-mini-2025-08-07")
+PROGRESS_EVERY = 20
+
+
+# ---------- IO helpers ----------
+
+def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def _atomic_write_jsonl(path: Path, rows: List[Dict[str, Any]]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    with tmp.open("w") as f:
+        for r in rows:
+            f.write(json.dumps(r) + "\n")
+    os.replace(tmp, path)
+
+
+def _is_correct(row: Dict[str, Any]) -> bool:
+    sc = row.get("score") or {}
+    if not isinstance(sc, dict):
+        return False
+    return float(sc.get("score", 0) or 0) >= 0.5
+
+
+# ---------- GAIA dataset (task_id -> question, reference) ----------
+
+def _load_gaia_index() -> Dict[str, Tuple[str, str]]:
+    """Map bare GAIA task_id -> (question, reference) over the full val set."""
+    ds = GAIADataset()
+    ds.load()
+    index: Dict[str, Tuple[str, str]] = {}
+    for rec in ds.iter_records():
+        md = rec.metadata or {}
+        task_id = str(md.get("task_id") or rec.record_id)
+        question = str(md.get("question") or rec.problem or "")
+        index[task_id] = (question, str(rec.reference or ""))
+    return index
+
+
+# ---------- Re-score one row with retry ----------
+
+def _rescore_row(
+    scorer: GAIAScorer,
+    task_id: str,
+    question: str,
+    reference: str,
+    answer: str,
+    err_log: Path,
+    err_log_lock: Lock,
+) -> Optional[Dict[str, Any]]:
+    """Re-score one answer. Returns the new score dict, or None if the judge
+    kept failing (caller leaves the row untouched rather than writing a
+    bogus 0)."""
+    record = EvalRecord(
+        record_id=task_id,
+        problem=question,
+        reference=reference,
+        category="agentic",
+        metadata={"task_id": task_id},
+    )
+    last_err = ""
+    for attempt in range(1, RETRY_ATTEMPTS + 1):
+        try:
+            is_correct, details = scorer.score(record, answer or "")
+            details = dict(details or {})
+            # A failed judge call must NOT be written as a real 0.
+            if details.get("match_type") == "llm_fallback_error":
+                last_err = f"attempt={attempt} judge_error: {details.get('error')}"
+                time.sleep(min(2 ** attempt, 30))
+                continue
+            details.setdefault("reference", reference)
+            return {
+                "success": bool(is_correct),
+                "score": 1.0 if is_correct else 0.0,
+                "details": details,
+            }
+        except Exception as exc:  # noqa: BLE001
+            last_err = f"attempt={attempt} {type(exc).__name__}: {exc}"
+            time.sleep(min(2 ** attempt, 30))
+    with err_log_lock:
+        with err_log.open("a") as f:
+            f.write(f"{task_id}\t{last_err}\n")
+    return None
+
+
+# ---------- Per-cell processing ----------
+
+def _process_cell(
+    cell_dir: Path,
+    gaia_index: Dict[str, Tuple[str, str]],
+) -> Dict[str, Any]:
+    name = cell_dir.name
+    results_path = cell_dir / "results.jsonl"
+    if not results_path.exists():
+        print(f"[SKIP] {name} — no results.jsonl", flush=True)
+        return {"cell": name, "skipped": True}
+
+    rows = _read_jsonl(results_path)
+    n_total = len(rows)
+
+    tracker_path = cell_dir / "_rescored_gaia_ids.txt"
+    already_done: set[str] = set()
+    if tracker_path.exists():
+        already_done = {
+            ln.strip() for ln in tracker_path.read_text().splitlines() if ln.strip()
+        }
+
+    old_resolved = sum(1 for r in rows if _is_correct(r))
+    old_acc = old_resolved / n_total if n_total else 0.0
+
+    # Worklist: non-error rows not yet rescored.
+    worklist: List[Tuple[int, str, str, str, str]] = []
+    skipped_err = 0
+    for i, r in enumerate(rows):
+        task_id = str(r.get("task_id") or "")
+        if r.get("error"):
+            skipped_err += 1
+            continue
+        if task_id in already_done:
+            continue
+        question, ref = gaia_index.get(task_id, ("", ""))
+        if not ref:
+            # fall back to the reference the old scorer stored
+            ref = str(((r.get("score") or {}).get("details") or {}).get("reference") or "")
+        worklist.append((i, task_id, question, ref, str(r.get("answer") or "")))
+
+    print(
+        f"[{name}] start: rows={n_total} error_rows={skipped_err} "
+        f"already_rescored={len(already_done)} todo={len(worklist)} "
+        f"old_acc={old_acc:.3f}",
+        flush=True,
+    )
+
+    # One-time backup before the first rewrite.
+    bak = cell_dir / f"results.jsonl.bak-gaiarescore-{int(time.time())}"
+    if worklist and not any(cell_dir.glob("results.jsonl.bak-gaiarescore-*")):
+        bak.write_text(results_path.read_text())
+
+    scorer = GAIAScorer(JarvisDirectBackend(engine_key="cloud"), JUDGE_MODEL)
+    err_log = cell_dir / "_rescore_gaia_errors.log"
+    err_log_lock = Lock()
+    tracker_lock = Lock()
+    rows_lock = Lock()
+    n_done = n_failed = n_flipped = 0
+
+    def _flush_tracker(task_id: str) -> None:
+        with tracker_lock:
+            with tracker_path.open("a") as f:
+                f.write(task_id + "\n")
+            already_done.add(task_id)
+
+    def _snapshot() -> None:
+        with rows_lock:
+            _atomic_write_jsonl(results_path, rows)
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        fut_to_meta = {
+            pool.submit(
+                _rescore_row, scorer, tid, q, ref, ans, err_log, err_log_lock
+            ): (i, tid)
+            for (i, tid, q, ref, ans) in worklist
+        }
+        for fut in as_completed(fut_to_meta):
+            i, tid = fut_to_meta[fut]
+            new_score = fut.result()
+            n_done += 1
+            if new_score is None:
+                n_failed += 1
+            else:
+                was = _is_correct(rows[i])
+                with rows_lock:
+                    rows[i]["score"] = new_score
+                _flush_tracker(tid)
+                if new_score["success"] and not was:
+                    n_flipped += 1
+            if n_done % PROGRESS_EVERY == 0:
+                _snapshot()
+                print(
+                    f"[{name}] rescored={n_done}/{len(worklist)} "
+                    f"newly_correct={n_flipped} judge_failed={n_failed}",
+                    flush=True,
+                )
+
+    _snapshot()
+
+    # Rebuild summary.json accuracy.
+    summary_path = cell_dir / "summary.json"
+    summary: Dict[str, Any] = {}
+    if summary_path.exists():
+        try:
+            summary = json.loads(summary_path.read_text())
+        except Exception:
+            summary = {}
+    new_resolved = sum(1 for r in rows if _is_correct(r))
+    n_done_summary = summary.get("n_done", n_total) or n_total
+    new_acc = new_resolved / n_done_summary if n_done_summary else 0.0
+    summary["accuracy"] = new_acc
+    summary_path.write_text(json.dumps(summary, indent=2))
+
+    print(
+        f"[DONE] {name} acc {old_acc:.3f} -> {new_acc:.3f} "
+        f"(resolved {old_resolved} -> {new_resolved}, +{n_flipped} flipped) "
+        f"judge_failed={n_failed}/{len(worklist)}",
+        flush=True,
+    )
+    return {
+        "cell": name,
+        "old_acc": old_acc,
+        "new_acc": new_acc,
+        "old_resolved": old_resolved,
+        "new_resolved": new_resolved,
+        "judge_failed": n_failed,
+        "n_done": n_done_summary,
+    }
+
+
+# ---------- results-table.md updater (auto-gen section only) ----------
+
+def _update_results_table(summaries: List[Dict[str, Any]]) -> None:
+    """Swap the accuracy in the auto-generated `| `cell` | GAIA | acc · $... `
+    rows. Curated tables (different format) are left for a manual pass."""
+    if not DOCS_TABLE.exists():
+        print(f"[WARN] {DOCS_TABLE} missing — skipping table update")
+        return
+    lines = DOCS_TABLE.read_text().splitlines()
+    updated = 0
+    for s in summaries:
+        if s.get("skipped"):
+            continue
+        needle = f"| `{s['cell']}` | GAIA | "
+        for i, line in enumerate(lines):
+            if line.startswith(needle):
+                rest = line[len(needle):]
+                # rest looks like "0.060 · $15.28 | 21619s | ..."
+                parts = rest.split(" · ", 1)
+                if len(parts) == 2:
+                    lines[i] = f"{needle}{s['new_acc']:.3f} · {parts[1]}"
+                    updated += 1
+                break
+    DOCS_TABLE.write_text("\n".join(lines) + "\n")
+    print(f"[TABLE] updated {updated} auto-gen GAIA rows in {DOCS_TABLE.name}")
+
+
+# ---------- CLI ----------
+
+def _resolve_cells(args: argparse.Namespace) -> List[Path]:
+    if args.all_gaia:
+        cells = sorted(p for p in RUNS_DIR.glob("*-gaia-n100") if p.is_dir())
+        return [c for c in cells if (c / "results.jsonl").exists()]
+    if not args.cells:
+        raise SystemExit("pass --all-gaia or --cells a,b,c")
+    out = []
+    for name in args.cells.split(","):
+        name = name.strip()
+        if name:
+            out.append(RUNS_DIR / name)
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--all-gaia", action="store_true",
+                    help="rescore every *-gaia-n100 cell")
+    ap.add_argument("--cells", help="comma-separated cell names")
+    args = ap.parse_args()
+
+    cells = _resolve_cells(args)
+    print(f"[rescore_gaia] judge={JUDGE_MODEL}  cells={len(cells)}", flush=True)
+    print("[rescore_gaia] loading GAIA index...", flush=True)
+    gaia_index = _load_gaia_index()
+    print(f"[rescore_gaia] GAIA index: {len(gaia_index)} tasks", flush=True)
+
+    summaries: List[Dict[str, Any]] = []
+    for cell_dir in cells:
+        if not cell_dir.exists():
+            print(f"[SKIP] {cell_dir.name} — missing dir", flush=True)
+            continue
+        summaries.append(_process_cell(cell_dir, gaia_index))
+
+    _update_results_table(summaries)
+
+    print("\n=== rescore_gaia summary ===", flush=True)
+    for s in sorted((x for x in summaries if not x.get("skipped")),
+                     key=lambda x: x["new_acc"] - x["old_acc"], reverse=True):
+        delta = s["new_acc"] - s["old_acc"]
+        print(f"  {s['cell']:<46} {s['old_acc']:.3f} -> {s['new_acc']:.3f} "
+              f"({delta:+.3f})  judge_failed={s['judge_failed']}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/ablation/rescore_swe.py

@@ -0,0 +1,372 @@
+"""Re-grade SWE-bench cells whose patches scored 0 due to the old Modal
+cgroup-v2 sandbox bug.
+
+The agents already produced valid patches in ``<cell>/results.jsonl``; only
+the grading step was broken. This script reruns ``_run_harness`` for each
+row whose ``score.details.patch`` is non-empty and rewrites the row with
+the new score. Empty patches are preserved (correctly score 0). Updated
+``summary.json`` has ``accuracy = #(score==1) / n_done`` recomputed.
+
+Usage:
+    .venv/bin/python scripts/ablation/rescore_swe.py \\
+        --cells cloud-only-haiku45-swe-n100,cloud-only-gpt5mini-swe-n100
+
+    .venv/bin/python scripts/ablation/rescore_swe.py --all-swe
+
+Idempotent: each cell writes ``_rescored_ids.txt`` listing task_ids that
+have been successfully rescored; reruns skip those.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from threading import Lock
+from typing import Any, Dict, List, Optional, Tuple
+
+# Make `openjarvis` importable when running this script directly.
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(_REPO_ROOT / "src"))
+
+from openjarvis.evals.scorers.swebench_harness import _run_harness  # noqa: E402
+
+HYBRID_DIR = Path(os.path.expanduser("~/.openjarvis/experiments/hybrid"))
+RUNS_DIR = HYBRID_DIR / "runs"
+DOCS_TABLE = HYBRID_DIR / "docs" / "results-table.md"
+
+MAX_WORKERS = 8
+RETRY_ATTEMPTS = 3
+TIMEOUT_S = 1800
+PROGRESS_EVERY = 10
+
+
+# ---------- IO helpers ----------
+
+def _read_jsonl(path: Path) -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            rows.append(json.loads(line))
+    return rows
+
+
+def _atomic_write_jsonl(path: Path, rows: List[Dict[str, Any]]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    with tmp.open("w") as f:
+        for r in rows:
+            f.write(json.dumps(r) + "\n")
+    os.replace(tmp, path)
+
+
+def _patch_of(row: Dict[str, Any]) -> Optional[str]:
+    sc = row.get("score")
+    if not isinstance(sc, dict):
+        return None
+    det = sc.get("details") or {}
+    p = det.get("patch")
+    if isinstance(p, str) and p.strip():
+        return p
+    return None
+
+
+def _is_correct(row: Dict[str, Any]) -> bool:
+    sc = row.get("score") or {}
+    if not isinstance(sc, dict):
+        return False
+    return float(sc.get("score", 0) or 0) >= 1.0
+
+
+# ---------- Re-score one row with retry ----------
+
+def _rescore_row(
+    task_id: str,
+    patch: str,
+    err_log: Path,
+    err_log_lock: Lock,
+) -> Optional[Dict[str, Any]]:
+    """Call ``_run_harness`` with up to RETRY_ATTEMPTS retries.
+
+    Returns the new score dict ``{"success": bool, "score": float,
+    "details": {...}}`` on success. Returns ``None`` if all attempts
+    failed; the caller leaves the row untouched and logs the error.
+    """
+    last_err = ""
+    for attempt in range(1, RETRY_ATTEMPTS + 1):
+        try:
+            r = _run_harness(task_id, patch, TIMEOUT_S)
+            return {
+                "success": bool(r.get("success", False)),
+                "score": float(r.get("score", 0.0)),
+                "details": r.get("details", {}),
+            }
+        except Exception as exc:  # noqa: BLE001
+            last_err = f"attempt={attempt} {type(exc).__name__}: {exc}"
+            time.sleep(min(2 ** attempt, 30))
+    with err_log_lock:
+        with err_log.open("a") as f:
+            f.write(f"{task_id}\t{last_err}\n")
+    return None
+
+
+# ---------- Per-cell processing ----------
+
+def _process_cell(cell_dir: Path) -> Dict[str, Any]:
+    name = cell_dir.name
+    results_path = cell_dir / "results.jsonl"
+    if not results_path.exists():
+        print(f"[SKIP] {name} — no results.jsonl")
+        return {"cell": name, "skipped": True}
+
+    rows = _read_jsonl(results_path)
+    n_total = len(rows)
+
+    # Pre-load idempotency tracker.
+    tracker_path = cell_dir / "_rescored_ids.txt"
+    already_done: set[str] = set()
+    if tracker_path.exists():
+        already_done = {
+            line.strip() for line in tracker_path.read_text().splitlines()
+            if line.strip()
+        }
+
+    # Old accuracy from current rows.
+    old_resolved = sum(1 for r in rows if _is_correct(r))
+    old_acc = old_resolved / n_total if n_total else 0.0
+
+    # Build worklist of (idx, task_id, patch).
+    worklist: List[Tuple[int, str, str]] = []
+    for i, r in enumerate(rows):
+        task_id = r.get("task_id") or ""
+        patch = _patch_of(r)
+        if not patch:
+            continue
+        if task_id in already_done:
+            continue
+        worklist.append((i, task_id, patch))
+
+    print(
+        f"[{name}] start: rows={n_total} with_patch={sum(1 for r in rows if _patch_of(r))} "
+        f"already_rescored={len(already_done)} todo={len(worklist)} old_acc={old_acc:.3f}"
+    )
+
+    err_log = cell_dir / "_rescore_errors.log"
+    err_log_lock = Lock()
+    tracker_lock = Lock()
+    rows_lock = Lock()
+    n_done = 0
+    n_new_resolved = 0
+    n_failed = 0
+
+    def _flush_tracker(task_id: str) -> None:
+        with tracker_lock:
+            with tracker_path.open("a") as f:
+                f.write(task_id + "\n")
+            already_done.add(task_id)
+
+    # Periodic snapshotting: every PROGRESS_EVERY rescored rows, atomic-write
+    # the updated results.jsonl so partial progress is durable.
+    def _snapshot() -> None:
+        with rows_lock:
+            _atomic_write_jsonl(results_path, rows)
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        fut_to_meta = {
+            pool.submit(_rescore_row, tid, patch, err_log, err_log_lock):
+                (i, tid)
+            for (i, tid, patch) in worklist
+        }
+        for fut in as_completed(fut_to_meta):
+            i, tid = fut_to_meta[fut]
+            new_score = fut.result()
+            if new_score is None:
+                n_failed += 1
+                n_done += 1
+            else:
+                with rows_lock:
+                    old_score = rows[i].get("score") or {}
+                    new_reason = ((new_score.get("details") or {}).get("reason")
+                                  if isinstance(new_score.get("details"), dict) else None)
+                    if new_reason == "no_report":
+                        pass
+                    else:
+                        rows[i]["score"] = new_score
+                _flush_tracker(tid)
+                if new_score["success"]:
+                    n_new_resolved += 1
+                n_done += 1
+
+            if n_done % PROGRESS_EVERY == 0:
+                _snapshot()
+                print(
+                    f"[{name}] rescored={n_done}/{len(worklist)} "
+                    f"new_resolved_so_far={n_new_resolved} failed={n_failed}"
+                )
+
+    # Final snapshot.
+    _snapshot()
+
+    # Rebuild summary.json.
+    summary_path = cell_dir / "summary.json"
+    summary: Dict[str, Any] = {}
+    if summary_path.exists():
+        try:
+            summary = json.loads(summary_path.read_text())
+        except Exception:
+            summary = {}
+    new_resolved_total = sum(1 for r in rows if _is_correct(r))
+    n_done_summary = summary.get("n_done", n_total)
+    new_acc = new_resolved_total / n_done_summary if n_done_summary else 0.0
+    summary["accuracy"] = new_acc
+    summary_path.write_text(json.dumps(summary, indent=2))
+
+    err_share = 0.0
+    if worklist:
+        err_share = n_failed / len(worklist)
+
+    print(
+        f"[DONE] {name} acc_old={old_acc:.3f} -> acc_new={new_acc:.3f} "
+        f"(resolved: {old_resolved} -> {new_resolved_total}) "
+        f"rescore_errors={n_failed}/{len(worklist)} ({err_share*100:.1f}%)"
+    )
+    return {
+        "cell": name,
+        "n_total": n_total,
+        "old_acc": old_acc,
+        "new_acc": new_acc,
+        "old_resolved": old_resolved,
+        "new_resolved": new_resolved_total,
+        "rescored": len(worklist),
+        "failed": n_failed,
+        "cost_usd_total": summary.get("cost_usd_total", 0.0),
+        "wall_time_s": summary.get("wall_time_s", 0.0),
+        "tokens_local_total": summary.get("tokens_local_total", 0),
+        "tokens_cloud_total": summary.get("tokens_cloud_total", 0),
+        "n_done": n_done_summary,
+        "n_target": summary.get("n_target", n_total),
+        "bench": summary.get("bench", "swebench-verified"),
+    }
+
+
+# ---------- results-table.md updater ----------
+
+def _format_table_row(s: Dict[str, Any]) -> str:
+    bench = "SWE-bench"
+    return (
+        f"| `{s['cell']}` | {bench} | "
+        f"{s['new_acc']:.3f} · ${s['cost_usd_total']:.2f} | "
+        f"{int(s['wall_time_s'])}s | tools=— | "
+        f"tokens_local={int(s['tokens_local_total'])} | "
+        f"tokens_cloud={int(s['tokens_cloud_total'])} | "
+        f"{s['n_done']}/{s['n_target']} |"
+    )
+
+
+def _update_results_table(summaries: List[Dict[str, Any]]) -> bool:
+    if not DOCS_TABLE.exists():
+        print(f"[WARN] {DOCS_TABLE} missing — skipping table update")
+        return False
+    text = DOCS_TABLE.read_text()
+    lines = text.splitlines()
+    updated = 0
+    for s in summaries:
+        if s.get("skipped"):
+            continue
+        cell = s["cell"]
+        needle = f"| `{cell}` |"
+        new_row = _format_table_row(s)
+        replaced = False
+        for i, line in enumerate(lines):
+            if line.startswith(needle):
+                lines[i] = new_row
+                replaced = True
+                updated += 1
+                break
+        if not replaced:
+            # Append to end of file if section row missing.
+            lines.append(new_row)
+            updated += 1
+    DOCS_TABLE.write_text("\n".join(lines) + "\n")
+    print(f"[TABLE] updated {updated} rows in {DOCS_TABLE}")
+    return True
+
+
+# ---------- CLI ----------
+
+def _resolve_cells(args: argparse.Namespace) -> List[Path]:
+    if args.all_swe:
+        cells = sorted(p for p in RUNS_DIR.glob("*-swe-n100") if p.is_dir())
+        return [c for c in cells if (c / "results.jsonl").exists()]
+    if not args.cells:
+        raise SystemExit("Provide --cells or --all-swe")
+    out: List[Path] = []
+    for name in args.cells.split(","):
+        name = name.strip()
+        if not name:
+            continue
+        p = RUNS_DIR / name
+        if not p.exists():
+            print(f"[WARN] cell not found: {p}")
+            continue
+        out.append(p)
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--cells", type=str, default="",
+                    help="Comma-separated cell names under ~/.openjarvis/experiments/hybrid/runs/")
+    ap.add_argument("--all-swe", action="store_true",
+                    help="Auto-detect cells matching *-swe-n100")
+    args = ap.parse_args()
+
+    cells = _resolve_cells(args)
+    if not cells:
+        print("No cells to process")
+        return 1
+    print(f"Processing {len(cells)} cell(s): {[c.name for c in cells]}")
+
+    summaries: List[Dict[str, Any]] = []
+    t0 = time.time()
+    for cell in cells:
+        try:
+            summaries.append(_process_cell(cell))
+        except KeyboardInterrupt:
+            print(f"[INTERRUPT] aborting on {cell.name}")
+            raise
+        except Exception as exc:  # noqa: BLE001
+            print(f"[ERROR] {cell.name}: {type(exc).__name__}: {exc}")
+            summaries.append({"cell": cell.name, "error": str(exc)})
+    elapsed = time.time() - t0
+
+    # Final summary table.
+    print("\n=== FINAL SUMMARY ===")
+    print(f"{'cell':<48} {'old_acc':>8} {'new_acc':>8} {'resolved':>14} {'failed':>8}")
+    total_old = 0
+    total_new = 0
+    for s in summaries:
+        if "error" in s or s.get("skipped"):
+            print(f"{s['cell']:<48} {'-':>8} {'-':>8} {'-':>14} -")
+            continue
+        print(
+            f"{s['cell']:<48} {s['old_acc']:>8.3f} {s['new_acc']:>8.3f} "
+            f"{s['old_resolved']:>5} -> {s['new_resolved']:<5} {s['failed']:>8}"
+        )
+        total_old += s["old_resolved"]
+        total_new += s["new_resolved"]
+    print(f"\nTotal resolved: {total_old} -> {total_new}")
+    print(f"Wall time: {elapsed/60:.1f} min")
+
+    _update_results_table(summaries)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ablation/run_sweep.py

@@ -0,0 +1,965 @@
+#!/usr/bin/env python3
+"""Orchestrator for the n=100 hybrid-paradigm ablation sweep.
+
+Drives a batch of ``openjarvis.agents.hybrid.runner`` invocations with:
+
+  * per-tier concurrency caps (anthropic-opus / anthropic-haiku / openai / gemini)
+  * 30s monitor thread per cell, 120/300s heartbeat in main
+  * first-5 fail-fast (5/5 errored OR 5/5 scored-zero → kill the cell)
+  * exit-code + summary.json driven retry loop (resume mode, runner-handled)
+  * incremental atomic updates of ``results-table.md`` (auto-section only)
+  * Ctrl-C → SIGTERM the whole process group of each child cleanly
+"""
+
+from __future__ import annotations
+
+import argparse
+import fcntl
+import json
+import os
+import re
+import signal
+import subprocess
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VENV_PY = Path(os.environ.get("OPENJARVIS_VENV_PY", REPO_ROOT / ".venv" / "bin" / "python"))
+EXPERIMENTS_DIR = Path(
+    os.environ.get(
+        "OPENJARVIS_HYBRID_EXPERIMENTS_DIR",
+        Path.home() / ".openjarvis" / "experiments" / "hybrid",
+    )
+)
+RUNS_DIR = EXPERIMENTS_DIR / "runs"
+RESULTS_TABLE = EXPERIMENTS_DIR / "docs" / "results-table.md"
+LOG_DIR = Path(os.environ.get("HYBRID_SWEEP_LOG_DIR", "/tmp/hybrid-sweep-logs"))
+AUTO_SECTION_HEADER = "## n=100 sweep — auto-generated"
+
+# ---------------------------------------------------------------------------
+# Batch definitions (cell names exactly as in the registries)
+# ---------------------------------------------------------------------------
+
+WAVE_A = [
+    "skillorchestra-qwen-haiku45-gaia-n100",
+    "skillorchestra-qwen-haiku45-swe-n100",
+    "cloud-only-haiku45-gaia-n100",
+    "cloud-only-haiku45-swe-n100",
+    "skillorchestra-qwen-gpt5mini-gaia-n100",
+    "skillorchestra-qwen-gpt5mini-swe-n100",
+    "cloud-only-gpt5mini-gaia-n100",
+    "cloud-only-gpt5mini-swe-n100",
+    "skillorchestra-qwen-gemini25flash-gaia-n100",
+    "skillorchestra-qwen-gemini25flash-swe-n100",
+    "cloud-only-gemini25flash-gaia-n100",
+    "cloud-only-gemini25flash-swe-n100",
+]
+
+WAVE_B = [
+    "skillorchestra-qwen-gpt5-gaia-n100",
+    "skillorchestra-qwen-gpt5-swe-n100",
+    "cloud-only-gpt5-gaia-n100",
+    "cloud-only-gpt5-swe-n100",
+    "skillorchestra-qwen-gemini25pro-gaia-n100",
+    "skillorchestra-qwen-gemini25pro-swe-n100",
+    "cloud-only-gemini25pro-gaia-n100",
+    "cloud-only-gemini25pro-swe-n100",
+]
+
+WAVE_C = [
+    "skillorchestra-qwen-opus47-gaia-n100",
+    "skillorchestra-qwen-opus47-swe-n100",
+    "cloud-only-opus47-gaia-n100",
+    "cloud-only-opus47-swe-n100",
+]
+
+SMOKE = [
+    "mini-swe-agent-swebenchverified-opus-3",
+    "minions-swe-agent-swebenchverified-qwen27b-opus-3",
+    "conductor-swe-agent-swebenchverified-opus-3",
+    "advisors-swe-agent-swebenchverified-qwen9b-opus-3",
+    "skillorchestra-swe-agent-swebenchverified-qwen27b-opus-3",
+    "toolorchestra-swe-agent-swebenchverified-qwen27b-opus-3",
+    "archon-swe-agent-swebenchverified-qwen27b-opus-3",
+]
+
+ALL = WAVE_A + WAVE_B + WAVE_C
+
+BATCHES: Dict[str, List[str]] = {
+    "smoke": SMOKE,
+    "wave-a": WAVE_A,
+    "wave-b": WAVE_B,
+    "wave-c": WAVE_C,
+    "all": ALL,
+}
+
+# ---------------------------------------------------------------------------
+# Tiers / concurrency
+# ---------------------------------------------------------------------------
+
+TIER_CAPS: Dict[str, int] = {
+    "anthropic-opus": 2,
+    "anthropic-haiku": 4,
+    "openai": 4,
+    "gemini": 4,
+}
+
+
+def tier_of(cell: str) -> str:
+    """Map a cell name to a tier. Order matters: gpt5mini before gpt5."""
+    c = cell.lower()
+    if "opus47" in c or "-opus-" in c:
+        return "anthropic-opus"
+    if "haiku45" in c:
+        return "anthropic-haiku"
+    if "gpt5mini" in c:
+        return "openai"
+    if "gpt5" in c:
+        return "openai"
+    if "gemini25pro" in c or "gemini25flash" in c:
+        return "gemini"
+    return "anthropic-opus"  # safest cap for unknowns
+
+
+# ---------------------------------------------------------------------------
+# Cell-name parsing (for the table)
+# ---------------------------------------------------------------------------
+
+CLOUD_LABELS = {
+    "opus47": "claude-opus-4-7",
+    "haiku45": "claude-haiku-4-5",
+    "gpt5mini": "gpt-5-mini",
+    "gpt5": "gpt-5",
+    "gemini25pro": "gemini-2.5-pro",
+    "gemini25flash": "gemini-2.5-flash",
+}
+
+BENCH_LABELS = {"gaia": "GAIA", "swe": "SWE-bench"}
+
+
+def parse_cell(cell: str) -> Dict[str, str]:
+    """Pull (paradigm, local, cloud, bench) out of an ablation cell name.
+
+    Two naming conventions are supported:
+      * ``cloud-only-<cloud>-<bench>-n100``                (no local)
+      * ``<paradigm>-<local>-<cloud>-<bench>-n100``        (everything else,
+        e.g. ``skillorchestra-qwen-...``, ``advisors-qwen27b-...``)
+
+    We anchor on the ``-n100`` suffix and walk left to recover ``bench`` and
+    ``cloud`` so paradigm/local prefixes can vary without breaking parsing.
+    """
+    c = cell
+    if c.startswith("cloud-only-"):
+        paradigm = "cloud-only"
+        local = ""
+        rest = c[len("cloud-only-"):]
+        rparts = rest.split("-")
+        # rest is "<cloud>-<bench>-n100" (or "<cloud>-<bench>" historically)
+        cloud_key = rparts[0] if rparts else ""
+        bench_key = rparts[1] if len(rparts) > 1 else ""
+    else:
+        parts = c.split("-")
+        # Strip a trailing "n100"-style scale token so it can't be mistaken
+        # for the bench.
+        if parts and parts[-1].startswith("n") and parts[-1][1:].isdigit():
+            parts = parts[:-1]
+        # Expect at least <paradigm>-<local>-<cloud>-<bench>.
+        paradigm = parts[0] if parts else ""
+        local = parts[1] if len(parts) > 1 else ""
+        cloud_key = parts[2] if len(parts) > 2 else ""
+        bench_key = parts[3] if len(parts) > 3 else ""
+    return {
+        "paradigm": paradigm,
+        "local": local,
+        "cloud_key": cloud_key,
+        "cloud": CLOUD_LABELS.get(cloud_key, cloud_key),
+        "bench_key": bench_key,
+        "bench": BENCH_LABELS.get(bench_key, bench_key),
+    }
+
+
+# ---------------------------------------------------------------------------
+# results.jsonl scanning
+# ---------------------------------------------------------------------------
+
+
+def _read_jsonl(path: Path) -> List[dict]:
+    if not path.exists():
+        return []
+    try:
+        text = path.read_text()
+    except Exception:
+        return []
+    out: List[dict] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            out.append(json.loads(line))
+        except Exception:
+            # Partial / corrupt write — skip
+            continue
+    return out
+
+
+def _row_is_errored(row: dict) -> bool:
+    """Truth: runner writes top-level ``error`` (str or None) + ``score`` (dict or None).
+
+    A row is errored if ``error`` is non-null OR ``score`` is missing/null.
+    """
+    if row.get("error"):
+        return True
+    score = row.get("score")
+    if score is None:
+        return True
+    return False
+
+
+def _row_score(row: dict) -> Optional[float]:
+    score = row.get("score")
+    if not isinstance(score, dict):
+        return None
+    v = score.get("score")
+    if not isinstance(v, (int, float)):
+        return None
+    return float(v)
+
+
+def _row_last_error(row: dict) -> str:
+    err = row.get("error")
+    if isinstance(err, str) and err:
+        # First line only — runner tucks traceback in there.
+        return err.splitlines()[0][:160]
+    return ""
+
+
+def _check_first5_kill(rows: List[dict]) -> Optional[str]:
+    """Pure kill-decision for the first-5 fail-fast heuristic.
+
+    Returns:
+        - ``"killed-5error"`` if all of the first 5 rows are errored.
+        - ``"killed-5zero"`` if all of the first 5 rows scored 0.0 AND
+          have empty answers (= wiring broken, not legit poor performance).
+          A non-empty wrong answer is legit poor performance — e.g. GAIA's
+          first 5 may genuinely stump a weak model — so we do NOT kill.
+        - ``None`` if fewer than 5 rows or neither condition holds.
+
+    Threshold rationale: at n=100 on hard benchmarks (e.g. GAIA),
+    5 consecutive wrong-but-nonempty answers is plausible — the cell
+    can still end at acc≈0.18. Errors and empty-answers, by contrast,
+    indicate broken wiring (auth, parser, tool config) that won't recover.
+    """
+    if len(rows) < 5:
+        return None
+    first5 = rows[:5]
+    if all(_row_is_errored(r) for r in first5):
+        return "killed-5error"
+    all_zero_empty = all(
+        (not _row_is_errored(r))
+        and (_row_score(r) == 0.0)
+        and (not (r.get("answer") or "").strip())
+        for r in first5
+    )
+    if all_zero_empty:
+        return "killed-5zero"
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Cell state
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class CellState:
+    name: str
+    tier: str
+    expected_n: int = 100
+    round: int = 0
+    status: str = "queued"  # queued | running | done | killed-5error | killed-5zero | blocked-bug | error
+    proc: Optional[subprocess.Popen] = None
+    log_path: Optional[Path] = None
+    monitor_thread: Optional[threading.Thread] = None
+    stop_event: threading.Event = field(default_factory=threading.Event)
+    first5_checked: bool = False
+    kill_reason: str = ""
+    last_error: str = ""
+    started_at: float = 0.0
+    finished_at: float = 0.0
+    last_done: int = 0
+    last_acc: float = 0.0
+    last_err_count: int = 0
+    stall_ticks: int = 0  # consecutive 30s ticks where done didn't increase
+    final_summary: Optional[dict] = None
+    next_hb_at: float = 0.0  # earliest time we should emit a heartbeat line for this cell
+
+    @property
+    def out_dir(self) -> Path:
+        return RUNS_DIR / self.name
+
+    @property
+    def results_path(self) -> Path:
+        return self.out_dir / "results.jsonl"
+
+    @property
+    def summary_path(self) -> Path:
+        return self.out_dir / "summary.json"
+
+    @property
+    def lock_path(self) -> Path:
+        return self.out_dir / ".lock"
+
+
+# ---------------------------------------------------------------------------
+# Lock probe (read-only — never grabs)
+# ---------------------------------------------------------------------------
+
+
+def lock_is_held(lock_path: Path) -> Tuple[bool, str]:
+    """Return ``(held, holder_pid_str)``. Non-destructive: tries an LOCK_EX|NB,
+    immediately releases. If the lock file doesn't exist, treat as free."""
+    if not lock_path.exists():
+        return False, ""
+    try:
+        f = lock_path.open("a+")
+    except Exception:
+        return False, ""
+    try:
+        try:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+            return False, ""
+        except BlockingIOError:
+            f.seek(0)
+            pid = (f.read() or "?").strip() or "?"
+            return True, pid
+    finally:
+        try:
+            f.close()
+        except Exception:
+            pass
+
+
+# ---------------------------------------------------------------------------
+# Table updater (atomic, idempotent, hands-off other sections)
+# ---------------------------------------------------------------------------
+
+_TABLE_LOCK = threading.Lock()
+
+
+def _format_table_row(cell_name: str, summary: dict) -> str:
+    """One pipe-table row for the auto-generated section."""
+    acc = float(summary.get("accuracy", 0.0) or 0.0)
+    cost = float(summary.get("cost_usd_total", 0.0) or 0.0)
+    wall = float(summary.get("wall_time_s", 0.0) or 0.0)
+    n_done = int(summary.get("n_done", 0) or 0)
+    n_target = int(summary.get("n_target", 0) or 0)
+    tokens_local = int(summary.get("tokens_local_total", 0) or 0)
+    tokens_cloud = int(summary.get("tokens_cloud_total", 0) or 0)
+    bench_key = parse_cell(cell_name).get("bench_key", "")
+    bench = BENCH_LABELS.get(bench_key, bench_key or "?")
+    # tools_per_task: not in summary today — derive from results.jsonl if we have time.
+    # For now leave it as "—" so the column stays.
+    tools_str = "—"
+    return (
+        f"| `{cell_name}` | {bench} | {acc:.3f} · ${cost:.2f} | "
+        f"{wall:.0f}s | tools={tools_str} | tokens_local={tokens_local} | "
+        f"tokens_cloud={tokens_cloud} | {n_done}/{n_target} |"
+    )
+
+
+_AUTO_HEADER_LINE = AUTO_SECTION_HEADER  # exact match key
+_TABLE_HEADER = (
+    "| cell | bench | acc · $cost | wall | tools/task | tokens_local | "
+    "tokens_cloud | done/target |\n"
+    "|---|---|---|---|---|---|---|---|"
+)
+
+
+def update_table(cell_name: str, summary: dict) -> None:
+    """Splice (or insert) the row for ``cell_name`` under the auto section.
+
+    Atomic: write to ``.tmp``, then ``os.replace``. Idempotent: same cell
+    overwrites its own row in place.
+    """
+    with _TABLE_LOCK:
+        RESULTS_TABLE.parent.mkdir(parents=True, exist_ok=True)
+        if RESULTS_TABLE.exists():
+            text = RESULTS_TABLE.read_text()
+        else:
+            text = "# Results table\n"
+
+        new_row = _format_table_row(cell_name, summary)
+        cell_key = f"| `{cell_name}` |"
+
+        lines = text.splitlines()
+
+        # Find the auto-section header.
+        try:
+            hdr_idx = next(i for i, ln in enumerate(lines) if ln.strip() == _AUTO_HEADER_LINE)
+        except StopIteration:
+            # Append a fresh section.
+            if lines and lines[-1].strip():
+                lines.append("")
+            lines.append(_AUTO_HEADER_LINE)
+            lines.append("")
+            lines.append(_TABLE_HEADER)
+            lines.append(new_row)
+            _atomic_write_lines(RESULTS_TABLE, lines)
+            return
+
+        # Find the end of the auto-section: next H2 / H1 OR EOF.
+        end_idx = len(lines)
+        for i in range(hdr_idx + 1, len(lines)):
+            ln = lines[i]
+            if ln.startswith("## ") and ln.strip() != _AUTO_HEADER_LINE:
+                end_idx = i
+                break
+            if ln.startswith("# "):
+                end_idx = i
+                break
+
+        section = lines[hdr_idx:end_idx]
+
+        # Make sure section has the header line(s).
+        has_header = any("| cell | bench |" in ln for ln in section)
+        if not has_header:
+            # Re-seed: keep title, drop everything else inside section.
+            section = [_AUTO_HEADER_LINE, "", _TABLE_HEADER]
+
+        # Drop any prior row for this cell.
+        section = [ln for ln in section if not ln.startswith(cell_key)]
+
+        # Append new row.
+        section.append(new_row)
+
+        new_lines = lines[:hdr_idx] + section + lines[end_idx:]
+        _atomic_write_lines(RESULTS_TABLE, new_lines)
+
+
+def _atomic_write_lines(path: Path, lines: List[str]) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text("\n".join(lines).rstrip() + "\n")
+    os.replace(tmp, path)
+
+
+# ---------------------------------------------------------------------------
+# Monitor thread
+# ---------------------------------------------------------------------------
+
+
+def monitor_cell(state: CellState) -> None:
+    """30s polling on results.jsonl. First-5 fail-fast. Sets ``last_*`` on state."""
+    expected = state.expected_n
+    while not state.stop_event.is_set():
+        rows = _read_jsonl(state.results_path)
+        done = len(rows)
+        errs = sum(1 for r in rows if _row_is_errored(r))
+        scores = [_row_score(r) for r in rows if not _row_is_errored(r)]
+        scores = [s for s in scores if s is not None]
+        acc = (sum(scores) / len(scores)) if scores else 0.0
+
+        # stall detector (informational; we don't auto-kill on this)
+        if done == state.last_done:
+            state.stall_ticks += 1
+        else:
+            state.stall_ticks = 0
+
+        state.last_done = done
+        state.last_acc = acc
+        state.last_err_count = errs
+        # last_error: take from the latest errored row, if any.
+        for r in reversed(rows):
+            if _row_is_errored(r):
+                err = _row_last_error(r)
+                if err:
+                    state.last_error = err
+                break
+
+        # First-5 fail-fast — runs at most once per cell, on the same 5 rows.
+        # See _check_first5_kill for the kill-decision rules and rationale.
+        if (not state.first5_checked) and done >= 5:
+            state.first5_checked = True
+            first5 = rows[:5]
+            kill = _check_first5_kill(first5)
+            if kill == "killed-5error":
+                state.kill_reason = kill
+                _kill_proc(state)
+                _log_kill(state, first5, "all 5 errored")
+                state.stop_event.set()
+                return
+            if kill == "killed-5zero":
+                state.kill_reason = kill
+                _kill_proc(state)
+                _log_kill(state, first5, "all 5 scored 0.0 with empty answers")
+                state.stop_event.set()
+                return
+
+        # Subprocess died?
+        if state.proc is not None and state.proc.poll() is not None:
+            return
+
+        # Sleep 30s, but wake on stop.
+        state.stop_event.wait(30.0)
+
+
+def _kill_proc(state: CellState) -> None:
+    p = state.proc
+    if p is None:
+        return
+    try:
+        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    except (ProcessLookupError, PermissionError, OSError):
+        try:
+            p.terminate()
+        except Exception:
+            pass
+    try:
+        p.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+        except Exception:
+            try:
+                p.kill()
+            except Exception:
+                pass
+
+
+def _log_kill(state: CellState, rows: List[dict], reason: str) -> None:
+    print(
+        f"[KILL] {state.name}: {state.kill_reason} — {reason}",
+        flush=True,
+    )
+    for i, r in enumerate(rows):
+        tid = r.get("task_id", "?")
+        err = _row_last_error(r) or "(no error string)"
+        sc = _row_score(r)
+        print(f"  row[{i}] task={tid} score={sc} error={err}", flush=True)
+
+
+# ---------------------------------------------------------------------------
+# Launch / retry loop
+# ---------------------------------------------------------------------------
+
+
+def launch_cell(state: CellState) -> None:
+    """Spawn the runner subprocess for this cell (one round).
+
+    Sets ``proc``, ``log_path``, ``monitor_thread``, ``started_at``.
+    """
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    ts = int(time.time())
+    state.log_path = LOG_DIR / f"{state.name}.r{state.round}.{ts}.log"
+    state.stop_event = threading.Event()
+    state.first5_checked = False
+    state.kill_reason = ""
+    state.last_done = 0
+    state.stall_ticks = 0
+    state.started_at = time.time()
+
+    log_file = state.log_path.open("a")
+    cmd = [
+        str(VENV_PY),
+        "-m",
+        "openjarvis.agents.hybrid.runner",
+        "--cell",
+        state.name,
+    ]
+    log_file.write(f"# $ {' '.join(cmd)}\n# cwd={REPO_ROOT}\n# round={state.round}\n")
+    log_file.flush()
+    state.proc = subprocess.Popen(
+        cmd,
+        cwd=str(REPO_ROOT),
+        stdout=log_file,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,  # so we can SIGTERM the whole pgrp
+    )
+    state.status = "running"
+
+    state.monitor_thread = threading.Thread(
+        target=monitor_cell, args=(state,), daemon=True, name=f"mon-{state.name}"
+    )
+    state.monitor_thread.start()
+
+
+def collect_cell(state: CellState) -> str:
+    """Wait for ``state.proc`` to exit, then classify outcome.
+
+    Returns one of:
+      ``done``           — clean run, full summary, no errored tasks
+      ``retry``          — non-fatal: some errored rows / non-zero exit / partial
+      ``killed-5error``  — fail-fast triggered (5/5 errored on first batch)
+      ``killed-5zero``   — fail-fast triggered (5/5 scored 0)
+    """
+    assert state.proc is not None
+    rc = state.proc.wait()
+    state.stop_event.set()
+    if state.monitor_thread is not None:
+        state.monitor_thread.join(timeout=5)
+
+    if state.kill_reason in ("killed-5error", "killed-5zero"):
+        return state.kill_reason
+
+    # Inspect summary + jsonl
+    summary = None
+    if state.summary_path.exists():
+        try:
+            summary = json.loads(state.summary_path.read_text())
+        except Exception:
+            summary = None
+
+    rows = _read_jsonl(state.results_path)
+    err_rows = sum(1 for r in rows if _row_is_errored(r))
+
+    if (
+        rc == 0
+        and summary is not None
+        and int(summary.get("task_count", 0)) == state.expected_n
+        and int(summary.get("n_done", 0)) == state.expected_n
+        and err_rows == 0
+    ):
+        state.final_summary = summary
+        return "done"
+
+    return "retry"
+
+
+# ---------------------------------------------------------------------------
+# Heartbeat printer (main thread)
+# ---------------------------------------------------------------------------
+
+
+def _fmt_eta(eta_s: float) -> str:
+    if eta_s <= 0 or eta_s != eta_s:  # NaN guard
+        return "--:--"
+    m, s = divmod(int(eta_s), 60)
+    return f"{m:02d}:{s:02d}"
+
+
+def heartbeat_line(state: CellState) -> str:
+    elapsed = time.time() - state.started_at if state.started_at else 0.0
+    done = state.last_done
+    expected = state.expected_n
+    if done > 0:
+        eta_s = max(0.0, (expected - done) * (elapsed / done))
+    else:
+        eta_s = 0.0
+    status_tag = "OK"
+    tail = "OK"
+    # Detect issues
+    if state.kill_reason:
+        status_tag = "ALERT"
+        tail = state.kill_reason
+    elif state.last_err_count > 0:
+        tail = f"err_rows={state.last_err_count} last={state.last_error[:80]}"
+    elif state.stall_ticks >= 3:
+        status_tag = "ALERT"
+        tail = f"STALL ({state.stall_ticks} ticks no progress)"
+
+    return (
+        f"[{state.name}] round={state.round} done={done}/{expected} "
+        f"acc={state.last_acc:.3f} err={state.last_err_count} "
+        f"elapsed={_fmt_eta(elapsed)} eta={_fmt_eta(eta_s)} | "
+        f"{status_tag} | {tail}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Top-level orchestration
+# ---------------------------------------------------------------------------
+
+
+_SHUTDOWN = threading.Event()
+
+
+def _install_sigint(states: List[CellState]) -> None:
+    def _handler(signum, frame):
+        if _SHUTDOWN.is_set():
+            return
+        _SHUTDOWN.set()
+        print(
+            "\n[main] caught SIGINT — sending SIGTERM to all running cells.",
+            flush=True,
+        )
+        for s in states:
+            if s.proc is not None and s.proc.poll() is None:
+                _kill_proc(s)
+                s.stop_event.set()
+
+    signal.signal(signal.SIGINT, _handler)
+    signal.signal(signal.SIGTERM, _handler)
+
+
+def run_sweep(cells: List[str], max_rounds: int, smoke_n: bool = False) -> int:
+    """Drive the sweep. Returns process exit code (0 ok, 1 on any kill/blocked)."""
+    # Build initial states
+    states: Dict[str, CellState] = {}
+    for c in cells:
+        expected = 3 if smoke_n else 100
+        states[c] = CellState(name=c, tier=tier_of(c), expected_n=expected)
+
+    _install_sigint(list(states.values()))
+
+    # Build per-tier semaphores. We don't use threading.BoundedSemaphore because
+    # we drive launches from the main thread one-by-one; we just count active
+    # cells per tier and only launch when capacity allows.
+    active_per_tier: Dict[str, int] = {t: 0 for t in TIER_CAPS}
+
+    # Skip cells that already have a held lock (refuse to double-launch).
+    runnable: List[CellState] = []
+    for s in states.values():
+        held, pid = lock_is_held(s.lock_path)
+        if held:
+            print(
+                f"[skip] {s.name}: lock held by pid {pid}; refusing to launch a second instance.",
+                flush=True,
+            )
+            s.status = "skipped-locked"
+        else:
+            # If a complete summary already exists with full coverage, skip.
+            if s.summary_path.exists():
+                try:
+                    smry = json.loads(s.summary_path.read_text())
+                    if (
+                        int(smry.get("n_done", 0)) == s.expected_n
+                        and int(smry.get("task_count", 0)) == s.expected_n
+                        and int(smry.get("n_err", 0)) == 0
+                    ):
+                        s.status = "done"
+                        s.final_summary = smry
+                        update_table(s.name, smry)
+                        print(f"[skip] {s.name}: already complete (summary.json full).", flush=True)
+                        continue
+                except Exception:
+                    pass
+            runnable.append(s)
+
+    # Queue: list of CellState we still need to schedule (round 0 launches).
+    pending: List[CellState] = list(runnable)
+    running: List[CellState] = []
+
+    last_hb = 0.0
+
+    while pending or running:
+        if _SHUTDOWN.is_set():
+            break
+
+        # Launch as many as tier caps allow.
+        i = 0
+        while i < len(pending):
+            s = pending[i]
+            if active_per_tier[s.tier] < TIER_CAPS[s.tier]:
+                s.round += 1
+                if s.round > max_rounds:
+                    s.status = "blocked-bug"
+                    print(f"[blocked] {s.name}: exceeded max_rounds={max_rounds}", flush=True)
+                    pending.pop(i)
+                    continue
+                launch_cell(s)
+                active_per_tier[s.tier] += 1
+                running.append(s)
+                pending.pop(i)
+                print(
+                    f"[launch] {s.name} (round={s.round}, tier={s.tier}, "
+                    f"slot={active_per_tier[s.tier]}/{TIER_CAPS[s.tier]}, "
+                    f"log={s.log_path})",
+                    flush=True,
+                )
+            else:
+                i += 1
+
+        # Reap any that are done.
+        still_running: List[CellState] = []
+        for s in running:
+            if s.proc is None or s.proc.poll() is None:
+                # Also reap if monitor killed it
+                if s.kill_reason:
+                    pass  # will collect below
+                else:
+                    still_running.append(s)
+                    continue
+            outcome = collect_cell(s)
+            active_per_tier[s.tier] = max(0, active_per_tier[s.tier] - 1)
+            s.finished_at = time.time()
+
+            if outcome == "done":
+                s.status = "done"
+                assert s.final_summary is not None
+                try:
+                    update_table(s.name, s.final_summary)
+                except Exception as e:
+                    print(f"[table-error] {s.name}: {e}", flush=True)
+                elapsed = s.finished_at - s.started_at
+                print(
+                    f"[DONE] {s.name} acc={s.final_summary.get('accuracy', 0.0):.3f} "
+                    f"cost=${s.final_summary.get('cost_usd_total', 0.0):.2f} "
+                    f"done={s.final_summary.get('n_done', 0)}/{s.expected_n} "
+                    f"elapsed={_fmt_eta(elapsed)}",
+                    flush=True,
+                )
+            elif outcome in ("killed-5error", "killed-5zero"):
+                s.status = outcome
+                print(
+                    f"[KILLED] {s.name} reason={outcome} last_err={s.last_error[:120]}",
+                    flush=True,
+                )
+            else:  # retry
+                if s.round >= max_rounds:
+                    s.status = "blocked-bug"
+                    print(
+                        f"[BLOCKED] {s.name}: hit max_rounds={max_rounds} without clean exit. "
+                        f"last_err={s.last_error[:120]}",
+                        flush=True,
+                    )
+                else:
+                    print(
+                        f"[retry] {s.name} round {s.round} → re-queue (resume). "
+                        f"err_rows={s.last_err_count} last={s.last_error[:120]}",
+                        flush=True,
+                    )
+                    pending.append(s)
+        running = still_running
+
+        # Heartbeat every 120s (downshift to 300s past first-5 check).
+        now = time.time()
+        if now - last_hb >= 120.0 and running:
+            for s in running:
+                # Per-cell heartbeat cadence: 120s before first-5, 300s after.
+                cadence = 300.0 if s.first5_checked else 120.0
+                if now >= s.next_hb_at:
+                    print(heartbeat_line(s), flush=True)
+                    s.next_hb_at = now + cadence
+            last_hb = now
+
+        if running:
+            time.sleep(5.0)
+
+    # Final summary
+    print("\n" + "=" * 72, flush=True)
+    print("Final sweep summary:", flush=True)
+    print("=" * 72, flush=True)
+    any_bad = False
+    for s in states.values():
+        if s.status == "done":
+            smry = s.final_summary or {}
+            print(
+                f"DONE   {s.name}  acc={smry.get('accuracy', 0.0):.3f}  "
+                f"${smry.get('cost_usd_total', 0.0):.2f}  "
+                f"{smry.get('n_done', 0)}/{s.expected_n}",
+                flush=True,
+            )
+        elif s.status.startswith("killed-"):
+            any_bad = True
+            print(
+                f"KILL   {s.name:<60s} {s.status}  (last err: {s.last_error[:100]})",
+                flush=True,
+            )
+        elif s.status == "blocked-bug":
+            any_bad = True
+            print(
+                f"BLOCK  {s.name:<60s} blocked-bug  (last err: {s.last_error[:100]})",
+                flush=True,
+            )
+        elif s.status == "skipped-locked":
+            print(f"SKIP   {s.name:<60s} skipped-locked", flush=True)
+        else:
+            any_bad = True
+            print(f"?      {s.name:<60s} status={s.status}", flush=True)
+
+    return 1 if any_bad else 0
+
+
+# ---------------------------------------------------------------------------
+# Dry run
+# ---------------------------------------------------------------------------
+
+
+def dry_run(cells: List[str]) -> int:
+    print(f"Plan: {len(cells)} cells")
+    by_tier: Dict[str, List[str]] = {}
+    for c in cells:
+        by_tier.setdefault(tier_of(c), []).append(c)
+    print()
+    print("Cells (in order):")
+    for c in cells:
+        t = tier_of(c)
+        print(f"  {c:<60s} tier={t}")
+    print()
+    print("Concurrency plan (per-tier semaphores):")
+    for t, names in sorted(by_tier.items()):
+        cap = TIER_CAPS.get(t, "?")
+        print(f"  tier={t:<18s} cap={cap}  cells={len(names)}")
+    print()
+    print(f"Logs:    {LOG_DIR}/<cell>.r<round>.<ts>.log")
+    print(f"Outputs: {RUNS_DIR}/<cell>/{{results.jsonl,summary.json}}")
+    print(f"Table:   {RESULTS_TABLE}  (auto section: '{AUTO_SECTION_HEADER}')")
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    p = argparse.ArgumentParser(
+        prog="run_sweep.py",
+        description="Hybrid-paradigm ablation sweep orchestrator (n=100).",
+    )
+    p.add_argument(
+        "--batch",
+        choices=sorted(BATCHES.keys()),
+        default="wave-a",
+        help="Which preset batch to run (default: wave-a).",
+    )
+    p.add_argument(
+        "--cells",
+        default=None,
+        help="Comma-separated explicit cell list (overrides --batch).",
+    )
+    p.add_argument(
+        "--max-rounds",
+        type=int,
+        default=3,
+        help="Max retry rounds per cell (default: 3).",
+    )
+    p.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print plan and exit without launching anything.",
+    )
+    args = p.parse_args(argv)
+
+    if args.cells:
+        cells = [c.strip() for c in args.cells.split(",") if c.strip()]
+        smoke = False
+    else:
+        cells = list(BATCHES[args.batch])
+        smoke = args.batch == "smoke"
+
+    if not cells:
+        print("[error] no cells to run.", file=sys.stderr)
+        return 2
+
+    if args.dry_run:
+        return dry_run(cells)
+
+    if not VENV_PY.exists():
+        print(f"[error] python not found: {VENV_PY}", file=sys.stderr)
+        return 2
+
+    return run_sweep(cells, max_rounds=args.max_rounds, smoke_n=smoke)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ablation/test_run_sweep.py

@@ -0,0 +1,87 @@
+"""Tests for the ablation orchestrator's first-5 fail-fast kill heuristic.
+
+Background — the bug:
+    The old kill condition fired on 5/5 score=0 regardless of answer content.
+    GPT-5-mini on GAIA scored 0/5 with REAL wrong answers ("Herbert Simon" vs
+    ref "Claude Shannon"), got killed by the orchestrator, but the cell would
+    have ended at acc=0.18 over 100 tasks. The fix tightens the 5-zero kill
+    to require empty answers (= wiring broken), not merely wrong ones.
+
+Run:
+    .venv/bin/python -m pytest scripts/ablation/test_run_sweep.py -v
+"""
+from __future__ import annotations
+
+from scripts.ablation.run_sweep import _check_first5_kill
+
+
+def _errored_row(msg: str = "boom") -> dict:
+    return {"task_id": "t", "error": msg, "score": None, "answer": ""}
+
+
+def _scored_row(score: float, answer: str = "") -> dict:
+    return {
+        "task_id": "t",
+        "error": None,
+        "score": {"score": score},
+        "answer": answer,
+    }
+
+
+def test_kill_5_errors() -> None:
+    """5 errored rows -> killed-5error."""
+    rows = [_errored_row(f"err {i}") for i in range(5)]
+    assert _check_first5_kill(rows) == "killed-5error"
+
+
+def test_kill_5_empty_answers() -> None:
+    """5 rows with score=0.0 AND empty answer -> killed-5zero (wiring broken)."""
+    rows = [_scored_row(0.0, "") for _ in range(5)]
+    assert _check_first5_kill(rows) == "killed-5zero"
+
+
+def test_no_kill_5_wrong_answers() -> None:
+    """5 rows with score=0.0 AND non-empty wrong answer -> NO kill.
+
+    REGRESSION TEST for the original bug: GPT-5-mini answering "Herbert Simon"
+    where the reference was "Claude Shannon" is legit poor performance, not
+    broken wiring. The orchestrator must let it ride.
+    """
+    rows = [
+        _scored_row(0.0, "Herbert Simon"),
+        _scored_row(0.0, "Alan Turing"),
+        _scored_row(0.0, "John von Neumann"),
+        _scored_row(0.0, "Marvin Minsky"),
+        _scored_row(0.0, "Donald Knuth"),
+    ]
+    assert _check_first5_kill(rows) is None
+
+
+def test_no_kill_mixed() -> None:
+    """4 errors + 1 success -> no kill (not all errored, not all scored-zero)."""
+    rows = [_errored_row() for _ in range(4)] + [_scored_row(1.0, "right answer")]
+    assert _check_first5_kill(rows) is None
+
+
+def test_no_kill_under_5_rows() -> None:
+    """Fewer than 5 rows -> None (not yet evaluable)."""
+    rows = [_errored_row() for _ in range(4)]
+    assert _check_first5_kill(rows) is None
+
+
+def test_no_kill_4_empty_plus_1_nonempty() -> None:
+    """4 empty + 1 non-empty wrong answer -> no kill (the non-empty saves it)."""
+    rows = [_scored_row(0.0, "") for _ in range(4)] + [_scored_row(0.0, "wrong")]
+    assert _check_first5_kill(rows) is None
+
+
+def test_no_kill_whitespace_only_answer_treated_as_empty_but_with_score() -> None:
+    """Whitespace-only answer is treated as empty -> killed-5zero fires."""
+    rows = [_scored_row(0.0, "   \n\t") for _ in range(5)]
+    assert _check_first5_kill(rows) == "killed-5zero"
+
+
+def test_only_first_5_inspected() -> None:
+    """If the first 5 trigger a kill, later rows don't rescue it."""
+    rows = [_errored_row() for _ in range(5)] + [_scored_row(1.0, "ok")]
+    assert _check_first5_kill(rows) == "killed-5error"

src/openjarvis/agents/__init__.py

@@ -79,11 +79,6 @@ try:
 except ImportError:
     pass
 
-try:
-    import openjarvis.agents.proactive_agent  # noqa: F401
-except ImportError:
-    pass
-
 # Hybrid local+cloud paradigm agents (Minions, Conductor, Archon, Advisors,
 # SkillOrchestra, ToolOrchestra). Each module registers under its own name
 # via @AgentRegistry.register(). Optional deps may make some unavailable.

src/openjarvis/agents/hybrid/README.md

@@ -1,9 +1,10 @@
 # Hybrid local+cloud paradigm agents
 
-Six paradigms ported from
-[`/matx/u/aspark/hybrid-local-cloud-compute`](../../../../../) — each is
-registered as a standard OpenJarvis agent so the rest of the platform
-(SDK, CLI, distillation, evals) can use them like any other agent.
+Six paradigms ported from the original ``hybrid-local-cloud-compute``
+harness — each is registered as a standard OpenJarvis agent so the rest
+of the platform (SDK, CLI, distillation, evals) can use them like any
+other agent. Results live under ``$OPENJARVIS_HYBRID_EXPERIMENTS_DIR``
+(defaults to ``~/.openjarvis/experiments/hybrid/``).
 
 | Agent             | Plan shape      | Trains what?         | Workers                   |
 |-------------------|-----------------|----------------------|---------------------------|
@@ -45,15 +46,15 @@ structural scorer).
 ## Quickstart
 
 ```bash
-cd /matx/u/aspark/OpenJarvis
+cd OpenJarvis
 source .env                                           # API keys
 
-# 1. Start vLLM in another shell (see CLAUDE.md for the full recipe)
+# 1. Start vLLM in another shell (see your local launch recipe)
 #    CUDA_VISIBLE_DEVICES=0 .venv/bin/python -m vllm.entrypoints.openai.api_server \
 #       --model Qwen/Qwen3.5-27B-FP8 --port 8001 ...
 
 # 2. (Optional) for Minions: install the upstream library
-.venv/bin/uv pip install -e /matx/u/aspark/hybrid-local-cloud-compute/external/minions
+.venv/bin/uv pip install -e path/to/minions
 
 # 3. Run a smoke cell
 .venv/bin/python -m openjarvis.agents.hybrid.runner \
@@ -61,7 +62,7 @@ source .env                                           # API keys
 ```
 
 Outputs land in
-`$OPENJARVIS_HYBRID_EXPERIMENTS_DIR/<cell>/{results.jsonl,summary.json,config.json,logs/}`
+`$OPENJARVIS_HYBRID_EXPERIMENTS_DIR/runs/<cell>/{results.jsonl,summary.json,config.json,logs/}`
 (defaults to `~/.openjarvis-hybrid/experiments/`). The schema matches the
 hybrid harness so the existing rescore / dashboard scripts work
 unmodified.
@@ -80,7 +81,7 @@ That appends a `[cells.<name>]` block to
 ## How good is each paradigm?
 
 Numbers from the upstream hybrid harness
-(`/matx/u/aspark/hybrid-local-cloud-compute/docs/results.md`) at full N —
+(`~/.openjarvis/experiments/hybrid/docs/results.md`) at full N —
 GAIA val n=165, SWE-bench-Verified n=500. Local = Qwen-3.5-27B-FP8, cloud
 = Opus 4.7. Cloud-only baseline: GAIA 0.570 / $1.09, SWE 0.238 / $0.95.
 

src/openjarvis/agents/hybrid/__init__.py

@@ -6,7 +6,7 @@ Each module here registers one agent under ``@AgentRegistry.register("<name>")``
     conductor         — zero-shot planner emits a DAG of up to 5 worker calls
     minions           — supervisor (cloud) ↔ worker (local) reactive loop
     archon            — layered (generator → ranker → fuser) inference-time search
-    skillorchestra    — skill-aware router picks one agent from a pool
+    skillorchestra    — eval orchestrator: skill-routed search→reasoning→answer loop
     toolorchestra     — prompted multi-turn dispatcher over a mixed tool/model pool
 
 All agents share :class:`LocalCloudAgent` as the base. They are bench-agnostic:
@@ -15,9 +15,9 @@ or the bench's native formatter) and hands it in via ``run(input=...)``. Task
 metadata that the paradigm needs (a problem statement vs. a question, hints,
 etc.) goes through ``context.metadata``.
 
-The hybrid harness at ``/matx/u/aspark/hybrid-local-cloud-compute`` is the
-reference implementation and stays untouched — these ports are the
-OpenJarvis-native versions of the same paradigms.
+The original ``hybrid-local-cloud-compute`` harness is the reference
+implementation and stays untouched — these ports are the OpenJarvis-native
+versions of the same paradigms.
 """
 
 from __future__ import annotations
@@ -35,6 +35,8 @@ for _modname in (
     "skillorchestra",
     "toolorchestra",
     "mini_swe_agent",
+    "baseline_cloud",
+    "baseline_local",
 ):
     try:
         __import__(f"openjarvis.agents.hybrid.{_modname}")

src/openjarvis/agents/hybrid/_base.py

@@ -37,13 +37,16 @@ kwargs (``local_model``, ``local_endpoint``, ``cloud_endpoint``, …) follow.
 from __future__ import annotations
 
 import json
+import os
 import threading
 import time
 from abc import abstractmethod
+from collections import deque
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Deque, Dict, List, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext, AgentResult, BaseAgent
+from openjarvis.agents.hybrid._openai_retry import patch_openai_globally
 from openjarvis.agents.hybrid._prices import (
     NO_TEMP_PREFIXES,
     is_gpt5_family,
@@ -54,9 +57,29 @@ from openjarvis.agents.hybrid._prices import (
 )
 from openjarvis.engine._stubs import InferenceEngine
 
+# Install OpenAI SDK retry + per-org concurrency cap at import time so
+# every paradigm (advisors, conductor, minions, mini_swe_agent's cloud
+# loop, archon, …) inherits the hardening without each call site having
+# to remember to opt in. See ``_openai_retry.py`` for the env knobs
+# (default: 4 concurrent, 8 retries, 2s/60s exponential backoff w/ jitter).
+patch_openai_globally()
+
 # Anthropic server-side web_search: $10 per 1000 searches.
 WEB_SEARCH_COST_PER_CALL = 0.01
 
+# OpenAI Responses-API hosted web_search tool: $10 per 1000 calls
+# (2025-12 public list price for the `web_search` / `web_search_preview`
+# tool, billed per tool call). Same shape as Anthropic, so we reuse the
+# $0.01/call number — kept as a separate constant so it can drift.
+OPENAI_WEB_SEARCH_COST_PER_CALL = 0.01
+
+# Gemini Google-Search grounding: billed at $35 per 1000 grounded
+# *requests* (2025-12 public list price for the Grounding-with-Google-Search
+# tool, charged once per request that uses the tool regardless of how many
+# internal queries it issues). We charge per grounded request, not per
+# `web_search_queries` entry.
+GEMINI_SEARCH_COST_PER_CALL = 0.035
+
 ANTHROPIC_WEB_SEARCH_TOOL = {
     "type": "web_search_20250305",
     "name": "web_search",
@@ -64,6 +87,35 @@ ANTHROPIC_WEB_SEARCH_TOOL = {
 }
 
 
+def build_web_search_tool(max_uses: int = 8) -> Dict[str, Any]:
+    """Build an Anthropic server-side web_search tool block with a custom cap.
+
+    Web_search is server-side: Anthropic runs the searches internally before
+    returning, so ``max_uses`` is the only knob the caller has to bound cost
+    per task. Defaults to 8 (matches ``ANTHROPIC_WEB_SEARCH_TOOL``).
+    """
+    return {
+        "type": "web_search_20250305",
+        "name": "web_search",
+        "max_uses": int(max_uses),
+    }
+
+
+def web_search_cfg(method_cfg: Optional[Dict[str, Any]]) -> Tuple[bool, int]:
+    """Parse ``method_cfg.web_search = { enabled, max_uses }``.
+
+    Defaults: enabled=False, max_uses=8. ``enabled`` defaults to False so
+    existing cells stay one-shot (backwards compat). Cells opting in flip
+    ``enabled=true`` in the registry. Returns ``(enabled, max_uses)``.
+    """
+    if not method_cfg:
+        return False, 8
+    ws = method_cfg.get("web_search")
+    if not isinstance(ws, dict):
+        return False, 8
+    return bool(ws.get("enabled", False)), int(ws.get("max_uses", 8))
+
+
 # ---------- Thread-local trace buffer ----------
 #
 # Every call through ``_call_anthropic`` / ``_call_openai`` / ``_call_vllm``
@@ -96,6 +148,126 @@ def _close_trace() -> None:
         delattr(_TRACE_STATE, "events")
 
 
+# ---------- Thread-local LLM-call counter ----------
+#
+# Parallel to the trace buffer: every cloud SDK call (anthropic / openai /
+# gemini, including each turn of ``_call_anthropic_agent`` and each turn of
+# the mini-SWE multi-turn loops) bumps ``cloud``; every local vLLM call
+# bumps ``local``. ``run()`` opens a fresh pair per task, drops the totals
+# into ``meta["n_cloud_calls"]`` / ``meta["n_local_calls"]``, then closes.
+#
+# Why thread-local and not an instance counter: agents are shared across
+# the runner's ``ThreadPoolExecutor`` (one agent, N concurrent tasks). An
+# attribute on ``self`` would race; this matches the trace buffer's
+# pattern exactly so it composes with the rest of the per-task plumbing.
+
+_CALL_COUNTS = threading.local()
+
+
+def _call_counts() -> Optional[Dict[str, int]]:
+    return getattr(_CALL_COUNTS, "counts", None)
+
+
+def _bump_cloud_calls(n: int = 1) -> None:
+    counts = _call_counts()
+    if counts is not None:
+        counts["cloud"] += int(n)
+
+
+def _bump_local_calls(n: int = 1) -> None:
+    counts = _call_counts()
+    if counts is not None:
+        counts["local"] += int(n)
+
+
+def _open_call_counts() -> Dict[str, int]:
+    counts: Dict[str, int] = {"cloud": 0, "local": 0}
+    _CALL_COUNTS.counts = counts
+    return counts
+
+
+def _close_call_counts() -> None:
+    if hasattr(_CALL_COUNTS, "counts"):
+        delattr(_CALL_COUNTS, "counts")
+
+
+# ---------- OpenRouter in-process rate limiter ----------
+#
+# OpenRouter enforces per-account RPM and concurrency limits. A single agent
+# process running a wide ThreadPoolExecutor (Conductor's 7-worker pool fanned
+# across the runner's per-task threads) can trivially blow past those caps,
+# triggering 429s that the OpenAI SDK retries with backoff — wasted latency
+# and noisy traces.
+#
+# We gate every ``_call_openrouter`` call through two limits, **shared across
+# threads in this Python process** (singleton; lazy-initialized):
+#
+# - **Concurrency** — a ``threading.Semaphore``. Default 20 in-flight calls;
+#   override via ``OJ_OPENROUTER_MAX_CONCURRENT``. Held only around the SDK
+#   ``client.chat.completions.create`` invocation, not the bookkeeping.
+# - **RPM** — a sliding window of timestamps in a ``deque``. Default 60
+#   requests/minute; override via ``OJ_OPENROUTER_RPM``. If the deque has
+#   already accumulated ``RPM`` timestamps within the last 60 seconds, the
+#   caller sleeps until the oldest entry ages out, then proceeds. Every
+#   completed call appends ``time.time()`` so the call we just made is
+#   counted against the next window.
+#
+# Other cloud helpers (``_call_openai`` / ``_call_anthropic`` / ``_call_gemini``)
+# are NOT rate-limited here — OpenAI and Anthropic have their own concurrency
+# patch (``_openai_retry``) and Gemini's free-tier RPM is loose enough that
+# we haven't hit it yet. Add limiters for those one-by-one if needed.
+
+_OPENROUTER_LIMITER_LOCK = threading.Lock()
+_OPENROUTER_LIMITER: Optional["_OpenRouterLimiter"] = None
+
+
+class _OpenRouterLimiter:
+    """Process-wide concurrency + sliding-window RPM gate for OpenRouter."""
+
+    def __init__(self, max_concurrent: int, rpm: int) -> None:
+        self.max_concurrent = int(max_concurrent)
+        self.rpm = int(rpm)
+        self._sem = threading.Semaphore(self.max_concurrent)
+        self._window: Deque[float] = deque()
+        self._window_lock = threading.Lock()
+
+    def acquire_concurrency(self) -> None:
+        self._sem.acquire()
+
+    def release_concurrency(self) -> None:
+        self._sem.release()
+
+    def wait_for_rpm_slot(self) -> None:
+        """Block until making one more call would not exceed RPM in 60s."""
+        while True:
+            with self._window_lock:
+                now = time.time()
+                cutoff = now - 60.0
+                while self._window and self._window[0] < cutoff:
+                    self._window.popleft()
+                if len(self._window) < self.rpm:
+                    return
+                # Sleep until the oldest in-window call ages out, then recheck.
+                sleep_s = 60.0 - (now - self._window[0]) + 0.01
+            if sleep_s > 0:
+                time.sleep(sleep_s)
+
+    def record_call(self) -> None:
+        with self._window_lock:
+            self._window.append(time.time())
+
+
+def _openrouter_limiter() -> _OpenRouterLimiter:
+    global _OPENROUTER_LIMITER
+    if _OPENROUTER_LIMITER is None:
+        with _OPENROUTER_LIMITER_LOCK:
+            if _OPENROUTER_LIMITER is None:
+                max_concurrent = int(os.environ.get("OJ_OPENROUTER_MAX_CONCURRENT", "20") or 20)
+                rpm = int(os.environ.get("OJ_OPENROUTER_RPM", "60") or 60)
+                _OPENROUTER_LIMITER = _OpenRouterLimiter(max_concurrent, rpm)
+    return _OPENROUTER_LIMITER
+
+
 def _serialize_block(block: Any) -> Dict[str, Any]:
     """Turn an Anthropic content block (text / tool_use / server_tool_use /
     web_search_tool_result / thinking) into a JSON-safe dict.
@@ -223,13 +395,16 @@ class LocalCloudAgent(BaseAgent):
         tool_choice: Optional[dict] = None,
         output_config: Optional[dict] = None,
         timeout: float = 600.0,
-        max_retries: int = 5,
+        max_retries: int = 12,
         trace_role: str = "cloud",
     ) -> Tuple[str, int, int, int]:
         """Single Anthropic call. Returns (text, p_tok, c_tok, n_web_searches).
 
         Strips ``temperature`` for Opus 4.7+ (rejected by the API). Captures
-        the call into the active per-task trace if one is open.
+        the call into the active per-task trace if one is open. Bumped
+        default max_retries to 12 (~2 min of backoff) so cells survive
+        sustained Anthropic 529 "Overloaded" windows when many cells share
+        Opus quota.
         """
         import anthropic
 
@@ -251,6 +426,7 @@ class LocalCloudAgent(BaseAgent):
             kwargs["output_config"] = output_config
         t0 = time.time()
         msg = client.messages.create(**kwargs)
+        _bump_cloud_calls()
         latency = time.time() - t0
         text = "".join(b.text for b in msg.content if hasattr(b, "text"))
         srv = getattr(msg.usage, "server_tool_use", None)
@@ -321,6 +497,7 @@ class LocalCloudAgent(BaseAgent):
             kwargs["tool_choice"] = tool_choice
         t0 = time.time()
         resp = client.chat.completions.create(**kwargs)
+        _bump_cloud_calls()
         latency = time.time() - t0
         choice = resp.choices[0]
         message = choice.message
@@ -352,6 +529,181 @@ class LocalCloudAgent(BaseAgent):
         })
         return text, p, c
 
+    @staticmethod
+    def _call_openrouter(
+        model: str,
+        *,
+        user: str,
+        system: Optional[str] = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        timeout: float = 600.0,
+        trace_role: str = "cloud",
+        extra_body: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[str, int, int]:
+        """Single OpenRouter call. Returns (text, p_tok, c_tok).
+
+        OpenRouter is OpenAI-API-compatible; we use the OpenAI SDK with a
+        custom ``base_url`` and ``OPENROUTER_API_KEY``. ``model`` is the
+        OpenRouter slug ``"<provider>/<model>"`` (e.g.
+        ``"deepseek/deepseek-r1"``). For convenience the caller may also
+        pass the OpenJarvis-engine-style ``"openrouter/<provider>/<model>"``
+        prefix (see ``src/openjarvis/engine/cloud.py``) — we strip it here.
+
+        Note: unlike ``_call_openai``, we do NOT apply the GPT-5 family
+        ``max_completion_tokens`` rewrite or temperature stripping —
+        OpenRouter models have varied parameter support. We pass through
+        whatever the caller supplies; if a specific model errors on
+        ``temperature``, that's the cell's problem to handle.
+
+        ``extra_body`` is forwarded to the OpenAI SDK as the ``extra_body``
+        kwarg so callers can pass OpenRouter-specific fields (e.g.
+        ``{"reasoning": {"effort": "medium"}}`` to enable thinking on Qwen3,
+        or ``{"provider": {...}}`` to pin routing). The SDK serializes
+        ``extra_body`` into the JSON request body alongside the standard
+        fields.
+
+        Every call goes through the process-wide
+        ``_OpenRouterLimiter`` (concurrency semaphore + sliding-window RPM
+        deque) so a wide ThreadPoolExecutor can't blow past account caps.
+
+        Trace events use ``"kind": "openrouter"`` so the dashboard can
+        distinguish them from native OpenAI calls.
+        """
+        from openai import OpenAI
+
+        if model.startswith("openrouter/"):
+            model = model[len("openrouter/"):]
+        api_key = os.environ.get("OPENROUTER_API_KEY")
+        if not api_key:
+            raise RuntimeError(
+                "OPENROUTER_API_KEY is not set; cannot call OpenRouter."
+            )
+        client = OpenAI(
+            base_url="https://openrouter.ai/api/v1",
+            api_key=api_key,
+            timeout=timeout,
+        )
+        messages: list = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": user})
+        kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+        }
+        if extra_body:
+            kwargs["extra_body"] = extra_body
+        limiter = _openrouter_limiter()
+        limiter.wait_for_rpm_slot()
+        limiter.acquire_concurrency()
+        try:
+            t0 = time.time()
+            resp = client.chat.completions.create(**kwargs)
+        finally:
+            limiter.release_concurrency()
+            limiter.record_call()
+        _bump_cloud_calls()
+        latency = time.time() - t0
+        choice = resp.choices[0]
+        message = choice.message
+        text = message.content or ""
+        tool_calls = _serialize_openai_tool_calls(getattr(message, "tool_calls", None))
+        reasoning = getattr(message, "reasoning_content", None) or getattr(
+            message, "reasoning", None
+        )
+        u = resp.usage
+        p = getattr(u, "prompt_tokens", 0) if u else 0
+        c = getattr(u, "completion_tokens", 0) if u else 0
+        _record_event({
+            "kind": "openrouter",
+            "role": trace_role,
+            "model": model,
+            "system": system,
+            "user": user,
+            "response": text,
+            "tool_calls": tool_calls,
+            "reasoning_content": reasoning,
+            "tokens_in": p,
+            "tokens_out": c,
+            "finish_reason": getattr(choice, "finish_reason", None),
+            "latency_s": latency,
+            "ts": time.time(),
+        })
+        return text, p, c
+
+    @staticmethod
+    def _call_gemini(
+        model: str,
+        *,
+        user: str,
+        system: Optional[str] = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        timeout: float = 600.0,
+        trace_role: str = "cloud",
+    ) -> Tuple[str, int, int]:
+        """Single Gemini Developer-API call. Returns (text, p_tok, c_tok).
+
+        Tool-call parity with Anthropic is intentionally NOT implemented —
+        skillorchestra / baseline-cloud only need text generation, and the
+        google-genai tool-config plumbing diverges enough from the
+        Anthropic/OpenAI shape that wiring it would double the surface
+        area of this file. If a future paradigm needs Gemini tool use,
+        extend this helper rather than hand-rolling it in the agent.
+
+        Captures the call into the active per-task trace via the
+        ``"gemini"`` kind so the dashboard's trace renderer picks it up.
+        """
+        from google import genai
+        from google.genai import types
+
+        client = genai.Client(http_options=types.HttpOptions(timeout=int(timeout * 1000)))
+        cfg = types.GenerateContentConfig(
+            temperature=temperature,
+            max_output_tokens=max_tokens,
+        )
+        if system:
+            cfg.system_instruction = system
+        t0 = time.time()
+        resp = client.models.generate_content(
+            model=model,
+            contents=user,
+            config=cfg,
+        )
+        _bump_cloud_calls()
+        latency = time.time() - t0
+        # `resp.text` is the convenience accessor that concatenates every
+        # text part in the first candidate. Empty if the model emitted
+        # only non-text parts (which we don't request).
+        text = (resp.text or "") if hasattr(resp, "text") else ""
+        um = getattr(resp, "usage_metadata", None)
+        p = int(getattr(um, "prompt_token_count", 0) or 0) if um else 0
+        c = int(getattr(um, "candidates_token_count", 0) or 0) if um else 0
+        finish_reason = None
+        try:
+            finish_reason = str(resp.candidates[0].finish_reason)
+        except Exception:
+            pass
+        _record_event({
+            "kind": "gemini",
+            "role": trace_role,
+            "model": model,
+            "system": system,
+            "user": user,
+            "response": text,
+            "tokens_in": p,
+            "tokens_out": c,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+            "finish_reason": finish_reason,
+            "latency_s": latency,
+            "ts": time.time(),
+        })
+        return text, p, c
+
     @staticmethod
     def _call_vllm(
         model: str,
@@ -392,6 +744,7 @@ class LocalCloudAgent(BaseAgent):
             kwargs["tool_choice"] = tool_choice
         t0 = time.time()
         resp = client.chat.completions.create(**kwargs)
+        _bump_local_calls()
         latency = time.time() - t0
         choice = resp.choices[0]
         message = choice.message
@@ -426,6 +779,334 @@ class LocalCloudAgent(BaseAgent):
         })
         return text, p, c
 
+    @staticmethod
+    def _call_anthropic_agent(
+        model: str,
+        *,
+        user: str,
+        system: Optional[str] = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        tools: Optional[list] = None,
+        max_turns: int = 8,
+        timeout: float = 600.0,
+        max_retries: int = 5,
+        trace_role: str = "cloud",
+    ) -> Tuple[str, int, int, int, int]:
+        """Multi-turn Anthropic loop with optional tools.
+
+        Returns ``(final_text, prompt_tokens_sum, completion_tokens_sum,
+        n_web_searches_sum, turns)``. The loop appends the assistant
+        response (preserving server-tool blocks so Anthropic's continuation
+        is valid) and re-prompts until the model stops with ``end_turn``
+        or we hit ``max_turns``. Server-side tools (web_search) execute
+        inside a single message and don't require a tool_result echo —
+        the model just continues thinking with the search results
+        already in its context. Client-side tools aren't executed here;
+        if the model emits a client-side ``tool_use`` block we stop
+        (paradigms that want client tools should call ``_call_anthropic``
+        directly and handle their own dispatch).
+        """
+        import anthropic
+
+        client = anthropic.Anthropic(timeout=timeout, max_retries=max_retries)
+        # Build the conversation. We grow ``messages`` across turns so the
+        # model sees its own prior assistant content (text + server tool
+        # use + web_search_tool_result). For server tools Anthropic is
+        # happy as long as we feed the raw assistant content back; no
+        # synthetic user tool_result block is needed.
+        messages: List[Dict[str, Any]] = [{"role": "user", "content": user}]
+        p_total = 0
+        c_total = 0
+        n_searches_total = 0
+        last_text = ""
+        turns = 0
+        for turn in range(max(1, max_turns)):
+            turns = turn + 1
+            kwargs: Dict[str, Any] = {
+                "model": model,
+                "max_tokens": max_tokens,
+                "messages": messages,
+            }
+            if system:
+                kwargs["system"] = system
+            if supports_temperature(model):
+                kwargs["temperature"] = temperature
+            if tools:
+                kwargs["tools"] = tools
+            t0 = time.time()
+            msg = client.messages.create(**kwargs)
+            _bump_cloud_calls()
+            latency = time.time() - t0
+            text = "".join(b.text for b in msg.content if hasattr(b, "text"))
+            srv = getattr(msg.usage, "server_tool_use", None)
+            n_searches = getattr(srv, "web_search_requests", 0) if srv else 0
+            content_blocks = [_serialize_block(b) for b in msg.content]
+            tool_use_blocks = [
+                b for b in content_blocks
+                if b.get("type") in ("tool_use", "server_tool_use")
+            ]
+            tool_result_blocks = [
+                b for b in content_blocks
+                if b.get("type") in ("web_search_tool_result", "tool_result")
+            ]
+            stop_reason = getattr(msg, "stop_reason", None)
+            _record_event({
+                "kind": "anthropic",
+                "role": trace_role,
+                "model": model,
+                "system": system if turn == 0 else None,
+                "user": user if turn == 0 else None,
+                "turn": turn,
+                "response": text,
+                "content_blocks": content_blocks,
+                "tool_calls": tool_use_blocks,
+                "tool_results": tool_result_blocks,
+                "tokens_in": msg.usage.input_tokens,
+                "tokens_out": msg.usage.output_tokens,
+                "n_web_searches": n_searches,
+                "tools_declared": tools,
+                "stop_reason": stop_reason,
+                "latency_s": latency,
+                "ts": time.time(),
+            })
+            p_total += msg.usage.input_tokens
+            c_total += msg.usage.output_tokens
+            n_searches_total += n_searches
+            if text:
+                last_text = text
+            # If the model wants a client-side tool we don't dispatch
+            # here — break and let the caller (or future loop variant)
+            # handle it. Only ``server_tool_use`` blocks (web_search)
+            # are auto-continued by Anthropic itself.
+            client_tool_use = any(
+                b.get("type") == "tool_use" for b in content_blocks
+            )
+            if client_tool_use:
+                break
+            if stop_reason == "end_turn" or stop_reason is None:
+                break
+            # Otherwise: ``stop_reason`` like "max_tokens" or "tool_use"
+            # (server side) — Anthropic returned mid-thought. Append the
+            # assistant turn and ask it to continue.
+            messages.append({"role": "assistant", "content": msg.content})
+            messages.append({
+                "role": "user",
+                "content": "Continue.",
+            })
+        return last_text, p_total, c_total, n_searches_total, turns
+
+    @staticmethod
+    def _call_openai_agent(
+        model: str,
+        *,
+        user: str,
+        system: Optional[str] = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        tools: Optional[list] = None,
+        max_turns: int = 8,
+        timeout: float = 600.0,
+        trace_role: str = "cloud",
+    ) -> Tuple[str, int, int, int, int]:
+        """OpenAI hosted-web-search call via the Responses API.
+
+        Returns ``(final_text, prompt_tokens_sum, completion_tokens_sum,
+        n_web_searches_sum, turns)`` — the same 5-tuple shape as
+        ``_call_anthropic_agent`` so callers can dispatch uniformly.
+
+        OpenAI's hosted web search is exposed only through the **Responses
+        API** (``client.responses.create``), not chat completions. The tool
+        is declared as ``{"type": "web_search"}``; older SDK/model combos
+        only accept the legacy ``"web_search_preview"`` name, so we retry
+        once with that on a tool-type rejection. The Responses API runs the
+        search server-side and returns the post-search answer in one call,
+        so ``turns`` is 1 — the ``max_turns`` arg is accepted only for
+        signature parity with ``_call_anthropic_agent``.
+
+        ``n_web_searches`` counts ``web_search_call`` items in the response
+        output. Token usage is summed from ``response.usage``
+        (``input_tokens`` / ``output_tokens``).
+        """
+        from openai import OpenAI
+
+        del max_turns  # Responses API resolves search server-side in one call.
+        client = OpenAI(timeout=timeout)
+
+        search_tool_names = ["web_search", "web_search_preview"]
+        kwargs_base: Dict[str, Any] = {
+            "model": model,
+            "input": user,
+            "max_output_tokens": max_tokens,
+        }
+        if system:
+            kwargs_base["instructions"] = system
+        # GPT-5 family ignores `temperature` on the Responses API (reasoning
+        # models reject it); only pass it for non-gpt-5 models.
+        if not is_gpt5_family(model):
+            kwargs_base["temperature"] = temperature
+
+        resp = None
+        last_exc: Optional[BaseException] = None
+        used_tool_name = search_tool_names[0]
+        t0 = time.time()
+        for tool_name in search_tool_names:
+            try:
+                resp = client.responses.create(
+                    **kwargs_base,
+                    tools=[{"type": tool_name}],
+                )
+                used_tool_name = tool_name
+                break
+            except Exception as exc:  # noqa: BLE001
+                # Only fall through to the legacy name on what looks like a
+                # tool-type rejection; re-raise anything else immediately.
+                last_exc = exc
+                msg = str(exc).lower()
+                if "web_search" in msg or "tool" in msg or "unsupported" in msg:
+                    continue
+                raise
+        if resp is None:
+            raise last_exc if last_exc is not None else RuntimeError(
+                "openai responses.create failed for all web_search tool names"
+            )
+        _bump_cloud_calls()
+        latency = time.time() - t0
+
+        # Extract output text. Prefer the SDK convenience accessor; fall back
+        # to walking the output items for `output_text` content parts.
+        text = ""
+        try:
+            text = resp.output_text or ""
+        except Exception:  # noqa: BLE001
+            text = ""
+        output_items = list(getattr(resp, "output", None) or [])
+        if not text:
+            chunks: List[str] = []
+            for item in output_items:
+                if getattr(item, "type", None) != "message":
+                    continue
+                for part in getattr(item, "content", None) or []:
+                    if getattr(part, "type", None) in ("output_text", "text"):
+                        chunks.append(getattr(part, "text", "") or "")
+            text = "".join(chunks)
+
+        n_searches = sum(
+            1 for item in output_items
+            if getattr(item, "type", None) in (
+                "web_search_call", "web_search_tool_call",
+            )
+        )
+        u = getattr(resp, "usage", None)
+        p = int(getattr(u, "input_tokens", 0) or 0) if u else 0
+        c = int(getattr(u, "output_tokens", 0) or 0) if u else 0
+        _record_event({
+            "kind": "openai_agent",
+            "role": trace_role,
+            "model": model,
+            "system": system,
+            "user": user,
+            "response": text,
+            "output_items": _jsonable(output_items),
+            "tokens_in": p,
+            "tokens_out": c,
+            "n_web_searches": n_searches,
+            "tools_declared": [{"type": used_tool_name}],
+            "stop_reason": getattr(resp, "status", None),
+            "latency_s": latency,
+            "ts": time.time(),
+        })
+        return text, p, c, n_searches, 1
+
+    @staticmethod
+    def _call_gemini_agent(
+        model: str,
+        *,
+        user: str,
+        system: Optional[str] = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.0,
+        tools: Optional[list] = None,
+        max_turns: int = 8,
+        timeout: float = 600.0,
+        trace_role: str = "cloud",
+    ) -> Tuple[str, int, int, int, int]:
+        """Gemini call grounded with Google Search.
+
+        Returns ``(final_text, prompt_tokens_sum, completion_tokens_sum,
+        n_web_searches_sum, turns)`` — same shape as ``_call_anthropic_agent``.
+
+        Grounding is wired by adding ``Tool(google_search=GoogleSearch())``
+        to the ``GenerateContentConfig``. Gemini resolves the grounding
+        server-side inside a single ``generate_content`` call, so
+        ``turns`` is always 1 and ``max_turns`` / ``tools`` are accepted
+        only for signature parity with ``_call_anthropic_agent``.
+
+        ``n_web_searches`` is derived from ``candidates[0].grounding_metadata``
+        — we count the ``web_search_queries`` Gemini reports, falling back to
+        0 when no grounding metadata is present (the model answered without
+        searching).
+        """
+        from google import genai
+        from google.genai import types
+
+        del tools, max_turns  # grounding is config-level + single-call.
+        client = genai.Client(
+            http_options=types.HttpOptions(timeout=int(timeout * 1000))
+        )
+        cfg = types.GenerateContentConfig(
+            temperature=temperature,
+            max_output_tokens=max_tokens,
+            tools=[types.Tool(google_search=types.GoogleSearch())],
+        )
+        if system:
+            cfg.system_instruction = system
+        t0 = time.time()
+        resp = client.models.generate_content(
+            model=model,
+            contents=user,
+            config=cfg,
+        )
+        _bump_cloud_calls()
+        latency = time.time() - t0
+        text = (resp.text or "") if hasattr(resp, "text") else ""
+        um = getattr(resp, "usage_metadata", None)
+        p = int(getattr(um, "prompt_token_count", 0) or 0) if um else 0
+        c = int(getattr(um, "candidates_token_count", 0) or 0) if um else 0
+
+        # Derive search count from grounding metadata when present.
+        n_searches = 0
+        web_search_queries: List[str] = []
+        finish_reason = None
+        try:
+            cand0 = resp.candidates[0]
+            finish_reason = str(getattr(cand0, "finish_reason", None))
+            gm = getattr(cand0, "grounding_metadata", None)
+            if gm is not None:
+                queries = getattr(gm, "web_search_queries", None) or []
+                web_search_queries = [str(q) for q in queries]
+                n_searches = len(web_search_queries)
+        except Exception:  # noqa: BLE001
+            pass
+        _record_event({
+            "kind": "gemini_agent",
+            "role": trace_role,
+            "model": model,
+            "system": system,
+            "user": user,
+            "response": text,
+            "tokens_in": p,
+            "tokens_out": c,
+            "n_web_searches": n_searches,
+            "web_search_queries": web_search_queries,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+            "finish_reason": finish_reason,
+            "latency_s": latency,
+            "ts": time.time(),
+        })
+        return text, p, c, n_searches, 1
+
     def _call_cloud(
         self,
         *,
@@ -460,6 +1141,23 @@ class LocalCloudAgent(BaseAgent):
                 temperature=temperature,
                 **kwargs,
             )
+        if self._cloud_endpoint == "gemini":
+            # Gemini helper doesn't accept tools / response_format kwargs.
+            # Drop them rather than letting them surface as a TypeError so
+            # paradigms that opportunistically pass these for OpenAI /
+            # Anthropic still work when routed to Gemini.
+            kwargs.pop("tools", None)
+            kwargs.pop("tool_choice", None)
+            kwargs.pop("response_format", None)
+            kwargs.pop("output_config", None)
+            return self._call_gemini(
+                self._cloud_model,
+                user=user,
+                system=system,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                **kwargs,
+            )
         raise ValueError(f"unsupported cloud endpoint: {self._cloud_endpoint!r}")
 
     # ------------------------------------------------------------------
@@ -478,6 +1176,8 @@ class LocalCloudAgent(BaseAgent):
             "tokens_cloud": 0,
             "cost_usd": 0.0,
             "latency_s": 0.0,
+            "n_cloud_calls": 0,
+            "n_local_calls": 0,
             "soft_error": reason,
             "traces": {"soft_error": reason},
         }
@@ -499,6 +1199,7 @@ class LocalCloudAgent(BaseAgent):
         self._emit_turn_start(input)
         t0 = time.time()
         events = _open_trace()
+        counts = _open_call_counts()
         meta: Dict[str, Any]
         answer: str = ""
         soft_reason: Optional[str] = None
@@ -514,6 +1215,16 @@ class LocalCloudAgent(BaseAgent):
                 soft_reason = soft
                 meta = self._soft_fail_metadata(soft)
         finally:
+            # Snapshot call counts BEFORE closing the thread-local state.
+            # Subclasses don't track this themselves — every SDK call site
+            # bumps the thread-local in this file, so the totals are
+            # authoritative as of right now. Overwrites any value
+            # ``_run_paradigm`` happened to set (it shouldn't set them).
+            n_cloud = counts.get("cloud", 0)
+            n_local = counts.get("local", 0)
+            if "meta" in locals():
+                meta["n_cloud_calls"] = int(n_cloud)
+                meta["n_local_calls"] = int(n_local)
             # Persist the trace before the trace state is closed (and even on
             # hard failure, so we get a record of what we did before it broke).
             self._write_trace_log(
@@ -521,6 +1232,7 @@ class LocalCloudAgent(BaseAgent):
                 events, soft_reason, exc_obj,
             )
             _close_trace()
+            _close_call_counts()
         meta.setdefault("latency_s", time.time() - t0)
         if soft_reason is not None:
             self._emit_turn_end(soft_error=soft_reason)
@@ -606,10 +1318,16 @@ class LocalCloudAgent(BaseAgent):
 
 __all__ = [
     "ANTHROPIC_WEB_SEARCH_TOOL",
+    "GEMINI_SEARCH_COST_PER_CALL",
     "LocalCloudAgent",
     "NO_TEMP_PREFIXES",
+    "OPENAI_WEB_SEARCH_COST_PER_CALL",
     "WEB_SEARCH_COST_PER_CALL",
+    "_bump_cloud_calls",
+    "_bump_local_calls",
+    "build_web_search_tool",
     "estimate_cost",
     "is_gpt5_family",
     "supports_temperature",
+    "web_search_cfg",
 ]

src/openjarvis/agents/hybrid/_energy.py

@@ -0,0 +1,175 @@
+"""GPU energy collector for hybrid cell runs.
+
+Samples ``pynvml.nvmlDeviceGetPowerUsage`` at ~2Hz on a background thread,
+integrates power × dt → joules across the cell's wall-time. Used as a
+context manager wrapping ``_run_cell_locked`` so the whole cell's GPU
+energy is attributed to the cell (not per-task — concurrency makes
+per-task attribution very noisy).
+
+Failure modes are absorbed: if NVML isn't available (no GPU on this host,
+container without ``libnvidia-ml.so.1``, permission denied) the collector
+logs *once* and returns ``0.0``. It must never crash the run.
+
+Scope notes
+-----------
+* Samples **all visible NVIDIA GPUs on the host where the runner runs**.
+  In our setup that's the L40S node that also hosts vLLM
+  (``mkt1`` / ``matx2``), so this captures the local-model serving GPUs.
+  If the runner is invoked from a host without GPUs (e.g. a login node
+  with vLLM on a remote box), the collector logs and yields 0 — set
+  ``OPENJARVIS_HYBRID_ENERGY=0`` to silence the warning.
+* ``CUDA_VISIBLE_DEVICES`` is *honored* for parity with vLLM: only those
+  GPU indices are sampled. Unset = all GPUs on the host.
+* **Cloud energy is not measured** — cloud calls go over HTTPS to
+  Anthropic/OpenAI/Google, no measurable joules on our side. A future
+  pass could add a per-token J/token estimate (e.g. Patterson et al.
+  2021, Luccioni et al. 2022) but those numbers are vendor-opaque and
+  uncertain — leaving as a TODO until we explicitly decide to estimate.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+from typing import List, Optional
+
+
+_NVML_WARNED = False
+
+
+def _log_once(msg: str) -> None:
+    global _NVML_WARNED
+    if not _NVML_WARNED:
+        print(f"[energy] {msg}", flush=True)
+        _NVML_WARNED = True
+
+
+def _resolve_gpu_indices(total: int) -> List[int]:
+    """Honor CUDA_VISIBLE_DEVICES; default to every visible GPU."""
+    cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if not cvd:
+        return list(range(total))
+    out: List[int] = []
+    for s in cvd.split(","):
+        s = s.strip()
+        if not s:
+            continue
+        try:
+            idx = int(s)
+        except ValueError:
+            continue
+        if 0 <= idx < total:
+            out.append(idx)
+    return out or list(range(total))
+
+
+class EnergyCollector:
+    """Background NVML power sampler integrating to joules.
+
+    Use as a context manager::
+
+        with EnergyCollector() as ec:
+            ...                       # run the cell
+        joules = ec.energy_j_total    # also available after __exit__
+
+    Safe to instantiate even when NVML is unavailable: ``energy_j_total``
+    will simply be ``0.0`` and a one-time warning will be printed.
+    """
+
+    def __init__(self, sample_hz: float = 2.0) -> None:
+        self.sample_hz = float(sample_hz)
+        self.energy_j_total: float = 0.0
+        self.samples: int = 0
+        self.gpu_indices: List[int] = []
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+        self._t0: float = 0.0
+        self._t1: float = 0.0
+        self._enabled = os.environ.get("OPENJARVIS_HYBRID_ENERGY", "1") != "0"
+        self._pynvml = None
+        self._handles: list = []
+
+    # ---- context manager
+    def __enter__(self) -> "EnergyCollector":
+        self._t0 = time.time()
+        if not self._enabled:
+            return self
+        try:
+            import pynvml  # type: ignore[import-not-found]
+            pynvml.nvmlInit()
+            total = pynvml.nvmlDeviceGetCount()
+            self.gpu_indices = _resolve_gpu_indices(total)
+            self._handles = [
+                pynvml.nvmlDeviceGetHandleByIndex(i) for i in self.gpu_indices
+            ]
+            # Probe once so we fail fast if power query is unsupported.
+            for h in self._handles:
+                pynvml.nvmlDeviceGetPowerUsage(h)
+            self._pynvml = pynvml
+        except Exception as e:  # noqa: BLE001 — NVML failures must never crash the run
+            _log_once(
+                f"NVML unavailable ({type(e).__name__}: {e}); "
+                "energy_j_total will be 0. Set OPENJARVIS_HYBRID_ENERGY=0 to silence."
+            )
+            self._pynvml = None
+            return self
+
+        self._thread = threading.Thread(
+            target=self._sample_loop, name="energy-sampler", daemon=True
+        )
+        self._thread.start()
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        self._t1 = time.time()
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=5.0)
+        if self._pynvml is not None:
+            try:
+                self._pynvml.nvmlShutdown()
+            except Exception:  # noqa: BLE001
+                pass
+
+    # ---- sampler thread
+    def _sample_loop(self) -> None:
+        """Trapezoid-integrate Σ_gpu power_W × dt over the cell run.
+
+        ``nvmlDeviceGetPowerUsage`` returns milliwatts. Sample interval
+        is best-effort (~``1/sample_hz`` s); if the host is overloaded
+        and a tick lands late we still use the real ``dt`` so the
+        integral stays honest. Trapezoid rule (mean of consecutive
+        readings) damps jitter vs. left-Riemann.
+        """
+        pynvml = self._pynvml
+        assert pynvml is not None
+        period = 1.0 / max(self.sample_hz, 0.1)
+        last_t = time.time()
+        last_total_w: Optional[float] = None
+        while not self._stop.is_set():
+            try:
+                total_mw = 0
+                for h in self._handles:
+                    total_mw += pynvml.nvmlDeviceGetPowerUsage(h)
+                total_w = total_mw / 1000.0
+            except Exception:  # noqa: BLE001 — keep going even on transient NVML errors
+                self._stop.wait(period)
+                continue
+            now = time.time()
+            if last_total_w is not None:
+                dt = now - last_t
+                # Trapezoid rule: mean power across the interval × dt.
+                self.energy_j_total += 0.5 * (total_w + last_total_w) * dt
+            last_t = now
+            last_total_w = total_w
+            self.samples += 1
+            self._stop.wait(period)
+
+    # ---- inspection
+    @property
+    def wall_s(self) -> float:
+        return max(self._t1 - self._t0, 0.0) if self._t1 else (time.time() - self._t0)
+
+
+__all__ = ["EnergyCollector"]

src/openjarvis/agents/hybrid/_openai_retry.py

@@ -0,0 +1,355 @@
+"""Process-wide retry + concurrency hardening for cloud OpenAI calls.
+
+Why this exists
+---------------
+
+When we run the hybrid paradigms (Minions, Advisors, Conductor, …) at
+n=100 against ``gpt-5`` / ``gpt-5-mini`` over the prepaid OpenAI quota,
+sustained concurrency walls the org-level rate limit and the OpenAI SDK
+raises :class:`openai.RateLimitError`. The SDK's own retry path is short
+(default ``max_retries=2`` on a small backoff) — under sustained pressure
+every wave of retries hits the same wall and the runner records 19-69
+errored rows per cell, degenerating the result.
+
+Mirrors the existing ``_patch_anthropic_globally`` pattern in
+``minions.py``: monkey-patch the SDK at module level so it applies even
+to libraries that build their own ``openai.OpenAI()`` clients
+(HazyResearch Minions's ``OpenAIClient``, Archon's adapters,
+``mini_swe_agent``, etc.). One call to :func:`patch_openai_globally` from
+either ``_base.py`` or ``minions._apply_patches_once`` is enough — the
+patch is idempotent and process-wide.
+
+What it does
+------------
+
+1. Bumps ``openai.OpenAI()`` constructor defaults to
+   ``timeout=600.0`` / ``max_retries=8`` (the SDK's own backoff is fine
+   for transient blips; we layer our own loop on top for sustained walls).
+2. Wraps ``chat.completions.create`` with:
+
+   - A **per-org semaphore** (``OPENJARVIS_OPENAI_MAX_CONCURRENCY``,
+     default 4) that throttles sustained concurrency. Single bursts are
+     fine — prepaid quotas wall on sustained rate, not on a brief spike.
+     The semaphore is **only acquired for cloud calls** (``api.openai.com``);
+     vLLM calls routed through the OpenAI SDK (``base_url`` set to a
+     local endpoint or ``api_key="EMPTY"``) bypass it.
+   - Exponential backoff with jitter on :class:`openai.RateLimitError`,
+     :class:`openai.APITimeoutError`, :class:`openai.APIConnectionError`,
+     and 5xx :class:`openai.APIStatusError`. Starts at 2 s, caps at 60 s,
+     up to 8 attempts (~3.5 min worst case). Honors a ``Retry-After``
+     header when the SDK surfaces one.
+   - On exhaustion, re-raises the last exception (it bubbles up to the
+     runner, which records ``error="RateLimitError: ..."`` in
+     ``results.jsonl`` — no silent drop).
+
+Env knobs
+---------
+
+- ``OPENJARVIS_OPENAI_MAX_CONCURRENCY`` (default ``4``) — semaphore
+  capacity. Set to e.g. ``2`` if the wall is still hit; set to ``0`` to
+  disable throttling entirely (passes through to the SDK).
+- ``OPENJARVIS_OPENAI_MAX_RETRIES`` (default ``8``) — outer retry loop
+  cap (separate from the SDK's own ``max_retries``).
+- ``OPENJARVIS_OPENAI_RETRY_BASE`` (default ``2.0``) — base seconds for
+  exponential backoff. Schedule is ``min(60, base * 2**attempt) * jitter``.
+- ``OPENJARVIS_OPENAI_RETRY_CAP`` (default ``60.0``) — max single-step
+  sleep in seconds.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import threading
+import time
+from typing import Any, Callable, Optional, Tuple
+from urllib.parse import urlparse
+
+# ---------------------------------------------------------------------------
+# Tunables (read once at module load, can be overridden via env)
+# ---------------------------------------------------------------------------
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        v = int(os.environ.get(name, "") or default)
+        return max(0, v)
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, "") or default)
+    except ValueError:
+        return default
+
+
+_MAX_CONCURRENCY = _env_int("OPENJARVIS_OPENAI_MAX_CONCURRENCY", 4)
+_MAX_RETRIES = _env_int("OPENJARVIS_OPENAI_MAX_RETRIES", 8)
+_RETRY_BASE = _env_float("OPENJARVIS_OPENAI_RETRY_BASE", 2.0)
+_RETRY_CAP = _env_float("OPENJARVIS_OPENAI_RETRY_CAP", 60.0)
+
+
+# Single process-wide semaphore. ``BoundedSemaphore(0)`` would block
+# forever, so when the env knob is 0 we hand back a no-op context manager.
+class _NullSem:
+    def __enter__(self) -> "_NullSem":
+        return self
+
+    def __exit__(self, *a: Any) -> None:
+        return None
+
+
+_SEM: Any
+if _MAX_CONCURRENCY > 0:
+    _SEM = threading.BoundedSemaphore(_MAX_CONCURRENCY)
+else:
+    _SEM = _NullSem()
+
+
+_PATCHED = False
+_PATCH_LOCK = threading.Lock()
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _is_local_endpoint(client: Any) -> bool:
+    """True if this OpenAI client points at a local vLLM endpoint.
+
+    We detect via either ``api_key == "EMPTY"`` (the convention used by
+    our ``_call_vllm`` and ``mini_swe_agent``) or a ``base_url`` whose
+    hostname resolves to localhost. Either signal is enough; both are
+    cheap to read.
+    """
+    try:
+        api_key = getattr(client, "api_key", None)
+        if api_key == "EMPTY":
+            return True
+    except Exception:
+        pass
+    try:
+        base_url = str(getattr(client, "base_url", "") or "")
+        if not base_url:
+            return False
+        host = urlparse(base_url).hostname or ""
+        return host in ("localhost", "127.0.0.1", "0.0.0.0", "::1")
+    except Exception:
+        return False
+
+
+def _extract_retry_after(exc: BaseException) -> Optional[float]:
+    """Pull a Retry-After header off an APIStatusError if the SDK exposed it.
+
+    OpenAI's SDK keeps the underlying ``httpx.Response`` on
+    ``exc.response`` for ``APIStatusError`` subclasses. Header may be a
+    seconds-integer or an HTTP-date; we only handle the integer form
+    (the only thing OpenAI sends in practice).
+    """
+    resp = getattr(exc, "response", None)
+    if resp is None:
+        return None
+    headers = getattr(resp, "headers", None)
+    if not headers:
+        return None
+    for name in ("retry-after", "Retry-After", "x-ratelimit-reset-requests"):
+        val = headers.get(name) if hasattr(headers, "get") else None
+        if val is None:
+            continue
+        try:
+            secs = float(val)
+            if 0 <= secs <= 600:
+                return secs
+        except (TypeError, ValueError):
+            continue
+    return None
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    """Whether to retry this OpenAI exception class."""
+    try:
+        import openai
+    except ImportError:
+        return False
+    if isinstance(exc, (
+        openai.RateLimitError,
+        openai.APITimeoutError,
+        openai.APIConnectionError,
+        openai.InternalServerError,
+    )):
+        return True
+    if isinstance(exc, openai.APIStatusError):
+        status = getattr(exc, "status_code", None)
+        # 429 is RateLimitError already; 5xx is retryable; 408 is a
+        # timeout the SDK didn't classify (rare).
+        return status is not None and (status >= 500 or status in (408, 409, 429))
+    return False
+
+
+def _sleep_for(attempt: int, exc: BaseException) -> float:
+    """Backoff for attempt index ``attempt`` (0-based)."""
+    hinted = _extract_retry_after(exc)
+    if hinted is not None and hinted > 0:
+        # Respect a server-provided hint, but clamp to our cap so a
+        # pathological header can't stall the run for hours.
+        return min(_RETRY_CAP, hinted) + random.uniform(0, 0.5)
+    base = min(_RETRY_CAP, _RETRY_BASE * (2 ** attempt))
+    # Full jitter — better tail behavior than equal jitter when many
+    # workers wake at the same moment.
+    return random.uniform(0.0, base)
+
+
+# ---------------------------------------------------------------------------
+# Wrapping
+# ---------------------------------------------------------------------------
+
+
+def _wrap_create(orig: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrap a ``chat.completions.create`` (or ``responses.create``) bound
+    method's underlying function with retry + concurrency throttling.
+
+    The wrapper is a regular function that takes ``self`` as the first
+    arg, so it can replace ``Completions.create`` at the class level and
+    still see the bound client through ``self._client``.
+    """
+
+    def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
+        client = getattr(self, "_client", None)
+        local = client is not None and _is_local_endpoint(client)
+        # Local vLLM calls bypass the per-org throttle (no rate limit) and
+        # the long retry loop (vLLM is mostly either up or down — a 60s
+        # backoff just delays surfacing the failure). But brief
+        # ConnectionError blips do happen mid-sweep (socket queue, brief
+        # server warmup pause): give the local path a SHORT retry — 3
+        # attempts, 1s/2s/4s — so we don't error an entire row on a
+        # transient refused connection. Anything else (BadRequest etc.)
+        # still raises immediately.
+        if local:
+            local_last_exc: Optional[BaseException] = None
+            for attempt in range(3):
+                try:
+                    return orig(self, *args, **kwargs)
+                except BaseException as exc:  # noqa: BLE001
+                    try:
+                        import openai
+                    except ImportError:
+                        raise
+                    if not isinstance(exc, (
+                        openai.APIConnectionError,
+                        openai.APITimeoutError,
+                        openai.InternalServerError,
+                    )):
+                        raise
+                    local_last_exc = exc
+                    if attempt >= 2:
+                        break
+                    time.sleep(2 ** attempt)
+            assert local_last_exc is not None
+            raise local_last_exc
+
+        last_exc: Optional[BaseException] = None
+        for attempt in range(_MAX_RETRIES + 1):
+            try:
+                with _SEM:
+                    return orig(self, *args, **kwargs)
+            except BaseException as exc:  # noqa: BLE001
+                if not _is_retryable(exc):
+                    raise
+                last_exc = exc
+                if attempt >= _MAX_RETRIES:
+                    break
+                delay = _sleep_for(attempt, exc)
+                # Stderr, not stdout: heartbeat / progress lines must
+                # stay parseable in the runner log.
+                try:
+                    import sys
+                    print(
+                        f"[openai-retry] attempt {attempt + 1}/{_MAX_RETRIES} "
+                        f"{type(exc).__name__}: {str(exc)[:120]} — "
+                        f"sleeping {delay:.1f}s",
+                        file=sys.stderr,
+                        flush=True,
+                    )
+                except Exception:
+                    pass
+                time.sleep(delay)
+        # Exhausted. Re-raise so the runner records the row as errored.
+        assert last_exc is not None
+        raise last_exc
+
+    wrapped._hybrid_patched = True  # type: ignore[attr-defined]
+    wrapped.__wrapped__ = orig  # type: ignore[attr-defined]
+    return wrapped
+
+
+def patch_openai_globally() -> None:
+    """Idempotently monkey-patch the OpenAI SDK to add retry + throttling.
+
+    Safe to call from multiple modules — guarded by ``_PATCHED`` under a
+    lock. Patches both the ``Completions.create`` method and the
+    ``OpenAI.__init__`` defaults.
+    """
+    global _PATCHED
+    if _PATCHED:
+        return
+    with _PATCH_LOCK:
+        if _PATCHED:
+            return
+        try:
+            import openai
+            from openai.resources.chat import completions as _comp_mod
+        except ImportError:
+            return
+
+        # Bump constructor defaults so callers that don't pass timeout /
+        # max_retries explicitly still get sensible values. ``setdefault``
+        # so any explicit caller value wins.
+        if not getattr(openai.OpenAI.__init__, "_hybrid_patched", False):
+            _orig_init = openai.OpenAI.__init__
+
+            def _patched_init(self: Any, *args: Any, **kwargs: Any) -> None:
+                kwargs.setdefault("timeout", 600.0)
+                kwargs.setdefault("max_retries", _MAX_RETRIES)
+                return _orig_init(self, *args, **kwargs)
+
+            _patched_init._hybrid_patched = True  # type: ignore[attr-defined]
+            openai.OpenAI.__init__ = _patched_init  # type: ignore[assignment]
+
+        # Wrap chat.completions.create. The SDK exposes the bound method
+        # via ``Completions.create``; we replace the class attribute so
+        # every instance (including ones built inside external libs)
+        # sees the wrapped version.
+        if not getattr(_comp_mod.Completions.create, "_hybrid_patched", False):
+            _comp_mod.Completions.create = _wrap_create(  # type: ignore[assignment]
+                _comp_mod.Completions.create
+            )
+
+        # Also patch the async variant for completeness (none of our
+        # paradigms use it today, but Archon / future paradigms might).
+        try:
+            from openai.resources.chat import completions as _comp_mod_async
+
+            cls = getattr(_comp_mod_async, "AsyncCompletions", None)
+            if cls is not None and not getattr(
+                cls.create, "_hybrid_patched", False
+            ):
+                # Async wrapper is structurally different — only patch
+                # the bumped defaults via __init__; full retry loop on
+                # async would need an async wrapper. Leave that for the
+                # day a paradigm actually uses it.
+                pass
+        except ImportError:
+            pass
+
+        _PATCHED = True
+
+
+def current_settings() -> Tuple[int, int, float, float]:
+    """For tests / smoke runs: return (concurrency, retries, base, cap)."""
+    return _MAX_CONCURRENCY, _MAX_RETRIES, _RETRY_BASE, _RETRY_CAP
+
+
+__all__ = ["patch_openai_globally", "current_settings"]

src/openjarvis/agents/hybrid/_prices.py

@@ -19,7 +19,17 @@ PRICES: dict[str, tuple[float, float]] = {
     "gpt-5-mini":                  (0.25, 2.00),
     "gpt-5-mini-2025-08-07":       (0.25, 2.00),
     "gpt-4o":                      (0.15, 0.60),
+    # Gemini Developer API prices (USD per 1M tokens), 2025-12 list price.
+    # 2.5 Pro uses tiered pricing (>200K context = $2.50/$15); we charge the
+    # low-context tier since GAIA / SWE-bench prompts stay well under 200K.
     "gemini-2.5-pro":              (1.25, 10.0),
+    "gemini-2.5-flash":            (0.30, 2.50),
+    "gemini-2.5-flash-lite":       (0.10, 0.40),
+    # OpenRouter slugs (used by toolorchestra paper-match pool).
+    # Prices are OpenRouter list (USD/1M tokens), 2026-05 snapshot.
+    "qwen/qwen-2.5-coder-32b-instruct":  (0.08, 0.18),
+    "qwen/qwen3-32b":                    (0.10, 0.30),
+    "meta-llama/llama-3.3-70b-instruct": (0.13, 0.39),
 }
 
 # Models whose API rejects an explicit `temperature` param — callers should
@@ -44,3 +54,17 @@ def supports_temperature(model: str) -> bool:
 def is_gpt5_family(model: str) -> bool:
     """GPT-5 series requires ``max_completion_tokens`` and forced temp=1."""
     return model.startswith("gpt-5")
+
+
+def is_reasoning_model(model: str) -> bool:
+    """Models that consume the output-token budget on hidden chain-of-thought
+    before emitting visible answer text. At max_tokens=4096 these silently
+    truncate with empty answers on GAIA (26/100 GPT-5, 18/100 Gemini Pro)."""
+    m = (model or "").lower()
+    return is_gpt5_family(model) or "gemini-2.5-pro" in m
+
+
+def default_max_output_tokens(model: str) -> int:
+    """Sane default for ``max_tokens`` per cloud call. Reasoning models get
+    a larger budget so their hidden thinking doesn't crowd out the answer."""
+    return 16384 if is_reasoning_model(model) else 4096

src/openjarvis/agents/hybrid/advisors.py

@@ -28,7 +28,14 @@ import urllib.request
 from typing import Any, Dict, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent
+from openjarvis.agents.hybrid._base import (
+    GEMINI_SEARCH_COST_PER_CALL,
+    OPENAI_WEB_SEARCH_COST_PER_CALL,
+    WEB_SEARCH_COST_PER_CALL,
+    LocalCloudAgent,
+    build_web_search_tool,
+    web_search_cfg,
+)
 from openjarvis.agents.hybrid.mini_swe_agent import (
     run_swe_agent_loop,
 )
@@ -84,6 +91,21 @@ def _resolve_local_model(endpoint: str, registry_model: str) -> str:
     return served[0] if served else registry_model
 
 
+# Cloud endpoints that have a server-side web-search agent loop wired in
+# `_base.py`. Anything else (openrouter, vllm, unknown) cannot ground.
+_SEARCH_CAPABLE_ENDPOINTS = ("anthropic", "openai", "gemini")
+
+
+def _search_cost_per_call(endpoint: str) -> float:
+    """Per-search-call USD cost for the configured cloud endpoint."""
+    if endpoint == "openai":
+        return OPENAI_WEB_SEARCH_COST_PER_CALL
+    if endpoint == "gemini":
+        return GEMINI_SEARCH_COST_PER_CALL
+    # anthropic (and any caller that already validated the endpoint).
+    return WEB_SEARCH_COST_PER_CALL
+
+
 @AgentRegistry.register("advisors")
 class AdvisorsAgent(LocalCloudAgent):
     """Three-step executor ↔ advisor ↔ executor loop. See module docstring."""
@@ -112,13 +134,39 @@ class AdvisorsAgent(LocalCloudAgent):
         advisor_max_tokens = int(cfg.get("advisor_max_tokens", 1024))
         advisor_temperature = float(cfg.get("advisor_temperature", 0.2))
 
-        # 1. Initial executor pass
-        initial_resp, e1_in, e1_out = self._call_cloud(
-            user=f"Question:\n{question}",
-            system=EXECUTOR_INITIAL_SYS,
-            max_tokens=executor_max_tokens,
-            temperature=0.0,
-        )
+        ws_enabled, ws_max_uses = web_search_cfg(cfg)
+        if ws_enabled and self._cloud_endpoint not in _SEARCH_CAPABLE_ENDPOINTS:
+            raise ValueError(
+                f"web_search.enabled=true but cloud_endpoint={self._cloud_endpoint!r}; "
+                "server-side web_search is wired for anthropic / openai / gemini "
+                "executors only. Route this cell through one of those or disable "
+                "web_search — otherwise search would silently no-op and the "
+                "executor answers blind."
+            )
+        use_ws = ws_enabled
+        gaia_max_turns = int(cfg.get("gaia_max_turns", 8))
+        n_searches_total = 0
+
+        # 1. Initial executor pass — advisor (Qwen) doesn't get tools;
+        # only the cloud executor passes do. With web_search on, dispatch
+        # to the search-capable agent loop for the configured provider.
+        if use_ws:
+            initial_resp, e1_in, e1_out, n_s1, e1_turns = self._executor_search(
+                user=f"Question:\n{question}",
+                system=EXECUTOR_INITIAL_SYS,
+                max_tokens=executor_max_tokens,
+                ws_max_uses=ws_max_uses,
+                max_turns=gaia_max_turns,
+            )
+            n_searches_total += n_s1
+        else:
+            initial_resp, e1_in, e1_out = self._call_cloud(
+                user=f"Question:\n{question}",
+                system=EXECUTOR_INITIAL_SYS,
+                max_tokens=executor_max_tokens,
+                temperature=0.0,
+            )
+            e1_turns = 1
 
         # 2. Advisor pass (local)
         if not self._local_endpoint or not self._local_model:
@@ -147,30 +195,105 @@ class AdvisorsAgent(LocalCloudAgent):
             f"Produce your best final answer now, respecting the question's "
             f"answer-format rules."
         )
-        final_answer, e2_in, e2_out = self._call_cloud(
-            user=final_user,
-            system=EXECUTOR_FINAL_SYS,
-            max_tokens=executor_max_tokens,
-            temperature=0.0,
-        )
+        if use_ws:
+            final_answer, e2_in, e2_out, n_s2, e2_turns = self._executor_search(
+                user=final_user,
+                system=EXECUTOR_FINAL_SYS,
+                max_tokens=executor_max_tokens,
+                ws_max_uses=ws_max_uses,
+                max_turns=gaia_max_turns,
+            )
+            n_searches_total += n_s2
+        else:
+            final_answer, e2_in, e2_out = self._call_cloud(
+                user=final_user,
+                system=EXECUTOR_FINAL_SYS,
+                max_tokens=executor_max_tokens,
+                temperature=0.0,
+            )
+            e2_turns = 1
 
         tokens_local = adv_in + adv_out
         tokens_cloud = e1_in + e1_out + e2_in + e2_out
         cost = self.cost_usd(self._cloud_model, e1_in + e2_in, e1_out + e2_out)
+        cost += n_searches_total * _search_cost_per_call(self._cloud_endpoint)
 
         meta: Dict[str, Any] = {
             "tokens_local": tokens_local,
             "tokens_cloud": tokens_cloud,
             "cost_usd": cost,
-            "turns": 3,
+            # executor pass 1 + advisor pass (1) + executor pass 2. With
+            # web_search on, each executor pass is a multi-turn loop, so
+            # this is > 3; one-shot (no search) it's exactly 3.
+            "turns": e1_turns + 1 + e2_turns,
+            "web_search_uses": n_searches_total,
+            # GAIA: only the executor passes invoke a tool (web_search).
+            "tool_calls": int(n_searches_total),
             "traces": {
                 "initial_response": initial_resp,
                 "advisor_feedback": advisor_text,
+                "web_search_enabled": use_ws,
+                "n_web_searches": n_searches_total,
                 "note": "inference-only advisor (untrained); lower bound on the technique.",
             },
         }
         return final_answer, meta
 
+    # ------------------------------------------------------------------
+    # Web-search executor dispatch
+    # ------------------------------------------------------------------
+
+    def _executor_search(
+        self,
+        *,
+        user: str,
+        system: str,
+        max_tokens: int,
+        ws_max_uses: int,
+        max_turns: int,
+    ) -> Tuple[str, int, int, int, int]:
+        """Run a search-capable executor pass for the configured cloud.
+
+        Dispatches by ``self._cloud_endpoint`` to the matching ``_base``
+        agent loop. Returns the shared 5-tuple ``(text, p_tok, c_tok,
+        n_searches, turns)``. The endpoint is assumed already validated
+        against ``_SEARCH_CAPABLE_ENDPOINTS`` by the caller.
+        """
+        if self._cloud_endpoint == "anthropic":
+            return self._call_anthropic_agent(
+                self._cloud_model,
+                user=user,
+                system=system,
+                max_tokens=max_tokens,
+                temperature=0.0,
+                tools=[build_web_search_tool(ws_max_uses)],
+                max_turns=max_turns,
+            )
+        if self._cloud_endpoint == "openai":
+            return self._call_openai_agent(
+                self._cloud_model,
+                user=user,
+                system=system,
+                max_tokens=max_tokens,
+                temperature=0.0,
+                max_turns=max_turns,
+            )
+        if self._cloud_endpoint == "gemini":
+            return self._call_gemini_agent(
+                self._cloud_model,
+                user=user,
+                system=system,
+                max_tokens=max_tokens,
+                temperature=0.0,
+                max_turns=max_turns,
+            )
+        # Genuinely unsupported (openrouter / vllm / unknown). The caller
+        # guard should have caught this; raise defensively.
+        raise ValueError(
+            f"web_search executor pass requested but cloud_endpoint="
+            f"{self._cloud_endpoint!r} has no search wiring."
+        )
+
     # ------------------------------------------------------------------
     # SWE-bench variant: each executor pass is a full mini-SWE-agent run.
     # ------------------------------------------------------------------
@@ -265,6 +388,9 @@ class AdvisorsAgent(LocalCloudAgent):
             "tokens_cloud": tokens_cloud,
             "cost_usd": cost,
             "turns": initial_out["turns"] + 1 + final_out["turns"],
+            # SWE: sum bash turns from both executor passes; advisor pass
+            # is a local-model critique with no tools.
+            "tool_calls": int(initial_out["turns"] + final_out["turns"]),
             "traces": {
                 "swe_mode": True,
                 "initial_summary": initial_out["final_summary"],

src/openjarvis/agents/hybrid/archon.py

@@ -26,9 +26,9 @@ How the hybrid harness wires it (and what we mirror here):
 - ``ranker_model`` / ``fuser_model`` (default: ``cloud_model`` for both)
 - ``max_tokens`` (default 2048), ``temperature`` (default 0.7)
 
-Requires the Archon library (cloned at
-``hybrid-local-cloud-compute/external/Archon`` — add its ``src`` to
-``PYTHONPATH`` or pip-install editable). Import is lazy.
+Requires the Archon library from https://github.com/Stanford-ILIAD/Archon
+— either pip-install editable or set ``ARCHON_SRC`` to the checkout's
+``src/`` directory. Import is lazy.
 
 Ported from ``hybrid-local-cloud-compute/adapters/archon_adapter.py``.
 """
@@ -42,7 +42,15 @@ import types
 from typing import Any, Dict, List, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent, _record_event
+from openjarvis.agents.hybrid._base import (
+    WEB_SEARCH_COST_PER_CALL,
+    LocalCloudAgent,
+    _bump_cloud_calls,
+    _bump_local_calls,
+    _record_event,
+    build_web_search_tool,
+    web_search_cfg,
+)
 from openjarvis.agents.hybrid._prices import (
     NO_TEMP_PREFIXES,
 )
@@ -79,13 +87,11 @@ def _stub_archon_imports() -> None:
 
 
 def _add_archon_to_path() -> None:
-    """Locate Archon's ``src`` dir. The hybrid clone is the default location;
-    override with ``ARCHON_SRC`` env var if Archon is installed elsewhere."""
-    archon_src = os.environ.get(
-        "ARCHON_SRC",
-        "/matx/u/aspark/hybrid-local-cloud-compute/external/Archon/src",
-    )
-    if archon_src not in sys.path and os.path.isdir(archon_src):
+    """Locate Archon's ``src`` dir. Set ``ARCHON_SRC`` to point at your
+    Archon checkout (``<repo>/src``); otherwise we assume ``archon`` is on
+    ``sys.path`` already (e.g. ``pip install`` from a local clone)."""
+    archon_src = os.environ.get("ARCHON_SRC")
+    if archon_src and os.path.isdir(archon_src) and archon_src not in sys.path:
         sys.path.insert(0, archon_src)
 
 
@@ -125,8 +131,11 @@ def _tally() -> Dict[str, int]:
         counts = {
             "cloud_prompt": 0, "cloud_completion": 0,
             "local_prompt": 0, "local_completion": 0,
+            "n_web_searches": 0,
         }
         _TALLY_LOCAL.counts = counts
+    # Backfill for older threads that started before we added the key.
+    counts.setdefault("n_web_searches", 0)
     return counts
 
 
@@ -134,9 +143,24 @@ def _reset_tally() -> None:
     _TALLY_LOCAL.counts = {
         "cloud_prompt": 0, "cloud_completion": 0,
         "local_prompt": 0, "local_completion": 0,
+        "n_web_searches": 0,
     }
 
 
+# Per-thread web_search tool: when set, the Anthropic generator declares
+# it on every call. Set inside ``_run_paradigm`` before invoking Archon
+# so the ranker/fuser passes pick it up; cleared in ``finally``.
+_WS_LOCAL = threading.local()
+
+
+def _set_anthropic_web_search(tool: Optional[Dict[str, Any]]) -> None:
+    _WS_LOCAL.tool = tool
+
+
+def _get_anthropic_web_search() -> Optional[Dict[str, Any]]:
+    return getattr(_WS_LOCAL, "tool", None)
+
+
 def _make_local_generator(local_endpoint: str, local_model: str):
     """Archon custom-generator signature: (model, messages, max_tokens, temperature) -> str."""
     from openai import OpenAI
@@ -153,6 +177,7 @@ def _make_local_generator(local_endpoint: str, local_model: str):
                 max_tokens=max_tokens,
                 temperature=temperature,
             )
+            _bump_local_calls()
         except Exception as e:
             _record_event({
                 "kind": "archon_local_gen_error",
@@ -205,6 +230,7 @@ def _wrap_archon_cloud_generators() -> None:
             kwargs["max_completion_tokens"] = max_tokens
         t0 = _time.time()
         resp = client.chat.completions.create(**kwargs)
+        _bump_cloud_calls()
         u = resp.usage
         if u:
             _tally()["cloud_prompt"] += getattr(u, "prompt_tokens", 0) or 0
@@ -237,13 +263,20 @@ def _wrap_archon_cloud_generators() -> None:
         )
         if not model.startswith(NO_TEMP_PREFIXES):
             kwargs["temperature"] = temperature
+        ws_tool = _get_anthropic_web_search()
+        if ws_tool is not None:
+            kwargs["tools"] = [ws_tool]
         t0 = _time.time()
         resp = client.messages.create(**kwargs)
+        _bump_cloud_calls()
         text = "".join(b.text for b in resp.content if hasattr(b, "text"))
         u = resp.usage
         if u:
             _tally()["cloud_prompt"] += getattr(u, "input_tokens", 0) or 0
             _tally()["cloud_completion"] += getattr(u, "output_tokens", 0) or 0
+        srv = getattr(u, "server_tool_use", None) if u else None
+        n_searches = getattr(srv, "web_search_requests", 0) if srv else 0
+        _tally()["n_web_searches"] += int(n_searches)
         _record_event({
             "kind": "archon_cloud_anthropic",
             "model": model,
@@ -252,6 +285,8 @@ def _wrap_archon_cloud_generators() -> None:
             "response": text.strip(),
             "tokens_in": getattr(u, "input_tokens", 0) if u else 0,
             "tokens_out": getattr(u, "output_tokens", 0) if u else 0,
+            "n_web_searches": int(n_searches),
+            "tools_declared": kwargs.get("tools"),
             "latency_s": _time.time() - t0,
             "ts": _time.time(),
         })
@@ -355,14 +390,18 @@ def _presets():
                 "samples": 1,
             }],
         ],
-        "single_local": lambda K, local_model, *_a, **_kw: [
+        # ``single_local`` honors the cfg ``max_tokens`` (passed positionally
+        # like ``ensemble_rank_fuse``). Previously it hard-coded 2048, which
+        # cut Qwen off mid-reasoning before it could emit the GAIA
+        # ``FINAL ANSWER:`` line — the scorer then had nothing to extract.
+        "single_local": lambda K, local_model, ranker_model, fuser_model, max_tokens, temperature: [
             [{
                 "type": "generator",
                 "model": local_model,
                 "model_type": "vllm_local",
                 "top_k": 1,
                 "temperature": 0.0,
-                "max_tokens": 2048,
+                "max_tokens": max_tokens,
                 "samples": 1,
             }],
         ],
@@ -430,6 +469,14 @@ class ArchonAgent(LocalCloudAgent):
         archon_cfg = {"name": f"hybrid-archon-{arch}", "layers": layers}
 
         _reset_tally()
+        # Web_search opt-in: when enabled, declare the native server-side
+        # tool on Anthropic ranker/fuser passes (via thread-local). The
+        # local proposers run on vLLM and don't see it.
+        ws_enabled, ws_max_uses = web_search_cfg(cfg)
+        if ws_enabled:
+            _set_anthropic_web_search(build_web_search_tool(ws_max_uses))
+        else:
+            _set_anthropic_web_search(None)
         archon = Archon(archon_cfg)
 
         try:
@@ -437,8 +484,16 @@ class ArchonAgent(LocalCloudAgent):
                 {"role": "system", "content": "You are a helpful assistant."},
                 {"role": "user", "content": input},
             ])
-        except Exception as e:
-            answer = f"[archon error: {e!r}]"
+        except Exception:
+            # Re-raise so the base ``run()`` / runner's ``_run_one_inner``
+            # records this in the row's ``error`` field instead of stashing
+            # the exception string in ``answer`` (where it scores as a wrong
+            # answer and never counts toward ``n_err``). Anthropic 529
+            # overloads on the ranker/fuser pass were silently masked this
+            # way — they are infra failures, not model misses.
+            raise
+        finally:
+            _set_anthropic_web_search(None)
 
         if isinstance(answer, list):
             answer = answer[-1] if answer else ""
@@ -446,16 +501,21 @@ class ArchonAgent(LocalCloudAgent):
 
         cp = _tally()["cloud_prompt"]
         cc = _tally()["cloud_completion"]
+        n_searches = _tally().get("n_web_searches", 0)
         cost = _cost_cloud(ranker_model, cp, cc)
         if fuser_model != ranker_model:
             # Conservative: charge both at the more expensive of the two.
             cost = max(cost, _cost_cloud(fuser_model, cp, cc))
+        cost += n_searches * WEB_SEARCH_COST_PER_CALL
 
         meta = {
             "tokens_local": _tally()["local_prompt"] + _tally()["local_completion"],
             "tokens_cloud": cp + cc,
             "cost_usd": cost,
             "turns": (K + 2) if arch == "ensemble_rank_fuse" else 1,
+            "web_search_uses": n_searches,
+            # GAIA: only ranker/fuser can hit web_search; proposers don't.
+            "tool_calls": int(n_searches),
             "traces": {
                 "architecture": arch,
                 "n_samples":    K,
@@ -463,6 +523,8 @@ class ArchonAgent(LocalCloudAgent):
                 "fuser_model":  fuser_model,
                 "local_model":  self._local_model,
                 "tokens_breakdown": dict(_tally()),
+                "web_search_enabled": ws_enabled,
+                "n_web_searches": n_searches,
             },
         }
         return answer, meta
@@ -566,6 +628,9 @@ class ArchonAgent(LocalCloudAgent):
             "tokens_cloud": total_tokens_cloud,
             "cost_usd": total_cost,
             "turns": sum(c["turns"] for c in candidates) + 1,
+            # SWE: total bash turns across the K candidate runs; ranker
+            # is a single text call with no tools.
+            "tool_calls": int(sum(c["turns"] for c in candidates)),
             "traces": {
                 "swe_mode": True,
                 "K": K,

src/openjarvis/agents/hybrid/baseline_cloud.py

@@ -0,0 +1,178 @@
+"""BaselineCloudAgent — cloud-only reference for the hybrid ablation.
+
+Used as the "what does the cloud do alone?" row in the n=100 ablation
+matrix (see ``.openjarvis/experiments/hybrid/docs/results-table.md``).
+No local model is involved — ``local_*`` settings are ignored.
+
+On GAIA the agent makes one cloud call with the formatted prompt (which
+already carries the ``FINAL ANSWER:`` format reminder from
+``_prompts.format_gaia``) and returns the text. On SWE-bench-Verified
+the agent delegates to :func:`run_swe_agent_loop` with ``backbone="cloud"``
+so the model gets to run bash and read the repo — same wiring as the
+``mini-swe-agent-swebenchverified-opus-*`` cells. As of 2026-05-15
+``_loop_cloud`` dispatches to per-endpoint loops for Anthropic, OpenAI,
+and Gemini so all three cloud backbones get the proper bash-agent loop
+on SWE (previously OpenAI / Gemini SWE cells silently fell back to a
+one-shot blind patch — fixed).
+
+Construction args mirror :class:`LocalCloudAgent`. The ``cloud`` block
+in the cell registry determines the cloud model + endpoint; ``local``
+is accepted for schema compatibility but unused.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.agents.hybrid._base import (
+    WEB_SEARCH_COST_PER_CALL,
+    LocalCloudAgent,
+    build_web_search_tool,
+    web_search_cfg,
+)
+from openjarvis.agents.hybrid._prices import cost as estimate_cost
+from openjarvis.agents.hybrid._prices import default_max_output_tokens
+from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
+from openjarvis.core.registry import AgentRegistry
+
+
+@AgentRegistry.register("baseline_cloud")
+class BaselineCloudAgent(LocalCloudAgent):
+    """Cloud-only baseline used as a reference in the n=100 ablation.
+
+    Configurable knobs via ``cfg``:
+
+    - ``cloud_max_tokens`` (int, default 4096 / 16384 for reasoning models):
+      max_tokens per GAIA call and per turn of the SWE agent loop. Default
+      jumps to 16384 for GPT-5 family and Gemini 2.5 Pro because those models
+      burn the budget on hidden chain-of-thought before emitting visible
+      answer text; at 4096 they silently truncated 18–26% of GAIA cells with
+      empty answers. Override per-cell via ``method_cfg`` to opt out.
+    - ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
+    - ``swe_bash_timeout_s`` (int, default 120): SWE-bench bash timeout.
+    """
+
+    agent_id = "baseline_cloud"
+
+    def _run_paradigm(
+        self,
+        input: str,
+        context: Optional[AgentContext],
+        **kwargs: Any,
+    ) -> Tuple[str, Dict[str, Any]]:
+        cfg = self._cfg
+        task: Dict[str, Any] = {}
+        if context is not None:
+            task = context.metadata.get("task") or {}
+
+        is_swe = bool(
+            task.get("problem_statement")
+            and task.get("repo")
+            and task.get("base_commit")
+        )
+
+        if is_swe:
+            out = run_swe_agent_loop(
+                task,
+                backbone="cloud",
+                backbone_model=self._cloud_model,
+                cloud_endpoint=self._cloud_endpoint,
+                initial_prompt=input,
+                max_turns=int(cfg.get("swe_max_turns", 30)),
+                bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
+                output_cap=int(cfg.get("swe_output_cap", 10_000)),
+                turn_max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
+                trace_prefix="baseline_cloud",
+            )
+            meta = {
+                "tokens_local": 0,
+                "tokens_cloud": out["tokens_in"] + out["tokens_out"],
+                "cost_usd": out["cost_usd"],
+                "turns": out["turns"],
+                # SWE-bench: one bash invocation per agent turn.
+                "tool_calls": int(out["turns"]),
+                "traces": {
+                    "backbone": "cloud",
+                    "max_turns_hit": out["max_turns_hit"],
+                    "patch_chars": len(out["patch"]),
+                    "final_summary": out["final_summary"],
+                },
+            }
+            return out["answer"], meta
+
+        # GAIA branch. If `web_search.enabled` is true AND we're on
+        # Anthropic, run the multi-turn agent loop with the native
+        # server-side web_search tool. Otherwise fall back to the
+        # legacy one-shot call (preserves behavior of every existing
+        # non-opted-in cell).
+        ws_enabled, ws_max_uses = web_search_cfg(cfg)
+        gaia_max_turns = int(cfg.get("gaia_max_turns", 8))
+        if ws_enabled and self._cloud_endpoint == "anthropic":
+            text, p_tok, c_tok, n_searches, turns = self._call_anthropic_agent(
+                self._cloud_model,
+                user=input,
+                max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
+                temperature=0.0,
+                tools=[build_web_search_tool(ws_max_uses)],
+                max_turns=gaia_max_turns,
+            )
+            cost = (
+                estimate_cost(self._cloud_model, p_tok, c_tok)
+                + n_searches * WEB_SEARCH_COST_PER_CALL
+            )
+            meta = {
+                "tokens_local": 0,
+                "tokens_cloud": p_tok + c_tok,
+                "cost_usd": cost,
+                "turns": turns,
+                "web_search_uses": n_searches,
+                # GAIA: the only tool is web_search.
+                "tool_calls": int(n_searches),
+                "traces": {
+                    "mode": "anthropic_agent_loop",
+                    "is_swe": is_swe,
+                    "cloud_endpoint": self._cloud_endpoint,
+                    "web_search_enabled": True,
+                    "web_search_max_uses": ws_max_uses,
+                    "n_web_searches": n_searches,
+                },
+            }
+            return text, meta
+
+        if ws_enabled and self._cloud_endpoint != "anthropic":
+            # OpenAI / Gemini don't have a parity native web_search tool
+            # wired here. Skip cleanly rather than fake one. Cells that
+            # want web_search must run on Anthropic until those backends
+            # are wired.
+            self.record_trace_event({
+                "kind": "web_search_skipped",
+                "reason": "non_anthropic_endpoint",
+                "endpoint": self._cloud_endpoint,
+            })
+
+        # One-shot direct cloud call. GAIA only — SWE goes through the
+        # mini-SWE-agent loop above (now supports anthropic/openai/gemini).
+        text, p_tok, c_tok = self._call_cloud(
+            user=input,
+            max_tokens=int(cfg.get("cloud_max_tokens", default_max_output_tokens(self._cloud_model))),
+            temperature=0.0,
+        )
+        meta = {
+            "tokens_local": 0,
+            "tokens_cloud": p_tok + c_tok,
+            "cost_usd": estimate_cost(self._cloud_model, p_tok, c_tok),
+            "turns": 1,
+            "web_search_uses": 0,
+            # GAIA one-shot: zero tool calls (no bash, no web_search).
+            "tool_calls": 0,
+            "traces": {
+                "mode": "one_shot",
+                "is_swe": is_swe,
+                "cloud_endpoint": self._cloud_endpoint,
+            },
+        }
+        return text, meta
+
+
+__all__ = ["BaselineCloudAgent"]

src/openjarvis/agents/hybrid/baseline_local.py

@@ -0,0 +1,159 @@
+"""BaselineLocalAgent — local-only reference for the hybrid ablation.
+
+Mirror of :class:`BaselineCloudAgent` (`baseline_cloud.py`) but the entire
+trajectory runs on the local vLLM model. No cloud teacher / router / advisor
+is involved — this is the "what does the local model do by itself?" floor
+in the n=100 ablation matrix.
+
+On GAIA the agent makes one local call with the formatted prompt (which
+already carries the ``FINAL ANSWER:`` reminder from
+``_prompts.format_gaia``) and returns the text. On SWE-bench-Verified
+the agent delegates to :func:`run_swe_agent_loop` with ``backbone="local"``
+so the model gets to run bash and read the repo — same wiring as the
+``mini-swe-agent`` cells but driven by the local model.
+
+Construction args mirror :class:`LocalCloudAgent`. The ``local`` block in
+the cell registry determines the local model + endpoint; ``cloud`` is
+accepted for schema compatibility but unused (and ``cost_usd`` is always
+0 — local inference is free).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.agents.hybrid._base import LocalCloudAgent
+from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
+from openjarvis.core.registry import AgentRegistry
+
+
+@AgentRegistry.register("baseline_local")
+class BaselineLocalAgent(LocalCloudAgent):
+    """Local-only baseline. ``cloud_*`` fields are ignored.
+
+    Configurable knobs via ``cfg``:
+
+    - ``local_max_tokens`` (int, default 4096): max_tokens per GAIA call
+      and per turn of the SWE agent loop.
+    - ``local_temperature`` (float, default 0.0): sampling temperature
+      for the local model.
+    - ``swe_use_agent_loop`` (bool, default True for SWE): if False the
+      SWE branch falls back to a one-shot blind patch (not recommended;
+      kept for parity with other agents).
+    - ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
+    - ``swe_bash_timeout_s`` (int, default 120): bash timeout per turn.
+    - ``swe_turn_max_tokens`` (int, default 4096): max_tokens per agent
+      turn inside the SWE loop. Falls back to ``local_max_tokens``.
+    """
+
+    agent_id = "baseline_local"
+
+    def _run_paradigm(
+        self,
+        input: str,
+        context: Optional[AgentContext],
+        **kwargs: Any,
+    ) -> Tuple[str, Dict[str, Any]]:
+        cfg = self._cfg
+        task: Dict[str, Any] = {}
+        if context is not None:
+            task = context.metadata.get("task") or {}
+
+        is_swe = bool(
+            task.get("problem_statement")
+            and task.get("repo")
+            and task.get("base_commit")
+        )
+
+        local_max_tokens = int(cfg.get("local_max_tokens", 4096))
+        local_temperature = float(cfg.get("local_temperature", 0.0))
+
+        if not self._local_model or not self._local_endpoint:
+            raise ValueError(
+                "baseline_local requires `local.model` and `local.endpoint` "
+                "in the cell registry — got "
+                f"model={self._local_model!r}, endpoint={self._local_endpoint!r}"
+            )
+
+        if is_swe:
+            use_loop = bool(cfg.get("swe_use_agent_loop", True))
+            if use_loop:
+                out = run_swe_agent_loop(
+                    task,
+                    backbone="local",
+                    backbone_model=self._local_model,
+                    local_endpoint=self._local_endpoint,
+                    initial_prompt=input,
+                    max_turns=int(cfg.get("swe_max_turns", 30)),
+                    bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
+                    output_cap=int(cfg.get("swe_output_cap", 10_000)),
+                    turn_max_tokens=int(
+                        cfg.get("swe_turn_max_tokens", local_max_tokens)
+                    ),
+                    trace_prefix="baseline_local",
+                )
+                meta = {
+                    "tokens_local": out["tokens_in"] + out["tokens_out"],
+                    "tokens_cloud": 0,
+                    "cost_usd": 0.0,
+                    "turns": out["turns"],
+                    # SWE-bench: one bash invocation per agent turn.
+                    "tool_calls": int(out["turns"]),
+                    "traces": {
+                        "backbone": "local",
+                        "max_turns_hit": out["max_turns_hit"],
+                        "patch_chars": len(out["patch"]),
+                        "final_summary": out["final_summary"],
+                    },
+                }
+                return out["answer"], meta
+
+            # One-shot blind patch fallback on SWE (no bash).
+            text, p_tok, c_tok = self._call_vllm(
+                self._local_model,
+                self._local_endpoint,
+                user=input,
+                max_tokens=local_max_tokens,
+                temperature=local_temperature,
+            )
+            meta = {
+                "tokens_local": p_tok + c_tok,
+                "tokens_cloud": 0,
+                "cost_usd": 0.0,
+                "turns": 1,
+                "tool_calls": 0,
+                "traces": {
+                    "mode": "one_shot_swe",
+                    "backbone": "local",
+                },
+            }
+            return text, meta
+
+        # GAIA branch — one-shot local call. Matches baseline_cloud's
+        # GAIA one-shot path (no web_search; baseline_cloud's web_search
+        # is Anthropic server-side only and has no local equivalent).
+        text, p_tok, c_tok = self._call_vllm(
+            self._local_model,
+            self._local_endpoint,
+            user=input,
+            max_tokens=local_max_tokens,
+            temperature=local_temperature,
+        )
+        meta = {
+            "tokens_local": p_tok + c_tok,
+            "tokens_cloud": 0,
+            "cost_usd": 0.0,
+            "turns": 1,
+            "web_search_uses": 0,
+            "tool_calls": 0,
+            "traces": {
+                "mode": "one_shot",
+                "is_swe": is_swe,
+                "backbone": "local",
+            },
+        }
+        return text, meta
+
+
+__all__ = ["BaselineLocalAgent"]

src/openjarvis/agents/hybrid/conductor.py

@@ -31,6 +31,7 @@ from __future__ import annotations
 
 import ast
 import json
+import os
 import re
 import shutil
 import tempfile
@@ -39,8 +40,16 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent
+from openjarvis.agents.hybrid._base import (
+    GEMINI_SEARCH_COST_PER_CALL,
+    OPENAI_WEB_SEARCH_COST_PER_CALL,
+    WEB_SEARCH_COST_PER_CALL,
+    LocalCloudAgent,
+    build_web_search_tool,
+    web_search_cfg,
+)
 from openjarvis.agents.hybrid._prices import (
+    PRICES,
     is_gpt5_family,
     supports_temperature,
 )
@@ -175,57 +184,327 @@ def _vllm_alive(base_url: str) -> bool:
 
 
 def _default_pool(local_model: Optional[str], local_endpoint: Optional[str]) -> List[Dict[str, Any]]:
+    """Default worker pool — faithful to the Sakana Conductor paper (arXiv 2512.04388).
+
+    The paper composes a heterogeneous 7-worker pool spanning three frontier
+    cloud models (Gemini-2.5-Pro, Claude Sonnet-4, GPT-5) and four open-weights
+    workers routed via OpenRouter (DeepSeek-R1-Distill-Qwen-32B, Gemma3-27B-it,
+    Qwen3-32B with reasoning off, Qwen3-32B with reasoning on). No local vLLM
+    worker is in the paper's default — cells that want one should supply it
+    explicitly via ``cfg["worker_pool"]``.
+
+    Each provider is gated by an ``OJ_CONDUCTOR_DISABLE_*`` env var so a cell
+    that lacks one set of credentials can still run the rest of the pool. If
+    every provider is disabled the result is the empty list — the caller's
+    "empty worker pool" check will surface it.
+
+    ``local_model`` / ``local_endpoint`` are accepted for signature stability
+    (callers still pass them) but no longer consulted here; the local vLLM is
+    only included when the user opts in via ``cfg["worker_pool"]``.
+    """
+    del local_model, local_endpoint  # paper default carries no local worker
     pool: List[Dict[str, Any]] = []
-    if local_model and local_endpoint and _vllm_alive(local_endpoint):
+    if not os.environ.get("OJ_CONDUCTOR_DISABLE_GEMINI"):
         pool.append({
             "id": len(pool),
-            "name": "local-qwen",
-            "endpoint": "vllm",
-            "model": local_model,
-            "base_url": local_endpoint,
-            "api_key": "EMPTY",
+            "name": "gemini-pro",
+            "endpoint": "gemini",
+            "model": "gemini-2.5-pro",
             "description": (
-                "Open-weights Qwen3.5 served locally. Cheap and fast. Good at "
-                "concise extraction, formatting, arithmetic on given data; "
-                "weaker at open-domain factual recall and complex reasoning."
+                "Google Gemini 2.5 Pro. Frontier multimodal reasoner with a "
+                "very large context window. Strong at long-document synthesis, "
+                "multi-hop factual reasoning, and tasks that benefit from "
+                "wide retrieval. Slower and pricier than mid-tier workers."
             ),
         })
-    pool.append({
-        "id": len(pool),
-        "name": "frontier-anthropic",
-        "endpoint": "anthropic",
-        "model": "claude-opus-4-7",
-        "description": (
-            "Frontier reasoning model. Strongest at multi-step reasoning, "
-            "careful instruction following, code, and writing. Expensive; "
-            "use sparingly for hard or decisive steps."
-        ),
-    })
-    pool.append({
-        "id": len(pool),
-        "name": "frontier-openai-mini",
-        "endpoint": "openai",
-        "model": "gpt-5-mini",
-        "description": (
-            "Mid-tier OpenAI model. Solid general knowledge and reasoning at "
-            "a fraction of frontier cost. Good default for retrieval-style or "
-            "broad-knowledge questions."
-        ),
-    })
+    if not os.environ.get("OJ_CONDUCTOR_DISABLE_ANTHROPIC"):
+        pool.append({
+            "id": len(pool),
+            "name": "claude-sonnet-4",
+            "endpoint": "anthropic",
+            "model": "claude-sonnet-4-6",
+            "description": (
+                "Anthropic Claude Sonnet 4. Strong general-purpose reasoner "
+                "with careful instruction following and reliable formatting. "
+                "Good default for code, structured writing, and decisive "
+                "steps where accuracy matters more than raw throughput."
+            ),
+        })
+    if not os.environ.get("OJ_CONDUCTOR_DISABLE_OPENAI"):
+        pool.append({
+            "id": len(pool),
+            "name": "gpt-5",
+            "endpoint": "openai",
+            "model": "gpt-5",
+            "description": (
+                "OpenAI GPT-5. Frontier-tier broad-knowledge model. Best for "
+                "open-domain factual recall, creative generation, and "
+                "ambiguous questions where coverage matters. Expensive; use "
+                "for steps where breadth of world knowledge is the bottleneck."
+            ),
+        })
+    if not os.environ.get("OJ_CONDUCTOR_DISABLE_OPENROUTER"):
+        pool.append({
+            "id": len(pool),
+            "name": "deepseek-r1-distill-qwen-32b",
+            "endpoint": "openrouter",
+            "model": "deepseek/deepseek-r1-distill-qwen-32b",
+            "description": (
+                "DeepSeek R1 distilled into Qwen-32B (open weights via "
+                "OpenRouter). Specialized for chain-of-thought math, logic, "
+                "and competitive-programming-style problems. Verbose; "
+                "produces extensive reasoning traces before the final answer."
+            ),
+        })
+        pool.append({
+            "id": len(pool),
+            "name": "gemma3-27b-it",
+            "endpoint": "openrouter",
+            "model": "google/gemma-3-27b-it",
+            "description": (
+                "Google Gemma 3 27B Instruct (open weights via OpenRouter). "
+                "Mid-size instruction-tuned model. Cheap and fast; solid at "
+                "concise summarization, extraction, and short-form Q&A on "
+                "given context. Weaker than the frontier workers on multi-step "
+                "reasoning."
+            ),
+        })
+        pool.append({
+            "id": len(pool),
+            "name": "qwen3-32b",
+            "endpoint": "openrouter",
+            "model": "qwen/qwen3-32b",
+            "description": (
+                "Qwen3-32B in non-thinking mode (open weights via OpenRouter). "
+                "Fast general-purpose dialogue and instruction following. "
+                "Use when the step is straightforward generation, "
+                "summarization, or formatting — does NOT spend tokens on "
+                "internal reasoning."
+            ),
+        })
+        pool.append({
+            "id": len(pool),
+            "name": "qwen3-32b-thinking",
+            "endpoint": "openrouter",
+            "model": "qwen/qwen3-32b",
+            "extra_body": {"reasoning": {"effort": "medium"}},
+            "description": (
+                "Qwen3-32B with reasoning enabled (open weights via "
+                "OpenRouter). Same backbone as 'qwen3-32b' but spends tokens "
+                "on an internal chain of thought before answering. Stronger "
+                "on math, code, and multi-step logic; slower and consumes "
+                "more completion tokens. Prefer this for hard reasoning "
+                "steps; prefer the non-thinking variant for plain dialogue."
+            ),
+        })
+    # Reassign ids contiguously in case env-gates skipped some entries.
+    for new_id, entry in enumerate(pool):
+        entry["id"] = new_id
     return pool
 
 
+# Endpoints conductor's `_call_worker` actually knows how to dispatch to.
+# Web-search is NOT supported here — toolorchestra has the web-search
+# dispatcher. OpenRouter is OpenAI-compatible; Gemini is text-only (no
+# tool-call parity with Anthropic — see `_base._call_gemini`). Both are
+# opt-in via cfg["worker_pool"] (not added to _default_pool).
+_CONDUCTOR_VALID_ENDPOINTS = ("vllm", "openai", "anthropic", "openrouter", "gemini")
+
+
+def _resolve_worker_pool(
+    cfg: Dict[str, Any],
+    local_model: Optional[str],
+    local_endpoint: Optional[str],
+    cloud_model: str,
+) -> List[Dict[str, Any]]:
+    """Return the worker pool for this run.
+
+    Strict replace, not merge: if ``cfg["worker_pool"]`` is set, the
+    default pool is ignored entirely. Falls back to ``_default_pool`` when
+    the override is absent.
+
+    Each user-supplied entry must be a dict with keys ``id``, ``name``,
+    ``endpoint``, and ``model``. ``endpoint`` must be one of
+    ``vllm`` / ``openai`` / ``anthropic`` / ``openrouter`` / ``gemini`` —
+    conductor does not wire web-search workers. OpenRouter workers route
+    through the OpenAI-compatible OpenRouter proxy; Gemini workers call
+    the google-genai SDK (text-only, no tool use). Both are opt-in via
+    ``cfg["worker_pool"]`` (not in the default pool).
+
+    Substitution: ``model = "$local"`` (or ``"<local>"``) resolves to
+    ``local_model``; ``model = "$cloud"`` / ``"<cloud>"`` to ``cloud_model``.
+
+    On any validation failure, raises ``ValueError`` with the message
+    ``"Invalid worker_pool entry [<id>]: <reason>"``. Fails fast at agent
+    init rather than mid-task.
+    """
+    override = cfg.get("worker_pool")
+    if override is None:
+        return _default_pool(local_model, local_endpoint)
+    if not isinstance(override, list) or not override:
+        raise ValueError(
+            "Invalid worker_pool entry [-]: worker_pool must be a non-empty list"
+        )
+
+    resolved: List[Dict[str, Any]] = []
+    seen_ids: set = set()
+    has_non_search = False
+    for raw in override:
+        wid_repr = raw.get("id", "?") if isinstance(raw, dict) else "?"
+        if not isinstance(raw, dict):
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid_repr}]: entry must be a dict"
+            )
+        entry = dict(raw)
+        wid = entry.get("id")
+        if not isinstance(wid, int):
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid_repr}]: 'id' must be an int"
+            )
+        if wid in seen_ids:
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid}]: duplicate id"
+            )
+        seen_ids.add(wid)
+        if not entry.get("name") or not isinstance(entry["name"], str):
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid}]: 'name' must be a non-empty string"
+            )
+        endpoint = entry.get("endpoint") or entry.get("type")
+        if not isinstance(endpoint, str) or endpoint.lower() not in _CONDUCTOR_VALID_ENDPOINTS:
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid}]: 'endpoint' must be one of "
+                f"{_CONDUCTOR_VALID_ENDPOINTS} (got {endpoint!r})"
+            )
+        endpoint = endpoint.lower()
+        entry["endpoint"] = endpoint
+        # Substitute $local / $cloud placeholders.
+        model = entry.get("model")
+        if isinstance(model, str) and model in ("$local", "<local>"):
+            if not local_model:
+                raise ValueError(
+                    f"Invalid worker_pool entry [{wid}]: model='{model}' "
+                    "requires a local_model to be configured for this cell"
+                )
+            model = local_model
+            entry["model"] = model
+        elif isinstance(model, str) and model in ("$cloud", "<cloud>"):
+            model = cloud_model
+            entry["model"] = model
+        if not isinstance(model, str) or not model:
+            raise ValueError(
+                f"Invalid worker_pool entry [{wid}]: 'model' must be a non-empty string"
+            )
+        if endpoint == "vllm":
+            if not entry.get("base_url"):
+                # Default to the local endpoint if not specified — matches
+                # how _default_pool wires it.
+                if not local_endpoint:
+                    raise ValueError(
+                        f"Invalid worker_pool entry [{wid}]: vllm worker needs "
+                        "'base_url' (or a configured local_endpoint to fall back to)"
+                    )
+                entry["base_url"] = local_endpoint
+            entry.setdefault("api_key", "EMPTY")
+            # Local also counts as a non-search worker for the
+            # "must have at least one solver" check.
+            has_non_search = True
+        else:
+            # Cloud workers: model must be priced (any unknown model would
+            # silently cost $0, which masks billing mistakes downstream).
+            # OpenRouter is exempt — its model space is huge and varies
+            # per-provider; cost reporting for openrouter workers is 0
+            # (same as vllm). Cells that need accurate billing for an
+            # openrouter worker should add it to PRICES themselves.
+            if endpoint != "openrouter" and model not in PRICES:
+                raise ValueError(
+                    f"Invalid worker_pool entry [{wid}]: model {model!r} is "
+                    f"not in PRICES (known: {sorted(PRICES)})"
+                )
+            has_non_search = True
+        entry.setdefault(
+            "description",
+            f"User-supplied {endpoint} worker ({model}).",
+        )
+        resolved.append(entry)
+
+    if not has_non_search:
+        raise ValueError(
+            "Invalid worker_pool entry [-]: worker_pool must contain at least "
+            "one non-search worker (vllm / openai / anthropic / openrouter / gemini)"
+        )
+
+    # The planner picks a worker by its `id` (it sees "Model <id> (<name>)"),
+    # but execution dispatches via `workers[model_id]` — list *position*. If a
+    # config supplies non-contiguous or out-of-order ids those two diverge and
+    # a plan that names worker N silently runs a different worker. Enforce
+    # contiguous 0..N-1 ids that equal list position so id == index always.
+    expected = list(range(len(resolved)))
+    actual = [w["id"] for w in resolved]
+    if actual != expected:
+        raise ValueError(
+            f"Invalid worker_pool entry [-]: worker ids must be contiguous "
+            f"0..{len(resolved) - 1} in list order so the planner's model_id "
+            f"matches the dispatch index (got ids {actual}, expected {expected})"
+        )
+    return resolved
+
+
 def _format_worker_pool(workers: List[Dict[str, Any]]) -> str:
     return "\n".join(
         f"Model {w['id']} ({w['name']}): {w['description']}" for w in workers
     )
 
 
-def _build_conductor_prompt(question: str, workers: List[Dict[str, Any]]) -> str:
-    return (
+def _search_capable_indices(workers: List[Dict[str, Any]]) -> List[int]:
+    """Indices of workers whose endpoint can run server-side web search."""
+    return [
+        w["id"] for w in workers
+        if (w.get("endpoint") or "openai").lower()
+        in _SEARCH_CAPABLE_WORKER_ENDPOINTS
+    ]
+
+
+def _build_conductor_prompt(
+    question: str,
+    workers: List[Dict[str, Any]],
+    *,
+    web_search_enabled: bool = False,
+) -> str:
+    """Build the planner prompt.
+
+    When ``web_search_enabled`` is set (GAIA cells with web_search on),
+    append an explicit routing constraint: only the listed model indices
+    can perform web research, so any step that needs to look something up
+    on the web MUST be routed to one of them. Steps routed elsewhere
+    answer blind from parametric memory.
+    """
+    base = (
         f"Available models:\n{_format_worker_pool(workers)}\n\n"
         f"User question:\n{question}\n"
     )
+    if not web_search_enabled:
+        return base
+    capable = _search_capable_indices(workers)
+    if capable:
+        cap_str = ", ".join(str(i) for i in capable)
+        constraint = (
+            "\n\nWEB SEARCH CONSTRAINT:\n"
+            f"Only these model indices can perform live web search: [{cap_str}]. "
+            "Any step that needs to look up facts, current events, or other "
+            "information not reliably known from memory MUST be routed to one "
+            "of those indices. Steps routed to any other model can only use "
+            "their parametric memory and will answer such questions blind.\n"
+        )
+    else:
+        # No search-capable worker at all — the run-level guard raises
+        # before we get here, but keep the prompt honest just in case.
+        constraint = (
+            "\n\nWEB SEARCH CONSTRAINT:\n"
+            "No model in this pool can perform live web search; rely on the "
+            "models' own knowledge.\n"
+        )
+    return base + constraint
 
 
 def _build_step_prompt(
@@ -250,13 +529,42 @@ def _build_step_prompt(
 
 # ---------- Worker invocation ----------
 
+# Worker endpoints that can run server-side web search via a `_base`
+# agent loop. openrouter / vllm cannot ground.
+_SEARCH_CAPABLE_WORKER_ENDPOINTS = ("anthropic", "openai", "gemini")
+
+
+def _worker_search_cost_per_call(endpoint: str) -> float:
+    """Per-search-call USD cost for a worker's cloud endpoint."""
+    if endpoint == "openai":
+        return OPENAI_WEB_SEARCH_COST_PER_CALL
+    if endpoint == "gemini":
+        return GEMINI_SEARCH_COST_PER_CALL
+    return WEB_SEARCH_COST_PER_CALL
+
+
 def _call_worker(
-    worker: Dict[str, Any], prompt: str, cfg: Dict[str, Any]
-) -> Tuple[str, int, int, bool]:
-    """Returns (text, p_tok, c_tok, is_local)."""
+    worker: Dict[str, Any],
+    prompt: str,
+    cfg: Dict[str, Any],
+    *,
+    web_search_tool: Optional[Dict[str, Any]] = None,
+    web_search_max_uses: int = 8,
+) -> Tuple[str, int, int, bool, int]:
+    """Returns (text, p_tok, c_tok, is_local, n_web_searches).
+
+    ``web_search_tool``: a truthy marker that web_search is enabled for
+    this run. When set AND the worker endpoint is search-capable
+    (anthropic / openai / gemini), the worker call is routed through the
+    matching ``_base`` agent loop so the worker can ground its answer.
+    ``n_web_searches`` is the actual count the provider ran.
+    ``web_search_max_uses`` caps the Anthropic search tool.
+    Search-incapable workers (openrouter, vllm) ignore it and return 0.
+    """
     ep = (worker.get("endpoint") or "openai").lower()
     max_tok = int(cfg.get("worker_max_tokens", 4096))
     temp = float(cfg.get("worker_temperature", 0.2))
+    use_ws = web_search_tool is not None
 
     if ep == "vllm":
         text, p, c = LocalCloudAgent._call_vllm(
@@ -267,24 +575,71 @@ def _call_worker(
             temperature=temp,
             enable_thinking=False,
         )
-        return text, p, c, True
+        return text, p, c, True, 0
     if ep == "openai":
+        if use_ws:
+            text, p, c, n_searches, _ = LocalCloudAgent._call_openai_agent(
+                worker["model"],
+                user=prompt,
+                max_tokens=max_tok,
+                temperature=(1.0 if is_gpt5_family(worker["model"]) else temp),
+            )
+            return text, p, c, False, n_searches
         text, p, c = LocalCloudAgent._call_openai(
             worker["model"],
             user=prompt,
             max_tokens=max_tok,
             temperature=(1.0 if is_gpt5_family(worker["model"]) else temp),
         )
-        return text, p, c, False
+        return text, p, c, False, 0
+    if ep == "openrouter":
+        # OpenRouter is OpenAI-compatible; the helper handles the
+        # base_url + OPENROUTER_API_KEY plumbing. No server-side web
+        # search wired here. is_local=False so tokens count as cloud.
+        # ``worker["extra_body"]`` (e.g. {"reasoning": {"effort": "medium"}})
+        # is forwarded to the SDK so paper-faithful workers like
+        # "qwen3-32b-thinking" can toggle reasoning per-call.
+        extra_body = worker.get("extra_body")
+        text, p, c = LocalCloudAgent._call_openrouter(
+            worker["model"],
+            user=prompt,
+            max_tokens=max_tok,
+            temperature=temp,
+            extra_body=extra_body if isinstance(extra_body, dict) else None,
+        )
+        return text, p, c, False, 0
     if ep == "anthropic":
         eff_temp = temp if supports_temperature(worker["model"]) else 0.0
-        text, p, c, _ = LocalCloudAgent._call_anthropic(
-            worker["model"],
+        anthropic_kwargs: Dict[str, Any] = dict(
             user=prompt,
             max_tokens=max_tok,
             temperature=eff_temp,
         )
-        return text, p, c, False
+        if web_search_tool is not None:
+            anthropic_kwargs["tools"] = [web_search_tool]
+        text, p, c, n_searches = LocalCloudAgent._call_anthropic(
+            worker["model"], **anthropic_kwargs
+        )
+        return text, p, c, False, n_searches
+    if ep == "gemini":
+        # Gemini Developer API via google-genai. With web_search on, route
+        # through the Google-Search-grounded agent loop; otherwise plain
+        # text generation. is_local=False so tokens count as cloud.
+        if use_ws:
+            text, p, c, n_searches, _ = LocalCloudAgent._call_gemini_agent(
+                worker["model"],
+                user=prompt,
+                max_tokens=max_tok,
+                temperature=temp,
+            )
+            return text, p, c, False, n_searches
+        text, p, c = LocalCloudAgent._call_gemini(
+            worker["model"],
+            user=prompt,
+            max_tokens=max_tok,
+            temperature=temp,
+        )
+        return text, p, c, False, 0
     raise ValueError(f"unsupported worker endpoint: {ep!r}")
 
 
@@ -295,30 +650,35 @@ def _swe_worker_step(
     cfg: Dict[str, Any],
     workdir: Path,
     step_idx: int,
-) -> Tuple[str, int, int, bool]:
+) -> Tuple[str, int, int, bool, int, int]:
     """Run one Conductor worker step as a mini-SWE-agent subloop on a shared
-    workdir. Returns (final_summary_or_diff, tokens_in, tokens_out, is_local)
-    in the same shape as ``_call_worker``."""
+    workdir. Returns (final_summary_or_diff, tokens_in, tokens_out, is_local,
+    n_web_searches, bash_turns). SWE workers don't use web_search (the bash
+    tool is the only tool they need); ``bash_turns`` counts the agent-loop
+    turns so the caller can surface ``tool_calls``."""
     ep = (worker.get("endpoint") or "openai").lower()
     if ep == "vllm":
         backbone, model, endpoint, is_local = (
             "local", worker["model"], worker.get("base_url"), True,
         )
+        cloud_endpoint = "anthropic"  # unused on the local path
     elif ep == "anthropic":
         backbone, model, endpoint, is_local = (
             "cloud", worker["model"], None, False,
         )
+        cloud_endpoint = "anthropic"
     else:
         # OpenAI workers (gpt-5-mini etc.) aren't supported as agent-loop
         # backbones today (the loop's tool-call format is Anthropic- or
         # OpenAI-via-vllm-shaped only). Fall back to one-shot for those —
         # SWE-bench-wise they were already weak; this preserves behavior.
-        return _call_worker(worker, prompt, cfg)
+        text, p, c, is_local, n_searches = _call_worker(worker, prompt, cfg)
+        return text, p, c, is_local, n_searches, 0
     out = run_swe_agent_loop(
         task,
         backbone=backbone,
         backbone_model=model,
-        cloud_endpoint="anthropic" if backbone == "cloud" else "anthropic",
+        cloud_endpoint=cloud_endpoint,
         local_endpoint=endpoint,
         initial_prompt=prompt,
         max_turns=int(cfg.get("swe_max_turns", 30)),
@@ -328,7 +688,10 @@ def _swe_worker_step(
         trace_prefix=f"conductor_step{step_idx}",
         workdir=workdir,
     )
-    return out["final_summary"] or out["answer"], out["tokens_in"], out["tokens_out"], is_local
+    return (
+        out["final_summary"] or out["answer"],
+        out["tokens_in"], out["tokens_out"], is_local, 0, int(out["turns"]),
+    )
 
 
 @AgentRegistry.register("conductor")
@@ -337,6 +700,20 @@ class ConductorAgent(LocalCloudAgent):
 
     agent_id = "conductor"
 
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        # Validate `method_cfg.worker_pool` early — surfaces config errors
+        # at agent construction rather than on the first task. No-op when
+        # the override is absent (default pool is built later, lazily,
+        # because `_vllm_alive` needs a live network probe).
+        if self._cfg.get("worker_pool") is not None:
+            _resolve_worker_pool(
+                self._cfg,
+                self._local_model,
+                self._local_endpoint,
+                self._cloud_model,
+            )
+
     def _run_paradigm(
         self,
         input: str,
@@ -345,14 +722,45 @@ class ConductorAgent(LocalCloudAgent):
     ) -> Tuple[str, Dict[str, Any]]:
         question = input
         cfg = self._cfg
-        workers = cfg.get("workers") or _default_pool(
-            self._local_model, self._local_endpoint
-        )
+        # Resolution order (strict replace, no merge):
+        #   1. `cfg["workers"]` — legacy direct override, used by tests.
+        #   2. `cfg["worker_pool"]` — cell-config override; validated +
+        #      $local/$cloud substituted.
+        #   3. `_default_pool(...)` — heterogeneous default (Opus +
+        #      gpt-5-mini + optional local Qwen).
+        if cfg.get("workers"):
+            workers = cfg["workers"]
+        else:
+            workers = _resolve_worker_pool(
+                cfg,
+                self._local_model,
+                self._local_endpoint,
+                self._cloud_model,
+            )
         if not workers:
             raise RuntimeError("conductor: empty worker pool")
 
-        # 1. Plan
-        user = _build_conductor_prompt(question, workers)
+        # Determine swe_mode up front — needed so the planner prompt can
+        # carry the GAIA web-search routing constraint (Task 3). SWE tasks
+        # use the bash tool, not web_search, so the constraint is GAIA-only.
+        task_meta_early = (
+            context.metadata.get("task") if context is not None else {}
+        ) or {}
+        swe_mode_early = (
+            bool(cfg.get("swe_use_agent_loop"))
+            and bool(task_meta_early.get("problem_statement"))
+            and bool(task_meta_early.get("repo"))
+            and bool(task_meta_early.get("base_commit"))
+        )
+        ws_enabled, ws_max_uses = web_search_cfg(cfg)
+        planner_ws = ws_enabled and not swe_mode_early
+
+        # 1. Plan — when web_search is on (GAIA), the prompt names which
+        # worker indices can actually search, so the planner routes
+        # research steps to a search-capable worker.
+        user = _build_conductor_prompt(
+            question, workers, web_search_enabled=planner_ws,
+        )
         plan_text, p_in, p_out = self._call_cloud(
             user=user,
             system=CONDUCTOR_SYS,
@@ -400,19 +808,54 @@ class ConductorAgent(LocalCloudAgent):
         # every worker step runs through run_swe_agent_loop on a SHARED
         # workdir so step N+1 builds on step N's edits. The final patch is
         # whatever `git diff` produces after the last step.
-        task_meta = (context.metadata.get("task") if context is not None else {}) or {}
-        swe_mode = (
-            bool(cfg.get("swe_use_agent_loop"))
-            and bool(task_meta.get("problem_statement"))
-            and bool(task_meta.get("repo"))
-            and bool(task_meta.get("base_commit"))
-        )
+        # ``task_meta`` / ``swe_mode`` were computed up front (see Task-3
+        # planner constraint above) — reuse them.
+        task_meta = task_meta_early
+        swe_mode = swe_mode_early
         steps: List[Dict[str, Any]] = []
         tokens_local = 0
         tokens_cloud = 0
         cost = 0.0
         final_answer = ""
         shared_workdir: Optional[Path] = None
+        n_web_searches_total = 0
+        # tool_calls aggregator: bash turns on SWE (per worker subloop) +
+        # web_search uses on GAIA. Conductor planner is text-only.
+        tool_calls = 0
+
+        # Web_search opt-in: when enabled, search-capable workers
+        # (anthropic / openai / gemini) route through their `_base` agent
+        # loop in `_call_worker` and ground their answers. GAIA-only — SWE
+        # workers use bash not web_search. openrouter / vllm workers can't
+        # ground and ignore the flag. If the worker pool has NO
+        # search-capable worker, web_search can never run on any step —
+        # every step would answer the GAIA question blind from parametric
+        # memory. Fail loud instead of degrading silently.
+        # ``ws_enabled`` / ``ws_max_uses`` computed up front for the planner
+        # constraint — reuse them here.
+        if ws_enabled and not swe_mode:
+            search_workers = [
+                w for w in workers
+                if (w.get("endpoint") or "openai").lower()
+                in _SEARCH_CAPABLE_WORKER_ENDPOINTS
+            ]
+            if not search_workers:
+                endpoints = sorted({
+                    (w.get("endpoint") or "openai").lower() for w in workers
+                })
+                raise ValueError(
+                    f"web_search.enabled=true but the worker pool has no "
+                    f"search-capable worker (endpoints present: {endpoints}); "
+                    "server-side web_search is wired only for anthropic / "
+                    "openai / gemini workers in conductor's _call_worker. "
+                    "Add one of those to the pool or disable web_search — "
+                    "otherwise every step answers blind from parametric memory."
+                )
+        # ``ws_tool`` doubles as the enable marker passed to `_call_worker`
+        # (truthy => route search-capable workers through their agent loop).
+        ws_tool = (
+            build_web_search_tool(ws_max_uses) if ws_enabled else None
+        )
 
         try:
             if swe_mode:
@@ -444,18 +887,52 @@ class ConductorAgent(LocalCloudAgent):
                     "swe_mode": swe_mode,
                 })
 
+                worker_ep = (worker.get("endpoint") or "openai").lower()
+                # Post-hoc routing check: if web_search is on but the
+                # planner routed this step to a search-incapable worker,
+                # record a warning into the trace (don't crash — the step
+                # may legitimately not need search; see Task-3 planner
+                # constraint that tries to prevent this upfront).
+                if (
+                    ws_enabled and not swe_mode
+                    and worker_ep not in _SEARCH_CAPABLE_WORKER_ENDPOINTS
+                ):
+                    self.record_trace_event({
+                        "kind": "conductor_search_routing_warning",
+                        "step_idx": i,
+                        "worker_id": mid,
+                        "worker_name": worker["name"],
+                        "worker_endpoint": worker_ep,
+                        "warning": (
+                            f"web_search enabled but step {i} routed to "
+                            f"search-incapable worker {worker['name']!r} "
+                            f"(endpoint {worker_ep!r}); this step cannot "
+                            "ground and may answer blind."
+                        ),
+                    })
+
                 if swe_mode:
-                    text, w_in, w_out, is_local = _swe_worker_step(
-                        worker, task_meta, prompt, cfg, shared_workdir, i,
+                    text, w_in, w_out, is_local, n_searches, bash_turns = (
+                        _swe_worker_step(
+                            worker, task_meta, prompt, cfg, shared_workdir, i,
+                        )
                     )
+                    tool_calls += bash_turns
                 else:
-                    text, w_in, w_out, is_local = _call_worker(worker, prompt, cfg)
+                    text, w_in, w_out, is_local, n_searches = _call_worker(
+                        worker, prompt, cfg,
+                        web_search_tool=ws_tool,
+                        web_search_max_uses=ws_max_uses,
+                    )
 
                 if is_local:
                     tokens_local += w_in + w_out
                 else:
                     tokens_cloud += w_in + w_out
                     cost += self.cost_usd(worker["model"], w_in, w_out)
+                    cost += n_searches * _worker_search_cost_per_call(worker_ep)
+                n_web_searches_total += n_searches
+                tool_calls += n_searches
                 steps.append({
                     "step_idx": i,
                     "model_id": mid,
@@ -497,10 +974,14 @@ class ConductorAgent(LocalCloudAgent):
             "tokens_cloud": tokens_cloud,
             "cost_usd": cost,
             "turns": len(steps) + 1,  # planner + N execution steps
+            "web_search_uses": n_web_searches_total,
+            "tool_calls": int(tool_calls),
             "traces": {
                 "steps": traces,
                 "plan": plan,
                 "fallback_used": fallback_used,
+                "web_search_enabled": ws_enabled,
+                "n_web_searches": n_web_searches_total,
                 "parse_attempts": parse_attempts,
                 "workers": [
                     {k: v for k, v in w.items() if k != "api_key"}

src/openjarvis/agents/hybrid/mini_swe_agent.py

@@ -35,7 +35,10 @@ Differences vs. the upstream
 from __future__ import annotations
 
 import json
+import os
+import re
 import shutil
+import signal
 import subprocess
 import tempfile
 import time
@@ -43,15 +46,39 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent, _record_event
+from openjarvis.agents.hybrid._base import (
+    LocalCloudAgent,
+    _bump_cloud_calls,
+    _bump_local_calls,
+    _record_event,
+)
 from openjarvis.agents.hybrid._prices import (
     cost as estimate_cost,
 )
 from openjarvis.agents.hybrid._prices import (
+    is_gpt5_family,
     supports_temperature,
 )
 from openjarvis.core.registry import AgentRegistry
 
+
+# Gemini's FunctionDeclaration.parameters expects a Schema-shaped dict (or
+# Schema object) with capitalized type strings ("OBJECT", "STRING"). The
+# OpenAI/Anthropic JSON-Schema lower-case form is silently dropped — the
+# model then can't call the tool. Build a fresh dict instead of reusing
+# BASH_TOOL_OPENAI's parameters.
+BASH_TOOL_GEMINI_PARAMETERS: Dict[str, Any] = {
+    "type": "OBJECT",
+    "properties": {
+        "command": {
+            "type": "STRING",
+            "description": "The bash command to run.",
+        },
+    },
+    "required": ["command"],
+}
+
+
 SYSTEM_PROMPT = """\
 You are an expert software engineer fixing a bug in a Python repository. \
 You have one tool, `bash`, that runs a shell command and returns stdout, \
@@ -111,6 +138,36 @@ BASH_TOOL_OPENAI = {
 
 # ---------- Workdir / bash plumbing ----------
 
+# Models trained on SWE-bench Docker images (Qwen especially) reflexively
+# prefix commands with ``cd /testbed`` — the standard container repo path.
+# Our harness has no ``/testbed``; the repo is cloned into a per-task
+# tempdir and bash already runs with ``cwd`` set to it. An un-rewritten
+# ``cd /testbed`` errors with "No such file or directory" and, chained with
+# ``&&``, aborts the whole command — so the agent burns every turn and
+# never lands an edit. We rewrite ``/testbed`` references to the real
+# workdir so those commands run as intended.
+_TESTBED_CD_RE = re.compile(r"^\s*cd\s+/testbed(?:/\S*)?\s*(?:&&|;)\s*")
+
+
+def _rewrite_testbed_paths(command: str, workdir: Path) -> str:
+    """Neutralize hard-coded ``/testbed`` paths in a model-issued command.
+
+    - A leading ``cd /testbed && ...`` (or ``;``) is stripped — bash already
+      runs in the repo root, so the rest of the command is correct as-is.
+    - Any remaining ``/testbed`` occurrences (e.g. ``cat /testbed/foo.py``)
+      are rewritten to the real workdir.
+    """
+    wd = str(workdir)
+    new = _TESTBED_CD_RE.sub("", command)
+    # Bare ``cd /testbed`` with nothing after it → no-op into the workdir.
+    if re.fullmatch(r"\s*cd\s+/testbed/?\s*", new):
+        new = f"cd {wd}"
+    # Replace any other /testbed path references (word-boundary so we don't
+    # clobber e.g. /testbedrock).
+    new = re.sub(r"/testbed(?=/|\b)", wd, new)
+    return new
+
+
 def _clone_repo(repo: str, base_commit: str, dest: Path) -> None:
     """Shallow-fetch the SWE-bench repo at the right commit into ``dest``."""
     url = f"https://github.com/{repo}.git"
@@ -124,27 +181,89 @@ def _clone_repo(repo: str, base_commit: str, dest: Path) -> None:
     )
 
 
+def _decode_bash_output(raw: bytes, exit_code: int) -> str:
+    """Safely decode bash stdout/stderr bytes into a str the LLM can read.
+
+    The model sometimes runs commands that produce binary output (``cat``
+    on a ``.pyc`` / ``.png`` / packed extension, a ``find`` that pipes a
+    binary blob, ``xxd`` on a libc, ...). Decoding those with strict UTF-8
+    raises ``UnicodeDecodeError`` deep inside ``Popen.communicate()`` and
+    crashes the whole agent loop (1 errored row in the n=100 sweep per
+    such command). We:
+
+    1. Decode with ``errors="replace"`` so partial / mixed output is
+       always recoverable as a str.
+    2. If the result looks predominantly binary — contains a NUL byte OR
+       has more than ~5% U+FFFD replacement chars after decode — replace
+       it with a one-line stub so the model can keep working without
+       drowning in Mojibake. Threshold checked against the *bytes* length
+       (no NUL byte ⇒ probably text-ish; LLMs handle the occasional
+       replacement char fine).
+    """
+    if not raw:
+        return ""
+    decoded = raw.decode("utf-8", errors="replace")
+    if b"\x00" in raw or decoded.count("�") * 20 > len(decoded):
+        return f"[binary output: {len(raw)} bytes, exit={exit_code}]"
+    return decoded
+
+
 def _run_bash(
     command: str, workdir: Path, *, timeout: int = 120, output_cap: int = 10_000
 ) -> Dict[str, Any]:
     """Run one shell command in ``workdir``. Returns dict with stdout, stderr,
-    exit_code, and a ``truncated`` flag if output was clamped."""
+    exit_code, and a ``truncated`` flag if output was clamped.
+
+    The command is launched in its own process group (``start_new_session``)
+    so a model-issued command that backgrounds a long-lived child (a dev
+    server, ``sleep``, a hung test runner) can be killed *as a tree* on
+    timeout. Plain ``subprocess.run(..., capture_output=True, timeout=...)``
+    only kills the direct child and then re-blocks on ``communicate()``
+    draining the pipe — which a surviving grandchild holds open forever,
+    silently wedging the whole agent loop.
+    """
     t0 = time.time()
+    command = _rewrite_testbed_paths(command, workdir)
+    # Capture as bytes (no ``text=True``) so a tool invocation that emits
+    # binary output (compiled artifact, image, PDF, gzipped tarball) can't
+    # crash the loop on a strict UTF-8 decode mid-``communicate()``. We
+    # decode below with ``errors="replace"`` and, if the result looks
+    # binary (null byte or >5% replacement chars), substitute a stub so
+    # the model doesn't waste tokens / context on Mojibake.
+    proc = subprocess.Popen(
+        ["bash", "-lc", command],
+        cwd=str(workdir),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        start_new_session=True,
+    )
     try:
-        proc = subprocess.run(
-            ["bash", "-lc", command],
-            cwd=str(workdir),
-            capture_output=True, text=True, timeout=timeout,
-        )
-        stdout = proc.stdout
-        stderr = proc.stderr
+        stdout_b, stderr_b = proc.communicate(timeout=timeout)
         exit_code = proc.returncode
         timed_out = False
-    except subprocess.TimeoutExpired as e:
-        stdout = (e.stdout or "") if isinstance(e.stdout, str) else ""
-        stderr = (e.stderr or "") if isinstance(e.stderr, str) else ""
+    except subprocess.TimeoutExpired:
+        # Kill the whole process group so backgrounded grandchildren can't
+        # keep the stdout/stderr pipe open and deadlock the drain below.
+        for sig in (signal.SIGTERM, signal.SIGKILL):
+            try:
+                os.killpg(proc.pid, sig)
+            except (ProcessLookupError, PermissionError):
+                break
+            try:
+                proc.wait(timeout=5)
+                break
+            except subprocess.TimeoutExpired:
+                continue
+        try:
+            stdout_b, stderr_b = proc.communicate(timeout=10)
+        except subprocess.TimeoutExpired:
+            stdout_b, stderr_b = b"", b"";
+        stdout_b = stdout_b or b""
+        stderr_b = stderr_b or b""
         exit_code = -1
         timed_out = True
+    stdout = _decode_bash_output(stdout_b, exit_code)
+    stderr = _decode_bash_output(stderr_b, exit_code)
     truncated = False
     if len(stdout) > output_cap:
         stdout = stdout[:output_cap] + f"\n…[+{len(stdout) - output_cap} chars truncated]"
@@ -212,10 +331,13 @@ def run_swe_agent_loop(
     initial_prompt: Optional[str] = None,
     max_turns: int = 30,
     bash_timeout: int = 120,
+    bash_timeout_s: Optional[int] = None,
     output_cap: int = 10_000,
     turn_max_tokens: int = 4096,
     trace_prefix: str = "mini_swe",
     workdir: Optional[Path] = None,
+    compact_at_tokens: int = 24_000,
+    compact_keep_last: int = 4,
 ) -> Dict[str, Any]:
     """Run a mini-SWE-agent loop for one SWE-bench task. Returns:
 
@@ -254,6 +376,8 @@ def run_swe_agent_loop(
         want to chain multiple subloops over the same working tree can
         manage their own workdir.
     """
+    if bash_timeout_s is not None:
+        bash_timeout = int(bash_timeout_s)
     repo = task.get("repo") or ""
     base_commit = task.get("base_commit") or ""
     if not repo or not base_commit:
@@ -310,6 +434,8 @@ def run_swe_agent_loop(
                 output_cap=output_cap,
                 turn_max_tokens=turn_max_tokens,
                 trace_prefix=trace_prefix,
+                compact_at_tokens=compact_at_tokens,
+                compact_keep_last=compact_keep_last,
             )
         else:
             raise ValueError(f"unsupported backbone: {backbone!r}")
@@ -340,7 +466,7 @@ def run_swe_agent_loop(
             shutil.rmtree(workdir, ignore_errors=True)
 
 
-# ---------- Cloud loop (Anthropic multi-turn with tools) ----------
+# ---------- Cloud loop (dispatcher → per-endpoint multi-turn tool loops) ----------
 
 def _loop_cloud(
     problem: str,
@@ -354,11 +480,47 @@ def _loop_cloud(
     turn_max_tokens: int,
     trace_prefix: str,
 ) -> Dict[str, Any]:
-    if cloud_endpoint != "anthropic":
-        raise ValueError(
-            f"mini-SWE-agent cloud backbone currently supports anthropic only; "
-            f"got {cloud_endpoint!r}"
+    """Route to the right per-endpoint cloud loop. Anthropic path is the
+    original byte-identical implementation (16 cells in the n=100 sweep
+    depend on its exact behavior). OpenAI / Gemini paths added 2026-05-15
+    to unblock the 8 SWE cells that were stuck on Anthropic-only support."""
+    if cloud_endpoint == "anthropic":
+        return _loop_cloud_anthropic(
+            problem, workdir,
+            model=model, max_turns=max_turns,
+            bash_timeout=bash_timeout, output_cap=output_cap,
+            turn_max_tokens=turn_max_tokens, trace_prefix=trace_prefix,
         )
+    if cloud_endpoint == "openai":
+        return _loop_cloud_openai(
+            problem, workdir,
+            model=model, max_turns=max_turns,
+            bash_timeout=bash_timeout, output_cap=output_cap,
+            turn_max_tokens=turn_max_tokens, trace_prefix=trace_prefix,
+        )
+    if cloud_endpoint == "gemini":
+        return _loop_cloud_gemini(
+            problem, workdir,
+            model=model, max_turns=max_turns,
+            bash_timeout=bash_timeout, output_cap=output_cap,
+            turn_max_tokens=turn_max_tokens, trace_prefix=trace_prefix,
+        )
+    raise ValueError(
+        f"mini-SWE-agent cloud backbone unsupported endpoint: {cloud_endpoint!r}"
+    )
+
+
+def _loop_cloud_anthropic(
+    problem: str,
+    workdir: Path,
+    *,
+    model: str,
+    max_turns: int,
+    bash_timeout: int,
+    output_cap: int,
+    turn_max_tokens: int,
+    trace_prefix: str,
+) -> Dict[str, Any]:
     import anthropic
     client = anthropic.Anthropic(timeout=600.0, max_retries=5)
     messages: List[Dict[str, Any]] = [{"role": "user", "content": problem}]
@@ -380,6 +542,7 @@ def _loop_cloud(
             kwargs["temperature"] = 0.0
         t0 = time.time()
         msg = client.messages.create(**kwargs)
+        _bump_cloud_calls()
         latency = time.time() - t0
         tokens_in += msg.usage.input_tokens
         tokens_out += msg.usage.output_tokens
@@ -457,8 +620,596 @@ def _loop_cloud(
     }
 
 
+# ---------- Cloud loop (OpenAI multi-turn with function tools) ----------
+
+def _loop_cloud_openai(
+    problem: str,
+    workdir: Path,
+    *,
+    model: str,
+    max_turns: int,
+    bash_timeout: int,
+    output_cap: int,
+    turn_max_tokens: int,
+    trace_prefix: str,
+) -> Dict[str, Any]:
+    """OpenAI Chat Completions multi-turn loop. Mirrors the Anthropic
+    branch: each turn the model either calls ``bash`` (one or more parallel
+    tool_calls) or produces a no-tool-call final message that terminates
+    the loop. The final message's text is returned as ``final_summary``;
+    the patch comes from ``git diff`` on the workdir.
+
+    Quirks:
+    - GPT-5 family rejects ``temperature`` and uses ``max_completion_tokens``
+      instead of ``max_tokens``. We branch on ``is_gpt5_family`` for both.
+    - ``tool_calls`` arguments arrive as JSON-string blobs; we tolerate
+      malformed JSON by treating it as an empty arg dict (matches the
+      ``_loop_local`` behavior).
+    """
+    from openai import OpenAI
+    client = OpenAI(timeout=600.0)
+
+    messages: List[Dict[str, Any]] = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": problem},
+    ]
+    tokens_in = 0
+    tokens_out = 0
+    final_text = ""
+    turns = 0
+    for turn in range(1, max_turns + 1):
+        turns = turn
+        kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "tools": [BASH_TOOL_OPENAI],
+            "tool_choice": "auto",
+        }
+        if is_gpt5_family(model):
+            kwargs["max_completion_tokens"] = turn_max_tokens
+        else:
+            kwargs["max_tokens"] = turn_max_tokens
+            kwargs["temperature"] = 0.0
+        t0 = time.time()
+        resp = client.chat.completions.create(**kwargs)
+        _bump_cloud_calls()
+        latency = time.time() - t0
+        u = resp.usage
+        tokens_in += getattr(u, "prompt_tokens", 0) if u else 0
+        tokens_out += getattr(u, "completion_tokens", 0) if u else 0
+        choice = resp.choices[0]
+        message = choice.message
+        tool_calls = list(getattr(message, "tool_calls", None) or [])
+        text = message.content or ""
+
+        _record_event({
+            "kind": f"{trace_prefix}_turn",
+            "turn": turn,
+            "endpoint": "openai",
+            "finish_reason": choice.finish_reason,
+            "tokens_in": getattr(u, "prompt_tokens", 0) if u else 0,
+            "tokens_out": getattr(u, "completion_tokens", 0) if u else 0,
+            "latency_s": latency,
+            "text": text,
+            "tool_calls": [
+                {"id": tc.id, "name": tc.function.name, "arguments": tc.function.arguments}
+                for tc in tool_calls
+            ],
+            "ts": time.time(),
+        })
+
+        # Append the assistant turn (including any tool_calls) so the
+        # follow-up tool messages have the right call ids to reference.
+        # OpenAI Chat Completions rejects ``content: null`` with a 400
+        # ("expected a string, got null") on the next turn when this
+        # message gets replayed. Use ``""`` — explicitly allowed by the
+        # schema when ``tool_calls`` is present, and equivalent to
+        # "assistant had no visible text, only tool calls".
+        assistant_msg: Dict[str, Any] = {
+            "role": "assistant",
+            "content": text or "",
+        }
+        if tool_calls:
+            assistant_msg["tool_calls"] = [
+                {
+                    "id": tc.id, "type": "function",
+                    "function": {
+                        "name": tc.function.name,
+                        "arguments": tc.function.arguments,
+                    },
+                }
+                for tc in tool_calls
+            ]
+        messages.append(assistant_msg)
+
+        if not tool_calls:
+            # Silent-truncation recovery: ``finish_reason='length'`` means
+            # the response was cut mid-generation. If a tool call was
+            # forming when the cap hit, ``tool_calls`` is empty AND ``text``
+            # is empty/short — treating that as "model done" exits the loop
+            # with a useless empty final_summary (observed on gpt-5-mini SWE
+            # cells, 2026-05-15). Inject a one-shot recovery nudge and let
+            # the loop continue; only terminate naturally on ``stop``.
+            if (
+                choice.finish_reason == "length"
+                and not text.strip()
+                and turn < max_turns
+            ):
+                messages.append({
+                    "role": "user",
+                    "content": (
+                        "Your previous response was truncated by the token limit "
+                        "before producing a tool call or final summary. Retry: "
+                        "either issue ONE bash tool call (short command, no large "
+                        "output) or send a brief one-line final summary with no "
+                        "tool calls to end the loop."
+                    ),
+                })
+                _record_event({
+                    "kind": f"{trace_prefix}_recover",
+                    "turn": turn, "reason": "length_truncation_no_tool_call",
+                    "ts": time.time(),
+                })
+                continue
+            # No tool call → the model is done. Same termination rule as
+            # the Anthropic branch.
+            final_text = text.strip()
+            break
+
+        for tc in tool_calls:
+            try:
+                args = json.loads(tc.function.arguments or "{}")
+            except json.JSONDecodeError:
+                args = {}
+            if tc.function.name != "bash":
+                obs = f"unknown tool: {tc.function.name!r}"
+                _record_event({
+                    "kind": f"{trace_prefix}_unknown_tool",
+                    "turn": turn, "name": tc.function.name, "input": args,
+                    "ts": time.time(),
+                })
+            else:
+                command = str(args.get("command", ""))
+                result = _run_bash(
+                    command, workdir,
+                    timeout=bash_timeout, output_cap=output_cap,
+                )
+                _record_event({
+                    "kind": f"{trace_prefix}_bash",
+                    "turn": turn, "command": command,
+                    **result, "ts": time.time(),
+                })
+                obs = _format_observation(result)
+            messages.append({
+                "role": "tool",
+                "tool_call_id": tc.id,
+                "content": obs,
+            })
+
+    return {
+        "tokens_in": tokens_in,
+        "tokens_out": tokens_out,
+        "turns": turns,
+        "final_summary": final_text,
+        "max_turns_hit": turns == max_turns and not final_text,
+    }
+
+
+# ---------- Cloud loop (Gemini multi-turn with function tools) ----------
+
+def _loop_cloud_gemini(
+    problem: str,
+    workdir: Path,
+    *,
+    model: str,
+    max_turns: int,
+    bash_timeout: int,
+    output_cap: int,
+    turn_max_tokens: int,
+    trace_prefix: str,
+) -> Dict[str, Any]:
+    """google-genai multi-turn loop (Gemini Developer API, v1.64).
+
+    Tool-use plumbing diverges from OpenAI/Anthropic enough to warrant a
+    parallel branch:
+
+    - Parameters must be a Schema-shaped dict with capitalized type names
+      ("OBJECT", "STRING"). The lower-case JSON-Schema form is silently
+      dropped — the model never produces a call.
+    - We explicitly disable ``automatic_function_calling`` so the SDK
+      stops at the FunctionCall part and we drive the loop ourselves.
+    - Termination heuristic: stop when the model's response has zero
+      function_call parts. Same intent as Anthropic ("no tool_use blocks")
+      and OpenAI ("empty tool_calls"). Gemini occasionally emits a turn
+      with both text and function_call parts; we still treat that as a
+      tool turn (matches Anthropic's behavior with mixed content).
+    - System prompt goes on the config, not the contents list (Gemini
+      convention).
+    - The function-response part body is a free-form dict; we wrap the
+      bash observation in ``{"output": str}``.
+    """
+    from google import genai
+    from google.genai import types
+
+    client = genai.Client(http_options=types.HttpOptions(timeout=600_000))
+    bash_tool = types.Tool(function_declarations=[
+        types.FunctionDeclaration(
+            name="bash",
+            description=BASH_TOOL_ANTHROPIC["description"],
+            parameters=BASH_TOOL_GEMINI_PARAMETERS,
+        ),
+    ])
+
+    contents: List[types.Content] = [
+        types.Content(role="user", parts=[types.Part(text=problem)]),
+    ]
+    tokens_in = 0
+    tokens_out = 0
+    final_text = ""
+    turns = 0
+    for turn in range(1, max_turns + 1):
+        turns = turn
+        cfg = types.GenerateContentConfig(
+            temperature=0.0,
+            max_output_tokens=turn_max_tokens,
+            system_instruction=SYSTEM_PROMPT,
+            tools=[bash_tool],
+            automatic_function_calling=types.AutomaticFunctionCallingConfig(
+                disable=True,
+            ),
+        )
+        t0 = time.time()
+        resp = client.models.generate_content(
+            model=model, contents=contents, config=cfg,
+        )
+        _bump_cloud_calls()
+        latency = time.time() - t0
+        um = getattr(resp, "usage_metadata", None)
+        p = int(getattr(um, "prompt_token_count", 0) or 0) if um else 0
+        c = int(getattr(um, "candidates_token_count", 0) or 0) if um else 0
+        tokens_in += p
+        tokens_out += c
+
+        # Pull parts out of the first candidate. Defensive against the
+        # zero-candidate / safety-filtered case (returns empty parts and
+        # we'll terminate on the next branch).
+        cand_parts: List[Any] = []
+        try:
+            cand_content = resp.candidates[0].content
+            cand_parts = list(getattr(cand_content, "parts", None) or [])
+        except Exception:
+            cand_content = None
+            cand_parts = []
+
+        text_parts: List[str] = []
+        function_calls: List[Tuple[str, Dict[str, Any]]] = []
+        for part in cand_parts:
+            fc = getattr(part, "function_call", None)
+            if fc is not None:
+                fc_args = dict(getattr(fc, "args", None) or {})
+                function_calls.append((fc.name, fc_args))
+            elif getattr(part, "text", None):
+                text_parts.append(part.text)
+
+        finish_reason = None
+        try:
+            finish_reason = str(resp.candidates[0].finish_reason)
+        except Exception:
+            pass
+
+        _record_event({
+            "kind": f"{trace_prefix}_turn",
+            "turn": turn,
+            "endpoint": "gemini",
+            "finish_reason": finish_reason,
+            "tokens_in": p,
+            "tokens_out": c,
+            "latency_s": latency,
+            "text": "\n".join(text_parts),
+            "tool_calls": [
+                {"name": name, "arguments": args}
+                for name, args in function_calls
+            ],
+            "ts": time.time(),
+        })
+
+        # Append the model's content as-is so the next turn sees its own
+        # prior function_call parts (Gemini requires this for the
+        # function_response to bind).
+        if cand_content is not None:
+            contents.append(cand_content)
+        else:
+            # Safety-filtered / empty — synthesize an empty model turn so
+            # the conversation stays well-formed and exit the loop.
+            contents.append(types.Content(role="model", parts=[]))
+
+        if not function_calls:
+            # Silent-failure recovery for Gemini's quirky finish reasons:
+            #   MALFORMED_FUNCTION_CALL — model wanted to call bash but
+            #     produced unparseable args (24/100 of broken n=100 SWE
+            #     cells, 2026-05-15). Empty text + empty function_calls →
+            #     loop would exit with no final summary.
+            #   MAX_TOKENS — same shape as OpenAI's ``length``; truncated
+            #     mid-generation, no tool call landed.
+            # Inject a recovery nudge and let the loop continue; only
+            # treat genuine ``STOP`` with text as a final answer.
+            fr_str = str(finish_reason or "")
+            empty_text = not any(t.strip() for t in text_parts)
+            recoverable = empty_text and turn < max_turns and (
+                "MALFORMED_FUNCTION_CALL" in fr_str
+                or "MAX_TOKENS" in fr_str
+            )
+            if recoverable:
+                contents.append(types.Content(
+                    role="user",
+                    parts=[types.Part(text=(
+                        "Your previous response had no parsable function call "
+                        "and no final text (finish_reason="
+                        f"{fr_str}). Retry: either issue ONE well-formed "
+                        "`bash` function call (short command, valid JSON-ish "
+                        "args) or send a brief final text message with no "
+                        "function call to end the loop."
+                    ))],
+                ))
+                _record_event({
+                    "kind": f"{trace_prefix}_recover",
+                    "turn": turn,
+                    "reason": f"empty_response_{fr_str}",
+                    "ts": time.time(),
+                })
+                continue
+            final_text = "\n".join(text_parts).strip()
+            break
+
+        response_parts: List[Any] = []
+        for name, args in function_calls:
+            if name != "bash":
+                obs = f"unknown tool: {name!r}"
+                _record_event({
+                    "kind": f"{trace_prefix}_unknown_tool",
+                    "turn": turn, "name": name, "input": args,
+                    "ts": time.time(),
+                })
+            else:
+                command = str(args.get("command", ""))
+                result = _run_bash(
+                    command, workdir,
+                    timeout=bash_timeout, output_cap=output_cap,
+                )
+                _record_event({
+                    "kind": f"{trace_prefix}_bash",
+                    "turn": turn, "command": command,
+                    **result, "ts": time.time(),
+                })
+                obs = _format_observation(result)
+            response_parts.append(types.Part.from_function_response(
+                name=name, response={"output": obs},
+            ))
+        contents.append(types.Content(role="user", parts=response_parts))
+
+    return {
+        "tokens_in": tokens_in,
+        "tokens_out": tokens_out,
+        "turns": turns,
+        "final_summary": final_text,
+        "max_turns_hit": turns == max_turns and not final_text,
+    }
+
+
 # ---------- Local loop (vLLM, OpenAI-compatible multi-turn with tools) ----------
 
+_COMPACT_PROMPT = (
+    "Summarize the SWE-bench agent trajectory so far in under 2000 characters. "
+    "Preserve: filenames touched, hypotheses tested, what worked, what failed, "
+    "and the current plan. Be terse — no preamble, no quoted output, just facts."
+)
+
+_TIKTOKEN_ENC = None
+_TIKTOKEN_WARNED = False
+
+
+def _get_tiktoken_enc() -> Any:
+    global _TIKTOKEN_ENC, _TIKTOKEN_WARNED
+    if _TIKTOKEN_ENC is not None:
+        return _TIKTOKEN_ENC
+    try:
+        import tiktoken
+        _TIKTOKEN_ENC = tiktoken.get_encoding("cl100k_base")
+    except Exception as exc:
+        if not _TIKTOKEN_WARNED:
+            print(f"[mini_swe_agent] tiktoken unavailable ({exc!r}); falling back to len(s)//4", flush=True)
+            _TIKTOKEN_WARNED = True
+        _TIKTOKEN_ENC = False
+    return _TIKTOKEN_ENC
+
+
+def _estimate_prompt_tokens(messages: List[Dict[str, Any]]) -> int:
+    enc = _get_tiktoken_enc()
+    total = 0
+    for m in messages:
+        total += 4  # per-message overhead
+        c = m.get("content")
+        if isinstance(c, str):
+            s = c
+        elif isinstance(c, list):
+            parts = []
+            for block in c:
+                if isinstance(block, dict):
+                    parts.append(str(block.get("content") or block.get("text") or ""))
+            s = "\n".join(parts)
+        else:
+            s = ""
+        for tc in (m.get("tool_calls") or []):
+            try:
+                s += "\n" + (tc["function"]["arguments"] or "")
+                s += "\n" + (tc["function"].get("name") or "")
+            except (KeyError, TypeError):
+                pass
+        tcid = m.get("tool_call_id")
+        if tcid:
+            s += "\n" + str(tcid)
+        if enc:
+            total += len(enc.encode(s, disallowed_special=()))
+        else:
+            total += len(s) // 4
+    return total
+
+
+_EXIT_PATTERNS = (
+    re.compile(r"exit_code\s*[=:]\s*(-?\d+)"),
+    re.compile(r"returncode\s*[=:]\s*(-?\d+)"),
+    re.compile(r"\bexit\s+(-?\d+)\b"),
+)
+
+
+def _parse_exit_code(content: Any) -> str:
+    if not isinstance(content, str):
+        return "?"
+    for pat in _EXIT_PATTERNS:
+        m = pat.search(content)
+        if m:
+            return m.group(1)
+    return "?"
+
+
+def _identify_turns(messages: List[Dict[str, Any]]) -> List[Tuple[int, int]]:
+    """Return list of (start_idx, end_idx_exclusive) for each assistant+tools turn.
+
+    A turn = one assistant message (with or without tool_calls) plus any
+    immediately-following tool messages. System + initial user are skipped.
+    """
+    turns: List[Tuple[int, int]] = []
+    i = 0
+    n = len(messages)
+    while i < n:
+        role = messages[i].get("role")
+        if role == "assistant":
+            j = i + 1
+            while j < n and messages[j].get("role") == "tool":
+                j += 1
+            turns.append((i, j))
+            i = j
+        else:
+            i += 1
+    return turns
+
+
+def _compact_local_messages(
+    messages: List[Dict[str, Any]],
+    *,
+    client: Any,
+    model: str,
+    keep_last: int,
+    trace_prefix: str,
+    compact_at_tokens: int = 24_000,
+) -> List[Dict[str, Any]]:
+    if len(messages) < 2:
+        return messages
+    system_msg = messages[0]
+    initial_user = messages[1]
+
+    turns = _identify_turns(messages)
+    if len(turns) <= keep_last:
+        return messages
+
+    keep_turns = turns[-keep_last:]
+    old_turns = turns[:-keep_last]
+    keep_start = keep_turns[0][0]
+
+    # Stage 1: elide tool observations in old turns.
+    before_tokens = _estimate_prompt_tokens(messages)
+    new_messages: List[Dict[str, Any]] = list(messages)
+    n_tool_elided = 0
+    for (s, e) in old_turns:
+        for k in range(s, e):
+            m = new_messages[k]
+            if m.get("role") != "tool":
+                continue
+            orig = m.get("content")
+            if not isinstance(orig, str):
+                continue
+            n_chars = len(orig)
+            if n_chars <= 200:
+                continue
+            exit_code = _parse_exit_code(orig)
+            stub = f"[tool output elided: {n_chars} chars, exit={exit_code}]"
+            new_messages[k] = {
+                "role": "tool",
+                "tool_call_id": m.get("tool_call_id"),
+                "content": stub,
+            }
+            n_tool_elided += 1
+
+    after_stage1_tokens = _estimate_prompt_tokens(new_messages)
+    _record_event({
+        "kind": f"{trace_prefix}_compact",
+        "stage": "1",
+        "msgs_before": len(messages),
+        "msgs_after": len(new_messages),
+        "before_tokens": before_tokens,
+        "after_tokens": after_stage1_tokens,
+        "n_tool_elided": n_tool_elided,
+        "n_turns_folded": 0,
+        "ts": time.time(),
+    })
+
+    if after_stage1_tokens <= compact_at_tokens:
+        return new_messages
+
+    # Stage 2: fold old turns into a single synthetic system summary.
+    middle = new_messages[2:keep_start]
+    tail = new_messages[keep_start:]
+    if not middle:
+        return new_messages
+
+    summary_input = [
+        {"role": "system", "content": _COMPACT_PROMPT},
+        {"role": "user", "content": json.dumps(
+            [{"role": m.get("role"),
+              "content": m.get("content") if isinstance(m.get("content"), str) else str(m.get("content"))[:4000]}
+             for m in middle],
+            default=str,
+        )[:60_000]},
+    ]
+    summary = ""
+    try:
+        if client is not None:
+            resp = client.chat.completions.create(
+                model=model,
+                messages=summary_input,
+                temperature=0.0,
+                max_tokens=1024,
+                extra_body={"chat_template_kwargs": {"enable_thinking": False}},
+            )
+            _bump_local_calls()
+            summary = (resp.choices[0].message.content or "").strip()[:2000]
+    except Exception as exc:
+        summary = f"[compaction summary failed: {exc!r}; older turns dropped]"
+    if not summary:
+        summary = "[no summary produced; older turns dropped]"
+
+    n_turns_folded = len(old_turns)
+    synthetic = {
+        "role": "system",
+        "content": f"[turns 1–{n_turns_folded} elided: {summary}]",
+    }
+    folded = [system_msg, initial_user, synthetic, *tail]
+    after_stage2_tokens = _estimate_prompt_tokens(folded)
+    _record_event({
+        "kind": f"{trace_prefix}_compact",
+        "stage": "2",
+        "msgs_before": len(new_messages),
+        "msgs_after": len(folded),
+        "before_tokens": after_stage1_tokens,
+        "after_tokens": after_stage2_tokens,
+        "n_tool_elided": n_tool_elided,
+        "n_turns_folded": n_turns_folded,
+        "summary_chars": len(summary),
+        "ts": time.time(),
+    })
+    return folded
+
+
 def _loop_local(
     problem: str,
     workdir: Path,
@@ -470,7 +1221,16 @@ def _loop_local(
     output_cap: int,
     turn_max_tokens: int,
     trace_prefix: str,
+    compact_at_tokens: int = 22_000,
+    compact_keep_last: int = 3,
 ) -> Dict[str, Any]:
+    # Qwen-27B has a 32k context. With ``max_tokens=turn_max_tokens`` reserved
+    # for output (default 4096) plus ~1k for the bash tool schema + system
+    # prompt + format overhead, the practical input ceiling is ~27k. We
+    # compact at 22k so there's slack for one more tool result before the
+    # next turn's pre-call check fires again. Earlier we used 24k + keep=4
+    # but still saw 28k-input 400s on the n=100 SWE sweep (the keep window
+    # alone routinely exceeded the budget once bash outputs piled up).
     from openai import OpenAI
     client = OpenAI(base_url=endpoint, api_key="EMPTY", timeout=600.0)
 
@@ -484,16 +1244,60 @@ def _loop_local(
     turns = 0
     for turn in range(1, max_turns + 1):
         turns = turn
+        if compact_at_tokens > 0 and _estimate_prompt_tokens(messages) > compact_at_tokens:
+            messages = _compact_local_messages(
+                messages, client=client, model=model,
+                keep_last=compact_keep_last, trace_prefix=trace_prefix,
+                compact_at_tokens=compact_at_tokens,
+            )
         t0 = time.time()
-        resp = client.chat.completions.create(
-            model=model,
-            messages=messages,
-            temperature=0.0,
-            max_tokens=turn_max_tokens,
-            tools=[BASH_TOOL_OPENAI],
-            tool_choice="auto",
-            extra_body={"chat_template_kwargs": {"enable_thinking": False}},
-        )
+        try:
+            resp = client.chat.completions.create(
+                model=model,
+                messages=messages,
+                temperature=0.0,
+                max_tokens=turn_max_tokens,
+                tools=[BASH_TOOL_OPENAI],
+                tool_choice="auto",
+                extra_body={"chat_template_kwargs": {"enable_thinking": False}},
+            )
+        except Exception as exc:
+            # Emergency compaction on a context-length 400 from vLLM /
+            # OpenAI ("maximum context length is N tokens"). Our pre-call
+            # estimator can undercount when tool args / tool_call_ids /
+            # template overhead spike, so the budget check missed and the
+            # server walled the call. Compact aggressively (keep_last=1)
+            # and retry once. Re-raise on anything else or on a second
+            # failure — the runner records the row as errored.
+            msg = str(exc)
+            is_ctx = (
+                "maximum context length" in msg
+                or "context length" in msg.lower() and "exceed" in msg.lower()
+            )
+            if not is_ctx:
+                raise
+            _record_event({
+                "kind": f"{trace_prefix}_emergency_compact",
+                "turn": turn,
+                "error": msg[:300],
+                "tokens_before": _estimate_prompt_tokens(messages),
+                "ts": time.time(),
+            })
+            messages = _compact_local_messages(
+                messages, client=client, model=model,
+                keep_last=1, trace_prefix=trace_prefix,
+                compact_at_tokens=max(8_000, compact_at_tokens // 2),
+            )
+            resp = client.chat.completions.create(
+                model=model,
+                messages=messages,
+                temperature=0.0,
+                max_tokens=turn_max_tokens,
+                tools=[BASH_TOOL_OPENAI],
+                tool_choice="auto",
+                extra_body={"chat_template_kwargs": {"enable_thinking": False}},
+            )
+        _bump_local_calls()
         latency = time.time() - t0
         u = resp.usage
         tokens_in += getattr(u, "prompt_tokens", 0) if u else 0
@@ -518,10 +1322,17 @@ def _loop_local(
             "ts": time.time(),
         })
 
-        messages.append({
+        # Match the OpenAI cloud branch: content="" (not None) when only
+        # tool_calls are present; omit ``tool_calls`` entirely when there
+        # are none (vs. setting it to None) so the message validates
+        # against the strict OpenAI schema if it ever gets replayed by
+        # the compactor's summarizer call.
+        assistant_local_msg: Dict[str, Any] = {
             "role": "assistant",
-            "content": text or None,
-            "tool_calls": [
+            "content": text or "",
+        }
+        if tool_calls:
+            assistant_local_msg["tool_calls"] = [
                 {
                     "id": tc.id, "type": "function",
                     "function": {
@@ -530,8 +1341,8 @@ def _loop_local(
                     },
                 }
                 for tc in tool_calls
-            ] if tool_calls else None,
-        })
+            ]
+        messages.append(assistant_local_msg)
 
         if not tool_calls:
             final_text = text.strip()

src/openjarvis/agents/hybrid/minions.py

@@ -17,9 +17,9 @@ acc / $0.67 (vs $1.09).
 
 Requires the ``minions`` library from
 https://github.com/HazyResearch/minions installed in the same env (e.g.
-``uv pip install -e /matx/u/aspark/hybrid-local-cloud-compute/external/minions``).
-Import is lazy — the agent class registers without ``minions`` available,
-and the import error only fires on ``run()``.
+``uv pip install -e path/to/minions``). Import is lazy — the agent class
+registers without ``minions`` available, and the import error only fires
+on ``run()``.
 
 Compatibility patches applied at first ``run()`` (idempotent):
 
@@ -45,9 +45,13 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from openjarvis.agents._stubs import AgentContext
 from openjarvis.agents.hybrid._base import (
-    ANTHROPIC_WEB_SEARCH_TOOL,
     WEB_SEARCH_COST_PER_CALL,
     LocalCloudAgent,
+    build_web_search_tool,
+    web_search_cfg,
+)
+from openjarvis.agents.hybrid._openai_retry import (
+    patch_openai_globally as _patch_openai_globally,
 )
 from openjarvis.agents.hybrid._prices import NO_TEMP_PREFIXES
 from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
@@ -200,6 +204,13 @@ def _patch_anthropic_globally() -> None:
 
         def make_patched(orig):  # type: ignore[no-untyped-def]
             def patched(self, **kwargs):  # type: ignore[no-untyped-def]
+                # External Minions's AnthropicClient.chat passes
+                # `cache_control={"type":"ephemeral"}` as a top-level kwarg
+                # (clients/anthropic.py:207). Newer Anthropic SDKs reject that
+                # — cache_control belongs on individual content blocks, not on
+                # Messages.create itself. Strip it; we're not relying on the
+                # ephemeral hint for correctness in these short minions turns.
+                kwargs.pop("cache_control", None)
                 model = kwargs.get("model", "")
                 if model.startswith(NO_TEMP_PREFIXES):
                     kwargs.pop("temperature", None)
@@ -215,6 +226,96 @@ def _patch_anthropic_globally() -> None:
         cls.create = make_patched(orig)  # type: ignore[assignment]
 
 
+def _patch_gemini_client_usage() -> None:
+    """Patch vendored ``GeminiClient.schat`` for Gemini 2.5 quirks.
+
+    Two issues in the upstream client:
+
+    1. ``response.usage_metadata.candidates_token_count`` is sometimes ``None``
+       (empty / thinking-only responses on 2.5 Pro), and the upstream code
+       does ``total_token_count - candidates_token_count`` raw → ``TypeError``.
+
+    2. ``response.text`` raises if the model only emitted a non-text part
+       (e.g. safety block, thinking-only). We swallow it as empty.
+
+    Both fixes are idempotent and bypass the original ``schat`` body only
+    on the value-extraction lines — the API call itself is unchanged.
+    """
+    from minions.clients.gemini import GeminiClient  # type: ignore[import-not-found]
+    from minions.usage import Usage  # type: ignore[import-not-found]
+
+    if getattr(GeminiClient.schat, "_hybrid_patched", False):
+        return
+
+    _orig_schat = GeminiClient.schat
+
+    def _safe_int(x):  # type: ignore[no-untyped-def]
+        try:
+            return int(x) if x is not None else 0
+        except (TypeError, ValueError):
+            return 0
+
+    def patched_schat(self, messages, **kwargs):  # type: ignore[no-untyped-def]
+        # Mirror the upstream "native" branch by hand, but defensively.
+        # Skip the OpenAI-compat branch — Minions paradigm never sets that.
+        if self.use_openai_api:
+            return _orig_schat(self, messages, **kwargs)
+        if isinstance(messages, dict):
+            messages = [messages]
+        contents, system_instruction = self._format_content(messages)
+        if not system_instruction:
+            system_instruction = self.system_instruction
+        tools = self._prepare_tools(messages=messages)
+        config_kwargs = {
+            "temperature": self.temperature,
+            "max_output_tokens": self.max_tokens,
+        }
+        if self.thinking_budget is not None or self.thinking_level is not None:
+            tc = {}
+            if self.thinking_budget is not None:
+                tc["thinking_budget"] = self.thinking_budget
+            if self.thinking_level is not None:
+                tc["thinking_level"] = self.thinking_level
+            config_kwargs["thinking_config"] = self.types.ThinkingConfig(**tc)
+        if tools:
+            config_kwargs["tools"] = tools
+        config_kwargs["system_instruction"] = system_instruction
+        config = self.types.GenerateContentConfig(**config_kwargs)
+
+        response = self.client.models.generate_content(
+            model=self.model_name,
+            contents=contents,
+            config=config,
+        )
+
+        # Defensive text accessor — upstream `response.text` can raise when
+        # the model only emitted a non-text part.
+        try:
+            text = response.text or ""
+        except Exception:
+            try:
+                parts = response.candidates[0].content.parts or []
+                text = "".join(getattr(p, "text", "") or "" for p in parts)
+            except Exception:
+                text = ""
+
+        um = getattr(response, "usage_metadata", None)
+        total = _safe_int(getattr(um, "total_token_count", 0)) if um else 0
+        comp = _safe_int(getattr(um, "candidates_token_count", 0)) if um else 0
+        prompt = _safe_int(getattr(um, "prompt_token_count", 0)) if um else 0
+        # Prefer the explicit prompt count if present; fall back to (total - comp).
+        if not prompt and total:
+            prompt = max(total - comp, 0)
+        usage = Usage(prompt_tokens=prompt, completion_tokens=comp)
+
+        if self.local:
+            return [text], usage, ["stop"]
+        return [text], usage
+
+    patched_schat._hybrid_patched = True  # type: ignore[attr-defined]
+    GeminiClient.schat = patched_schat  # type: ignore[assignment]
+
+
 def _patch_minions_extract_json() -> None:
     """Minions's ``_extract_json`` uses a non-greedy regex that grabs the
     first short bracket pair and prefers ```json``` fences. With structured
@@ -245,13 +346,23 @@ def _apply_patches_once() -> None:
         return
     _stub_missing_imports()
     _patch_anthropic_globally()
+    # Mirror the Anthropic patch for OpenAI so the Minions library's own
+    # ``OpenAIClient`` instances pick up retry + per-org concurrency caps.
+    # Idempotent — also applied at ``_base`` import time.
+    _patch_openai_globally()
+    _patch_gemini_client_usage()
     _patch_minions_extract_json()
     _PATCHES_APPLIED = True
 
 
 # ---------- Pre-fetch helper (GAIA only) ----------
 
-def _prefetch_context(question: str, cloud_endpoint: str, cloud_model: str) -> Dict[str, Any]:
+def _prefetch_context(
+    question: str,
+    cloud_endpoint: str,
+    cloud_model: str,
+    max_uses: int = 8,
+) -> Dict[str, Any]:
     """Use Anthropic web_search to fetch real source material the worker can read.
 
     Minions's premise is "worker reads a doc, asks cloud for help" — but GAIA
@@ -278,7 +389,7 @@ def _prefetch_context(question: str, cloud_endpoint: str, cloud_model: str) -> D
             cloud_model,
             user=prompt,
             max_tokens=8192,
-            tools=[ANTHROPIC_WEB_SEARCH_TOOL],
+            tools=[build_web_search_tool(max_uses)],
             tool_choice={"type": "any"},
         )
         from openjarvis.agents.hybrid._prices import cost as _cost_usd
@@ -362,6 +473,9 @@ class MinionsAgent(LocalCloudAgent):
         from minions.clients.anthropic import (
             AnthropicClient,  # type: ignore[import-not-found]
         )
+        from minions.clients.gemini import (
+            GeminiClient,  # type: ignore[import-not-found]
+        )
         from minions.clients.openai import (
             OpenAIClient,  # type: ignore[import-not-found]
         )
@@ -397,6 +511,17 @@ class MinionsAgent(LocalCloudAgent):
                 temperature=0.0,
                 max_tokens=4096,
             )
+        elif self._cloud_endpoint == "gemini":
+            # The vendored Minion library already special-cases GeminiClient
+            # in minion.py: it passes response_mime_type=application/json plus
+            # a Pydantic response_schema so the supervisor reply parses with
+            # the same {decision, message, answer} shape Opus/GPT use. We just
+            # have to hand it a GeminiClient instance — no extra plumbing.
+            cloud_client = GeminiClient(
+                model_name=self._cloud_model,
+                temperature=0.0,
+                max_tokens=4096,
+            )
         else:
             raise ValueError(f"unsupported cloud endpoint: {self._cloud_endpoint!r}")
 
@@ -412,12 +537,29 @@ class MinionsAgent(LocalCloudAgent):
         # GAIA-shape only: prefetch a web_search digest so the worker has
         # something real to read. SWE-bench (problem_statement only) already
         # ships its own doc.
+        #
+        # Honors the new ``method_cfg.web_search`` schema:
+        #   - omitted          → prefetch ON (legacy default for minions GAIA)
+        #   - enabled = false  → prefetch OFF
+        #   - enabled = true   → prefetch ON (honors max_uses)
         prefetch: Dict[str, Any] = {
             "text": "", "tokens": 0, "cost_usd": 0.0, "n_searches": 0,
         }
-        if task_meta.get("question"):
+        ws_block = cfg.get("web_search") if isinstance(cfg.get("web_search"), dict) else None
+        ws_enabled, ws_max_uses = web_search_cfg(cfg)
+        # If the cell explicitly set web_search.enabled = false, honor that.
+        # If it set web_search.enabled = true, honor max_uses. If it didn't
+        # set a web_search block at all, keep legacy prefetch ON.
+        prefetch_on = (
+            ws_block is None  # legacy default
+            or ws_enabled
+        )
+        if task_meta.get("question") and prefetch_on:
             prefetch = _prefetch_context(
-                task_meta["question"], self._cloud_endpoint, self._cloud_model
+                task_meta["question"],
+                self._cloud_endpoint,
+                self._cloud_model,
+                max_uses=ws_max_uses,
             )
 
         if prefetch.get("text"):
@@ -462,6 +604,10 @@ class MinionsAgent(LocalCloudAgent):
             "tokens_cloud": (rp + rc) + prefetch["tokens"],
             "cost_usd": self.cost_usd(self._cloud_model, rp, rc) + prefetch["cost_usd"],
             "turns": cfg.get("max_rounds", 3),
+            "web_search_uses": prefetch["n_searches"],
+            # GAIA: only countable tool surface is the prefetch web_search.
+            # The Minions protocol itself is supervisor↔worker text, no tools.
+            "tool_calls": int(prefetch["n_searches"]),
             "traces": {
                 "mode": mode,
                 "supervisor_messages": out.get("supervisor_messages"),
@@ -540,6 +686,8 @@ class MinionsAgent(LocalCloudAgent):
             "tokens_cloud": p_in + p_out,
             "cost_usd": supervisor_cost,
             "turns": 1 + out["turns"],
+            # SWE: only the worker invokes tools (bash); supervisor is text-only.
+            "tool_calls": int(out["turns"]),
             "traces": {
                 "swe_mode": True,
                 "supervisor_plan": plan_text,

src/openjarvis/agents/hybrid/registry/ablation_advisors.toml

@@ -0,0 +1,107 @@
+# Winner-confirm cells for the n=100 ablation.
+# Re-run the per-axis cloud winner on advisors. Local fixed at Qwen-3.5-27B-FP8.
+
+[cells.advisors-qwen27b-opus47-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024, web_search = { enabled = true, max_uses = 8 } }
+
+[cells.advisors-qwen27b-opus47-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+
+[cells.advisors-qwen27b-gpt5-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024 }
+
+[cells.advisors-qwen27b-gpt5-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+
+[cells.advisors-qwen27b-gpt5mini-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024 }
+
+[cells.advisors-qwen27b-gpt5mini-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+
+[cells.advisors-qwen27b-gemini25pro-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024 }
+
+[cells.advisors-qwen27b-gemini25flash-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+
+[cells.advisors-qwen27b-gemini25pro-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+
+# ============================================================
+# Anthropic Haiku 4.5 winner-confirm (added 2026-05-18)
+# ============================================================
+
+[cells.advisors-qwen27b-haiku45-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024, web_search = { enabled = true, max_uses = 8 } }
+concurrency = 2
+
+[cells.advisors-qwen27b-haiku45-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+concurrency = 2

src/openjarvis/agents/hybrid/registry/ablation_cloud_only.toml

@@ -0,0 +1,160 @@
+# n=100 ablation sweep — cloud-only reference (no local model).
+#
+# Paradigm is baseline_cloud (see baseline_cloud.py). On GAIA the agent
+# makes one cloud call with the FINAL-ANSWER-formatted prompt. On SWE-
+# bench, Anthropic backbones run the full mini-SWE-agent bash loop;
+# OpenAI / Gemini fall back to a one-shot blind call (the SWE agent
+# loop is Anthropic-only — `swe_agent.py` enforces this). Those
+# fallback cells exist primarily to anchor the ablation rather than to
+# produce competitive SWE numbers.
+#
+# Cell `local` is set to a dummy vLLM endpoint per server purely so the
+# hybrid runner doesn't choke on missing fields; `baseline_cloud`
+# ignores `local_*` entirely. The :8001/:8002 split mirrors
+# ablation_skillorchestra.toml — same pairings, so a cloud worker shows
+# up on the same vLLM whether or not the local agent is in play.
+
+# ============================================================
+# :8001 — Opus 4.7
+# ============================================================
+
+[cells.cloud-only-opus47-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-opus47-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { cloud_max_tokens = 4096, swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8001 — Haiku 4.5
+# ============================================================
+
+[cells.cloud-only-haiku45-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-haiku45-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { cloud_max_tokens = 4096, swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8001 — GPT-5
+# ============================================================
+
+[cells.cloud-only-gpt5-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-gpt5-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8002 — GPT-5 mini
+# ============================================================
+
+[cells.cloud-only-gpt5mini-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-gpt5mini-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8002 — Gemini 2.5 Pro
+# (Doc says "Gemini 3.1 Pro" but the SDK doesn't serve a 3.x model.)
+# ============================================================
+
+[cells.cloud-only-gemini25pro-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-gemini25pro-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8002 — Gemini 2.5 Flash
+# ============================================================
+
+[cells.cloud-only-gemini25flash-gaia-n100]
+method = "baseline_cloud"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.cloud-only-gemini25flash-swe-n100]
+method = "baseline_cloud"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3

src/openjarvis/agents/hybrid/registry/ablation_gemma.toml

@@ -0,0 +1,120 @@
+# n=100 ablation — Gemma-4-31B as local, cloud = Opus 4.7.
+# Local endpoint :8004 (single Gemma TP=4 instance). All 8 cells share
+# the same vLLM; keep per-cell concurrency moderate.
+
+# ============================================================
+# Minions × Gemma-4-31B
+# ============================================================
+
+[cells.minions-gemma31b-opus47-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+concurrency = 2
+
+[cells.minions-gemma31b-opus47-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+concurrency = 2
+
+# ============================================================
+# Advisors × Gemma-4-31B
+# ============================================================
+
+[cells.advisors-gemma31b-opus47-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024 }
+concurrency = 2
+
+[cells.advisors-gemma31b-opus47-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+concurrency = 2
+
+# ============================================================
+# Conductor × Gemma-4-31B
+# ============================================================
+
+# Deterministic 3-worker pool: 0=local (Gemma-4-31B via vLLM), 1=Opus 4.7
+# (cloud), 2=gpt-5-mini. `$local` / `$cloud` resolve against the cell's
+# `local` / `cloud` blocks. Explicit `worker_pool` is race-free (no
+# per-task vLLM probe) and pins a correct id->worker mapping.
+[cells.conductor-gemma31b-opus47-gaia-n100]
+method = "conductor"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 2
+[cells.conductor-gemma31b-opus47-gaia-n100.method_cfg]
+worker_max_tokens = 4096
+worker_temperature = 0.2
+conductor_max_tokens = 2048
+web_search = { enabled = true, max_uses = 8 }
+worker_pool = [
+  { id = 0, name = "local-gemma", endpoint = "vllm", model = "$local", description = "Open-weights Gemma-4-31B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic on given data; weaker at open-domain factual recall and complex multi-step reasoning. Cannot perform live web search." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, code, and writing. Can perform live web search. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Can perform live web search. Good default for retrieval-style or broad-knowledge questions." },
+]
+
+[cells.conductor-gemma31b-opus47-swe-n100]
+method = "conductor"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 2
+[cells.conductor-gemma31b-opus47-swe-n100.method_cfg]
+worker_max_tokens = 8192
+worker_temperature = 0.2
+conductor_max_tokens = 4096
+worker_pool = [
+  { id = 0, name = "local-gemma", endpoint = "vllm", model = "$local", description = "Open-weights Gemma-4-31B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic; weaker at complex multi-step reasoning and large code patches." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, and code. Writes tight, well-formed unified diffs. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Good default for broad-knowledge or straightforward code steps." },
+]
+
+# ============================================================
+# SkillOrchestra × Gemma-4-31B
+# ============================================================
+
+[cells.skillorchestra-gemma31b-opus47-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 2
+
+[cells.skillorchestra-gemma31b-opus47-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096, swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 2

src/openjarvis/agents/hybrid/registry/ablation_minions.toml

@@ -0,0 +1,115 @@
+# Winner-confirm cells for the n=100 ablation.
+# Re-run the per-axis cloud winner on minions to check the ranking
+# isn't a skillorchestra-only artifact. Same subset, same n.
+#
+# Local is fixed at Qwen-3.5-27B-FP8 (matches the n=100 sweep local lane).
+# NOTE: minions × Gemini is fully supported as of 2026-05-19 — the
+# vendored Minions library ships a GeminiClient that minion.py already
+# special-cases (Pydantic response_schema for the supervisor JSON
+# decision/message/answer shape). Wired into MinionsAgent via the
+# `cloud_endpoint == "gemini"` branch.
+
+[cells.minions-qwen27b-opus47-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+
+[cells.minions-qwen27b-opus47-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+
+[cells.minions-qwen27b-gpt5-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+
+[cells.minions-qwen27b-gpt5-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+
+[cells.minions-qwen27b-gpt5mini-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+
+[cells.minions-qwen27b-gpt5mini-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+
+[cells.minions-qwen27b-gemini25flash-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+
+[cells.minions-qwen27b-gemini25pro-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+
+[cells.minions-qwen27b-gemini25pro-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+
+# ============================================================
+# Anthropic Haiku 4.5 winner-confirm (added 2026-05-18)
+# ============================================================
+
+[cells.minions-qwen27b-haiku45-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+concurrency = 2
+
+[cells.minions-qwen27b-haiku45-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+concurrency = 2

src/openjarvis/agents/hybrid/registry/ablation_qwen36.toml

@@ -0,0 +1,121 @@
+# n=100 ablation — Qwen-3.6-27B-FP8 as local, cloud = Opus 4.7.
+# Two PP=2 replicas live on matx2: :8002 and :8005 (each on 2 L40S).
+# GAIA cells → :8002, SWE cells → :8005 to keep per-replica load balanced
+# (SWE is ~5–10× longer per task but has fewer concurrent slots).
+
+# ============================================================
+# Minions × Qwen-3.6-27B-FP8
+# ============================================================
+
+[cells.minions-qwen36-opus47-gaia-n100]
+method = "minions"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
+concurrency = 3
+
+[cells.minions-qwen36-opus47-swe-n100]
+method = "minions"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8005/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
+concurrency = 3
+
+# ============================================================
+# Advisors × Qwen-3.6-27B-FP8
+# ============================================================
+
+[cells.advisors-qwen36-opus47-gaia-n100]
+method = "advisors"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { executor_max_tokens = 4096, advisor_max_tokens = 1024 }
+concurrency = 3
+
+[cells.advisors-qwen36-opus47-swe-n100]
+method = "advisors"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8005/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
+concurrency = 3
+
+# ============================================================
+# Conductor × Qwen-3.6-27B-FP8
+# ============================================================
+
+# Deterministic 3-worker pool: 0=local (Qwen3.6-27B via vLLM), 1=Opus 4.7
+# (cloud), 2=gpt-5-mini. `$local` / `$cloud` resolve against the cell's
+# `local` / `cloud` blocks. Local endpoint is :8002 (the live Qwen3.6
+# server); explicit `worker_pool` is race-free and pins id->worker.
+[cells.conductor-qwen36-opus47-gaia-n100]
+method = "conductor"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 4
+[cells.conductor-qwen36-opus47-gaia-n100.method_cfg]
+worker_max_tokens = 4096
+worker_temperature = 0.2
+conductor_max_tokens = 2048
+web_search = { enabled = true, max_uses = 8 }
+worker_pool = [
+  { id = 0, name = "local-qwen", endpoint = "vllm", model = "$local", description = "Open-weights Qwen3.6-27B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic on given data; weaker at open-domain factual recall and complex multi-step reasoning. Cannot perform live web search." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, code, and writing. Can perform live web search. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Can perform live web search. Good default for retrieval-style or broad-knowledge questions." },
+]
+
+[cells.conductor-qwen36-opus47-swe-n100]
+method = "conductor"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 4
+[cells.conductor-qwen36-opus47-swe-n100.method_cfg]
+worker_max_tokens = 8192
+worker_temperature = 0.2
+conductor_max_tokens = 4096
+worker_pool = [
+  { id = 0, name = "local-qwen", endpoint = "vllm", model = "$local", description = "Open-weights Qwen3.6-27B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic; weaker at complex multi-step reasoning and large code patches." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, and code. Writes tight, well-formed unified diffs. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Good default for broad-knowledge or straightforward code steps." },
+]
+
+# ============================================================
+# SkillOrchestra × Qwen-3.6-27B-FP8
+# ============================================================
+
+[cells.skillorchestra-qwen36-opus47-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen36-opus47-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://matx2:8005/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096, swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3

src/openjarvis/agents/hybrid/registry/ablation_skillorchestra.toml

@@ -0,0 +1,167 @@
+# n=100 ablation sweep — skillorchestra paradigm, vary cloud worker.
+#
+# All cells use the fixed n=100 subset
+# (<experiments>/subsets/{gaia,swebench_verified}_n100_seed42.json) so
+# results are directly comparable across cloud workers. Paradigm is held
+# constant at skillorchestra (the Pareto winner from the n=165/n=500
+# study); local is Qwen/Qwen3.5-27B-FP8 throughout.
+#
+# vLLM load is split across the two servers running on localhost:8001
+# and :8002. Cells are paired by cloud worker so one cloud's (gaia, swe)
+# pair sits entirely on one vLLM and the two servers see comparable
+# request mixes:
+#
+#   :8001 → opus-4-7, haiku-4-5, gpt-5      (6 cells)
+#   :8002 → gpt-5-mini, gemini-2.5-pro,
+#           gemini-2.5-flash                (6 cells)
+#
+# The router model is pinned to Anthropic Opus across every cell — the
+# ablation isolates "which cloud worker the router escalates to", not
+# "which router". Router endpoint defaults to anthropic; routing prompt
+# uses Anthropic's output_config schema for strict JSON.
+
+# ============================================================
+# :8001 — Opus 4.7 worker
+# ============================================================
+
+[cells.skillorchestra-qwen-opus47-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-opus47-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_use_agent_loop = true , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8001 — Haiku 4.5 worker
+# ============================================================
+
+[cells.skillorchestra-qwen-haiku45-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-haiku45-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_use_agent_loop = true , swe_max_turns = 50, swe_bash_timeout_s = 120 }
+concurrency = 3
+
+# ============================================================
+# :8001 — GPT-5 worker
+# ============================================================
+
+[cells.skillorchestra-qwen-gpt5-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-gpt5-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 , swe_use_agent_loop = true }
+concurrency = 3
+
+# ============================================================
+# :8002 — GPT-5 mini worker
+# ============================================================
+
+[cells.skillorchestra-qwen-gpt5mini-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-gpt5mini-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 , swe_use_agent_loop = true }
+concurrency = 3
+
+# ============================================================
+# :8002 — Gemini 2.5 Pro worker
+# (NOTE: results-table.md says "Gemini 3.1 Pro" but no such SDK model
+# exists. Using the real 2.5 Pro id. Flagged in deliverable report.)
+# ============================================================
+
+[cells.skillorchestra-qwen-gemini25pro-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-gemini25pro-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 , swe_use_agent_loop = true }
+concurrency = 3
+
+# ============================================================
+# :8002 — Gemini 2.5 Flash worker
+# ============================================================
+
+[cells.skillorchestra-qwen-gemini25flash-gaia-n100]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 }
+concurrency = 4
+
+[cells.skillorchestra-qwen-gemini25flash-swe-n100]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { router_model = "claude-opus-4-7", router_endpoint = "anthropic", cloud_max_tokens = 4096 , swe_max_turns = 50, swe_bash_timeout_s = 120 , swe_use_agent_loop = true }
+concurrency = 3

src/openjarvis/agents/hybrid/registry/advisors.toml

@@ -11,7 +11,7 @@ bench  = "gaia"
 n      = 3
 local  = { model = "Qwen/Qwen3.5-9B", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { web_search = { enabled = true, max_uses = 8 } }
 
 [cells.advisors-swebenchverified-qwen9b-opus-3]
 method = "advisors"
@@ -28,7 +28,7 @@ bench  = "gaia"
 n      = 30
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { web_search = { enabled = true, max_uses = 8 } }
 concurrency = 4
 
 [cells.advisors-swebenchverified-qwen9b-opus-30]

src/openjarvis/agents/hybrid/registry/archon.toml

@@ -25,7 +25,7 @@ bench  = "gaia"
 n      = 3
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
-method_cfg = { architecture = "single_local", max_tokens = 1024 }
+method_cfg = { architecture = "single_local", max_tokens = 8192 }
 
 [cells.archon-swebenchverified-qwen27b-opus-ensemble-K3-3]
 method = "archon"
@@ -54,7 +54,7 @@ bench  = "gaia"
 n      = 30
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
-method_cfg = { architecture = "single_local", max_tokens = 2048 }
+method_cfg = { architecture = "single_local", max_tokens = 8192 }
 concurrency = 4
 
 [cells.archon-gaia-gemma31b-opus-ensemble-K5-30]

src/openjarvis/agents/hybrid/registry/baseline_local.toml

@@ -0,0 +1,82 @@
+# n=100 ablation — local-only baseline (no cloud teacher / router).
+#
+# Paradigm is `baseline_local` (see baseline_local.py). On GAIA the agent
+# makes one local vLLM call with the FINAL-ANSWER-formatted prompt. On
+# SWE-bench-Verified the agent runs the mini-SWE bash agent loop with
+# `backbone="local"` so the local model drives turns directly.
+#
+# Cell `cloud` is set to a dummy model purely so the hybrid runner doesn't
+# choke on missing fields; `baseline_local` ignores `cloud_*` entirely
+# and cost_usd is always 0.0 (local inference is free).
+
+# ============================================================
+# :8004 — Gemma-4-31B (TP=4 on GPUs 0-3)
+# ============================================================
+
+[cells.baseline-local-gemma31b-gaia-n100]
+method = "baseline_local"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096 }
+concurrency = 4
+
+[cells.baseline-local-gemma31b-swe-n100]
+method = "baseline_local"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "google/gemma-4-31B-it", endpoint = "http://localhost:8004/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 4
+
+# ============================================================
+# :8002 — Qwen-3.6-27B-FP8 (PP=2 on GPUs 5-6)
+# ============================================================
+
+[cells.baseline-local-qwen36-gaia-n100]
+method = "baseline_local"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096 }
+concurrency = 4
+
+[cells.baseline-local-qwen36-swe-n100]
+method = "baseline_local"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.6-27B-FP8", endpoint = "http://localhost:8002/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 4
+
+# ============================================================
+# :8001 — Qwen-3.5-27B-FP8 (GPU 7) — the Qwen-3.5 solo floor
+# ============================================================
+
+[cells.baseline-local-qwen27b-gaia-n100]
+method = "baseline_local"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096 }
+concurrency = 4
+
+[cells.baseline-local-qwen27b-swe-n100]
+method = "baseline_local"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { local_max_tokens = 4096, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 4

src/openjarvis/agents/hybrid/registry/conductor.toml

@@ -1,9 +1,15 @@
 # Conductor cells. Inference-only repro of arXiv 2512.04388 (Sakana, 2025).
 #
 # Stage-1 substitutes the (untrained) Qwen2.5-7B conductor with a zero-shot
-# frontier planner (Opus). Workers default to local Qwen3.5-27B + Opus +
-# gpt-5-mini; the adapter auto-drops the local worker if vLLM is down.
-# Override `method_cfg.workers` to customize the pool.
+# frontier planner (Opus). Workers come from `method_cfg.worker_pool` — an
+# explicit, deterministic pool. The n100 cells below pin a 3-worker pool
+# (0=local, 1=Opus, 2=gpt-5-mini) via `$local` / `$cloud` substitution so
+# the named local + cloud models are actually exercised.
+#
+# DO NOT rely on the implicit default pool: `_default_pool` is the paper's
+# 7-worker cloud-only pool (no local, no Opus). Cells that want hybrid
+# local+cloud MUST set `worker_pool` explicitly (race-free, no per-task
+# vLLM probe).
 #
 # Naming: conductor-<bench>-<conductor-short>-<N>
 
@@ -52,3 +58,48 @@ local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1"
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { worker_max_tokens = 8192, worker_temperature = 0.2, conductor_max_tokens = 4096 }
 concurrency = 8
+
+# ============================================================
+# n=100 ablation cells (added 2026-05-18)
+# ============================================================
+
+# Deterministic 3-worker pool: 0=local (Qwen3.5-27B via vLLM), 1=Opus 4.7
+# (cloud), 2=gpt-5-mini. `$local` / `$cloud` resolve against the cell's
+# `local` / `cloud` blocks. GAIA cells enable web_search so the search-
+# capable workers (anthropic / openai) can ground their answers.
+[cells.conductor-qwen27b-opus47-gaia-n100]
+method = "conductor"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 4
+[cells.conductor-qwen27b-opus47-gaia-n100.method_cfg]
+worker_max_tokens = 4096
+worker_temperature = 0.2
+conductor_max_tokens = 2048
+web_search = { enabled = true, max_uses = 8 }
+worker_pool = [
+  { id = 0, name = "local-qwen", endpoint = "vllm", model = "$local", description = "Open-weights Qwen3.5-27B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic on given data; weaker at open-domain factual recall and complex multi-step reasoning. Cannot perform live web search." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, code, and writing. Can perform live web search. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Can perform live web search. Good default for retrieval-style or broad-knowledge questions." },
+]
+
+[cells.conductor-qwen27b-opus47-swe-n100]
+method = "conductor"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+concurrency = 4
+[cells.conductor-qwen27b-opus47-swe-n100.method_cfg]
+worker_max_tokens = 8192
+worker_temperature = 0.2
+conductor_max_tokens = 4096
+worker_pool = [
+  { id = 0, name = "local-qwen", endpoint = "vllm", model = "$local", description = "Open-weights Qwen3.5-27B served locally via vLLM. Cheap and fast. Good at concise extraction, formatting, and arithmetic; weaker at complex multi-step reasoning and large code patches." },
+  { id = 1, name = "frontier-anthropic", endpoint = "anthropic", model = "$cloud", description = "Anthropic Claude Opus 4.7. Frontier reasoning model — strongest at multi-step reasoning, careful instruction following, and code. Writes tight, well-formed unified diffs. Expensive; use for hard or decisive steps." },
+  { id = 2, name = "frontier-openai-mini", endpoint = "openai", model = "gpt-5-mini", description = "OpenAI gpt-5-mini. Mid-tier model with solid general knowledge and reasoning at a fraction of frontier cost. Good default for broad-knowledge or straightforward code steps." },
+]

src/openjarvis/agents/hybrid/registry/skillorchestra.toml

@@ -1,15 +1,27 @@
 # SkillOrchestra cells (arXiv:2602.19672).
 #
-# Deployment-time skill-aware routing only. The paper's offline pipeline
-# (explore → learn → select over an SGLang model pool + FAISS wiki index)
-# isn't reproduced here — we don't have the SGLang serving stack, the
-# retriever index, or a training split to learn a routing policy on. What
-# we DO reproduce is the inference-time orchestrator: an Opus router
-# analyzes the question into skill weights, picks one of two agents
-# (local Qwen-27B vs cloud Opus) under an explicit cost trade-off, then
-# the chosen agent answers. Mirrors the eval_orchestrator paradigm in
-# external/SkillOrchestra/skillorchestra/prompts/eval_orchestrator.py.
-# See adapters/skillorchestra_adapter.py for full notes.
+# Faithful port of the eval-orchestrator runtime — the multi-round
+# search -> reasoning -> answer ReAct loop, verbatim eval_orchestrator
+# prompts, the StageSkillHandbook + 5 routing strategies, real Python
+# subprocess execution, and a model-alias pool. See the package
+# README at agents/hybrid/skillorchestra/README.md.
+#
+# method_cfg knobs (all optional):
+#   routing_strategy  none | router_decides | analyze_model_decide
+#                     | weighted_avg | weakest_skill | strongest_skill
+#   handbook_path     StageSkillHandbook JSON; relative -> resolved inside
+#                     the skillorchestra package. weighted/weakest/strongest
+#                     need one; router_decides / none do not.
+#   max_rounds        orchestrator loop cap (default 6).
+#   retriever_url     FAISS wiki retriever; absent -> Anthropic web_search.
+#   model_pool        per-alias { model, endpoint } overrides.
+#   orchestrator_model / orchestrator_endpoint   pin the orchestrator LLM.
+#
+# GAIA cells run the full skill-routing loop against the hand-authored
+# seed handbook (handbook_seed.json — NOT learned; swap in a learned one
+# when the explore->learn->select pipeline has run). SWE-bench is out of
+# scope for the original QA orchestrator: those cells run the cloud
+# backbone through the shared mini SWE agent loop.
 
 [cells.skillorchestra-gaia-qwen27b-opus-3]
 method = "skillorchestra"
@@ -17,7 +29,7 @@ bench  = "gaia"
 n      = 3
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { routing_strategy = "weighted_avg", handbook_path = "handbook_seed.json", max_rounds = 6 }
 
 [cells.skillorchestra-swebenchverified-qwen27b-opus-3]
 method = "skillorchestra"
@@ -25,7 +37,7 @@ bench  = "swebench-verified"
 n      = 3
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120 }
 concurrency = 3
 
 [cells.skillorchestra-gaia-qwen27b-opus-30]
@@ -34,7 +46,7 @@ bench  = "gaia"
 n      = 30
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { routing_strategy = "weighted_avg", handbook_path = "handbook_seed.json", max_rounds = 6 }
 concurrency = 4
 
 [cells.skillorchestra-swebenchverified-qwen27b-opus-30]
@@ -43,5 +55,5 @@ bench  = "swebench-verified"
 n      = 30
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
-method_cfg = {}
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 50, swe_bash_timeout_s = 120 }
 concurrency = 3

src/openjarvis/agents/hybrid/registry/swe_agent_loop.toml

@@ -35,7 +35,7 @@ method_cfg = { swe_use_agent_loop = true, conductor_max_tokens = 2048, swe_max_t
 method = "advisors"
 bench  = "swebench-verified"
 n      = 3
-local  = { model = "Qwen/Qwen3.5-9B", endpoint = "http://localhost:8001/v1" }
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { swe_use_agent_loop = true, swe_max_turns = 25, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, advisor_max_tokens = 2048 }
 
@@ -45,7 +45,7 @@ method_cfg = { swe_use_agent_loop = true, swe_max_turns = 25, swe_bash_timeout_s
 method = "skillorchestra"
 bench  = "swebench-verified"
 n      = 3
-local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { swe_use_agent_loop = true, router_max_tokens = 1024, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
 
@@ -55,7 +55,7 @@ method_cfg = { swe_use_agent_loop = true, router_max_tokens = 1024, swe_max_turn
 method = "toolorchestra"
 bench  = "swebench-verified"
 n      = 3
-local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { swe_use_agent_loop = true, max_turns = 4, orchestrator_max_tokens = 1024, swe_max_turns = 25, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
 
@@ -65,6 +65,6 @@ method_cfg = { swe_use_agent_loop = true, max_turns = 4, orchestrator_max_tokens
 method = "archon"
 bench  = "swebench-verified"
 n      = 3
-local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8002/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { swe_use_agent_loop = true, n_samples = 3, swe_max_turns = 25, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, ranker_max_tokens = 1024 }

src/openjarvis/agents/hybrid/registry/toolorchestra.toml

@@ -1,9 +1,29 @@
-# ToolOrchestra cells. Prompted port — uses a cloud model (Opus) as the
-# orchestrator, NOT the RL-trained Nemotron-Orchestrator-8B from the paper
-# (arXiv:2511.21689). Treat results as preliminary until a real
-# Orchestrator-8B deployment is wired up.
+# ToolOrchestra cells (arXiv:2511.21689).
 #
-# Naming: toolorchestra-<bench>-<local-short>-<cloud-short>-<N>
+# Two modes via `method_cfg.orchestrator_mode`:
+#
+#   "prompted" (default)  -- cloud model (Opus etc.) plays the orchestrator
+#                            over a heterogeneous worker pool. Useful as a
+#                            prompted upper-bound; NOT what the paper does.
+#
+#   "rl"                  -- paper-faithful path. The RL-trained
+#                            `nvidia/Orchestrator-8B` served on a local vLLM
+#                            (default :8003) is the orchestrator and emits
+#                            OpenAI-style tool_calls for the three NVlabs
+#                            tools (enhance_reasoning / answer / search).
+#                            Expert pool maps tool-`model` slots to our
+#                            workers: *-1 -> cloud, *-2 -> gpt-5-mini,
+#                            *-3 -> local vLLM, search -> Anthropic web_search.
+#                            See `toolorchestra.py` module docstring for
+#                            what's collapsed vs. the upstream (no Tavily /
+#                            FAISS-wiki / Qwen-Coder code-interpreter).
+#
+# Naming: toolorchestra-<orch-short>-<cloud-short>-<bench>-<N>
+#         (legacy prompted cells keep their original naming scheme.)
+
+# ============================================================
+# Legacy prompted-mode cells (cloud-as-orchestrator).
+# ============================================================
 
 [cells.toolorchestra-gaia-qwen27b-opus-3]
 method = "toolorchestra"
@@ -28,3 +48,182 @@ n      = 30
 local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
 cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
 method_cfg = { max_turns = 8, orchestrator_max_tokens = 1024, worker_max_tokens = 4096 }
+
+# ============================================================
+# RL-mode cells -- Orchestrator-8B drives the loop (n=100).
+# ============================================================
+#
+# `local` carries the orchestrator's vLLM (nvidia/Orchestrator-8B on :8003).
+# Tier-3 expert calls (`answer-3`, `reasoner-3`, …) route to this same model.
+# `cloud` is the frontier worker invoked for tier-1 slots.
+# `gpt-5-mini` is hard-coded as the tier-2 mid worker (matches the paper's
+# `gpt-5-mini` slots in `eval_hle.py` MODEL_MAPPING).
+
+[cells.toolorchestra-orch8b-opus47-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 4
+
+[cells.toolorchestra-orch8b-opus47-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 3
+
+# ============================================================
+# Paper-match pool smoke (2026-05-19). n=5 GAIA. Opts into the
+# closer-to-paper worker pool via `method_cfg.pool = "paper"`:
+#   - search       -> Tavily API (TAVILY_API_KEY in OpenJarvis/.env)
+#   - enhance_..   -> Qwen2.5-Coder-32B (OpenRouter) + Modal Python sandbox
+#   - answer-1     -> GPT-5             answer-math-* -> Qwen2.5-Coder-32B
+#   - answer-2     -> GPT-5-mini        answer-3     -> Llama-3.3-70B (OR)
+#   - answer-4     -> local Qwen (the Orchestrator-8B endpoint at :8003)
+# Skipped vs. paper: FAISS RAG, Qwen2.5-Math-{72B,7B} (not on OpenRouter).
+[cells.toolorchestra-papermatch-orch8b-opus47-gaia-n5]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 5
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2, pool = "paper", modal_python_timeout_s = 60, tavily_max_results = 5 }
+concurrency = 1
+
+# Smoke cell for the RL-mode SWE wiring (2026-05-19). n=2, no subset so it
+# just grabs the first two SWE-bench Verified tasks. Used to verify
+# end-to-end: workdir clone, _swe_call_worker dispatch through Orchestrator-8B,
+# diff extraction, Modal-harness scoring. Promote to n=100 once green.
+[cells.toolorchestra-orch8b-opus47-swe-smoke2]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 2
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-opus-4-7", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 6, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 15, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 1
+
+# ============================================================
+# ToolOrchestra × cloud-worker ablation (n=100, 2026-05-19)
+# Cloud-axis sweep: Haiku 4.5, GPT-5, GPT-5 mini, Gemini 2.5 Pro,
+# Gemini 2.5 Flash. Mirrors the skillorchestra cloud-axis cells.
+# All share Orchestrator-8B on :8003 as the local orchestrator.
+# concurrency=3 for Anthropic/Google, 2 for OpenAI (prepaid-quota wall).
+# ============================================================
+
+# ---------- Anthropic Haiku 4.5 ----------
+
+[cells.toolorchestra-orch8b-haiku45-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 3
+
+[cells.toolorchestra-orch8b-haiku45-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "claude-haiku-4-5", endpoint = "anthropic" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 3
+
+# ---------- OpenAI GPT-5 ----------
+
+[cells.toolorchestra-orch8b-gpt5-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 2
+
+[cells.toolorchestra-orch8b-gpt5-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gpt-5", endpoint = "openai" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 2
+
+# ---------- OpenAI GPT-5 mini ----------
+
+[cells.toolorchestra-orch8b-gpt5mini-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 2
+
+[cells.toolorchestra-orch8b-gpt5mini-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gpt-5-mini", endpoint = "openai" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 2
+
+# ---------- Google Gemini 2.5 Pro ----------
+
+[cells.toolorchestra-orch8b-gemini25pro-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 3
+
+[cells.toolorchestra-orch8b-gemini25pro-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gemini-2.5-pro", endpoint = "gemini" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 3
+
+# ---------- Google Gemini 2.5 Flash ----------
+
+[cells.toolorchestra-orch8b-gemini25flash-gaia-n100]
+method = "toolorchestra"
+bench  = "gaia"
+n      = 100
+subset = "gaia_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 4096, worker_temperature = 0.2 }
+concurrency = 3
+
+[cells.toolorchestra-orch8b-gemini25flash-swe-n100]
+method = "toolorchestra"
+bench  = "swebench-verified"
+n      = 100
+subset = "swebench_verified_n100_seed42.json"
+local  = { model = "nvidia/Orchestrator-8B", endpoint = "http://localhost:8003/v1" }
+cloud  = { model = "gemini-2.5-flash", endpoint = "gemini" }
+method_cfg = { orchestrator_mode = "rl", orchestrator_endpoint = "http://localhost:8003/v1", orchestrator_model = "nvidia/Orchestrator-8B", max_turns = 8, orchestrator_max_tokens = 4096, worker_max_tokens = 8192, worker_temperature = 0.2, swe_use_agent_loop = true, swe_max_turns = 30, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096 }
+concurrency = 3

src/openjarvis/agents/hybrid/runner.py

@@ -8,7 +8,7 @@ Reads a cell definition from ``registry/<method>.toml`` (bundled with this
 package or pointed at by ``OPENJARVIS_HYBRID_REGISTRY_DIR``), constructs
 the registered agent, loads bench tasks via OpenJarvis's existing dataset
 providers, runs every task, scores it, and writes
-``<EXPERIMENTS_DIR>/<cell>/results.jsonl`` + ``summary.json``.
+``<EXPERIMENTS_DIR>/runs/<cell>/results.jsonl`` + ``summary.json``.
 
 The output schema matches ``hybrid-local-cloud-compute/runner.py`` so the
 existing rescore / dashboard scripts can read OpenJarvis cells without
@@ -26,6 +26,7 @@ import threading
 import time
 import traceback
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import TimeoutError as FuturesTimeoutError
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -36,6 +37,7 @@ except ModuleNotFoundError:
     import tomli as tomllib  # type: ignore[import-not-found,no-redef]
 
 from openjarvis.agents._stubs import AgentContext, AgentResult
+from openjarvis.agents.hybrid._energy import EnergyCollector
 from openjarvis.agents.hybrid._prompts import format_prompt as _format_prompt
 
 PACKAGE_DIR = Path(__file__).parent
@@ -43,13 +45,61 @@ DEFAULT_REGISTRY_DIR = PACKAGE_DIR / "registry"
 DEFAULT_EXPERIMENTS_DIR = Path(
     os.environ.get(
         "OPENJARVIS_HYBRID_EXPERIMENTS_DIR",
-        str(Path.home() / ".openjarvis-hybrid" / "experiments"),
+        Path.home() / ".openjarvis" / "experiments" / "hybrid",
     )
 )
+DEFAULT_SUBSETS_DIR = DEFAULT_EXPERIMENTS_DIR / "subsets"
+DEFAULT_RUNS_DIR = DEFAULT_EXPERIMENTS_DIR / "runs"
+
+# Hard per-task wall-clock cap. Even when every individual network /
+# subprocess call has its own timeout, a pathological chain (SDK retries
+# stacking on top of a hung connection, a Modal harness subprocess whose
+# grandchildren keep its stdout pipe open, etc.) can leave one task
+# blocked indefinitely — and with the runner's ThreadPoolExecutor that
+# wedges the whole cell (`as_completed` never advances). SWE-bench tasks
+# legitimately run ~15-20 min, so the default cap is 30 min: long enough
+# never to abort a healthy task, short enough that a frozen one is
+# abandoned and recorded as an error row (which the resume logic re-runs)
+# instead of silently killing the process. Override with
+# ``OPENJARVIS_HYBRID_TASK_TIMEOUT_S`` (0 / negative disables).
+DEFAULT_TASK_TIMEOUT_S = float(
+    os.environ.get("OPENJARVIS_HYBRID_TASK_TIMEOUT_S", "1800") or 1800
+)
 
 
 # ---------- Registry ----------
 
+_SWE_BENCHES = {"swebench-verified", "swebench_verified", "swebench"}
+
+
+def _validate_cells(cells: Dict[str, Dict[str, Any]]) -> None:
+    """Catch registry mistakes that would silently degrade behaviour.
+
+    Currently: skillorchestra on a SWE bench MUST have
+    ``method_cfg.swe_use_agent_loop = true``. Without the flag,
+    skillorchestra.py falls back to a one-shot cloud call even for
+    SWE-bench tasks, which is rarely what the experimenter wants and is
+    invisible at runtime (Bug 5, 2026-05-15).
+    """
+    bad: List[str] = []
+    for name, cell in cells.items():
+        if cell.get("method") != "skillorchestra":
+            continue
+        if cell.get("bench") not in _SWE_BENCHES:
+            continue
+        mcfg = cell.get("method_cfg") or {}
+        if not bool(mcfg.get("swe_use_agent_loop")):
+            bad.append(name)
+    if bad:
+        raise ValueError(
+            "skillorchestra SWE cells missing required "
+            "`method_cfg.swe_use_agent_loop = true`: "
+            + ", ".join(sorted(bad))
+            + ". Without this flag the cell silently falls back to a "
+            "one-shot cloud call for SWE-bench tasks."
+        )
+
+
 def load_registry(registry_dir: Optional[Path] = None) -> Dict[str, Dict[str, Any]]:
     """Merge every ``<registry_dir>/*.toml``. Cell names must be unique."""
     base = registry_dir or DEFAULT_REGISTRY_DIR
@@ -67,6 +117,7 @@ def load_registry(registry_dir: Optional[Path] = None) -> Dict[str, Dict[str, An
                     f"duplicate cell {name!r} (already defined before {p.name})"
                 )
             cells[name] = cell
+    _validate_cells(cells)
     return cells
 
 
@@ -81,12 +132,17 @@ def _load_gaia_tasks(n: Optional[int]) -> List[Dict[str, Any]]:
     out: List[Dict[str, Any]] = []
     for rec in ds.iter_records():
         # rec.problem is the formatted question prompt; rec.metadata carries
-        # the GAIA-specific fields including any reference answer.
+        # the GAIA-specific fields including any reference answer. Prefer the
+        # upstream GAIA `task_id` field (bare uuid) over rec.record_id (which
+        # OpenJarvis prefixes with `gaia-`) so subsets keyed by the upstream
+        # id round-trip.
+        md = rec.metadata or {}
+        task_id = md.get("task_id") or rec.record_id
         out.append({
-            "task_id": rec.record_id,
-            "question": rec.metadata.get("question", rec.problem),
+            "task_id": task_id,
+            "question": md.get("question", rec.problem),
             "reference": rec.reference,
-            "metadata": dict(rec.metadata),
+            "metadata": dict(md),
         })
     return out
 
@@ -95,7 +151,7 @@ def _load_swebench_tasks(n: Optional[int]) -> List[Dict[str, Any]]:
     """SWE-bench-Verified test. Each task carries patch-evaluation fields."""
     from openjarvis.evals.datasets.swebench import SWEBenchDataset
 
-    ds = SWEBenchDataset()
+    ds = SWEBenchDataset(variant="verified")
     ds.load(max_samples=n)
     out: List[Dict[str, Any]] = []
     for rec in ds.iter_records():
@@ -124,36 +180,151 @@ def load_tasks(bench: str, n: Optional[int]) -> List[Dict[str, Any]]:
     raise ValueError(f"unknown bench: {bench!r}")
 
 
+def _load_subset_file(subset_path: str) -> Dict[str, Any]:
+    """Resolve a cell's ``subset`` field to a parsed JSON dict.
+
+    Resolution order:
+      1. Absolute path → use as-is.
+      2. Bare filename / relative path → look up under
+         ``<experiments>/subsets/`` (matches where ``make_subset.py``
+         writes its output).
+
+    Accepts both list-of-ids and dict-with-task_ids shapes; the legacy
+    harness wrote the dict shape and we preserve that. Returns a dict
+    with at least a ``task_ids`` list so callers don't have to branch.
+    """
+    p = Path(subset_path)
+    if not p.is_absolute():
+        p = DEFAULT_SUBSETS_DIR / p
+    if not p.exists():
+        raise FileNotFoundError(f"subset file not found: {p}")
+    data = json.loads(p.read_text())
+    if isinstance(data, list):
+        return {"task_ids": list(data)}
+    if isinstance(data, dict):
+        if "task_ids" not in data:
+            raise ValueError(
+                f"subset {p.name} has no 'task_ids' field; got keys {list(data.keys())}"
+            )
+        return data
+    raise ValueError(f"subset {p.name} must be a list or dict; got {type(data).__name__}")
+
+
+def _apply_subset(
+    tasks: List[Dict[str, Any]],
+    subset: Dict[str, Any],
+    cell: Dict[str, Any],
+) -> List[Dict[str, Any]]:
+    """Filter ``tasks`` to the subset's task IDs, preserving subset order.
+
+    Hard-errors if the cell's ``n`` doesn't equal ``len(task_ids)`` so a
+    typo in the registry can't silently shrink the eval. Also errors if
+    any subset ID is missing from the dataset (caller's bench wiring is
+    broken).
+    """
+    ids: List[str] = list(subset["task_ids"])
+    cell_n = int(cell["n"])
+    if cell_n != len(ids):
+        raise ValueError(
+            f"subset n={len(ids)} ≠ cell n={cell_n} — refusing to silently "
+            "change scope. Fix the registry's `n` to match the subset file."
+        )
+    if "bench" in subset and subset["bench"] != cell["bench"]:
+        raise ValueError(
+            f"subset bench={subset['bench']!r} ≠ cell bench={cell['bench']!r}"
+        )
+    order = {tid: i for i, tid in enumerate(ids)}
+    allow = set(ids)
+    kept = [t for t in tasks if t["task_id"] in allow]
+    kept.sort(key=lambda t: order[t["task_id"]])
+    missing = allow - {t["task_id"] for t in kept}
+    if missing:
+        raise ValueError(
+            f"subset references {len(missing)} task_ids not in dataset "
+            f"(e.g. {next(iter(missing))!r})"
+        )
+    return kept
+
+
 # ---------- Scoring ----------
 
-def _score_gaia(task: Dict[str, Any], answer: str) -> Dict[str, Any]:
-    """Exact-match-with-format-normalization GAIA scorer.
+_GAIA_SCORER = None
+_GAIA_SCORER_LOCK = threading.Lock()
 
-    Lightweight version: extracts the final-answer line and string-compares
-    against the reference. Use the OpenJarvis gaia_exact scorer for the
-    judge-tiebreaker path.
+
+def _get_gaia_scorer():
+    """Lazily build the shared GAIA scorer (normalized exact-match + LLM judge).
+
+    Judge model defaults to ``gpt-5-mini-2025-08-07`` (override via
+    ``OPENJARVIS_GAIA_JUDGE_MODEL``); the judge backend is the ``cloud``
+    engine, so ``OPENJARVIS_CONFIG`` needs a ``[engine.cloud]`` section.
     """
-    import re
+    global _GAIA_SCORER
+    if _GAIA_SCORER is None:
+        with _GAIA_SCORER_LOCK:
+            if _GAIA_SCORER is None:
+                from openjarvis.evals.backends.jarvis_direct import (
+                    JarvisDirectBackend,
+                )
+                from openjarvis.evals.scorers.gaia_exact import GAIAScorer
+
+                judge_model = os.environ.get(
+                    "OPENJARVIS_GAIA_JUDGE_MODEL", "gpt-5-mini-2025-08-07"
+                )
+                try:
+                    backend = JarvisDirectBackend(engine_key="cloud")
+                except Exception:  # noqa: BLE001
+                    backend = None
+                _GAIA_SCORER = GAIAScorer(backend, judge_model)
+    return _GAIA_SCORER
+
+
+def _score_gaia(task: Dict[str, Any], answer: str) -> Dict[str, Any]:
+    """GAIA scorer — normalized exact-match with an LLM-judge fallback.
+
+    Uses the shared OpenJarvis :class:`GAIAScorer`. The previous version
+    only credited answers that emitted a literal ``FINAL ANSWER:`` line and
+    string-matched it; a verbose answer that stated the right answer in
+    prose silently scored 0. Opus emits the marker ~92% of the time but
+    GPT-5-mini / Haiku almost never do, so their GAIA cells were badly
+    undercounted. The judge recovers the answer from prose instead.
+    """
+    from openjarvis.evals.core.types import EvalRecord
 
     ref = (task.get("reference") or "").strip()
     if not ref:
         return {"success": False, "score": 0.0, "details": {"reason": "no_reference"}}
-    m = re.search(
-        r"FINAL\s*ANSWER\s*:\s*(.+?)\s*$",
-        answer,
-        re.IGNORECASE | re.MULTILINE,
+
+    record = EvalRecord(
+        record_id=str(task.get("task_id") or ""),
+        problem=str(task.get("question") or ""),
+        reference=ref,
+        category="agentic",
+        metadata=dict(task.get("metadata") or {}),
     )
-    pred = (m.group(1).strip() if m else answer.strip()).rstrip(".").strip()
-    success = pred.lower() == ref.lower()
+    is_correct, details = _get_gaia_scorer().score(record, answer or "")
+    details = dict(details or {})
+    details.setdefault("reference", ref)
     return {
-        "success": success,
-        "score": 1.0 if success else 0.0,
-        "details": {"prediction": pred, "reference": ref},
+        "success": bool(is_correct),
+        "score": 1.0 if is_correct else 0.0,
+        "details": details,
     }
 
 
-def _score_swebench(task: Dict[str, Any], answer: str) -> Dict[str, Any]:
-    """Modal-backed SWE-bench Verified harness scorer."""
+def _score_swebench(
+    task: Dict[str, Any],
+    answer: str,
+    cell_name: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Modal-backed SWE-bench Verified harness scorer.
+
+    ``cell_name`` is passed through to :class:`SWEBenchHarnessScorer` so the
+    underlying ``run_id`` is unique per (cell, instance). Without it,
+    concurrent hybrid cells scoring the same task collide on the shared
+    swebench harness cache and the second cell silently scores 0 with
+    ``reason: no_report`` (or reads the first cell's verdict).
+    """
     from openjarvis.evals.core.types import EvalRecord
     from openjarvis.evals.scorers.swebench_harness import (
         SWEBenchHarnessScorer,
@@ -171,7 +342,10 @@ def _score_swebench(task: Dict[str, Any], answer: str) -> Dict[str, Any]:
         category="agentic",
         metadata={"instance_id": task["task_id"]},
     )
-    scorer = SWEBenchHarnessScorer(timeout_s=int(os.environ.get("SWEBENCH_TIMEOUT_S", "1800")))
+    scorer = SWEBenchHarnessScorer(
+        timeout_s=int(os.environ.get("SWEBENCH_TIMEOUT_S", "1800")),
+        cell_name=cell_name,
+    )
     is_correct, details = scorer.score(record, answer)
     return {
         "success": bool(is_correct),
@@ -180,11 +354,16 @@ def _score_swebench(task: Dict[str, Any], answer: str) -> Dict[str, Any]:
     }
 
 
-def score(bench: str, task: Dict[str, Any], answer: str) -> Dict[str, Any]:
+def score(
+    bench: str,
+    task: Dict[str, Any],
+    answer: str,
+    cell_name: Optional[str] = None,
+) -> Dict[str, Any]:
     if bench == "gaia":
         return _score_gaia(task, answer)
     if bench in ("swebench-verified", "swebench_verified", "swebench"):
-        return _score_swebench(task, answer)
+        return _score_swebench(task, answer, cell_name=cell_name)
     raise ValueError(f"unknown bench: {bench!r}")
 
 
@@ -256,7 +435,26 @@ def _build_agent(cell: Dict[str, Any]):
     )
 
 
-def _run_one(agent, bench: str, task: Dict[str, Any], log_dir: str) -> Dict[str, Any]:
+def _error_row(task: Dict[str, Any], t0: float, error: str) -> Dict[str, Any]:
+    """Build a hybrid-shape error row (kept null-shaped like the catch path
+    in :func:`_run_one` so the resume logic re-runs it)."""
+    return {
+        "task_id": task["task_id"],
+        "answer": "",
+        "tokens_local": 0, "tokens_cloud": 0,
+        "cost_usd": 0.0, "latency_s": time.time() - t0,
+        "web_search_uses": 0,
+        "tool_calls": 0,
+        "n_cloud_calls": 0,
+        "n_local_calls": 0,
+        "traces": {},
+        "error": error,
+    }
+
+
+def _run_one_inner(
+    agent, bench: str, task: Dict[str, Any], log_dir: str
+) -> Dict[str, Any]:
     """Run the agent on one task. Returns a hybrid-shape row."""
     prompt = _format_prompt(task)
     ctx = AgentContext(metadata={
@@ -275,20 +473,80 @@ def _run_one(agent, bench: str, task: Dict[str, Any], log_dir: str) -> Dict[str,
             "tokens_cloud": int(meta.get("tokens_cloud", 0)),
             "cost_usd": float(meta.get("cost_usd", 0.0)),
             "latency_s": float(meta.get("latency_s", time.time() - t0)),
+            "web_search_uses": int(meta.get("web_search_uses", 0)),
+            "tool_calls": int(meta.get("tool_calls", 0)),
+            "n_cloud_calls": int(meta.get("n_cloud_calls", 0)),
+            "n_local_calls": int(meta.get("n_local_calls", 0)),
             "traces": meta.get("traces", {}),
         }
         if "soft_error" in meta:
             out["soft_error"] = meta["soft_error"]
         return {**out, "error": None}
     except Exception as e:
-        return {
-            "task_id": task["task_id"],
-            "answer": "",
-            "tokens_local": 0, "tokens_cloud": 0,
-            "cost_usd": 0.0, "latency_s": time.time() - t0,
-            "traces": {},
-            "error": f"{type(e).__name__}: {e}\n{traceback.format_exc()}",
-        }
+        return _error_row(
+            task, t0, f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
+        )
+
+
+def _run_one(
+    agent,
+    bench: str,
+    task: Dict[str, Any],
+    log_dir: str,
+    *,
+    task_timeout_s: float = DEFAULT_TASK_TIMEOUT_S,
+) -> Dict[str, Any]:
+    """Run one task under a hard wall-clock cap.
+
+    ``_run_one_inner`` runs on a dedicated **daemon** thread; if it doesn't
+    finish within ``task_timeout_s`` we give up on it and record a
+    ``TaskTimeout`` error row. The worker thread is then *abandoned* — a
+    truly wedged task (hung socket read with no enforced timeout, a Modal
+    harness subprocess deadlocked draining pipes) cannot be killed
+    cooperatively in CPython, so leaking the thread is the only safe
+    option. It's a daemon thread, so it never blocks process exit, and the
+    leak is bounded (one per timed-out task) — far cheaper than letting the
+    whole cell freeze on the runner's ``as_completed`` join. The error row
+    makes the resume logic re-run the task on the next invocation.
+
+    ``task_timeout_s <= 0`` disables the cap (runs inline, legacy behavior).
+    """
+    t0 = time.time()
+    if task_timeout_s <= 0:
+        return _run_one_inner(agent, bench, task, log_dir)
+
+    box: Dict[str, Any] = {}
+
+    def _target() -> None:
+        try:
+            box["row"] = _run_one_inner(agent, bench, task, log_dir)
+        except BaseException as e:  # noqa: BLE001 — never let the worker die silently
+            box["row"] = _error_row(
+                task, t0, f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
+            )
+
+    worker = threading.Thread(
+        target=_target,
+        name=f"hybrid-task-{task['task_id']}",
+        daemon=True,
+    )
+    worker.start()
+    worker.join(timeout=task_timeout_s)
+    if worker.is_alive():
+        print(
+            f"[timeout] task={task['task_id']} exceeded "
+            f"{task_timeout_s/60:.1f}m — abandoning worker, recording error row",
+            flush=True,
+        )
+        return _error_row(
+            task, t0,
+            f"TaskTimeout: task exceeded the {task_timeout_s:.0f}s hybrid "
+            "per-task wall-clock cap (likely a hung network or Modal-harness "
+            "call); worker thread abandoned, task left for resume.",
+        )
+    return box.get("row") or _error_row(
+        task, t0, "TaskError: worker thread exited without producing a row."
+    )
 
 
 def _heartbeat(done: int, total: int, row: Dict[str, Any], t_start: float) -> None:
@@ -312,6 +570,8 @@ def _write_summary(
     cell: Dict[str, Any],
     tasks: List[Dict[str, Any]],
     t_start: float,
+    n_processed: int = -1,
+    energy_j_session: float = 0.0,
 ) -> None:
     results_path = out_dir / "results.jsonl"
     rows = [
@@ -326,8 +586,38 @@ def _write_summary(
     total_cost = sum(r.get("cost_usd", 0.0) for r in rows)
     total_local = sum(r.get("tokens_local", 0) for r in rows)
     total_cloud = sum(r.get("tokens_cloud", 0) for r in rows)
+    total_web_searches = sum(int(r.get("web_search_uses", 0) or 0) for r in rows)
+    total_tool_calls = sum(int(r.get("tool_calls", 0) or 0) for r in rows)
+    total_cloud_calls = sum(int(r.get("n_cloud_calls", 0) or 0) for r in rows)
+    total_local_calls = sum(int(r.get("n_local_calls", 0) or 0) for r in rows)
     elapsed = time.time() - t_start
 
+    # Preserve prior wall_time_s on no-op resume so we don't clobber the
+    # original run's runtime. A resume that did zero work (everything was
+    # already cached in results.jsonl) records ~seconds of elapsed time;
+    # writing that as wall_time_s makes the cell look 20× faster than it
+    # really was. If we did process anything this session, accumulate so
+    # partial-resumes still report total wall time honestly.
+    summary_path = out_dir / "summary.json"
+    prior_wall = 0.0
+    prior_energy = 0.0
+    if summary_path.exists():
+        try:
+            prior = json.loads(summary_path.read_text())
+            prior_wall = float(prior.get("wall_time_s", 0.0) or 0.0)
+            prior_energy = float(prior.get("energy_j_total", 0.0) or 0.0)
+        except Exception:
+            prior_wall = 0.0
+            prior_energy = 0.0
+    if n_processed == 0 and prior_wall > 0:
+        wall = prior_wall
+        # No work done this session → keep prior energy total (don't add a
+        # spurious idle-load reading from a resume that processed nothing).
+        energy_j = prior_energy
+    else:
+        wall = prior_wall + elapsed
+        energy_j = prior_energy + float(energy_j_session or 0.0)
+
     summary = {
         "cell": cell_name,
         "method": cell["method"],
@@ -338,14 +628,27 @@ def _write_summary(
         "accuracy": acc,
         "tokens_local_total": total_local,
         "tokens_cloud_total": total_cloud,
+        "web_search_uses_total": total_web_searches,
+        "tool_calls_total": total_tool_calls,
+        "n_cloud_calls_total": total_cloud_calls,
+        "n_local_calls_total": total_local_calls,
         "cost_usd_total": total_cost,
-        "wall_time_s": elapsed,
+        "wall_time_s": wall,
+        # GPU energy integrated over the cell's wall-time across the GPUs
+        # visible to the runner host. Cloud energy is **not** included —
+        # see ``_energy.py``. Joules; sum of session + any prior resumes.
+        # TODO: decide whether to add a cloud J/token estimate; for now 0
+        # cloud contribution. (Patterson 2021 / Luccioni 2022 are options.)
+        "energy_j_total": energy_j,
         "task_count": len(tasks),
     }
-    (out_dir / "summary.json").write_text(json.dumps(summary, indent=2))
+    summary_path.write_text(json.dumps(summary, indent=2))
     print(
         f"[summary] {cell_name}: n={n_done}/{cell['n']} err={n_err} "
-        f"acc={acc:.3f} cost=${total_cost:.2f} time={elapsed/60:.1f}m",
+        f"acc={acc:.3f} cost=${total_cost:.2f} time={wall/60:.1f}m "
+        f"energy={energy_j/1000:.1f}kJ "
+        f"(session +{elapsed/60:.1f}m +{energy_j_session/1000:.1f}kJ, "
+        f"processed={n_processed})",
         flush=True,
     )
 
@@ -358,7 +661,7 @@ def run_cell(
     resume: bool = True,
     root: Optional[Path] = None,
 ) -> None:
-    out_root = root or DEFAULT_EXPERIMENTS_DIR
+    out_root = root or DEFAULT_RUNS_DIR
     out_dir = _cell_dir(cell_name, out_root)
     with _cell_lock(out_dir, cell_name):
         _run_cell_locked(
@@ -398,49 +701,93 @@ def _run_cell_locked(
             flush=True,
         )
 
-    tasks = load_tasks(cell["bench"], n=cell["n"])
-    print(f"[load] {cell['bench']} → {len(tasks)} tasks", flush=True)
+    subset_path = cell.get("subset")
+    if subset_path:
+        subset = _load_subset_file(subset_path)
+        # Load the full bench (n=None) so we can pick the exact subset IDs.
+        # The dataset providers are cached, so this isn't a re-fetch.
+        all_tasks = load_tasks(cell["bench"], n=None)
+        tasks = _apply_subset(all_tasks, subset, cell)
+        print(
+            f"[load] {cell['bench']} subset={Path(subset_path).name} "
+            f"→ {len(tasks)} tasks",
+            flush=True,
+        )
+    else:
+        tasks = load_tasks(cell["bench"], n=cell["n"])
+        print(f"[load] {cell['bench']} → {len(tasks)} tasks", flush=True)
 
     pending = [t for t in tasks if t["task_id"] not in done_ids]
     concurrency = max(1, int(cell.get("concurrency", 1)))
     if concurrency > 1:
         print(f"[concurrency] {concurrency} workers", flush=True)
 
+    # Hard per-task wall-clock cap. A cell may override it via the registry
+    # (``method_cfg.task_timeout_s``); otherwise the process-wide default
+    # (env ``OPENJARVIS_HYBRID_TASK_TIMEOUT_S``, 1800s) applies. 0 disables.
+    mcfg = cell.get("method_cfg") or {}
+    task_timeout_s = float(mcfg.get("task_timeout_s", DEFAULT_TASK_TIMEOUT_S))
+    if task_timeout_s > 0:
+        print(f"[task-timeout] {task_timeout_s/60:.1f}m per task", flush=True)
+
     agent = _build_agent(cell)
 
     t_start = time.time()
     write_lock = threading.Lock()
     completed = [0]
+    written_ok_ids: set = set()
     log_dir = str(out_dir / "logs")
 
     def _process(task: Dict[str, Any]) -> None:
-        row = _run_one(agent, cell["bench"], task, log_dir)
+        row = _run_one(
+            agent, cell["bench"], task, log_dir,
+            task_timeout_s=task_timeout_s,
+        )
         scored: Optional[Dict[str, Any]] = None
         if do_score and row.get("error") is None:
             try:
-                scored = score(cell["bench"], task, row["answer"])
+                scored = score(
+                    cell["bench"], task, row["answer"], cell_name=cell_name,
+                )
             except Exception as e:
                 scored = {
                     "success": False, "score": 0.0,
                     "details": {"score_error": str(e)},
                 }
         full_row = {**row, "score": scored}
-        with write_lock, results_path.open("a") as f:
-            f.write(json.dumps(full_row) + "\n")
-            f.flush()
+        with write_lock:
+            # Idempotency guard: a Modal retry can re-run the same task within
+            # one process. Skip appending once a non-error row exists for this
+            # task_id so results.jsonl never carries duplicate rows.
+            if full_row["task_id"] in written_ok_ids:
+                return
+            with results_path.open("a") as f:
+                f.write(json.dumps(full_row) + "\n")
+                f.flush()
+            if full_row.get("error") is None:
+                written_ok_ids.add(full_row["task_id"])
             completed[0] += 1
             _heartbeat(completed[0], len(tasks), full_row, t_start)
 
-    if concurrency == 1:
-        for task in pending:
-            _process(task)
-    else:
-        with ThreadPoolExecutor(max_workers=concurrency) as ex:
-            futures = [ex.submit(_process, t) for t in pending]
-            for fut in as_completed(futures):
-                fut.result()
+    # GPU energy sampler covers the same wall-clock window as ``wall_time_s``
+    # so the two numbers can be divided into an effective Watts figure.
+    # Sampler is best-effort: NVML failures degrade to ``energy_j_total=0``
+    # without crashing the run (see ``_energy.py``).
+    with EnergyCollector() as energy:
+        if concurrency == 1:
+            for task in pending:
+                _process(task)
+        else:
+            with ThreadPoolExecutor(max_workers=concurrency) as ex:
+                futures = [ex.submit(_process, t) for t in pending]
+                for fut in as_completed(futures):
+                    fut.result()
 
-    _write_summary(out_dir, cell_name, cell, tasks, t_start)
+    _write_summary(
+        out_dir, cell_name, cell, tasks, t_start,
+        n_processed=len(pending),
+        energy_j_session=energy.energy_j_total,
+    )
 
 
 # ---------- CLI ----------
@@ -492,6 +839,7 @@ if __name__ == "__main__":
 
 __all__ = [
     "DEFAULT_EXPERIMENTS_DIR",
+    "DEFAULT_RUNS_DIR",
     "DEFAULT_REGISTRY_DIR",
     "load_registry",
     "load_tasks",

src/openjarvis/agents/hybrid/skillorchestra/README.md

@@ -0,0 +1,48 @@
+# skillorchestra — faithful port of SkillOrchestra (arXiv:2602.19672)
+
+This package replaces the old single-file `skillorchestra.py` (a 2-agent
+JSON router that shared none of the original's prompts or structure). It
+restructures the agent to **be** the original's eval-orchestrator runtime.
+
+## What's faithful
+
+- **Prompts** — `prompts/` is a verbatim copy of the upstream
+  `skillorchestra/prompts/` package (`eval_orchestrator.py`,
+  `model_routing.py`, `learning.py`).
+- **Handbook + routing** — `stage_router.py` and `types.py` are verbatim
+  copies of upstream `adapters/stage_router.py` and `core/types.py`:
+  `StageSkillHandbook`, `parse_skill_analysis`, and all 5 routing
+  strategies (`router_decides`, `analyze_model_decide`, `weighted_avg`,
+  `weakest_skill`, `strongest_skill`).
+- **Loop** — `orchestrator.py` ports `eval_frames.py:run_single`: the
+  multi-round `search -> reasoning -> answer` ReAct loop, the verbatim
+  worker prompts, `<skill_analysis>` parsing, alias-based model routing,
+  last-round forced answer.
+- **Code tool** — `tools.run_code` runs model-generated Python in a real
+  `subprocess` with a timeout, exactly as upstream.
+
+## What can't match without infrastructure
+
+The original runtime needs three things this cluster doesn't have. Each
+degrades gracefully and is configurable:
+
+| Original | Here |
+|---|---|
+| FAISS wiki retriever for `search` | `method_cfg.retriever_url` POSTs the same `/retrieve` payload; absent → Anthropic `web_search` |
+| 6+ SGLang-served pool models | alias tiers collapse onto the cell's local/cloud pair; override per alias via `method_cfg.model_pool` |
+| Learned `handbook.json` from explore→learn→select | `method_cfg.handbook_path` (a hand-authored `handbook_seed.json` ships here); absent → `routing_strategy="none"`, the original's baseline mode |
+
+The offline explore→learn→select pipeline is **not** ported — it needs
+the served model pool + FRAMES/NQ datasets to run. The handbook *schema*
+it produces is fully supported by `StageSkillHandbook.load`, so a learned
+handbook can be dropped in later with no code change.
+
+SWE-bench is out of scope for the original (a QA orchestrator). SWE cells
+run the cloud backbone through the shared `mini_swe_agent` loop.
+
+## method_cfg
+
+`routing_strategy`, `handbook_path`, `max_rounds`, `retriever_url`,
+`model_pool`, `orchestrator_model` / `orchestrator_endpoint`,
+`code_timeout_s`, `answer_max_tokens`, `context_char_cap`. `router_model`
+/ `router_endpoint` are accepted as back-compat aliases.

src/openjarvis/agents/hybrid/skillorchestra/__init__.py

@@ -0,0 +1,22 @@
+"""SkillOrchestra — faithful port of the eval-orchestrator (arXiv:2602.19672).
+
+Importing this package registers the ``skillorchestra`` agent. Layout
+mirrors the upstream repo (``external/SkillOrchestra``):
+
+* :mod:`.prompts`       — verbatim ``eval_orchestrator`` / ``model_routing``
+                          / ``learning`` prompt templates.
+* :mod:`.stage_router`  — verbatim ``StageSkillHandbook``,
+                          ``parse_skill_analysis``, the 5 routing strategies.
+* :mod:`.types`         — verbatim learning-time data types (BetaCompetence,
+                          Skill, AgentProfile, ...).
+* :mod:`.pool`          — model-alias -> local/cloud resolution.
+* :mod:`.tools`         — search / enhance_reasoning / answer executors.
+* :mod:`.orchestrator`  — the multi-round search->code->answer loop.
+* :mod:`.agent`         — :class:`SkillOrchestraAgent`, the harness entry.
+"""
+
+from __future__ import annotations
+
+from .agent import SkillOrchestraAgent
+
+__all__ = ["SkillOrchestraAgent"]

src/openjarvis/agents/hybrid/skillorchestra/agent.py

@@ -0,0 +1,153 @@
+"""SkillOrchestraAgent — the OpenJarvis harness entry point.
+
+A faithful port of the SkillOrchestra eval orchestrator (arXiv:2602.19672,
+``orchestration/eval_frames.py``). The agent runs the multi-round
+search -> reasoning -> answer loop in :mod:`.orchestrator`, using the
+verbatim ``eval_orchestrator`` prompts, the ``StageSkillHandbook`` +
+``RoutingStrategy`` machinery, real Python subprocess execution, and a
+model-alias pool collapsed onto the cell's local/cloud pair.
+
+Three things the original needs that this environment does not have, and
+how each is handled (see ``README.md`` in this package for the full
+note):
+
+* **Learned handbook** — produced offline by the explore->learn->select
+  pipeline. With no handbook the orchestrator runs ``routing_strategy =
+  "none"`` (the original's baseline mode). Point ``method_cfg.handbook_path``
+  at a ``StageSkillHandbook`` JSON to enable skill routing.
+* **FAISS wiki retriever** — the ``search`` tool POSTs to it when
+  ``method_cfg.retriever_url`` is set; otherwise it falls back to
+  Anthropic ``web_search``.
+* **6+ model pool** — the alias tiers collapse onto the cell's local +
+  cloud models; override per alias with ``method_cfg.model_pool``.
+
+SWE-bench cells are out of scope for the original (it is a QA
+orchestrator). They run the cloud backbone through the shared mini SWE
+agent loop instead.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.core.registry import AgentRegistry
+
+from .._base import LocalCloudAgent
+from ..mini_swe_agent import run_swe_agent_loop
+from .orchestrator import run_orchestrator
+from .stage_router import StageSkillHandbook
+
+_VALID_STRATEGIES = {
+    "none", "router_decides", "analyze_model_decide",
+    "weighted_avg", "weakest_skill", "strongest_skill",
+}
+
+
+@AgentRegistry.register("skillorchestra")
+class SkillOrchestraAgent(LocalCloudAgent):
+    """Inference-time skill-aware orchestrator. See module docstring."""
+
+    agent_id = "skillorchestra"
+
+    # ------------------------------------------------------------------
+
+    def _is_soft_failure(self, exc: BaseException) -> Optional[str]:
+        # Malformed orchestrator / router JSON -> soft-fail row, matching
+        # the rest of the hybrid family.
+        if isinstance(exc, (ValueError, json.JSONDecodeError)):
+            return f"{type(exc).__name__}: {str(exc)[:120]}"
+        return None
+
+    def _load_handbook(self) -> Optional[StageSkillHandbook]:
+        """Load the StageSkillHandbook from ``method_cfg.handbook_path``.
+
+        A relative path resolves against this package directory so the
+        shipped ``handbook_seed.json`` works out of the box. Any load
+        failure degrades to ``None`` (orchestrator runs baseline mode).
+        """
+        path = self._cfg.get("handbook_path")
+        if not path:
+            return None
+        p = Path(path)
+        if not p.is_absolute():
+            p = Path(__file__).parent / p
+        if not p.exists():
+            return None
+        try:
+            return StageSkillHandbook.load(str(p))
+        except Exception:  # noqa: BLE001
+            return None
+
+    # ------------------------------------------------------------------
+
+    def _run_paradigm(
+        self,
+        input: str,
+        context: Optional[AgentContext],
+        **kwargs: Any,
+    ) -> Tuple[str, Dict[str, Any]]:
+        cfg = self._cfg
+        task_meta = (context.metadata.get("task") if context is not None else {}) or {}
+
+        # SWE-bench: the original SkillOrchestra has no code-repo mode.
+        # Run the cloud backbone through the shared mini SWE agent loop.
+        swe_mode = (
+            bool(cfg.get("swe_use_agent_loop"))
+            and bool(task_meta.get("problem_statement"))
+            and bool(task_meta.get("repo"))
+            and bool(task_meta.get("base_commit"))
+        )
+        if swe_mode:
+            out = run_swe_agent_loop(
+                task_meta,
+                backbone="cloud",
+                backbone_model=self._cloud_model,
+                cloud_endpoint=self._cloud_endpoint,
+                initial_prompt=input,
+                max_turns=int(cfg.get("swe_max_turns", 30)),
+                bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
+                output_cap=int(cfg.get("swe_output_cap", 10_000)),
+                turn_max_tokens=int(cfg.get("swe_turn_max_tokens", 4096)),
+                trace_prefix="skillorch_swe",
+            )
+            meta = {
+                "tokens_local": 0,
+                "tokens_cloud": out["tokens_in"] + out["tokens_out"],
+                "cost_usd": out["cost_usd"],
+                "turns": int(out["turns"]),
+                "tool_calls": int(out["turns"]),
+                "web_search_uses": 0,
+                "traces": {
+                    "mode": "swe_agent_loop",
+                    "backbone_model": self._cloud_model,
+                    "note": "original SkillOrchestra is QA-only; SWE uses the cloud backbone",
+                },
+            }
+            return out["answer"], meta
+
+        # QA path — the faithful eval orchestrator.
+        handbook = self._load_handbook()
+        strategy = str(cfg.get("routing_strategy", "none"))
+        if strategy not in _VALID_STRATEGIES:
+            raise ValueError(
+                f"routing_strategy {strategy!r} unknown; valid: "
+                f"{sorted(_VALID_STRATEGIES)}"
+            )
+        if handbook is None and strategy in ("router_decides", "analyze_model_decide"):
+            # These two strategies just honor the orchestrator's own model
+            # pick — they need no learned skill data, so an empty handbook
+            # is enough to run the skill-orchestrator prompt + loop.
+            handbook = StageSkillHandbook()
+        if handbook is None:
+            # No handbook -> baseline routing, exactly like the original.
+            strategy = "none"
+
+        return run_orchestrator(
+            self, input, cfg=cfg, handbook=handbook, strategy=strategy,
+        )
+
+
+__all__ = ["SkillOrchestraAgent"]

src/openjarvis/agents/hybrid/skillorchestra/handbook_seed.json

@@ -0,0 +1,500 @@
+{
+  "version": "0.0.1-seed",
+  "created_at": "2026-05-19",
+  "updated_at": "2026-05-19",
+  "_note": "HAND-AUTHORED SEED \u2014 not produced by the explore->learn->select pipeline. Priors are calibrated to Qwen-27B vs Opus on GAIA-style QA. Replace with a learned StageSkillHandbook when available.",
+  "skills": {
+    "search": {
+      "search.entity_lookup": {
+        "skill_id": "search.entity_lookup",
+        "name": "Entity Lookup",
+        "description": "Find a specific named entity, date, or attribute (person, place, work, organization).",
+        "stage": "search",
+        "examples": [
+          "Who directed the 1997 film Titanic?"
+        ],
+        "discovered_from_problems": []
+      },
+      "search.recent_events": {
+        "skill_id": "search.recent_events",
+        "name": "Recent Events",
+        "description": "Find recent or time-sensitive facts unlikely to be in a small model's parametric memory.",
+        "stage": "search",
+        "examples": [
+          "Which team won the 2025 Cricket World Cup?"
+        ],
+        "discovered_from_problems": []
+      },
+      "search.multi_hop": {
+        "skill_id": "search.multi_hop",
+        "name": "Multi Hop",
+        "description": "Locate intermediate facts that chain together to answer a compositional question.",
+        "stage": "search",
+        "examples": [
+          "What is the capital of the country that won the most 2024 Olympic gold medals?"
+        ],
+        "discovered_from_problems": []
+      }
+    },
+    "code": {
+      "code.arithmetic": {
+        "skill_id": "code.arithmetic",
+        "name": "Arithmetic",
+        "description": "Exact numeric computation, unit conversion, and counting over given values.",
+        "stage": "code",
+        "examples": [
+          "How many days are between 1999-03-01 and 2001-07-15?"
+        ],
+        "discovered_from_problems": []
+      },
+      "code.data_transform": {
+        "skill_id": "code.data_transform",
+        "name": "Data Transform",
+        "description": "Parse, filter, sort, or aggregate structured data to derive an intermediate result.",
+        "stage": "code",
+        "examples": [
+          "Given this table, which row has the third-highest revenue?"
+        ],
+        "discovered_from_problems": []
+      }
+    },
+    "answer": {
+      "answer.synthesis": {
+        "skill_id": "answer.synthesis",
+        "name": "Synthesis",
+        "description": "Combine retrieved documents and code results into a single correct answer.",
+        "stage": "answer",
+        "examples": [
+          "Combine the search results to state the final figure."
+        ],
+        "discovered_from_problems": []
+      },
+      "answer.format_compliance": {
+        "skill_id": "answer.format_compliance",
+        "name": "Format Compliance",
+        "description": "Emit the answer in the exact required format (no units, exact casing, list shape).",
+        "stage": "answer",
+        "examples": [
+          "Answer with a comma-separated list, no units."
+        ],
+        "discovered_from_problems": []
+      },
+      "answer.long_context": {
+        "skill_id": "answer.long_context",
+        "name": "Long Context",
+        "description": "Read a long accumulated context and extract the precise answer span.",
+        "stage": "answer",
+        "examples": [
+          "Extract the requested value from the document above."
+        ],
+        "discovered_from_problems": []
+      }
+    }
+  },
+  "model_profiles": {
+    "search-1": {
+      "model_alias": "search-1",
+      "actual_model": "claude-opus-4-7",
+      "stage": "search",
+      "skill_scores": {
+        "search.entity_lookup": 0.82,
+        "search.recent_events": 0.78,
+        "search.multi_hop": 0.8
+      },
+      "skill_attempts": {
+        "search.entity_lookup": 24,
+        "search.recent_events": 24,
+        "search.multi_hop": 24
+      },
+      "skill_successes": {
+        "search.entity_lookup": 20,
+        "search.recent_events": 19,
+        "search.multi_hop": 19
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.014,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "search-2": {
+      "model_alias": "search-2",
+      "actual_model": "claude-opus-4-7",
+      "stage": "search",
+      "skill_scores": {
+        "search.entity_lookup": 0.82,
+        "search.recent_events": 0.78,
+        "search.multi_hop": 0.8
+      },
+      "skill_attempts": {
+        "search.entity_lookup": 24,
+        "search.recent_events": 24,
+        "search.multi_hop": 24
+      },
+      "skill_successes": {
+        "search.entity_lookup": 20,
+        "search.recent_events": 19,
+        "search.multi_hop": 19
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.014,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "search-3": {
+      "model_alias": "search-3",
+      "actual_model": "Qwen/Qwen3.5-27B-FP8",
+      "stage": "search",
+      "skill_scores": {
+        "search.entity_lookup": 0.55,
+        "search.recent_events": 0.3,
+        "search.multi_hop": 0.42
+      },
+      "skill_attempts": {
+        "search.entity_lookup": 24,
+        "search.recent_events": 24,
+        "search.multi_hop": 24
+      },
+      "skill_successes": {
+        "search.entity_lookup": 13,
+        "search.recent_events": 7,
+        "search.multi_hop": 10
+      },
+      "overall_success_rate": 0.34,
+      "total_attempts": 24,
+      "total_successes": 8,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 280.0,
+      "avg_cost_usd": 0.0,
+      "strengths": [
+        "strict formatting",
+        "arithmetic on given values"
+      ],
+      "weaknesses": [
+        "recent / rare facts",
+        "long-context extraction"
+      ]
+    },
+    "reasoner-1": {
+      "model_alias": "reasoner-1",
+      "actual_model": "claude-opus-4-7",
+      "stage": "code",
+      "skill_scores": {
+        "code.arithmetic": 0.85,
+        "code.data_transform": 0.84
+      },
+      "skill_attempts": {
+        "code.arithmetic": 24,
+        "code.data_transform": 24
+      },
+      "skill_successes": {
+        "code.arithmetic": 20,
+        "code.data_transform": 20
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.014,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "reasoner-2": {
+      "model_alias": "reasoner-2",
+      "actual_model": "claude-opus-4-7",
+      "stage": "code",
+      "skill_scores": {
+        "code.arithmetic": 0.85,
+        "code.data_transform": 0.84
+      },
+      "skill_attempts": {
+        "code.arithmetic": 24,
+        "code.data_transform": 24
+      },
+      "skill_successes": {
+        "code.arithmetic": 20,
+        "code.data_transform": 20
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.014,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "reasoner-3": {
+      "model_alias": "reasoner-3",
+      "actual_model": "Qwen/Qwen3.5-27B-FP8",
+      "stage": "code",
+      "skill_scores": {
+        "code.arithmetic": 0.78,
+        "code.data_transform": 0.66
+      },
+      "skill_attempts": {
+        "code.arithmetic": 24,
+        "code.data_transform": 24
+      },
+      "skill_successes": {
+        "code.arithmetic": 19,
+        "code.data_transform": 16
+      },
+      "overall_success_rate": 0.34,
+      "total_attempts": 24,
+      "total_successes": 8,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 280.0,
+      "avg_cost_usd": 0.0,
+      "strengths": [
+        "strict formatting",
+        "arithmetic on given values"
+      ],
+      "weaknesses": [
+        "recent / rare facts",
+        "long-context extraction"
+      ]
+    },
+    "answer-1": {
+      "model_alias": "answer-1",
+      "actual_model": "claude-opus-4-7",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.84,
+        "answer.format_compliance": 0.8,
+        "answer.long_context": 0.86
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 20,
+        "answer.format_compliance": 19,
+        "answer.long_context": 21
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.018,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "answer-2": {
+      "model_alias": "answer-2",
+      "actual_model": "claude-opus-4-7",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.84,
+        "answer.format_compliance": 0.8,
+        "answer.long_context": 0.86
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 20,
+        "answer.format_compliance": 19,
+        "answer.long_context": 21
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.018,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "answer-3": {
+      "model_alias": "answer-3",
+      "actual_model": "Qwen/Qwen3.5-27B-FP8",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.52,
+        "answer.format_compliance": 0.74,
+        "answer.long_context": 0.58
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 12,
+        "answer.format_compliance": 18,
+        "answer.long_context": 14
+      },
+      "overall_success_rate": 0.34,
+      "total_attempts": 24,
+      "total_successes": 8,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 280.0,
+      "avg_cost_usd": 0.0,
+      "strengths": [
+        "strict formatting",
+        "arithmetic on given values"
+      ],
+      "weaknesses": [
+        "recent / rare facts",
+        "long-context extraction"
+      ]
+    },
+    "answer-4": {
+      "model_alias": "answer-4",
+      "actual_model": "Qwen/Qwen3.5-27B-FP8",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.52,
+        "answer.format_compliance": 0.74,
+        "answer.long_context": 0.58
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 12,
+        "answer.format_compliance": 18,
+        "answer.long_context": 14
+      },
+      "overall_success_rate": 0.34,
+      "total_attempts": 24,
+      "total_successes": 8,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 280.0,
+      "avg_cost_usd": 0.0,
+      "strengths": [
+        "strict formatting",
+        "arithmetic on given values"
+      ],
+      "weaknesses": [
+        "recent / rare facts",
+        "long-context extraction"
+      ]
+    },
+    "answer-math-1": {
+      "model_alias": "answer-math-1",
+      "actual_model": "claude-opus-4-7",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.84,
+        "answer.format_compliance": 0.8,
+        "answer.long_context": 0.86
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 20,
+        "answer.format_compliance": 19,
+        "answer.long_context": 21
+      },
+      "overall_success_rate": 0.6,
+      "total_attempts": 24,
+      "total_successes": 14,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 320.0,
+      "avg_cost_usd": 0.018,
+      "strengths": [
+        "multi-hop reasoning",
+        "rare-fact recall"
+      ],
+      "weaknesses": [
+        "higher cost"
+      ]
+    },
+    "answer-math-2": {
+      "model_alias": "answer-math-2",
+      "actual_model": "Qwen/Qwen3.5-27B-FP8",
+      "stage": "answer",
+      "skill_scores": {
+        "answer.synthesis": 0.52,
+        "answer.format_compliance": 0.74,
+        "answer.long_context": 0.58
+      },
+      "skill_attempts": {
+        "answer.synthesis": 24,
+        "answer.format_compliance": 24,
+        "answer.long_context": 24
+      },
+      "skill_successes": {
+        "answer.synthesis": 12,
+        "answer.format_compliance": 18,
+        "answer.long_context": 14
+      },
+      "overall_success_rate": 0.34,
+      "total_attempts": 24,
+      "total_successes": 8,
+      "avg_prompt_tokens": 1200.0,
+      "avg_completion_tokens": 280.0,
+      "avg_cost_usd": 0.0,
+      "strengths": [
+        "strict formatting",
+        "arithmetic on given values"
+      ],
+      "weaknesses": [
+        "recent / rare facts",
+        "long-context extraction"
+      ]
+    }
+  },
+  "usage_patterns": {
+    "stages": {},
+    "guidelines": {},
+    "models": {},
+    "raw": {}
+  },
+  "learning_history": [],
+  "routing_insights": [
+    "Route recent/rare-fact searches to a cloud search model; the local model lacks that knowledge.",
+    "Arithmetic and strict-format answers can go to the cheaper local model without losing accuracy.",
+    "Switch to the answer stage once retrieved documents cover every sub-fact of the question."
+  ]
+}

src/openjarvis/agents/hybrid/skillorchestra/orchestrator.py

@@ -0,0 +1,359 @@
+"""The SkillOrchestra eval-orchestrator loop.
+
+Faithful port of ``orchestration/eval_frames.py:run_single`` — the
+multi-round search -> reasoning -> answer ReAct loop. Each round:
+
+1. Build a context string from accumulated docs / code results / attempts.
+2. Ask the orchestrator model (with the 3 tools) for the next stage. The
+   prompt is the verbatim ``build_skill_orchestrator_prompt`` when a
+   handbook is loaded, else the baseline ``"Problem: ... Choose an
+   appropriate tool."`` string.
+3. Parse the tool call + any ``<skill_analysis>`` block; route the worker
+   model alias through the configured ``RoutingStrategy``.
+4. Execute the tool. ``answer`` ends the loop; the last round force-calls
+   ``answer``.
+
+The orchestrator step does raw SDK calls (Anthropic / OpenAI) because it
+needs the parsed ``tool_use`` blocks back — the same thing
+``extract_response_content_and_tool_calls`` does in the original.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+from .._prices import is_gpt5_family, supports_temperature
+from .pool import ModelSpec, build_pool
+from .prompts import build_skill_orchestrator_prompt
+from .stage_router import (
+    StageSkillHandbook,
+    get_routing_strategy,
+    parse_skill_analysis,
+)
+from .tools import anthropic_tools, openai_tools, run_answer, run_code, run_search
+
+# tool name -> routing stage (stage_router uses "reasoning" for code).
+_TOOL_STAGE = {
+    "search": "search",
+    "enhance_reasoning": "reasoning",
+    "code": "reasoning",
+    "answer": "answer",
+}
+_STAGE_DEFAULT_ALIAS = {
+    "search": "search-1",
+    "reasoning": "reasoner-1",
+    "answer": "answer-1",
+}
+
+
+# ---------------------------------------------------------------------------
+# Orchestrator decision step (raw SDK — needs tool_use blocks back)
+# ---------------------------------------------------------------------------
+
+def _orchestrate_step(
+    agent: Any,
+    *,
+    user: str,
+    model: str,
+    endpoint: str,
+    max_tokens: int,
+) -> Tuple[str, List[Dict[str, Any]], int, int, float]:
+    """One orchestrator turn. Returns (text, tool_calls, p_tok, c_tok, cost).
+
+    ``tool_calls`` is a list of ``{"name", "input"}`` dicts.
+    """
+    endpoint = endpoint.lower()
+    if endpoint == "anthropic":
+        import anthropic
+
+        client = anthropic.Anthropic(timeout=600.0, max_retries=12)
+        kwargs: Dict[str, Any] = dict(
+            model=model,
+            max_tokens=max_tokens,
+            messages=[{"role": "user", "content": user}],
+            tools=anthropic_tools(),
+        )
+        if supports_temperature(model):
+            kwargs["temperature"] = 1.0
+        msg = client.messages.create(**kwargs)
+        text = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
+        tool_calls = [
+            {"name": b.name, "input": dict(b.input or {})}
+            for b in msg.content
+            if getattr(b, "type", "") == "tool_use"
+        ]
+        p = getattr(msg.usage, "input_tokens", 0)
+        c = getattr(msg.usage, "output_tokens", 0)
+
+    elif endpoint == "openai":
+        from openai import OpenAI
+
+        client = OpenAI(timeout=600.0)
+        kwargs = dict(
+            model=model,
+            messages=[{"role": "user", "content": user}],
+            tools=openai_tools(),
+            tool_choice="auto",
+        )
+        if is_gpt5_family(model):
+            kwargs["max_completion_tokens"] = max_tokens
+            kwargs["temperature"] = 1.0
+        else:
+            kwargs["max_tokens"] = max_tokens
+            kwargs["temperature"] = 1.0
+        resp = client.chat.completions.create(**kwargs)
+        choice = resp.choices[0].message
+        text = choice.content or ""
+        tool_calls = []
+        for tc in getattr(choice, "tool_calls", None) or []:
+            try:
+                args = json.loads(tc.function.arguments or "{}")
+            except json.JSONDecodeError:
+                args = {}
+            tool_calls.append({"name": tc.function.name, "input": args})
+        u = resp.usage
+        p = getattr(u, "prompt_tokens", 0) if u else 0
+        c = getattr(u, "completion_tokens", 0) if u else 0
+    else:
+        raise ValueError(
+            f"orchestrator endpoint {endpoint!r} unsupported — route the "
+            "orchestrator through anthropic/openai (set method_cfg."
+            "orchestrator_endpoint)."
+        )
+
+    cost = agent.cost_usd(model, p, c)
+    agent.record_trace_event({
+        "kind": "skillorchestra_orchestrate",
+        "model": model,
+        "endpoint": endpoint,
+        "prompt": user,
+        "response": text,
+        "tool_calls": tool_calls,
+        "tokens_in": p,
+        "tokens_out": c,
+    })
+    return text, tool_calls, p, c, cost
+
+
+# ---------------------------------------------------------------------------
+# Context assembly  —  eval_frames.py:1305-1351
+# ---------------------------------------------------------------------------
+
+def _build_context(
+    doc_list: List[Tuple[str, str]],
+    code_list: List[Tuple[str, str]],
+    attempt_list: List[Tuple[str, str]],
+    *,
+    char_cap: int,
+) -> str:
+    parts: List[str] = []
+    if doc_list:
+        blk = ["## Retrieved Information"]
+        for i, (query, txt) in enumerate(doc_list):
+            blk.append(f"### Search {i + 1} — query: {query}\n{txt}")
+        parts.append("\n\n".join(blk))
+    if code_list:
+        blk = ["## Code Execution Results"]
+        for i, (code, out) in enumerate(code_list):
+            blk.append(
+                f"### Code {i + 1}\n```python\n{code}\n```\n"
+                f"Output:\n{out if out else '(no output)'}"
+            )
+        parts.append("\n\n".join(blk))
+    if attempt_list:
+        blk = ["## Previous Answer Attempts"]
+        for who, ans in attempt_list:
+            blk.append(f"- {who}: {ans}")
+        parts.append("\n".join(blk))
+    ctx = "\n\n".join(parts)
+    if len(ctx) > char_cap:
+        # Keep the tail — most recent docs/code/attempts matter most.
+        ctx = "...[earlier context truncated]...\n" + ctx[-char_cap:]
+    return ctx
+
+
+# ---------------------------------------------------------------------------
+# Main loop  —  eval_frames.py:run_single
+# ---------------------------------------------------------------------------
+
+def run_orchestrator(
+    agent: Any,
+    problem: str,
+    *,
+    cfg: Dict[str, Any],
+    handbook: Optional[StageSkillHandbook],
+    strategy: str,
+) -> Tuple[str, Dict[str, Any]]:
+    """Run the eval orchestrator on one problem. Returns (answer, metadata)."""
+    max_rounds = int(cfg.get("max_rounds", 6))
+    char_cap = int(cfg.get("context_char_cap", 24000))
+    retriever_url = cfg.get("retriever_url")
+    code_timeout = int(cfg.get("code_timeout_s", 60))
+    answer_max_tokens = int(cfg.get("answer_max_tokens", 40000))
+    ws_max_uses = int(cfg.get("web_search_max_uses", 5))
+
+    # The orchestrator model: a fixed model per run (the original's
+    # MODEL_NAME). Defaults to the cell's cloud model when that endpoint
+    # supports tool calls, else Opus. ``router_model`` / ``router_endpoint``
+    # are accepted as back-compat aliases (pre-restructure cfg key names).
+    orch_endpoint = (cfg.get("orchestrator_endpoint")
+                     or cfg.get("router_endpoint")
+                     or agent._cloud_endpoint).lower()
+    orch_model = (cfg.get("orchestrator_model")
+                  or cfg.get("router_model")
+                  or agent._cloud_model)
+    if orch_endpoint not in ("anthropic", "openai"):
+        orch_endpoint, orch_model = "anthropic", "claude-opus-4-7"
+    orch_max_tokens = int(cfg.get("orchestrator_max_tokens", 4096))
+
+    pool = build_pool(
+        local_model=agent._local_model,
+        local_endpoint=agent._local_endpoint,
+        cloud_model=agent._cloud_model,
+        cloud_endpoint=agent._cloud_endpoint,
+        overrides=cfg.get("model_pool"),
+    )
+
+    doc_list: List[Tuple[str, str]] = []
+    code_list: List[Tuple[str, str]] = []
+    attempt_list: List[Tuple[str, str]] = []
+    route_log: List[Dict[str, Any]] = []
+
+    tokens_local = 0
+    tokens_cloud = 0
+    cost_usd = 0.0
+    tool_calls_n = 0
+    web_uses = 0
+    final_pred = ""
+    used_rounds = 0
+
+    def _route(stage: str, tool_alias: Optional[str], orch_text: str) -> str:
+        """Resolve the worker alias for ``stage`` via the routing strategy."""
+        if handbook is not None and strategy != "none":
+            sa = parse_skill_analysis(orch_text)
+            rr = get_routing_strategy(strategy, handbook).select_model(
+                stage, sa, tool_call_model=tool_alias,
+            )
+            return rr.model_alias
+        return tool_alias or _STAGE_DEFAULT_ALIAS[stage]
+
+    for step in range(max_rounds):
+        used_rounds = step + 1
+        is_last = step == max_rounds - 1
+        context_str = _build_context(
+            doc_list, code_list, attempt_list, char_cap=char_cap,
+        )
+
+        if handbook is not None and strategy != "none":
+            user = build_skill_orchestrator_prompt(
+                problem=problem,
+                context_str=context_str,
+                strategy=strategy,
+                handbook=handbook,
+            )
+        else:
+            user = (
+                f"Problem: {problem}\n\n{context_str}\n\n"
+                "Choose an appropriate tool."
+            )
+
+        text, tcalls, p, c, ocost = _orchestrate_step(
+            agent, user=user, model=orch_model,
+            endpoint=orch_endpoint, max_tokens=orch_max_tokens,
+        )
+        tokens_cloud += p + c
+        cost_usd += ocost
+
+        # The orchestrator may answer directly in <answer> tags.
+        if not tcalls and "<answer>" in text and "</answer>" in text:
+            final_pred = text.split("<answer>")[-1].split("</answer>")[0].strip()
+            break
+
+        # Last round: force the answer tool (eval_frames.py:1373-1380).
+        if is_last:
+            ans_alias = None
+            for tc in tcalls:
+                if tc["name"] == "answer":
+                    ans_alias = (tc.get("input") or {}).get("model")
+            tcalls = [{"name": "answer", "input": {"model": ans_alias or "answer-1"}}]
+        elif not tcalls:
+            # No tool, no answer — record the text and continue.
+            if text.strip():
+                attempt_list.append(("orchestrator", text.strip()[:2000]))
+            continue
+
+        finish = False
+        for tc in tcalls:
+            tool = tc["name"]
+            tool_alias = (tc.get("input") or {}).get("model")
+            stage = _TOOL_STAGE.get(tool, "answer")
+            chosen_alias = _route(stage, tool_alias, text)
+            spec: ModelSpec = pool.get(chosen_alias) or pool[
+                _STAGE_DEFAULT_ALIAS[stage]
+            ]
+            route_log.append({
+                "step": step,
+                "tool": tool,
+                "orchestrator_alias": tool_alias,
+                "routed_alias": chosen_alias,
+                "routed_model": spec.model,
+                "is_local": spec.is_local,
+            })
+            tool_calls_n += 1
+
+            if tool == "search":
+                res = run_search(
+                    agent, spec, context_str=context_str, problem=problem,
+                    retriever_url=retriever_url, web_search_max_uses=ws_max_uses,
+                )
+                docs = res["search_results_data"]
+                joined = "\n---\n".join(d for d in docs if d)[:char_cap]
+                doc_list.append((res["query"], joined or "(no results)"))
+                web_uses += res.get("web_search_uses", 0)
+            elif tool in ("enhance_reasoning", "code"):
+                res = run_code(
+                    agent, spec, context_str=context_str, problem=problem,
+                    bash_timeout_s=code_timeout,
+                )
+                code_list.append((res["generated_code"], res["exec_result"]))
+            else:  # answer
+                res = run_answer(
+                    agent, spec, context_str=context_str, problem=problem,
+                    max_tokens=answer_max_tokens,
+                )
+                final_pred = res["pred"]
+                attempt_list.append((res["alias"], final_pred))
+                finish = True
+
+            if res["is_local"]:
+                tokens_local += res["tokens_in"] + res["tokens_out"]
+            else:
+                tokens_cloud += res["tokens_in"] + res["tokens_out"]
+            cost_usd += res["cost_usd"]
+
+        if finish:
+            break
+
+    agent.record_trace_event({
+        "kind": "skillorchestra_route_log",
+        "strategy": strategy,
+        "rounds_used": used_rounds,
+        "routes": route_log,
+    })
+
+    meta = {
+        "tokens_local": tokens_local,
+        "tokens_cloud": tokens_cloud,
+        "cost_usd": cost_usd,
+        "turns": used_rounds,
+        "tool_calls": tool_calls_n,
+        "web_search_uses": web_uses,
+        "traces": {
+            "strategy": strategy,
+            "handbook_loaded": handbook is not None,
+            "rounds_used": used_rounds,
+            "routes": route_log,
+        },
+    }
+    return final_pred, meta

src/openjarvis/agents/hybrid/skillorchestra/pool.py

@@ -0,0 +1,151 @@
+"""Model-alias pool for the SkillOrchestra eval orchestrator.
+
+The original SkillOrchestra (``config/models.py`` + ``config/pool_config.json``)
+maps stage aliases — ``search-1/2/3``, ``reasoner-1/2/3``,
+``answer-1/2/3/4``, ``answer-math-1/2`` — onto a pool of 6+ models served
+via SGLang. OpenJarvis runs a 2-model world (one local vLLM student + one
+cloud model), so the default pool *collapses* the alias tiers onto
+local/cloud by cost rank: the dearer ``-1`` / ``-2`` aliases (and
+``answer-math-1``) route to the cloud model, the cheaper ``-3`` / ``-4``
+aliases (and ``answer-math-2``) route to the local model. This mirrors
+``stage_router.WeightedAverageStrategy.COST_TIERS``.
+
+A cell overrides any alias through ``method_cfg.model_pool``::
+
+    method_cfg.model_pool = {
+        "search-1" = { model = "claude-opus-4-7", endpoint = "anthropic" },
+        "search-3" = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" },
+        ...
+    }
+
+``endpoint`` is ``anthropic`` / ``openai`` / ``gemini`` for a cloud model,
+or an OpenAI-compatible base URL (``http://...``) for a local vLLM model.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+# Stage -> ordered alias list. Matches stage_router._get_models_for_stage
+# and orchestration/tools.json exactly.
+STAGE_ALIASES: Dict[str, List[str]] = {
+    "search": ["search-1", "search-2", "search-3"],
+    "reasoning": ["reasoner-1", "reasoner-2", "reasoner-3"],
+    "answer": ["answer-1", "answer-2", "answer-3", "answer-4",
+               "answer-math-1", "answer-math-2"],
+}
+
+# Every alias the orchestrator can emit, flat.
+ALL_ALIASES: List[str] = [a for aliases in STAGE_ALIASES.values() for a in aliases]
+
+# Default tier: which aliases collapse onto the cloud model vs the local
+# model. Dearer ``-1``/``-2`` (+ answer-math-1) -> cloud; cheaper -> local.
+_CLOUD_ALIASES = {
+    "search-1", "search-2",
+    "reasoner-1", "reasoner-2",
+    "answer-1", "answer-2", "answer-math-1",
+}
+
+
+@dataclass
+class ModelSpec:
+    """A resolved alias: which concrete model on which endpoint."""
+
+    alias: str
+    model: str
+    endpoint: str          # "anthropic" | "openai" | "gemini" | "http://..."
+    kind: str              # "cloud" | "local"
+
+    @property
+    def is_local(self) -> bool:
+        return self.kind == "local"
+
+
+def _endpoint_kind(endpoint: str) -> str:
+    return "local" if endpoint.startswith("http") else "cloud"
+
+
+def build_pool(
+    *,
+    local_model: Optional[str],
+    local_endpoint: Optional[str],
+    cloud_model: str,
+    cloud_endpoint: str,
+    overrides: Optional[Dict[str, Dict[str, str]]] = None,
+) -> Dict[str, ModelSpec]:
+    """Resolve every alias to a :class:`ModelSpec`.
+
+    Default mapping collapses the alias tiers onto the cell's local/cloud
+    pair; ``overrides`` (from ``method_cfg.model_pool``) wins per alias.
+    """
+    pool: Dict[str, ModelSpec] = {}
+    have_local = bool(local_model and local_endpoint)
+    for alias in ALL_ALIASES:
+        to_cloud = alias in _CLOUD_ALIASES or not have_local
+        if to_cloud:
+            pool[alias] = ModelSpec(alias, cloud_model, cloud_endpoint, "cloud")
+        else:
+            pool[alias] = ModelSpec(
+                alias, local_model, local_endpoint, "local"  # type: ignore[arg-type]
+            )
+
+    for alias, spec in (overrides or {}).items():
+        if alias not in pool:
+            continue
+        model = spec.get("model")
+        endpoint = spec.get("endpoint")
+        if not model or not endpoint:
+            continue
+        pool[alias] = ModelSpec(alias, model, endpoint, _endpoint_kind(endpoint))
+    return pool
+
+
+def call_alias(
+    agent: Any,
+    spec: ModelSpec,
+    *,
+    user: str,
+    system: Optional[str] = None,
+    max_tokens: int = 8000,
+    temperature: float = 1.0,
+) -> Tuple[str, int, int, float]:
+    """Single text-generation call through a resolved alias.
+
+    Returns ``(text, tokens_in, tokens_out, cost_usd)``. Dispatches to the
+    right :class:`LocalCloudAgent` SDK helper by endpoint. ``temperature``
+    defaults to 1.0 — the value the original ``call_tool`` uses for every
+    worker call (``eval_frames.py:657``).
+    """
+    if spec.is_local:
+        text, p, c = agent._call_vllm(
+            spec.model,
+            spec.endpoint,
+            user=user,
+            system=system,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            enable_thinking=False,
+            trace_role="local",
+        )
+        return text, p, c, 0.0
+
+    ep = spec.endpoint.lower()
+    if ep == "anthropic":
+        text, p, c, _ = agent._call_anthropic(
+            spec.model, user=user, system=system,
+            max_tokens=max_tokens, temperature=temperature, trace_role="cloud",
+        )
+    elif ep == "openai":
+        text, p, c = agent._call_openai(
+            spec.model, user=user, system=system,
+            max_tokens=max_tokens, temperature=temperature, trace_role="cloud",
+        )
+    elif ep == "gemini":
+        text, p, c = agent._call_gemini(
+            spec.model, user=user, system=system,
+            max_tokens=max_tokens, temperature=temperature, trace_role="cloud",
+        )
+    else:
+        raise ValueError(f"unsupported pool endpoint: {spec.endpoint!r}")
+    return text, p, c, agent.cost_usd(spec.model, p, c)

src/openjarvis/agents/hybrid/skillorchestra/prompts/__init__.py

@@ -0,0 +1,49 @@
+"""Prompt templates for SkillOrchestra.
+
+Centralized prompts for:
+- eval_orchestrator: FRAMES orchestrator (search/code/answer)
+- learning: handbook discovery, refinement, profiler
+- model_routing: QA benchmarks (skill-based and baseline routing)
+"""
+
+from .eval_orchestrator import (
+    build_skill_orchestrator_prompt,
+    format_baseline_tool_info,
+    SKILL_ORCHESTRATOR_PROMPT,
+    SKILL_ANALYSIS_ORCHESTRATOR_PROMPT,
+)
+from .learning import (
+    SKILL_DISCOVERY_PROMPT,
+    AGENT_ORCHESTRATION_DISCOVERY_PROMPT,
+    SKILL_IDENTIFICATION_PROMPT,
+    MODE_INSIGHT_PROMPT,
+    PROFILE_SUMMARY_PROMPT,
+    SKILL_SPLIT_PROMPT,
+    SKILL_MERGE_PROMPT,
+    AGENT_ORCHESTRATION_SPLIT_PROMPT,
+    AGENT_ORCHESTRATION_MERGE_PROMPT,
+    FAILURE_DRIVEN_REFINEMENT_PROMPT,
+)
+from .model_routing import SKILL_ANALYSIS_PROMPT, BASELINE_PROMPT
+
+__all__ = [
+    # Eval orchestrator
+    "build_skill_orchestrator_prompt",
+    "format_baseline_tool_info",
+    "SKILL_ORCHESTRATOR_PROMPT",
+    "SKILL_ANALYSIS_ORCHESTRATOR_PROMPT",
+    # Learning
+    "SKILL_DISCOVERY_PROMPT",
+    "AGENT_ORCHESTRATION_DISCOVERY_PROMPT",
+    "SKILL_IDENTIFICATION_PROMPT",
+    "MODE_INSIGHT_PROMPT",
+    "PROFILE_SUMMARY_PROMPT",
+    "SKILL_SPLIT_PROMPT",
+    "SKILL_MERGE_PROMPT",
+    "AGENT_ORCHESTRATION_SPLIT_PROMPT",
+    "AGENT_ORCHESTRATION_MERGE_PROMPT",
+    "FAILURE_DRIVEN_REFINEMENT_PROMPT",
+    # Model routing
+    "SKILL_ANALYSIS_PROMPT",
+    "BASELINE_PROMPT",
+]

src/openjarvis/agents/hybrid/skillorchestra/prompts/eval_orchestrator.py

@@ -0,0 +1,233 @@
+"""
+Eval orchestrator prompts for skill-based agent orchestration.
+"""
+
+from typing import Any, Optional
+
+# =============================================================================
+# Baseline Tool Info
+# =============================================================================
+
+
+def format_baseline_tool_info() -> str:
+    """Format baseline tool information for the orchestrator prompt."""
+    return """- Tool: search, Models: search-1 ($10/M output), search-2 ($2/M output), search-3 ($0.8/M output)
+Description: Search for missing information
+- Tool: code|enhance_reasoning, Models: reasoner-1 ($10/M output), reasoner-2 ($2/M output), reasoner-3 ($0.8/M output)
+Description: Write and execute Python code to solve the problem
+- Tool: answer, Models: answer-1 ($10/M output), answer-2 ($2/M output), answer-3 ($0.9/M output), answer-4 ($0.8/M output), answer-math-1 ($0.9/M output), answer-math-2 ($0.2/M output)
+Description: Extract the final answer if you think you have enough information to answer the problem"""
+
+
+# =============================================================================
+# Skill-Enhanced Orchestrator Prompt (router_decides strategy)
+# =============================================================================
+
+SKILL_ORCHESTRATOR_PROMPT = """You are a skill-based orchestrator for multi-step question answering. Choose the best tool and model for each step.
+
+## Available Tools and Models (Baseline)
+
+{baseline_tool_info}
+
+## Learned Skill Definitions
+
+### Search Skills
+{search_skills}
+
+### Code|Enhance Reasoning Skills  
+{reasoning_skills}
+
+### Answer Skills
+{answer_skills}
+
+## Model Performance (learned from validation)
+
+### Search Models
+{search_model_performance}
+
+### Code|Enhance Reasoning Models
+{reasoning_model_performance}
+
+### Answer Models
+{answer_model_performance}
+
+## Your Task
+1. Analyze the problem and current context
+2. Identify which skills are needed for the next step
+3. Choose the appropriate tool (search/enhance_reasoning/answer)
+4. Select the best model for that tool based on skill match and cost
+
+Consider cost-efficiency: if multiple models can handle it, prefer cheaper ones.
+
+You must first reason inside <think>...</think> about:
+- What information is missing or what computation is needed
+- Which skills from the catalog are required
+- Which model is best suited based on performance data
+
+**IMPORTANT**: When calling a tool, you MUST specify the model parameter using the model alias (e.g., "answer-1", "search-1", "reasoner-1"). Use the exact model names from the Available Models section above.
+
+Problem: {problem}
+
+{context_str}
+
+Choose an appropriate tool."""
+
+
+# =============================================================================
+# Skill Analysis Prompt (for weighted_avg, analyze_model_decide strategies)
+# =============================================================================
+
+SKILL_ANALYSIS_ORCHESTRATOR_PROMPT = """You are a skill-based orchestrator for multi-step question answering. You select the best tool (search|code|answer) and model by analyzing required skills.
+
+## Problem to Solve
+{problem}
+
+## Current Context
+{context_str}
+
+---
+
+## Quick Reference: What You Need to Do
+
+**CRITICAL REQUIREMENT**: Before making ANY tool call, you MUST:
+1. Inside <think>...</think>, analyze required skills and output in <skill_analysis> tags 
+2. Then choose the appropriate tool with the selected model based on the skill analysis.
+
+**The context above may be long - scroll back to see the problem and context, then follow the instructions below.**
+
+---
+
+## Available Tools and Models (Baseline)
+
+{baseline_tool_info}
+
+## Learned Skill Definitions
+
+### Search Skills
+{search_skills}
+
+### Reasoning Skills  
+{reasoning_skills}
+
+### Answer Skills
+{answer_skills}
+Use general performance of answer models to select the best model for the answer stage if you think you have enough information to answer the problem.
+
+## Model Performance (learned from validation)
+
+### Search Models
+{search_model_performance}
+
+### Reasoning Models
+{reasoning_model_performance}
+
+### Answer Models
+{answer_model_performance}
+---
+
+## Detailed Instructions
+
+**STEP 1 - REQUIRED**: Based on the Problem and Context shown at the top, think about what should be the next stage (search|code|answer).
+Search stage is to find missing information that you think is needed to answer the problem.
+Code stage is to write and execute Python code to solve the problem.
+Answer stage is to synthesize all gathered information into a final answer.
+
+**STEP 2 - REQUIRED FORMAT**:
+After deciding the next stage, analyze the skills needed for the next stage and provide the detailed skill analysis needed for the next stage.
+Reason inside <think>...</think> about why these skills are needed and their relative importance. We will use this skill analysis to select the best model for the next stage.
+Then output your analysis in the following format inside <skill_analysis> tags:
+<skill_analysis>
+{{ "required_skills": [ {{"skill_id": "skill.id", "percentage": 50}}, {{"skill_id": "skill.id", "percentage": 30}}, ... ], "reasoning": "Brief explanation of why these skills are needed" }}
+</skill_analysis>
+
+**STEP 3**: Choose the appropriate tool with the selected model based on the skill analysis.
+
+---
+
+## Final Reminders
+
+**CRITICAL**: The <skill_analysis> block is MANDATORY and must appear BEFORE your tool call. Without it, the routing system cannot function properly.
+
+**IMPORTANT**: When calling a tool, you MUST specify the model parameter using the model alias (e.g., "answer-1", "search-1", "reasoner-1"). Use the exact model names from the Available Models section above.
+
+Now, based on the Problem and Context shown at the top, analyze what should be the next stage (search|code|answer), provide the detailed skill analysis needed for the next stage in the <skill_analysis> tags and then choose an appropriate tool.
+"""
+
+
+# =============================================================================
+# Prompt Builder
+# =============================================================================
+
+
+def build_skill_orchestrator_prompt(
+    problem: str,
+    context_str: str,
+    strategy: str = "router_decides",
+    handbook: Any = None,
+    search_skills: Optional[str] = None,
+    reasoning_skills: Optional[str] = None,
+    answer_skills: Optional[str] = None,
+    search_model_performance: Optional[str] = None,
+    reasoning_model_performance: Optional[str] = None,
+    answer_model_performance: Optional[str] = None,
+    baseline_tool_info: Optional[str] = None,
+) -> str:
+    """
+    Build enhanced orchestrator prompt with skill catalog and model performance.
+
+    Args:
+        problem: The question/problem to solve
+        context_str: Current context (documents, code results, etc.)
+        strategy: Routing strategy - "router_decides" or "analyze_model_decide" etc.
+        handbook: SkillHandbook object with format_skills(stage/mode) and format_model_performance(stage/mode)
+        search_skills: Override - skill definitions for search stage
+        reasoning_skills: Override - skill definitions for reasoning stage
+        answer_skills: Override - skill definitions for answer stage
+        search_model_performance: Override - model performance for search
+        reasoning_model_performance: Override - model performance for reasoning
+        answer_model_performance: Override - model performance for answer
+        baseline_tool_info: Override - baseline tool descriptions
+
+    Returns:
+        Formatted prompt string
+    """
+    if baseline_tool_info is None:
+        baseline_tool_info = format_baseline_tool_info()
+
+    if handbook:
+        if search_skills is None:
+            search_skills = handbook.format_skills("search")
+        if reasoning_skills is None:
+            reasoning_skills = handbook.format_skills("code")
+        if answer_skills is None:
+            answer_skills = handbook.format_skills("answer")
+        if search_model_performance is None:
+            search_model_performance = handbook.format_model_performance("search")
+        if reasoning_model_performance is None:
+            reasoning_model_performance = handbook.format_model_performance("code")
+        if answer_model_performance is None:
+            answer_model_performance = handbook.format_model_performance("answer")
+
+    search_skills = search_skills or "No skills defined"
+    reasoning_skills = reasoning_skills or "No skills defined"
+    answer_skills = answer_skills or "No skills defined"
+    search_model_performance = search_model_performance or "No performance data"
+    reasoning_model_performance = reasoning_model_performance or "No performance data"
+    answer_model_performance = answer_model_performance or "No performance data"
+
+    if strategy == "router_decides":
+        template = SKILL_ORCHESTRATOR_PROMPT
+    else:
+        template = SKILL_ANALYSIS_ORCHESTRATOR_PROMPT
+
+    return template.format(
+        baseline_tool_info=baseline_tool_info,
+        search_skills=search_skills,
+        reasoning_skills=reasoning_skills,
+        answer_skills=answer_skills,
+        search_model_performance=search_model_performance,
+        reasoning_model_performance=reasoning_model_performance,
+        answer_model_performance=answer_model_performance,
+        problem=problem,
+        context_str=context_str if context_str else "(No context yet)",
+    )

src/openjarvis/agents/hybrid/skillorchestra/prompts/learning.py

@@ -0,0 +1,448 @@
+"""
+LLM prompt templates for Skill Handbook learning.
+
+All prompts used during the learning pipeline:
+- Skill discovery from trajectory contrast
+- Skill identification for a query
+- Mode-level insight distillation
+- Profile summarization (strengths/weaknesses)
+- Skill split/merge analysis
+- Failure-driven refinement
+"""
+
+# ---------------------------------------------------------------------------
+# Phase 1a: Skill Discovery - Model Routing
+# ---------------------------------------------------------------------------
+
+SKILL_DISCOVERY_PROMPT = """You are a skill taxonomist analyzing QA problems and model performance data to discover what skills are needed for effective model routing.
+
+## Task
+Analyze the sample problems below along with per-model success/failure data. Your goal: Propose a HIERARCHICAL skill taxonomy with:
+1. HIGH-LEVEL CATEGORIES (3-6): Broad skill areas that differentiate problems
+2. FINE-GRAINED SKILLS (2-4 per category): Specific capabilities within each category
+
+## Requirements
+- Skills should capture what makes problems DIFFERENT from each other
+- Skills should explain what makes MODELS perform DIFFERENTLY on those problems
+- Skills should be SPECIFIC and MEASURABLE (not vague like "intelligence" or "reasoning")
+- Include INDICATORS (keywords/patterns that suggest a skill is needed)
+- Include EXAMPLES from the sample problems
+- Skill IDs should follow the pattern: category_name.specific_skill_name
+
+## Sample Problems with Model Performance
+{sample_problems}
+
+## Contrastive Evidence (where models disagree)
+{contrastive_evidence}
+
+## Existing Skills (avoid duplicates)
+{existing_skills}
+
+## Output Format
+Return a JSON object with:
+{{
+  "categories": [
+    {{
+      "name": "category_name",
+      "description": "What this category covers",
+      "skills": [
+        {{
+          "skill_id": "category_name.skill_name",
+          "name": "Human-readable Skill Name",
+          "description": "What this specific skill involves and why models differ on it",
+          "indicators": ["keyword1", "pattern2", "phrase3"],
+          "examples": ["Example query requiring this skill"],
+          "mode": "answer"
+        }}
+      ]
+    }}
+  ]
+}}
+
+Aim for a taxonomy that covers the FULL DIVERSITY of the sample problems, not just one narrow topic. If problems span temporal facts, entity lookups, numeric data, relational facts, etc., the taxonomy should reflect all of those."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 1a: Skill Discovery - Agent Orchestration
+# ---------------------------------------------------------------------------
+
+AGENT_ORCHESTRATION_DISCOVERY_PROMPT = """You are a skill taxonomist analyzing QA problems to discover the underlying skills required to solve them.
+
+## Stage Information
+We have 3 stages in our pipeline:
+- **search**: Web search to retrieve factual information (tool: search)
+- **code**: Code generation and execution for calculations (tool: enhance_reasoning)
+- **answer**: Generate final answer from context (tool: answer)
+
+## Sample Problems
+{sample_problems}
+
+## Task
+Analyze these problems and DISCOVER what skills are needed.
+Propose a HIERARCHICAL skill taxonomy with:
+1. HIGH-LEVEL CATEGORIES (3-5): Broad skill areas that differentiate problems
+2. FINE-GRAINED SKILLS (2-4 per category): Specific capabilities within each category
+
+## Requirements
+- Skills should capture what makes problems DIFFERENT and what makes MODELS perform differently
+- Skills should be SPECIFIC and MEASURABLE (not vague like "intelligence")
+- Include INDICATORS (keywords/patterns that suggest a skill is needed)
+- Use hierarchical IDs: stage.category.specific_skill
+
+## Output Format (JSON)
+```json
+{{
+  "categories": [
+    {{
+      "stage": "search|code|answer",
+      "name": "category_name",
+      "description": "What this category covers",
+      "skills": [
+        {{
+          "id": "stage.category.skill_name",
+          "name": "Human Readable Name",
+          "description": "What this specific skill involves",
+          "indicators": ["keyword1", "pattern2", "phrase3"],
+          "examples": ["Example query requiring this skill"]
+        }}
+      ]
+    }}
+  ]
+}}
+```
+
+Respond with JSON only."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 1b: Skill Identification (which skills are active for a query)
+# ---------------------------------------------------------------------------
+
+SKILL_IDENTIFICATION_PROMPT = """You are an expert at identifying which skills from a catalog are required to handle a given query.
+
+## Query
+{query}
+
+## Ground Truth
+{ground_truth}
+
+## Operational Mode
+{mode}
+
+## Model Results (all models' outputs and whether they succeeded)
+Use this contrastive evidence: where models differ in success/failure, the outputs reveal what skills matter for this query.
+
+{model_results}
+
+## Available Skills for this Mode
+{mode_skills}
+
+## Output Format
+Return a JSON object:
+{{
+  "active_skills": [
+    {{
+      "skill_id": "the skill id",
+      "weight": 0.0 to 1.0,
+      "reasoning": "brief explanation"
+    }}
+  ]
+}}
+
+Weights should sum to approximately 1.0. Only include skills that are genuinely relevant to this specific query and mode. Use the model outputs to infer which skills differentiate successful vs failed attempts."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 1b: Mode-level Insight Distillation
+# ---------------------------------------------------------------------------
+
+MODE_INSIGHT_PROMPT = """You are an expert at analyzing execution patterns to derive reusable routing insights.
+
+## Task
+Analyze the execution patterns below and derive mode-level routing insights. These insights should help an orchestrator decide WHEN to use each mode and HOW to transition between modes.
+
+## Execution Patterns
+{execution_patterns}
+
+## Modes
+{modes}
+
+## Output Format
+Return a JSON object:
+{{
+  "insights": [
+    {{
+      "mode": "search|code|answer",
+      "content": "The routing insight as a clear, actionable rule",
+      "insight_type": "transition|usage|constraint",
+      "confidence": 0.0 to 1.0
+    }}
+  ]
+}}
+
+Focus on patterns that generalize across queries, not query-specific observations. Examples:
+- "If multiple arithmetic operations are needed, switch to code mode instead of search"
+- "Prefer search-3 for multi-hop queries requiring entity tracking"
+- "Switch to answer mode once all required facts have been gathered"
+"""
+
+
+# ---------------------------------------------------------------------------
+# Phase 1b: Agent Profile Summarization
+# ---------------------------------------------------------------------------
+
+PROFILE_SUMMARY_PROMPT = """You are an expert at summarizing agent capabilities from performance data.
+
+## Agent
+{agent_id} (model: {model_name}, mode: {mode})
+
+## Performance Data
+{performance_data}
+
+## Output Format
+Return a JSON object:
+{{
+  "strengths": ["strength1", "strength2"],
+  "weaknesses": ["weakness1", "weakness2"],
+  "routing_signals": ["when to use this agent", "when to avoid"]
+}}
+
+Be specific and evidence-based. Reference skill categories where applicable."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 2: Skill Split Analysis
+# ---------------------------------------------------------------------------
+
+SKILL_SPLIT_PROMPT = """You are an expert at analyzing whether a skill should be split into finer-grained sub-skills.
+
+## Skill Under Review
+ID: {skill_id}
+Name: {skill_name}
+Description: {skill_description}
+Mode: {mode}
+
+## Evidence for Splitting
+The agents have highly VARIABLE performance on this skill, suggesting it may conflate distinct capabilities:
+
+{performance_evidence}
+
+## Sample Queries Where Agents Disagree
+{sample_queries}
+
+## Output Format
+Return a JSON object:
+{{
+  "should_split": true/false,
+  "rationale": "explanation",
+  "proposed_splits": [
+    {{
+      "skill_id": "mode.category.new_name",
+      "name": "New Skill Name",
+      "description": "What this sub-skill captures",
+      "indicators": ["indicator1", "indicator2"],
+      "distinguishing_feature": "What separates this from sibling skills"
+    }}
+  ]
+}}
+
+Only recommend splitting if there is clear evidence that the skill conflates genuinely different capabilities. The splits should be actionable for routing decisions."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 2: Skill Merge Analysis
+# ---------------------------------------------------------------------------
+
+SKILL_MERGE_PROMPT = """You are an expert at analyzing whether two skills should be merged.
+
+## Skills Under Review
+
+### Skill 1
+ID: {skill_1_id}
+Name: {skill_1_name}
+Description: {skill_1_description}
+
+### Skill 2
+ID: {skill_2_id}
+Name: {skill_2_name}
+Description: {skill_2_description}
+
+## Evidence for Merging
+All agents have statistically INDISTINGUISHABLE performance between these two skills, suggesting they are redundant for routing purposes:
+
+{performance_evidence}
+
+## Output Format
+Return a JSON object:
+{{
+  "should_merge": true/false,
+  "rationale": "Why merge or not",
+  "merged_skill": {{
+    "skill_id": "mode.category.merged_name",
+    "name": "Merged Skill Name",
+    "description": "Combined description",
+    "indicators": ["combined indicators"]
+  }},
+  "alternative_explanation": "If not merging, explain why they should remain separate"
+}}
+
+Only recommend merging if the skills truly capture the same capability from a routing perspective, even if they differ semantically."""
+
+
+# ---------------------------------------------------------------------------
+# Phase 2: Agent Orchestration Split/Merge
+# ---------------------------------------------------------------------------
+
+AGENT_ORCHESTRATION_SPLIT_PROMPT = """You are analyzing whether a skill should be split into more fine-grained skills.
+
+## Skill to Analyze
+{skill_definition}
+
+## Performance Data
+{performance_data}
+
+## Sample Queries
+### High Performance Queries (models succeeded)
+{high_perf_queries}
+
+### Low Performance Queries (models failed)
+{low_perf_queries}
+
+### Divergent Performance Queries (some models succeeded, others failed)
+{divergent_queries}
+
+## Sample Trajectories (if available)
+### Successful Trajectories
+{success_trajectories}
+
+### Failed Trajectories
+{failure_trajectories}
+
+## Task
+Analyze whether this skill should be split:
+1. Does the skill have high variance across models? (suggests splitting)
+2. Do different query types show different performance patterns? (suggests splitting)
+3. Are there clear sub-skills that could be distinguished? (suggests splitting)
+
+## Output Format (JSON)
+```json
+{{
+  "should_split": true/false,
+  "rationale": "Why split or not",
+  "proposed_splits": [
+    {{
+      "skill_id": "stage.category.subskill1",
+      "name": "Sub-skill Name",
+      "description": "What this sub-skill covers",
+      "indicators": ["indicator1", "indicator2"],
+      "distinguishing_feature": "What distinguishes this from sibling skills"
+    }}
+  ]
+}}
+```
+
+Respond with JSON only."""
+
+
+AGENT_ORCHESTRATION_MERGE_PROMPT = """You are analyzing whether two skills should be merged.
+
+## Skills to Analyze
+{skills_definitions}
+
+## Performance Correlation
+{performance_correlation}
+
+## Sample Queries
+### Skill 1 Queries
+{skill1_queries}
+
+### Skill 2 Queries
+{skill2_queries}
+
+## Task
+Analyze whether these skills should be merged:
+1. Do they have nearly identical performance patterns across models? (suggests merge)
+2. Are they conceptually similar or overlapping? (suggests merge)
+3. Would merging simplify routing without losing important distinctions? (suggests merge)
+
+## Output Format (JSON)
+```json
+{{
+  "should_merge": true/false,
+  "rationale": "Why merge or not",
+  "merged_skill": {{
+    "skill_id": "stage.category.merged_skill",
+    "name": "Merged Skill Name",
+    "description": "Combined description",
+    "indicators": ["indicator1", "indicator2"]
+  }},
+  "alternative_explanation": "If not merging, explain why they should remain separate"
+}}
+```
+
+Respond with JSON only."""
+
+
+# ---------------------------------------------------------------------------
+# Failure-driven refinement (when skill routing < oracle on training set)
+# ---------------------------------------------------------------------------
+
+FAILURE_DRIVEN_REFINEMENT_PROMPT = """You are an expert at analyzing why skill-based model routing fails and how to improve the skill taxonomy.
+
+## Context
+We have a skill-based routing system that selects which LLM to call based on identified skills. On the training set, we achieved:
+- **Oracle accuracy**: {oracle_accuracy:.1%} (best possible — if we always picked the correct model per query)
+- **Skill-based accuracy**: {skill_accuracy:.1%} (our current routing)
+
+We failed to achieve oracle-level performance. This suggests either:
+1. **Missing skills**: Queries require skills not in our catalog
+2. **Skills too coarse**: Existing skills conflate distinct capabilities and lead to wrong model selection
+3. **Skill identification gaps**: The router fails to identify the right skills for some query types
+
+## Current Skill Catalog
+{skill_catalog}
+
+## Failed Queries (oracle would have been correct, but we routed wrong)
+For each failed query we show: the question, what model(s) would have been correct (oracle), what model we routed to, and whether any model got it right.
+
+{failed_queries}
+
+## Task
+Reflect on why the routing failed for these queries. Consider:
+1. What skills are **missing** from the catalog that would have helped route correctly?
+2. Which existing skills might need to be **split** into finer-grained sub-skills?
+3. What **indicators** or patterns in the failed queries suggest new or refined skills?
+
+## Output Format
+Return a JSON object:
+{{
+  "rationale": "Your overall reflection on why routing failed and what the main gaps are",
+  "proposed_new_skills": [
+    {{
+      "skill_id": "category.specific_skill_name",
+      "name": "Human-readable name",
+      "description": "What this skill captures and why it matters for routing",
+      "indicators": ["keyword1", "pattern2", "phrase3"],
+      "example_queries": ["Example from failed queries that would match this skill"]
+    }}
+  ],
+  "proposed_splits": [
+    {{
+      "parent_skill_id": "existing.skill.id",
+      "rationale": "Why this skill should be split",
+      "proposed_sub_skills": [
+        {{
+          "skill_id": "existing.new_sub_name",
+          "name": "Sub-skill name",
+          "description": "What this sub-skill captures",
+          "indicators": ["indicator1"],
+          "distinguishing_feature": "What separates this from sibling sub-skills"
+        }}
+      ]
+    }}
+  ]
+}}
+
+- Only propose new skills or splits that are clearly supported by the failed queries
+- Skill IDs should follow category.specific_name pattern
+- Be specific: tie each proposal to concrete failed queries"""

src/openjarvis/agents/hybrid/skillorchestra/prompts/model_routing.py

@@ -0,0 +1,134 @@
+"""Prompt templates for model routing (skill-based and baseline)."""
+
+SKILL_ANALYSIS_PROMPT = """You are a skill-based model router. You are selecting the best model to answer a question by analyzing a question to identify required skills and their importance related to this question.
+
+## Learned Skill Definitions (from validation)
+{skill_catalog}
+
+## Model Performance (learned from validation)
+{model_performance}
+
+## Cost Tiers (cheapest to most expensive)
+- Cheap: Qwen2.5-7B-Instruct, LLaMA-3.1-8B-Instruct, Mistral-7B-Instruct
+- Medium: Gemma-2-27B-Instruct
+- Expensive: LLaMA-3.1-70B-Instruct, Mixtral-8x22B-Instruct
+
+## Task
+1. First, analyze the question below and identify which skills are needed, along with the percentage/weight of each skill (how important each skill is for answering this question).
+
+**IMPORTANT: Output your skill analysis FIRST, before any <think> tags.** Use the exact skill_id values from the catalog above (e.g. "disambiguation_and_scope.ambiguous_media_title_resolution").
+
+<skill_analysis>
+{{
+  "required_skills": [
+    {{"skill_id": "category.skill_id", "percentage": 50}},
+    {{"skill_id": "category.skill_id", "percentage": 30}},
+    ...
+  ],
+  "reasoning": "Brief explanation of why these skills are needed"
+}}
+</skill_analysis>
+
+The percentages should sum to approximately 100 (they don't need to be exact, but should reflect relative importance).
+
+2. After providing the skill analysis, reflect on which model is best suited based on the skills required and model performance data above.
+
+3. Route to that model using <search> tags and provide final answer in <answer>...</answer>
+
+Every time you receive new information, you must first conduct reasoning inside <think> ... </think>. \
+After reasoning, if you find you lack some knowledge, you can call a specialized LLM by writing a query inside <search> LLM-Name:Your-Query </search>. \
+
+!!! STRICT FORMAT RULES for <search>: !!!
+    + You MUST replace LLM-Name with the EXACT name of a model selected from [Qwen2.5-7B-Instruct, LLaMA-3.1-8B-Instruct, LLaMA-3.1-70B-Instruct, Mistral-7B-Instruct, Mixtral-8x22B-Instruct, Gemma-2-27B-Instruct]. \
+    + You MUST replace Your-Query with the EXACT same question as the original question below (DO NOT CHANGE IT). \
+    + NEVER copy or paste model descriptions into <search>.
+    + NEVER output the placeholder format <search> LLM-Name:Your-Query </search>. Always replace both parts correctly. \
+
+Before each LLM call, you MUST explicitly reason inside <think> ... </think> about: \
+    + Why external information is needed. \
+    + Which skills from the catalog are required for this question. \
+    + Which model is best suited based on the model performance data above. \
+
+When you call an LLM, the response will be returned between <information> and </information>. \
+You are encouraged to explore and utilize different LLMs to better understand their respective strengths and weaknesses. \
+
+If you find that no further external knowledge is needed, you can directly provide your final answer to the original question inside <answer> ... </answer>, without additional explanation or illustration. \
+For example: <answer> Beijing </answer>. \
+    + Important: You must not output the placeholder text "<answer> and </answer>" alone. \
+    + You must insert your actual answer between <answer> and </answer>, following the correct format. \
+    + You must not output the model name or query between <answer> and </answer>. \
+
+If you think none of the models listed have the necessary skills to answer this question directly, you can route to the model with the highest overall pass rate of models in the pool to get more information.
+
+Question: {question}
+"""
+
+
+BASELINE_PROMPT = """Answer the given question. \
+Every time you receive new information, you must first conduct reasoning inside <think> ... </think>. \
+After reasoning, if you find you lack some knowledge, you can call a specialized LLM by writing a query inside <search> LLM-Name:Your-Query </search>. \
+
+!!! STRICT FORMAT RULES for <search>: !!!
+    + You MUST replace LLM-Name with the EXACT name of a model selected from [Qwen2.5-7B-Instruct, LLaMA-3.1-8B-Instruct, LLaMA-3.1-70B-Instruct, Mistral-7B-Instruct, Mixtral-8x22B-Instruct, Gemma-2-27B-Instruct]. \
+    + You MUST replace Your-Query with a CONCRETE QUESTION that helps answer the original question below. \
+    + NEVER copy or paste model descriptions into <search>.
+    + NEVER output the placeholder format <search> LLM-Name:Your-Query </search>. Always replace both parts correctly. \
+
+Before each LLM call, you MUST explicitly reason inside <think> ... </think> about: \
+    + Why external information is needed. \
+    + Which model is best suited for answering it, based on the LLMs' abilities (described below). \
+
+When you call an LLM, the response will be returned between <information> and </information>. \
+You must not limit yourself to repeatedly calling a single LLM (unless its provided information is consistently the most effective and informative). \
+You are encouraged to explore and utilize different LLMs to better understand their respective strengths and weaknesses. \
+It is also acceptable—and recommended—to call different LLMs multiple times for the same input question to gather more comprehensive information. \
+
+
+#### The Descriptions of Each LLM \
+
+Qwen2.5-7B-Instruct:\
+Qwen2.5-7B-Instruct is a powerful Chinese-English instruction-tuned large language model designed for tasks in language, \
+coding, mathematics, and reasoning. As part of the Qwen2.5 series, it features enhanced knowledge, stronger coding and \
+math abilities, improved instruction following, better handling of long and structured texts, and supports up to 128K \
+context tokens. It also offers multilingual capabilities across over 29 languages.\
+
+
+LLaMA-3.1-8B-Instruct:\
+LLaMA-3.1-8B-Instruct is an 8-billion-parameter instruction-tuned language model optimized for multilingual dialogue. \
+It provides strong language understanding, reasoning, and text generation performance, outperforming many open-source \
+and closed-source models on standard industry benchmarks.\
+
+
+LLaMA-3.1-70B-Instruct:\
+LLaMA-3.1-70B-Instruct is a 70-billion-parameter state-of-the-art language model designed for advanced multilingual \
+dialogue tasks. It excels in language comprehension, complex reasoning, and high-quality text generation, setting a new \
+standard against both open and closed models in benchmark evaluations.\
+
+
+Mistral-7B-Instruct:\
+Mistral-7B-Instruct is a fine-tuned version of the Mistral-7B-v0.3 language model designed to follow instructions, \
+complete user requests, and generate creative text. It was trained on diverse public conversation datasets to enhance \
+its ability to handle interactive tasks effectively.\
+
+
+Mixtral-8x22B-Instruct:\
+Mixtral-8x22B-Instruct is a cutting-edge sparse Mixture-of-Experts (SMoE) large language model from MistralAI. It \
+efficiently uses 39B active parameters out of 141B total, delivering high performance at lower costs. The model excels \
+at following instructions, completing tasks, and generating creative text, with strong skills in multiple languages \
+(English, French, Italian, German, Spanish), mathematics, and coding. It also supports native function calling and \
+handles long contexts up to 64K tokens for better information recall.\
+
+
+Gemma-2-27B-Instruct:\
+Gemma-2-27B-Instruct is a cutting-edge, instruction-tuned text generation model developed by Google. Built using the \
+same technology as Gemini, it excels at text understanding, transformation, and code generation. As a lightweight, \
+decoder-only model with open weights, it is ideal for tasks like question answering, summarization, and reasoning. \
+Its compact size enables deployment on laptops, desktops, or private cloud setups, making powerful AI more accessible.\
+
+
+If you find that no further external knowledge is needed, you can directly provide your final answer inside <answer> ... </answer>, without additional explanation or illustration. \
+For example: <answer> Beijing </answer>. \
+    + Important: You must not output the placeholder text "<answer> and </answer>" alone. \
+    + You must insert your actual answer between <answer> and </answer>, following the correct format. \
+Question: {question}
+"""

src/openjarvis/agents/hybrid/skillorchestra/stage_router.py

@@ -0,0 +1,420 @@
+"""
+Adapter for orchestration eval script.
+
+Provides: StageSkillHandbook (load from JSON), parse_skill_analysis,
+get_routing_strategy. Compatible with JSON produced by to_stage_router.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+# =============================================================================
+# Handbook dataclasses (compatible with stage_router JSON)
+# =============================================================================
+
+
+@dataclass
+class Skill:
+    """A skill that models can have."""
+
+    skill_id: str
+    name: str
+    description: str
+    stage: str
+    examples: List[str] = field(default_factory=list)
+    discovered_from_problems: List[Dict[str, str]] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Skill":
+        return cls(
+            skill_id=data.get("skill_id", ""),
+            name=data.get("name", ""),
+            description=data.get("description", ""),
+            stage=data.get("stage", ""),
+            examples=data.get("examples", []),
+            discovered_from_problems=data.get("discovered_from_problems", []),
+        )
+
+
+@dataclass
+class ModelProfile:
+    """Performance profile for a model alias."""
+
+    model_alias: str
+    actual_model: str
+    stage: str
+    skill_scores: Dict[str, float] = field(default_factory=dict)
+    skill_attempts: Dict[str, int] = field(default_factory=dict)
+    skill_successes: Dict[str, int] = field(default_factory=dict)
+    overall_success_rate: float = 0.5
+    total_attempts: int = 0
+    total_successes: int = 0
+    avg_prompt_tokens: float = 0.0
+    avg_completion_tokens: float = 0.0
+    avg_cost_usd: float = 0.0
+    strengths: List[str] = field(default_factory=list)
+    weaknesses: List[str] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ModelProfile":
+        return cls(
+            model_alias=data.get("model_alias", ""),
+            actual_model=data.get("actual_model", ""),
+            stage=data.get("stage", ""),
+            skill_scores=data.get("skill_scores", {}),
+            skill_attempts=data.get("skill_attempts", {}),
+            skill_successes=data.get("skill_successes", {}),
+            overall_success_rate=data.get("overall_success_rate", 0.5),
+            total_attempts=data.get("total_attempts", 0),
+            total_successes=data.get("total_successes", 0),
+            avg_prompt_tokens=data.get("avg_prompt_tokens", 0.0),
+            avg_completion_tokens=data.get("avg_completion_tokens", 0.0),
+            avg_cost_usd=data.get("avg_cost_usd", 0.0),
+            strengths=data.get("strengths", []),
+            weaknesses=data.get("weaknesses", []),
+        )
+
+
+# =============================================================================
+# StageSkillHandbook
+# =============================================================================
+
+
+class StageSkillHandbook:
+    """Handbook for eval routing. Load from JSON produced by to_stage_router."""
+
+    def __init__(self, load_defaults: bool = True):
+        self.skills: Dict[str, Dict[str, Skill]] = {
+            "search": {},
+            "code": {},
+            "answer": {},
+        }
+        self.model_profiles: Dict[str, ModelProfile] = {}
+        self.usage_patterns: Dict[str, Any] = {"stages": {}, "guidelines": {}, "models": {}, "raw": {}}
+        self.routing_insights: List[str] = []
+        self.learning_history: List[Dict[str, Any]] = []
+        self.version = "1.0.0"
+        self.created_at = ""
+        self.updated_at = ""
+
+    def get_model_skill_scores(self) -> Dict[str, Dict[str, float]]:
+        return {alias: profile.skill_scores for alias, profile in self.model_profiles.items()}
+
+    def get_models_for_stage(self, stage: str) -> List[ModelProfile]:
+        return [p for p in self.model_profiles.values() if p.stage == stage]
+
+    def format_skills(self, stage: str) -> str:
+        catalog_skills = self.skills.get(stage, {})
+        models = self.get_models_for_stage(stage)
+        skills_with_performance = set()
+        for model in models:
+            skills_with_performance.update(model.skill_scores.keys())
+
+        lines = []
+        shown_skills = set()
+        for skill_id, skill in catalog_skills.items():
+            if skill_id in skills_with_performance:
+                shown_skills.add(skill_id)
+                lines.append(f"- {skill_id}: {skill.description}")
+                if skill.examples:
+                    lines.append(f"  Examples: {', '.join(skill.examples[:2])}")
+
+        orphaned = skills_with_performance - set(catalog_skills.keys())
+        if orphaned:
+            if shown_skills:
+                lines.append("")
+                lines.append("# Additional skills with performance data available:")
+            for skill_id in sorted(orphaned):
+                lines.append(f"- {skill_id}")
+
+        return "\n".join(lines) if lines else "No skills defined"
+
+    def format_model_performance(self, stage: str) -> str:
+        profiles = self.get_models_for_stage(stage)
+        valid_prefixes = {"search": ["search-"], "code": ["reasoner-", "code-"], "answer": ["answer-"]}
+        prefixes = valid_prefixes.get(stage, [])
+
+        lines = []
+        for p in profiles:
+            if not any(p.model_alias.startswith(prefix) for prefix in prefixes):
+                continue
+            has_data = (p.skill_scores and len(p.skill_scores) > 0) or p.strengths or p.weaknesses
+            if p.total_attempts > 0 or has_data:
+                lines.append(f"\n### {p.model_alias} ({p.actual_model})")
+                if p.total_attempts > 0:
+                    rate = p.total_successes / p.total_attempts
+                    lines.append(f"Overall: {rate:.0%} success ({p.total_successes}/{p.total_attempts})")
+                else:
+                    lines.append("Overall: 0% overall")
+                if p.skill_scores:
+                    lines.append("Skill scores:")
+                    stage_skill_scores = {
+                        sid: s
+                        for sid, s in p.skill_scores.items()
+                        if (sid.split(".")[0] if "." in sid else sid) in ("code", stage)
+                    }
+                    for skill_id, score in sorted(stage_skill_scores.items(), key=lambda x: x[1], reverse=True):
+                        lines.append(f"  - {skill_id}: {score:.0%}")
+                if p.strengths:
+                    lines.append(f"Strengths: {', '.join(p.strengths[:3])}")
+                if p.weaknesses:
+                    lines.append(f"Weaknesses: {', '.join(p.weaknesses[:3])}")
+
+        return "\n".join(lines) if lines else "No model performance data learned yet."
+
+    @classmethod
+    def load(cls, path: str) -> "StageSkillHandbook":
+        with open(path) as f:
+            data = json.load(f)
+        handbook = cls(load_defaults=False)
+        handbook.version = data.get("version", "1.0.0")
+        handbook.created_at = data.get("created_at", "")
+        handbook.updated_at = data.get("updated_at", "")
+        for stage, skills in data.get("skills", {}).items():
+            if stage not in handbook.skills:
+                handbook.skills[stage] = {}
+            for sid, sdata in skills.items():
+                handbook.skills[stage][sid] = Skill.from_dict(sdata)
+        for alias, pdata in data.get("model_profiles", {}).items():
+            handbook.model_profiles[alias] = ModelProfile.from_dict(pdata)
+        raw_usage = data.get("usage_patterns", {})
+        handbook.usage_patterns = {
+            "stages": raw_usage.get("stages", {}),
+            "guidelines": raw_usage.get("guidelines", {}),
+            "models": raw_usage.get("models", {}),
+            "raw": raw_usage.get("raw", {}),
+        }
+        handbook.learning_history = data.get("learning_history", [])
+        handbook.routing_insights = data.get("routing_insights", [])
+        return handbook
+
+
+# =============================================================================
+# Skill analysis parsing
+# =============================================================================
+
+
+@dataclass
+class SkillWeight:
+    skill_id: str
+    percentage: float
+
+
+@dataclass
+class SkillAnalysis:
+    stage: str
+    required_skills: List[SkillWeight] = field(default_factory=list)
+    reasoning: str = ""
+    raw_json: Dict[str, Any] = field(default_factory=dict)
+
+
+def parse_skill_analysis(output: str) -> Optional[SkillAnalysis]:
+    pattern = r"<skill_analysis>\s*(.*?)\s*</skill_analysis>"
+    match = re.search(pattern, output, re.DOTALL)
+    if not match:
+        return None
+    try:
+        data = json.loads(match.group(1).strip())
+        required_skills = [
+            SkillWeight(skill_id=s.get("skill_id", ""), percentage=float(s.get("percentage", 0)))
+            for s in data.get("required_skills", [])
+        ]
+        return SkillAnalysis(
+            stage=data.get("stage", ""),
+            required_skills=required_skills,
+            reasoning=data.get("reasoning", ""),
+            raw_json=data,
+        )
+    except (json.JSONDecodeError, KeyError, ValueError):
+        return None
+
+
+# =============================================================================
+# Routing strategies
+# =============================================================================
+
+
+@dataclass
+class ModelRoutingResult:
+    model_alias: str
+    decision_logic: str
+    confidence: float = 0.0
+    all_scores: Dict[str, float] = field(default_factory=dict)
+
+
+class RoutingStrategy:
+    def __init__(self, handbook: Optional[StageSkillHandbook] = None):
+        self.handbook = handbook
+        self._model_skill_scores: Dict[str, Dict[str, float]] = {}
+        if handbook:
+            self._model_skill_scores = handbook.get_model_skill_scores()
+
+    def _find_skill_id(self, stage: str, skill_id_or_name: str) -> Optional[str]:
+        if not self.handbook:
+            return None
+        handbook_stage = "code" if stage == "reasoning" else stage
+        stage_skills = self.handbook.skills.get(handbook_stage, {})
+        if skill_id_or_name in stage_skills:
+            return skill_id_or_name
+        lower = skill_id_or_name.lower()
+        for sid, skill in stage_skills.items():
+            if skill.name.lower() == lower or lower in skill.name.lower():
+                return sid
+        return None
+
+    def _get_models_for_stage(self, stage: str) -> List[str]:
+        if stage == "search":
+            return ["search-1", "search-2", "search-3"]
+        if stage == "reasoning":
+            return ["reasoner-1", "reasoner-2", "reasoner-3"]
+        if stage == "answer":
+            return ["answer-1", "answer-2", "answer-3", "answer-4", "answer-math-1", "answer-math-2"]
+        return []
+
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        raise NotImplementedError
+
+
+class RouterDecidesStrategy(RoutingStrategy):
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        if tool_call_model:
+            return ModelRoutingResult(tool_call_model, "router_decides_from_tool_call", 1.0)
+        defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+        return ModelRoutingResult(defaults.get(stage, "answer-1"), "router_decides_fallback", 0.5)
+
+
+class AnalyzeModelDecideStrategy(RoutingStrategy):
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        if tool_call_model:
+            return ModelRoutingResult(tool_call_model, "analyze_model_decide_with_skill_analysis", 1.0)
+        defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+        return ModelRoutingResult(defaults.get(stage, "answer-1"), "analyze_model_decide_fallback", 0.5)
+
+
+class WeightedAverageStrategy(RoutingStrategy):
+    COST_TIERS = {
+        "search-3": 1, "search-2": 2, "search-1": 3,
+        "reasoner-3": 1, "reasoner-2": 2, "reasoner-1": 3,
+        "answer-math-2": 1, "answer-4": 1, "answer-3": 2,
+        "answer-math-1": 2, "answer-2": 3, "answer-1": 4,
+    }
+
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        if not skill_analysis or not skill_analysis.required_skills:
+            if tool_call_model:
+                return ModelRoutingResult(tool_call_model, "weighted_avg_no_skills_use_tool_call", 0.7)
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "weighted_avg_no_skills_fallback", 0.5)
+
+        models = self._get_models_for_stage(stage)
+        model_scores = {}
+        for model in models:
+            scores = self._model_skill_scores.get(model, {})
+            weighted_sum = total_weight = 0.0
+            for sw in skill_analysis.required_skills:
+                weight = sw.percentage / 100.0
+                sid = self._find_skill_id(stage, sw.skill_id) or sw.skill_id
+                score = scores.get(sid, 0.0)
+                weighted_sum += weight * score
+                total_weight += weight
+            model_scores[model] = weighted_sum / total_weight if total_weight > 0 else 0.5
+
+        if not model_scores:
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "weighted_avg_no_model_scores", 0.5)
+        max_score = max(model_scores.values())
+        best = [m for m, s in model_scores.items() if abs(s - max_score) < 0.001]
+        best.sort(key=lambda m: self.COST_TIERS.get(m, 999))
+        return ModelRoutingResult(best[0], "weighted_avg_from_skill_analysis", max_score, model_scores)
+
+
+class WeakestSkillStrategy(RoutingStrategy):
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        if not skill_analysis or not skill_analysis.required_skills:
+            if tool_call_model:
+                return ModelRoutingResult(tool_call_model, "weakest_skill_no_skills_use_tool_call", 0.7)
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "weakest_skill_no_skills_fallback", 0.5)
+        weakest = min(skill_analysis.required_skills, key=lambda s: s.percentage)
+        sid = self._find_skill_id(stage, weakest.skill_id) or weakest.skill_id
+        models = self._get_models_for_stage(stage)
+        model_scores = {m: self._model_skill_scores.get(m, {}).get(sid, 0.5) for m in models}
+        if not model_scores:
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "weakest_skill_no_model_scores", 0.5)
+        best = max(model_scores, key=model_scores.get)
+        return ModelRoutingResult(best, f"weakest_skill_{weakest.skill_id}", model_scores[best], model_scores)
+
+
+class StrongestSkillStrategy(RoutingStrategy):
+    def select_model(
+        self,
+        stage: str,
+        skill_analysis: Optional[SkillAnalysis] = None,
+        tool_call_model: Optional[str] = None,
+    ) -> ModelRoutingResult:
+        if not skill_analysis or not skill_analysis.required_skills:
+            if tool_call_model:
+                return ModelRoutingResult(tool_call_model, "strongest_skill_no_skills_use_tool_call", 0.7)
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "strongest_skill_no_skills_fallback", 0.5)
+        strongest = max(skill_analysis.required_skills, key=lambda s: s.percentage)
+        sid = self._find_skill_id(stage, strongest.skill_id) or strongest.skill_id
+        models = self._get_models_for_stage(stage)
+        model_scores = {m: self._model_skill_scores.get(m, {}).get(sid, 0.5) for m in models}
+        if not model_scores:
+            defaults = {"search": "search-1", "reasoning": "reasoner-1", "answer": "answer-1"}
+            return ModelRoutingResult(defaults.get(stage, "answer-1"), "strongest_skill_no_model_scores", 0.5)
+        best = max(model_scores, key=model_scores.get)
+        return ModelRoutingResult(best, f"strongest_skill_{strongest.skill_id}", model_scores[best], model_scores)
+
+
+ROUTING_STRATEGIES = {
+    "router_decides": RouterDecidesStrategy,
+    "analyze_model_decide": AnalyzeModelDecideStrategy,
+    "weighted_avg": WeightedAverageStrategy,
+    "weakest_skill": WeakestSkillStrategy,
+    "strongest_skill": StrongestSkillStrategy,
+}
+
+
+def get_routing_strategy(
+    strategy_name: str, handbook: Optional[StageSkillHandbook] = None
+) -> RoutingStrategy:
+    if strategy_name not in ROUTING_STRATEGIES:
+        raise ValueError(
+            f"Unknown routing strategy: {strategy_name}. "
+            f"Available: {list(ROUTING_STRATEGIES.keys())}"
+        )
+    return ROUTING_STRATEGIES[strategy_name](handbook)

src/openjarvis/agents/hybrid/skillorchestra/tools.py

@@ -0,0 +1,376 @@
+"""The three SkillOrchestra tools: search, enhance_reasoning (code), answer.
+
+Faithful port of ``orchestration/eval_frames.py:call_tool`` — same worker
+prompts, same extraction, same Python subprocess execution. Two deltas,
+both forced by the OpenJarvis environment and documented inline:
+
+* ``search`` — the original POSTs to a FAISS wiki retriever service. We
+  honor ``method_cfg.retriever_url`` and POST the exact same payload when
+  it's set; with no retriever configured we fall back to Anthropic's
+  server-side ``web_search`` tool so the stage still grounds.
+* in-tool correctness check — the original ``answer`` tool LLM-judges the
+  prediction against the gold answer inside ``call_tool``. OpenJarvis
+  scores with its own harness judge downstream, so we only return the
+  prediction; no gold answer is threaded in.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .._base import (
+    GEMINI_SEARCH_COST_PER_CALL,
+    OPENAI_WEB_SEARCH_COST_PER_CALL,
+    WEB_SEARCH_COST_PER_CALL,
+    build_web_search_tool,
+)
+from .pool import ModelSpec, call_alias
+
+# Cloud endpoints with a server-side web-search agent loop wired in
+# `_base.py`. Anything else (openrouter, vllm, unknown) can't ground.
+_SEARCH_CAPABLE_ENDPOINTS = ("anthropic", "openai", "gemini")
+
+# ---------------------------------------------------------------------------
+# Tool schemas — orchestration/tools.json, in Anthropic + OpenAI shapes.
+# ---------------------------------------------------------------------------
+
+_SEARCH_DESC = "Search for missing information."
+_CODE_DESC = (
+    "Write and execute Python code to compute intermediate results for "
+    "the problem."
+)
+_ANSWER_DESC = (
+    "Extract the final answer when you have gathered enough information "
+    "to answer the problem."
+)
+
+_ENUMS = {
+    "search": ["search-1", "search-2", "search-3"],
+    "enhance_reasoning": ["reasoner-1", "reasoner-2", "reasoner-3"],
+    "answer": ["answer-1", "answer-2", "answer-3", "answer-4",
+               "answer-math-1", "answer-math-2"],
+}
+
+
+def _model_prop(tool: str) -> Dict[str, Any]:
+    return {
+        "type": "string",
+        "description": (
+            f"Model alias for the {tool} tool. Choose one of: "
+            + ", ".join(_ENUMS[tool])
+        ),
+        "enum": _ENUMS[tool],
+    }
+
+
+def anthropic_tools() -> List[Dict[str, Any]]:
+    """The 3 orchestrator tools in Anthropic ``input_schema`` shape."""
+    out = []
+    for name, desc in (
+        ("search", _SEARCH_DESC),
+        ("enhance_reasoning", _CODE_DESC),
+        ("answer", _ANSWER_DESC),
+    ):
+        out.append({
+            "name": name,
+            "description": desc,
+            "input_schema": {
+                "type": "object",
+                "properties": {"model": _model_prop(name)},
+                "required": ["model"],
+            },
+        })
+    return out
+
+
+def openai_tools() -> List[Dict[str, Any]]:
+    """The 3 orchestrator tools in OpenAI ``function`` shape."""
+    out = []
+    for name, desc in (
+        ("search", _SEARCH_DESC),
+        ("enhance_reasoning", _CODE_DESC),
+        ("answer", _ANSWER_DESC),
+    ):
+        out.append({
+            "type": "function",
+            "function": {
+                "name": name,
+                "description": desc,
+                "parameters": {
+                    "type": "object",
+                    "properties": {"model": _model_prop(name)},
+                    "required": ["model"],
+                },
+            },
+        })
+    return out
+
+
+# ---------------------------------------------------------------------------
+# enhance_reasoning / code  —  eval_frames.py:659-812
+# ---------------------------------------------------------------------------
+
+def run_code(
+    agent: Any,
+    spec: ModelSpec,
+    *,
+    context_str: str,
+    problem: str,
+    bash_timeout_s: int = 60,
+) -> Dict[str, Any]:
+    """Generate self-contained Python with ``spec``, execute it, return stdout.
+
+    Mirrors the original worker prompt and ``subprocess.run(['python', ...],
+    timeout=60)`` verbatim. Execution failures yield empty ``exec_result``
+    rather than raising — the orchestrator learns the model can't code.
+    """
+    prompt = (
+        context_str.strip() + "\n\n"
+        + f"Question: {problem}\nInstead of directly answering the question, "
+        "please write additional python code that will give intermidiate "
+        "results after execution. Wrap the code within ```python and ```. "
+        "The code should be self-contained with all the import and "
+        "initialization."
+    )
+    text, p, c, cost = call_alias(
+        agent, spec, user=prompt, max_tokens=8000, temperature=1.0,
+    )
+    generated_code = ""
+    if "```python" in text:
+        generated_code = text.split("```python")[-1].split("```")[0]
+
+    exec_result = ""
+    if generated_code.strip():
+        with tempfile.TemporaryDirectory() as td:
+            code_path = Path(td) / "exec_code.py"
+            code_path.write_text(generated_code)
+            try:
+                proc = subprocess.run(
+                    [sys.executable, str(code_path)],
+                    timeout=bash_timeout_s,
+                    capture_output=True,
+                    text=True,
+                )
+                exec_result = proc.stdout
+            except Exception:
+                exec_result = ""
+    return {
+        "tool": "enhance_reasoning",
+        "model": spec.model,
+        "alias": spec.alias,
+        "generated_code": generated_code,
+        "exec_result": exec_result,
+        "response": text,
+        "tokens_in": p,
+        "tokens_out": c,
+        "cost_usd": cost,
+        "is_local": spec.is_local,
+    }
+
+
+# ---------------------------------------------------------------------------
+# answer  —  eval_frames.py:814-997
+# ---------------------------------------------------------------------------
+
+def run_answer(
+    agent: Any,
+    spec: ModelSpec,
+    *,
+    context_str: str,
+    problem: str,
+    max_tokens: int = 40000,
+) -> Dict[str, Any]:
+    """Generate the final answer with ``spec`` and extract the prediction.
+
+    The original branches the prompt by model family: Qwen-3 / Qwen-math
+    get a ``\\boxed{}`` system prompt; GPT-5 / Claude (and we extend this
+    to every other model) get the ``<think>/<answer>`` instruction. The
+    in-tool LLM correctness check is dropped — OpenJarvis scores
+    downstream.
+    """
+    base = context_str.strip() + "\n\n" + problem
+    model_l = spec.model.lower()
+    system: Optional[str] = None
+    boxed = False
+
+    if "qwen3" in model_l and "235" not in model_l:
+        system = "Please reason step by step, and put your final answer within \\boxed{}."
+        user = base
+        boxed = True
+    elif "qwen2.5-math" in model_l or "qwen-2.5-math" in model_l:
+        system = "Please reason step by step, and put your final answer within \\boxed{}."
+        user = base
+        boxed = True
+    else:
+        user = base + (
+            "\n\nTake a deep breath and think hard with high reasoning, wrap "
+            "the thoughts within <think> and </think>, and wrap only the "
+            "exact answer without any explanation within <answer> and "
+            "</answer>.Output using the following format:\n<think>\n...\n"
+            "</think>\n<answer>\n...\n</answer>"
+        )
+
+    text, p, c, cost = call_alias(
+        agent, spec, user=user, system=system,
+        max_tokens=max_tokens, temperature=1.0,
+    )
+
+    pred = ""
+    if boxed and "\\boxed{" in text:
+        pred = "}".join(text.split("\\boxed{")[-1].split("}")[:-1]).strip()
+    elif "<answer>" in text:
+        pred = text.split("<answer>")[-1].split("</answer>")[0].strip()
+    else:
+        pred = text.strip()
+    # Original: a >500-word "answer" is treated as a non-answer.
+    if len(pred.split()) > 500:
+        pred = ""
+
+    return {
+        "tool": "answer",
+        "model": spec.model,
+        "alias": spec.alias,
+        "pred": pred,
+        "response": text,
+        "tokens_in": p,
+        "tokens_out": c,
+        "cost_usd": cost,
+        "is_local": spec.is_local,
+    }
+
+
+# ---------------------------------------------------------------------------
+# search  —  eval_frames.py:999-1096
+# ---------------------------------------------------------------------------
+
+def run_search(
+    agent: Any,
+    spec: ModelSpec,
+    *,
+    context_str: str,
+    problem: str,
+    retriever_url: Optional[str] = None,
+    topk: int = 150,
+    web_search_max_uses: int = 5,
+) -> Dict[str, Any]:
+    """Write a search query with ``spec``, then retrieve documents.
+
+    Query generation is the original verbatim worker prompt. Retrieval:
+    if ``retriever_url`` is set we POST the original ``/retrieve`` payload;
+    otherwise we fall back to Anthropic ``web_search`` (the documented
+    OpenJarvis substitution for the missing FAISS wiki index).
+    """
+    prompt = (
+        context_str.strip() + "\n\n"
+        + f"Question: {problem}\nInstead of directly answering the question, "
+        "please think hard and write a concise query to search Wikipedia. "
+        "Wrap the query within <query> and </query>."
+    )
+    text, p, c, cost = call_alias(
+        agent, spec, user=prompt, max_tokens=8000, temperature=1.0,
+    )
+    if "<query>" in text:
+        query = text.split("<query>")[-1].split("</query>")[0].strip()
+    else:
+        query = ""
+    if len(query) < 10:
+        query = problem
+
+    contents: List[str] = []
+    search_uses = 0
+
+    if retriever_url:
+        # Faithful path — the original FAISS retriever service.
+        import requests
+
+        payload = {
+            "queries": [query[:390]],
+            "topk": topk,
+            "return_scores": True,
+        }
+        try:
+            results = requests.post(
+                f"{retriever_url.rstrip('/')}/retrieve", json=payload, timeout=120,
+            ).json()
+            for r in results[0]:
+                doc = r.get("document", {})
+                if "content" in doc:
+                    contents.append(doc["content"])
+                elif "contents" in doc:
+                    contents.append(doc["contents"])
+        except Exception as exc:  # noqa: BLE001
+            contents.append(f"[retriever error: {exc}]")
+    else:
+        # Substitution path — server-side web search via the cloud's
+        # `_base` agent loop. The search-capable helpers all talk to
+        # ``agent._cloud_model`` with their provider SDK. If this cell
+        # routes the cloud through an endpoint with no search wiring
+        # (openrouter / vllm), the search stage would produce nothing and
+        # the orchestrator answers the GAIA question blind — fail loud.
+        endpoint = agent._cloud_endpoint
+        if endpoint not in _SEARCH_CAPABLE_ENDPOINTS:
+            raise ValueError(
+                f"skillorchestra search fell back to web_search but "
+                f"cloud_endpoint={endpoint!r}; server-side web_search is "
+                "wired for anthropic / openai / gemini executors only. "
+                "Set method_cfg.retriever_url to a FAISS retriever, route "
+                "this cell's cloud through one of those endpoints, or "
+                "override the search-* aliases in method_cfg.model_pool — "
+                "otherwise the search stage produces nothing and the "
+                "orchestrator answers blind."
+            )
+        search_user = f"Search the web and report findings for: {query}"
+        try:
+            if endpoint == "anthropic":
+                ws_text, wp, wc, n_searches, _ = agent._call_anthropic_agent(
+                    agent._cloud_model,
+                    user=search_user,
+                    max_tokens=4096,
+                    temperature=1.0,
+                    tools=[build_web_search_tool(web_search_max_uses)],
+                    max_turns=4,
+                )
+                ws_cost_per_call = WEB_SEARCH_COST_PER_CALL
+            elif endpoint == "openai":
+                ws_text, wp, wc, n_searches, _ = agent._call_openai_agent(
+                    agent._cloud_model,
+                    user=search_user,
+                    max_tokens=4096,
+                    temperature=1.0,
+                    max_turns=4,
+                )
+                ws_cost_per_call = OPENAI_WEB_SEARCH_COST_PER_CALL
+            else:  # gemini
+                ws_text, wp, wc, n_searches, _ = agent._call_gemini_agent(
+                    agent._cloud_model,
+                    user=search_user,
+                    max_tokens=4096,
+                    temperature=1.0,
+                    max_turns=4,
+                )
+                ws_cost_per_call = GEMINI_SEARCH_COST_PER_CALL
+            contents.append(ws_text)
+            p += wp
+            c += wc
+            search_uses = n_searches
+            cost += agent.cost_usd(agent._cloud_model, wp, wc)
+            cost += n_searches * ws_cost_per_call
+        except Exception as exc:  # noqa: BLE001
+            contents.append(f"[web_search error: {exc}]")
+
+    return {
+        "tool": "search",
+        "model": spec.model,
+        "alias": spec.alias,
+        "query": query,
+        "search_results_data": contents,
+        "tokens_in": p,
+        "tokens_out": c,
+        "cost_usd": cost,
+        "web_search_uses": search_uses,
+        "is_local": spec.is_local,
+    }

src/openjarvis/agents/hybrid/skillorchestra/types.py

@@ -0,0 +1,422 @@
+"""
+Core data types for SkillOrchestra.
+
+- Skill
+- AgentProfile
+- BetaCompetence
+- ModeMetadata
+- RoutingInsight
+- CostStats
+"""
+
+from __future__ import annotations
+
+import math
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+# ---------------------------------------------------------------------------
+# BetaCompetence
+# ---------------------------------------------------------------------------
+
+@dataclass
+class BetaCompetence:
+    """Bayesian competence estimate for an agent on a specific skill.
+
+    skill_scores / get_competence use empirical_rate (successes/attempts)
+    """
+
+    alpha: float = 1.0
+    beta: float = 1.0
+
+    @property
+    def mean(self) -> float:
+        return self.alpha / (self.alpha + self.beta)
+
+    @property
+    def empirical_rate(self) -> float:
+        """Empirical success rate: successes/attempts."""
+        n = self.total_observations
+        if n <= 0:
+            return 0.0
+        successes = max(0, int(self.alpha - 1))
+        return successes / n
+
+    @property
+    def variance(self) -> float:
+        """Posterior variance."""
+        total = self.alpha + self.beta
+        return (self.alpha * self.beta) / (total * total * (total + 1))
+
+    @property
+    def std(self) -> float:
+        return math.sqrt(self.variance)
+
+    @property
+    def total_observations(self) -> int:
+        """Total observations (excluding prior)"""
+        return max(0, int(self.alpha + self.beta - 2))
+
+    def update(self, success: bool) -> None:
+        if success:
+            self.alpha += 1.0
+        else:
+            self.beta += 1.0
+
+    def update_batch(self, successes: int, failures: int) -> None:
+        self.alpha += successes
+        self.beta += failures
+
+    def to_dict(self) -> Dict[str, float]:
+        return {"alpha": self.alpha, "beta": self.beta}
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, float]) -> BetaCompetence:
+        return cls(alpha=d["alpha"], beta=d["beta"])
+
+
+# ---------------------------------------------------------------------------
+# CostStats
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CostStats:
+    """Execution cost statistics for an agent under a specific mode.
+
+    Tracks both total cost (prompt + completion) and completion-only cost
+    separately, since completion cost is the variable component that
+    differs most between models (prompt cost is roughly constant for the
+    same query).
+    """
+
+    avg_prompt_tokens: float = 0.0
+    avg_completion_tokens: float = 0.0
+    avg_latency_s: float = 0.0
+    avg_cost_usd: float = 0.0
+    avg_completion_cost_usd: float = 0.0
+    avg_prompt_cost_usd: float = 0.0
+    total_executions: int = 0
+
+    def update(
+        self,
+        prompt_tokens: float,
+        completion_tokens: float,
+        latency_s: float,
+        cost_usd: float,
+        completion_cost_usd: float = 0.0,
+        prompt_cost_usd: float = 0.0,
+    ) -> None:
+        """Incremental running-average update."""
+        n = self.total_executions
+        self.avg_prompt_tokens = (self.avg_prompt_tokens * n + prompt_tokens) / (n + 1)
+        self.avg_completion_tokens = (self.avg_completion_tokens * n + completion_tokens) / (n + 1)
+        self.avg_latency_s = (self.avg_latency_s * n + latency_s) / (n + 1)
+        self.avg_cost_usd = (self.avg_cost_usd * n + cost_usd) / (n + 1)
+        self.avg_completion_cost_usd = (self.avg_completion_cost_usd * n + completion_cost_usd) / (n + 1)
+        self.avg_prompt_cost_usd = (self.avg_prompt_cost_usd * n + prompt_cost_usd) / (n + 1)
+        self.total_executions = n + 1
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "avg_prompt_tokens": self.avg_prompt_tokens,
+            "avg_completion_tokens": self.avg_completion_tokens,
+            "avg_latency_s": self.avg_latency_s,
+            "avg_cost_usd": self.avg_cost_usd,
+            "avg_completion_cost_usd": self.avg_completion_cost_usd,
+            "avg_prompt_cost_usd": self.avg_prompt_cost_usd,
+            "total_executions": self.total_executions,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> CostStats:
+        return cls(**{k: d[k] for k in cls.__dataclass_fields__ if k in d})
+
+
+# ---------------------------------------------------------------------------
+# RoutingInsight
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RoutingInsight:
+    """A single routing insight learned from execution traces"""
+
+    insight_id: str = field(default_factory=lambda: uuid.uuid4().hex[:8])
+    content: str = ""
+    insight_type: str = ""  # "transition", "usage", "constraint", "agent_preference"
+    evidence_query_ids: List[str] = field(default_factory=list)
+    confidence: float = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "insight_id": self.insight_id,
+            "content": self.content,
+            "insight_type": self.insight_type,
+            "evidence_query_ids": self.evidence_query_ids,
+            "confidence": self.confidence,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> RoutingInsight:
+        return cls(**{k: d[k] for k in cls.__dataclass_fields__ if k in d})
+
+
+# ---------------------------------------------------------------------------
+# ModeMetadata
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ModeMetadata:
+    """Mode-level routing metadata."""
+
+    mode: str = ""
+    description: str = ""
+    insights: List[RoutingInsight] = field(default_factory=list)
+
+    def add_insight(self, insight: RoutingInsight) -> None:
+        self.insights.append(insight)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "mode": self.mode,
+            "description": self.description,
+            "insights": [i.to_dict() for i in self.insights],
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> ModeMetadata:
+        insights = [RoutingInsight.from_dict(i) for i in d.get("insights", [])]
+        return cls(
+            mode=d.get("mode", ""),
+            description=d.get("description", ""),
+            insights=insights,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Skill
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SkillProvenance:
+    """Tracks how and why a skill was discovered."""
+
+    discovered_from_queries: List[str] = field(default_factory=list)
+    positive_trajectories: List[str] = field(default_factory=list)
+    negative_trajectories: List[str] = field(default_factory=list)
+    discovery_round: int = 0
+    refinement_history: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "discovered_from_queries": self.discovered_from_queries,
+            "positive_trajectories": self.positive_trajectories,
+            "negative_trajectories": self.negative_trajectories,
+            "discovery_round": self.discovery_round,
+            "refinement_history": self.refinement_history,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> SkillProvenance:
+        return cls(**{k: d[k] for k in cls.__dataclass_fields__ if k in d})
+
+
+@dataclass
+class Skill:
+    """A reusable capability abstraction."""
+
+    skill_id: str = ""
+    name: str = ""
+    description: str = ""
+    indicators: List[str] = field(default_factory=list)
+    examples: List[str] = field(default_factory=list)
+    mode: str = ""
+    parent_skill_id: Optional[str] = None # for hierarchical skills
+    provenance: SkillProvenance = field(default_factory=SkillProvenance)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "skill_id": self.skill_id,
+            "name": self.name,
+            "description": self.description,
+            "indicators": self.indicators,
+            "examples": self.examples,
+            "mode": self.mode,
+            "parent_skill_id": self.parent_skill_id,
+            "provenance": self.provenance.to_dict(),
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> Skill:
+        provenance = SkillProvenance.from_dict(d.get("provenance", {}))
+        return cls(
+            skill_id=d.get("skill_id", ""),
+            name=d.get("name", ""),
+            description=d.get("description", ""),
+            indicators=d.get("indicators", []),
+            examples=d.get("examples", []),
+            mode=d.get("mode", ""),
+            parent_skill_id=d.get("parent_skill_id"),
+            provenance=provenance,
+        )
+
+    def get_children(self, all_skills: Dict[str, Skill]) -> List[Skill]:
+        """Get child skills in the hierarchy."""
+        return [s for s in all_skills.values() if s.parent_skill_id == self.skill_id]
+
+    def is_leaf(self, all_skills: Dict[str, Skill]) -> bool:
+        """True if this skill has no children."""
+        return len(self.get_children(all_skills)) == 0
+
+
+# ---------------------------------------------------------------------------
+# AgentProfile
+# ---------------------------------------------------------------------------
+
+@dataclass
+class AgentProfile:
+    """Agent profile for skill-aware orchestration."""
+
+    agent_id: str = ""
+    mode: str = ""
+    model_name: str = ""
+    tools: List[str] = field(default_factory=list)
+
+    skill_competence: Dict[str, BetaCompetence] = field(default_factory=dict)
+
+    total_attempts: int = 0
+    total_successes: int = 0
+
+    cost_stats: CostStats = field(default_factory=CostStats)
+
+    routing_signals: List[str] = field(default_factory=list)
+    strengths: List[str] = field(default_factory=list)
+    weaknesses: List[str] = field(default_factory=list)
+
+    def get_competence(self, skill_id: str) -> float:
+        """Get empirical success rate for a skill. Returns 0 if unseen."""
+        if skill_id in self.skill_competence:
+            return self.skill_competence[skill_id].empirical_rate
+        return 0.0
+
+    def get_competence_dist(self, skill_id: str) -> BetaCompetence:
+        """Get full Beta distribution for a skill, creating with prior if unseen."""
+        if skill_id not in self.skill_competence:
+            self.skill_competence[skill_id] = BetaCompetence()
+        return self.skill_competence[skill_id]
+
+    def update_competence(self, skill_id: str, success: bool) -> None:
+        """Update competence estimate for a skill."""
+        self.get_competence_dist(skill_id).update(success)
+
+    def weighted_competence(
+        self, skill_weights: Dict[str, float]
+    ) -> float:
+        """Compute weighted competence: sum w_{t,sigma} * alpha/(alpha+beta)."""
+        if not skill_weights:
+            return 0.5
+        total = 0.0
+        for skill_id, weight in skill_weights.items():
+            total += weight * self.get_competence(skill_id)
+        return total
+
+    def category_competence(self, category_prefix: str) -> float:
+        """Aggregate competence on all skills under a category (skill_id prefix).
+
+        E.g. category_competence('entertainment_knowledge') = avg of
+        get_competence(s) for all s where s.startswith('entertainment_knowledge.').
+        """
+        prefix = category_prefix.rstrip(".") + "."
+        scores = [
+            self.get_competence(sid)
+            for sid in self.skill_competence
+            if sid.startswith(prefix)
+        ]
+        return sum(scores) / len(scores) if scores else 0.0
+
+    def category_competence_for_skills(
+        self, active_skill_ids: List[str]
+    ) -> float:
+        """Category-level competence for hierarchical tie-breaking.
+
+        Extracts parent categories from active_skill_ids (e.g. 'entertainment_knowledge'
+        from 'entertainment_knowledge.episodic_competition_outcome'), computes
+        category_competence for each, returns average.
+        """
+        categories: set = set()
+        for sid in active_skill_ids:
+            cat = sid.rsplit(".", 1)[0] if "." in sid else sid
+            categories.add(cat)
+        if not categories:
+            return 0.0
+        return sum(self.category_competence(cat) for cat in categories) / len(categories)
+
+    @property
+    def overall_success_rate(self) -> float:
+        """Overall success rate (trajectory-level when available, else skill-level)."""
+        if self.total_attempts > 0:
+            return self.total_successes / self.total_attempts
+        total_attempts = 0
+        total_successes = 0
+        for bc in self.skill_competence.values():
+            n = bc.total_observations
+            s = max(0, int(bc.alpha - 1))
+            total_attempts += n
+            total_successes += s
+        return total_successes / total_attempts if total_attempts > 0 else 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        skill_scores = {}
+        skill_attempts = {}
+        skill_successes = {}
+        for sid, bc in self.skill_competence.items():
+            obs = bc.total_observations
+            successes = max(0, int(bc.alpha - 1))
+            skill_attempts[sid] = obs
+            skill_successes[sid] = successes
+            skill_scores[sid] = round(successes / obs, 4) if obs > 0 else 0.0
+
+        skill_total_attempts = sum(skill_attempts.values())
+        skill_total_successes = sum(skill_successes.values())
+
+        return {
+            "agent_id": self.agent_id,
+            "mode": self.mode,
+            "model_name": self.model_name,
+            "tools": self.tools,
+            "skill_competence": {
+                sid: bc.to_dict() for sid, bc in self.skill_competence.items()
+            },
+            "skill_scores": skill_scores,
+            "skill_attempts": skill_attempts,
+            "skill_successes": skill_successes,
+            "total_attempts": self.total_attempts if self.total_attempts > 0 else skill_total_attempts,
+            "total_successes": self.total_successes if self.total_attempts > 0 else skill_total_successes,
+            "cost_stats": self.cost_stats.to_dict(),
+            "routing_signals": self.routing_signals,
+            "strengths": self.strengths,
+            "weaknesses": self.weaknesses,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> AgentProfile:
+        skill_competence = {
+            sid: BetaCompetence.from_dict(bc)
+            for sid, bc in d.get("skill_competence", {}).items()
+        }
+        cost_stats = CostStats.from_dict(d.get("cost_stats", {}))
+        return cls(
+            agent_id=d.get("agent_id", ""),
+            mode=d.get("mode", ""),
+            model_name=d.get("model_name", ""),
+            tools=d.get("tools", []),
+            skill_competence=skill_competence,
+            total_attempts=d.get("total_attempts", 0),
+            total_successes=d.get("total_successes", 0),
+            cost_stats=cost_stats,
+            routing_signals=d.get("routing_signals", []),
+            strengths=d.get("strengths", []),
+            weaknesses=d.get("weaknesses", []),
+        )

src/openjarvis/evals/configs/distillation/m2/gaia-qwen-27b-distilled.toml

@@ -0,0 +1,31 @@
+# DISTILLED: gaia × Qwen/Qwen3.5-27B-FP8 — temp 0.6→0.3; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "gaia-qwen-27b-distilled"
+description = "Distilled gaia on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.3
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]

src/openjarvis/evals/configs/distillation/m2/gaia-qwen-2b-distilled.toml

@@ -0,0 +1,31 @@
+# DISTILLED: gaia × Qwen/Qwen3.5-2B — temp 0.6→0.3; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "gaia-qwen-2b-distilled"
+description = "Distilled gaia on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.3
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]

src/openjarvis/evals/configs/distillation/m2/livecodebench-qwen-27b-distilled.toml

@@ -0,0 +1,29 @@
+# DISTILLED: livecodebench × Qwen/Qwen3.5-27B-FP8 — CONTROL (no consensus edits applied)
+[meta]
+name = "livecodebench-qwen-27b-distilled"
+description = "Distilled livecodebench on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/livecodebench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20

src/openjarvis/evals/configs/distillation/m2/livecodebench-qwen-2b-distilled.toml

@@ -0,0 +1,29 @@
+# DISTILLED: livecodebench × Qwen/Qwen3.5-2B — CONTROL (no consensus edits applied)
+[meta]
+name = "livecodebench-qwen-2b-distilled"
+description = "Distilled livecodebench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/livecodebench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20

src/openjarvis/evals/configs/distillation/m2/liveresearch-qwen-27b-distilled.toml

@@ -0,0 +1,31 @@
+# DISTILLED: liveresearch × Qwen/Qwen3.5-27B-FP8 — temp 0.6→0.3; removed ['file_write']; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "liveresearch-qwen-27b-distilled"
+description = "Distilled liveresearch on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.3
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "code_interpreter", "think"]

src/openjarvis/evals/configs/distillation/m2/liveresearch-qwen-2b-distilled.toml

@@ -0,0 +1,31 @@
+# DISTILLED: liveresearch × Qwen/Qwen3.5-2B — temp 0.6→0.3; removed ['file_write']; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "liveresearch-qwen-2b-distilled"
+description = "Distilled liveresearch on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.3
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "code_interpreter", "think"]

src/openjarvis/evals/configs/distillation/m2/liveresearchbench-qwen-27b-distilled.toml

@@ -0,0 +1,29 @@
+# DISTILLED: liveresearchbench × Qwen/Qwen3.5-27B-FP8 — CONTROL (no consensus edits applied)
+[meta]
+name = "liveresearchbench-qwen-27b-distilled"
+description = "Distilled liveresearchbench on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/liveresearchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearchbench"
+backend = "jarvis-direct"
+max_samples = 50

src/openjarvis/evals/configs/distillation/m2/liveresearchbench-qwen-2b-distilled.toml

@@ -0,0 +1,29 @@
+# DISTILLED: liveresearchbench × Qwen/Qwen3.5-2B — CONTROL (no consensus edits applied)
+[meta]
+name = "liveresearchbench-qwen-2b-distilled"
+description = "Distilled liveresearchbench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.0
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/liveresearchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearchbench"
+backend = "jarvis-direct"
+max_samples = 50

src/openjarvis/evals/configs/distillation/m2/liveresearchbench-qwen-9b-distilled.toml

@@ -0,0 +1,29 @@
+# DISTILLED: liveresearchbench × Qwen/Qwen3.5-9B — CONTROL (no consensus edits applied)
+[meta]
+name = "liveresearchbench-qwen-9b-distilled"
+description = "Distilled liveresearchbench on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.0
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-9b/liveresearchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearchbench"
+backend = "jarvis-direct"
+max_samples = 50

src/openjarvis/evals/configs/distillation/m2/pinchbench-qwen-27b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: pinchbench × Qwen/Qwen3.5-27B-FP8 — temp 0.6→0.3; removed ['file_write']; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "pinchbench-qwen-27b-distilled"
+description = "Distilled pinchbench on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.3
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/pinchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = ["think", "file_read", "web_search", "shell_exec", "code_interpreter", "browser_navigate", "image_generate", "calculator", "http_request", "pdf_extract"]

src/openjarvis/evals/configs/distillation/m2/pinchbench-qwen-2b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: pinchbench × Qwen/Qwen3.5-2B — temp 0.6→0.3; removed ['file_write']; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "pinchbench-qwen-2b-distilled"
+description = "Distilled pinchbench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.3
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/pinchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = ["think", "file_read", "web_search", "shell_exec", "code_interpreter", "browser_navigate", "image_generate", "calculator", "http_request", "pdf_extract"]

src/openjarvis/evals/configs/distillation/m2/pinchbench-qwen-9b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: pinchbench × Qwen/Qwen3.5-9B — temp 0.6→0.3; removed ['file_write']; route[research]→qwen3.5:27b; system_prompt overrides (via $OPENJARVIS_HOME/agents/*); few_shot exemplars (via $OPENJARVIS_HOME/agents/*); tool descriptions (via $OPENJARVIS_HOME/tools/descriptions.toml)
+[meta]
+name = "pinchbench-qwen-9b-distilled"
+description = "Distilled pinchbench on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.3
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-9b/pinchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = ["think", "file_read", "web_search", "shell_exec", "code_interpreter", "browser_navigate", "image_generate", "calculator", "http_request", "pdf_extract"]

src/openjarvis/evals/configs/distillation/m2/taubench-qwen-27b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench × Qwen/Qwen3.5-27B-FP8 — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-qwen-27b-distilled"
+description = "Distilled taubench on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/taubench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"

src/openjarvis/evals/configs/distillation/m2/taubench-qwen-2b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench × Qwen/Qwen3.5-2B — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-qwen-2b-distilled"
+description = "Distilled taubench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/taubench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"

src/openjarvis/evals/configs/distillation/m2/taubench-qwen-9b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench × Qwen/Qwen3.5-9B — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-qwen-9b-distilled"
+description = "Distilled taubench on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-9b/taubench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"

src/openjarvis/evals/configs/distillation/m2/taubench-telecom-qwen-27b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench-telecom × Qwen/Qwen3.5-27B-FP8 — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-telecom-qwen-27b-distilled"
+description = "Distilled taubench-telecom on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"

src/openjarvis/evals/configs/distillation/m2/taubench-telecom-qwen-2b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench-telecom × Qwen/Qwen3.5-2B — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-telecom-qwen-2b-distilled"
+description = "Distilled taubench-telecom on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"

src/openjarvis/evals/configs/distillation/m2/taubench-telecom-qwen-9b-distilled.toml

@@ -0,0 +1,30 @@
+# DISTILLED: taubench-telecom × Qwen/Qwen3.5-9B — CONTROL (no consensus edits applied)
+[meta]
+name = "taubench-telecom-qwen-9b-distilled"
+description = "Distilled taubench-telecom on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-9b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"

src/openjarvis/evals/configs/distillation/m2/toolcall15-qwen-27b-distilled.toml

@@ -0,0 +1,28 @@
+# DISTILLED: toolcall15 × Qwen/Qwen3.5-27B-FP8 — CONTROL (no consensus edits applied)
+[meta]
+name = "toolcall15-qwen-27b-distilled"
+description = "Distilled toolcall15 on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-27b/toolcall15/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"

src/openjarvis/evals/configs/distillation/m2/toolcall15-qwen-2b-distilled.toml

@@ -0,0 +1,28 @@
+# DISTILLED: toolcall15 × Qwen/Qwen3.5-2B — CONTROL (no consensus edits applied)
+[meta]
+name = "toolcall15-qwen-2b-distilled"
+description = "Distilled toolcall15 on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-2b/toolcall15/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"

src/openjarvis/evals/configs/distillation/m2/toolcall15-qwen-9b-distilled.toml

@@ -0,0 +1,28 @@
+# DISTILLED: toolcall15 × Qwen/Qwen3.5-9B — CONTROL (no consensus edits applied)
+[meta]
+name = "toolcall15-qwen-9b-distilled"
+description = "Distilled toolcall15 on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/distilled/qwen-9b/toolcall15/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"

src/openjarvis/evals/datasets/swebench.py

@@ -63,6 +63,11 @@ class SWEBenchDataset(DatasetProvider):
     _default_split = "test"
 
     def __init__(self, variant: str = "verified_mini") -> None:
+        # NOTE: default is the 50-task mini variant. The full 500-task set is
+        # "verified" (princeton-nlp/SWE-bench_Verified). If your subset JSON
+        # references task_ids from the full set (e.g. subsets/swebench_*_n100*),
+        # pass variant="verified" explicitly — otherwise the 450 missing tasks
+        # are silently dropped. See agents/hybrid/runner.py:_load_swebench_tasks.
         if variant not in _HF_PATHS:
             raise ValueError(
                 f"Unknown SWE-bench variant {variant!r}; "

src/openjarvis/evals/scorers/swebench_harness.py

@@ -19,7 +19,12 @@ with two upstream-`swebench` patches applied at import time:
    ``swebench/harness/modal_eval/run_evaluation_modal.py:66`` writes to
    ``/sys/fs/cgroup/cpu/cpu.shares`` (cgroup v1). Modal v2 sandboxes use
    cgroup v2 — the path doesn't exist and every sandbox dies on the write.
-   Wrap the write in try/except.
+   Wrap the write in try/except. In swebench 4.x the call site is
+   ``ModalSandboxRuntime.__init__`` → ``self.write_file(...)`` →
+   ``self.sandbox.open(path, "w")``; in older swebench it was a free
+   ``set_cpu_quota`` function. We patch both: ``write_file`` swallows
+   FileNotFoundError for cgroup paths, and ``set_cpu_quota`` (if present)
+   is wrapped too.
 
 2. **Rescore `*_ids` fix**: older harness rescore code read
    ``resolved_instances`` / ``unresolved_instances`` / ``error_instances``
@@ -37,6 +42,7 @@ import json
 import logging
 import os
 import re
+import signal
 import subprocess
 import sys
 import tempfile
@@ -49,6 +55,71 @@ from openjarvis.evals.core.types import EvalRecord
 logger = logging.getLogger(__name__)
 
 
+def _run_subprocess_hard_timeout(
+    cmd: list,
+    *,
+    timeout_s: int,
+    cwd: str,
+) -> "subprocess.CompletedProcess":
+    """Run ``cmd`` with a timeout that is actually enforced.
+
+    ``subprocess.run(..., capture_output=True, timeout=...)`` has a
+    well-known deadlock: on timeout it kills only the *direct* child, then
+    calls ``communicate()`` again to drain the pipes — but if that child
+    spawned grandchildren that inherited the stdout/stderr fds (the Modal
+    ``swebench`` harness does exactly this), those grandchildren keep the
+    pipe open and the drain blocks **forever**. The nominal timeout never
+    fires; the runner freezes.
+
+    This helper avoids that by:
+
+    1. Launching the child in its own process group (``start_new_session``)
+       so we can signal the whole tree, not just the direct child.
+    2. On timeout, ``SIGTERM`` then ``SIGKILL`` the entire group so no
+       grandchild survives to hold a pipe open.
+    3. Draining output with a *bounded* ``communicate()`` after the kill so
+       even a stubborn drain can't hang us.
+
+    Raises :class:`subprocess.TimeoutExpired` (same contract as
+    ``subprocess.run``) so callers can keep their existing except clause.
+    """
+    proc = subprocess.Popen(
+        cmd,
+        cwd=cwd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        start_new_session=True,  # own process group → killable as a tree
+    )
+    try:
+        stdout, stderr = proc.communicate(timeout=timeout_s)
+        return subprocess.CompletedProcess(
+            cmd, proc.returncode, stdout, stderr
+        )
+    except subprocess.TimeoutExpired:
+        # Kill the whole group, not just the direct child — Modal harness
+        # subprocesses fork workers that would otherwise keep pipes open.
+        for sig in (signal.SIGTERM, signal.SIGKILL):
+            try:
+                os.killpg(proc.pid, sig)
+            except (ProcessLookupError, PermissionError):
+                break
+            try:
+                proc.wait(timeout=10)
+                break
+            except subprocess.TimeoutExpired:
+                continue
+        # Drain whatever is left, but never block on it again — the group
+        # is dead, so this returns promptly; the short cap is just paranoia.
+        try:
+            stdout, stderr = proc.communicate(timeout=10)
+        except subprocess.TimeoutExpired:
+            stdout, stderr = "", ""
+        raise subprocess.TimeoutExpired(
+            cmd, timeout_s, output=stdout, stderr=stderr
+        )
+
+
 # ---------- Patch tracking ----------
 
 _PATCHES_APPLIED = False
@@ -100,11 +171,126 @@ def _patch_modal_cgroup_v2() -> None:
     _m._hybrid_cgroup_patched = True  # type: ignore[attr-defined]
 
 
+_CGROUP_SOURCE_SENTINEL = "_OPENJARVIS_CGROUP_V2_PATCH_APPLIED"
+
+
+def _patch_modal_sandbox_source() -> None:
+    """Patch ``run_evaluation_modal.py`` on disk so subprocesses inherit it.
+
+    ``_run_harness`` shells out to ``python -m swebench.harness.run_evaluation``,
+    which means our in-process monkey-patches don't help. We do a one-time
+    idempotent textual rewrite of the swebench module file in the venv:
+
+    - Replace the bare ``self.write_file("/sys/fs/cgroup/cpu/cpu.shares", "2048")``
+      with a try/except FileNotFoundError. Marked with a sentinel so we
+      don't reapply on every call.
+
+    Only fires when the original unwrapped line is present and the sentinel
+    isn't — safe to run repeatedly. No-op if upstream ever fixes this.
+    """
+    try:
+        from swebench.harness.modal_eval import run_evaluation_modal as _m  # type: ignore[import-not-found]
+    except Exception:
+        return
+    src_path = getattr(_m, "__file__", None)
+    if not src_path:
+        return
+    try:
+        src = Path(src_path).read_text()
+    except Exception:
+        return
+    if _CGROUP_SOURCE_SENTINEL in src:
+        return
+    needle = '        self.write_file("/sys/fs/cgroup/cpu/cpu.shares", "2048")'
+    if needle not in src:
+        # Upstream changed the line — bail rather than apply blindly.
+        return
+    replacement = (
+        '        # ' + _CGROUP_SOURCE_SENTINEL + '\n'
+        '        try:\n'
+        '            self.write_file("/sys/fs/cgroup/cpu/cpu.shares", "2048")\n'
+        '        except FileNotFoundError:\n'
+        '            pass  # cgroup v2 Modal sandbox — path missing is fine\n'
+    )
+    new_src = src.replace(needle + "\n", replacement, 1)
+    try:
+        Path(src_path).write_text(new_src)
+    except Exception:
+        return
+
+
+def _patch_modal_sandbox_write_file() -> None:
+    """Make ``ModalSandboxRuntime.write_file`` survive cgroup-v2 sandboxes.
+
+    swebench 4.x removed ``set_cpu_quota`` and inlined the cgroup write in
+    ``ModalSandboxRuntime.__init__`` as
+    ``self.write_file("/sys/fs/cgroup/cpu/cpu.shares", "2048")``. Modal v1's
+    ``sandbox.open(path, "w")`` raises ``FileNotFoundError`` because the
+    sandbox image is cgroup-v2 and the parent dir doesn't exist, which kills
+    the whole constructor before any patch can be applied. We wrap
+    ``write_file`` to swallow that specific failure for cgroup paths, while
+    still letting real write failures (patch/eval script) surface.
+    """
+    try:
+        from swebench.harness.modal_eval import run_evaluation_modal as _m  # type: ignore[import-not-found]
+    except Exception:
+        return
+    runtime = getattr(_m, "ModalSandboxRuntime", None)
+    if runtime is None:
+        return
+    if getattr(runtime, "_hybrid_write_file_patched", False):
+        return
+    orig_write = runtime.write_file
+
+    def patched_write_file(self, file_path: str, content: str):  # type: ignore[no-untyped-def]
+        try:
+            return orig_write(self, file_path, content)
+        except FileNotFoundError:
+            # cgroup-v1 paths don't exist in Modal v2 sandboxes — skip
+            # silently for those, re-raise for everything else.
+            if isinstance(file_path, str) and file_path.startswith("/sys/fs/cgroup/"):
+                return None
+            raise
+
+    runtime.write_file = patched_write_file  # type: ignore[assignment]
+    runtime._hybrid_write_file_patched = True  # type: ignore[attr-defined]
+
+
+def _sentinel_present_on_disk() -> bool:
+    """Return True iff the cgroup-v2 sentinel is in the installed swebench file.
+
+    Used to detect that a ``uv sync`` / pip reinstall has reverted the textual
+    patch out from under us while the process is still running. The in-process
+    monkey-patches survive that (they live on the imported module object), but
+    the subprocess fork in :func:`_run_harness` reads the file fresh and would
+    silently regress to the broken version.
+    """
+    try:
+        from swebench.harness.modal_eval import run_evaluation_modal as _m  # type: ignore[import-not-found]
+    except Exception:
+        return False
+    src_path = getattr(_m, "__file__", None)
+    if not src_path:
+        return False
+    try:
+        return _CGROUP_SOURCE_SENTINEL in Path(src_path).read_text()
+    except Exception:
+        return False
+
+
 def _apply_patches_once() -> None:
+    """Apply all swebench patches; idempotent and resilient to disk reverts.
+
+    The in-process flag short-circuits the common case, but if the on-disk
+    sentinel is missing we force a re-apply (covers ``uv sync`` / pip
+    reinstall clobbering the textual rewrite while the process is alive).
+    """
     global _PATCHES_APPLIED
-    if _PATCHES_APPLIED:
+    if _PATCHES_APPLIED and _sentinel_present_on_disk():
         return
     _patch_modal_cgroup_v2()
+    _patch_modal_sandbox_write_file()
+    _patch_modal_sandbox_source()
     _PATCHES_APPLIED = True
 
 
@@ -151,8 +337,8 @@ def _find_report(cache: Path, instance_id: str, run_id: str) -> Optional[Dict[st
     """Find the harness's report JSON for one instance.
 
     swebench writes ``<model_name_or_path>.<run_id>.json`` inside the
-    subprocess CWD. We use ``model_name_or_path="openjarvis-harness"``,
-    ``run_id=f"oj-{instance_id}"`` in :func:`_run_harness`.
+    subprocess CWD. We use ``model_name_or_path="openjarvis-harness"``;
+    ``run_id`` is built by :func:`_build_run_id`.
     """
     fname = f"openjarvis-harness.{run_id}.json"
     p = cache / fname
@@ -164,7 +350,53 @@ def _find_report(cache: Path, instance_id: str, run_id: str) -> Optional[Dict[st
         return None
 
 
-def _run_harness(instance_id: str, patch: str, timeout_s: int) -> Dict[str, Any]:
+_RUN_ID_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def _sanitize_run_id_part(s: str) -> str:
+    """Reduce a free-form string to filesystem-safe ``[A-Za-z0-9._-]+``.
+
+    Both the harness summary filename (``<model>.<run_id>.json``) and the
+    per-instance log subtree (``logs/run_evaluation/<run_id>/...``) are
+    keyed on ``run_id``, so any character that breaks paths or globs will
+    silently corrupt the score. Strip leading/trailing dashes too — those
+    look fine but make filenames awkward to manage by hand.
+    """
+    return _RUN_ID_SAFE_RE.sub("-", s).strip("-")
+
+
+def _build_run_id(instance_id: str, cell_name: Optional[str]) -> str:
+    """Construct a swebench ``run_id`` unique per (cell, instance).
+
+    The harness keys both its "already run, skipping" cache and its report
+    file path on ``run_id`` alone, so two concurrent cells scoring the
+    same ``instance_id`` with the same ``run_id`` collide: the second
+    cell's harness invocation finds the first's report on disk, skips
+    actual execution, and our caller silently reads the wrong verdict (or
+    ``no_report`` if the two cells race on the summary file write). See
+    :func:`_run_harness` for the full failure mode.
+
+    With ``cell_name`` we emit ``oj-<cell>-<instance>``, which keeps the
+    intra-cell resume cache working (same cell + same instance → same
+    run_id → harness cache hit) while making inter-cell collisions
+    impossible. Without ``cell_name`` we fall back to the legacy
+    ``oj-<instance>`` form for backwards compat with single-cell callers.
+    """
+    safe_instance = _sanitize_run_id_part(instance_id)
+    if not cell_name:
+        return f"oj-{safe_instance}"
+    safe_cell = _sanitize_run_id_part(cell_name)
+    if not safe_cell:
+        return f"oj-{safe_instance}"
+    return f"oj-{safe_cell}-{safe_instance}"
+
+
+def _run_harness(
+    instance_id: str,
+    patch: str,
+    timeout_s: int,
+    cell_name: Optional[str] = None,
+) -> Dict[str, Any]:
     """Hand one prediction to ``python -m swebench.harness.run_evaluation``.
 
     Returns ``{"success": bool, "score": float, "details": dict}``.
@@ -172,7 +404,7 @@ def _run_harness(instance_id: str, patch: str, timeout_s: int) -> Dict[str, Any]
     _apply_patches_once()
     backend = os.environ.get("SWEBENCH_BACKEND", "modal").lower()
     cache = _harness_cache_dir()
-    run_id = f"oj-{instance_id}"
+    run_id = _build_run_id(instance_id, cell_name)
 
     # Defend against stale reports: ``run_id`` is deterministic per
     # instance, the cache dir is shared across runs, and ``_find_report``
@@ -206,10 +438,25 @@ def _run_harness(instance_id: str, patch: str, timeout_s: int) -> Dict[str, Any]
         if backend == "modal":
             cmd += ["--modal", "true"]
 
-        proc = subprocess.run(
-            cmd, capture_output=True, text=True,
-            timeout=timeout_s, cwd=str(cache),
-        )
+        try:
+            proc = _run_subprocess_hard_timeout(
+                cmd, timeout_s=timeout_s, cwd=str(cache),
+            )
+        except subprocess.TimeoutExpired as exc:
+            # The harness subprocess (and its Modal grandchildren) exceeded
+            # the cap and were force-killed as a process group. Record an
+            # error verdict rather than letting the exception bubble — the
+            # caller's row stays well-formed and the cell keeps moving.
+            return {
+                "success": False,
+                "score": 0.0,
+                "details": {
+                    "reason": "harness_timeout",
+                    "timeout_s": timeout_s,
+                    "stdout": (exc.stdout or "")[-2000:],
+                    "stderr": (exc.stderr or "")[-2000:],
+                },
+            }
 
         report = _find_report(cache, instance_id, run_id)
         if report is None:
@@ -261,10 +508,17 @@ class SWEBenchHarnessScorer(Scorer):
         self,
         *,
         timeout_s: int = 1800,
+        cell_name: Optional[str] = None,
         judge_backend: object = None,  # noqa: ARG002 — CLI factory compat
         judge_model: str = "",         # noqa: ARG002 — CLI factory compat
     ) -> None:
         self._timeout_s = int(timeout_s)
+        # ``cell_name`` namespaces the ``run_id`` so concurrent cells scoring
+        # the same SWE instance don't collide on the harness's shared cache.
+        # See :func:`_build_run_id` for the failure mode this prevents. Pass
+        # the hybrid cell name (e.g. ``"skillorchestra-qwen36-opus47-swe-n100"``)
+        # or leave as ``None`` for single-cell callers.
+        self._cell_name = cell_name
 
     def score(
         self,
@@ -286,10 +540,17 @@ class SWEBenchHarnessScorer(Scorer):
         if not instance_id:
             return False, {"reason": "missing_instance_id"}
 
-        result = _run_harness(instance_id, patch, self._timeout_s)
+        result = _run_harness(
+            instance_id, patch, self._timeout_s, cell_name=self._cell_name,
+        )
         details = dict(result.get("details", {}))
         details["patch"] = patch
         return bool(result["success"]), details
 
 
-__all__ = ["SWEBenchHarnessScorer", "extract_patch"]
+__all__ = [
+    "SWEBenchHarnessScorer",
+    "extract_patch",
+    "_build_run_id",
+    "_sanitize_run_id_part",
+]

tests/agents/hybrid/test_openai_adapter.py

@@ -0,0 +1,222 @@
+"""Regression tests for the mini-SWE-agent OpenAI cloud + bash adapter.
+
+Two failure modes caught in the n=100 hybrid SWE sweep (May 2026) — both
+deterministic enough to pin down here without hitting any real model or
+shelling out for real binary data:
+
+1. **Bug 1 — null assistant content.** ``_loop_cloud_openai`` used to
+   append ``{"role": "assistant", "content": text or None}`` on each
+   turn. On a tool-only turn (model produced no text alongside its
+   ``bash`` call) that wrote ``content: null`` into the message list;
+   the next ``chat.completions.create`` then 400'd with
+   ``Invalid value for 'content': expected a string, got null``. Fix
+   uses ``""`` (or omitted) per OpenAI's schema.
+
+2. **Bug 3 — binary bash output crashes the loop.** ``_run_bash`` used
+   to pass ``text=True`` to ``subprocess.Popen``, so any command that
+   emitted non-UTF-8 bytes (cat'ing a compiled artifact, an image, a
+   PDF) raised ``UnicodeDecodeError`` from inside
+   ``Popen.communicate()`` and killed the whole task. Fix captures
+   bytes and decodes via ``_decode_bash_output`` with ``errors="replace"``
+   plus a binary-detection stub.
+
+Run with:
+
+    .venv/bin/python -m pytest tests/agents/hybrid/test_openai_adapter.py -v
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List
+
+import pytest
+
+from openjarvis.agents.hybrid.mini_swe_agent import (
+    _decode_bash_output,
+    _loop_cloud_openai,
+    _run_bash,
+)
+
+
+# ---------------------------------------------------------------------------
+# Bug 3 — binary bash output
+# ---------------------------------------------------------------------------
+
+
+class TestDecodeBashOutput:
+    def test_pure_ascii_passes_through(self) -> None:
+        assert _decode_bash_output(b"hello world\n", 0) == "hello world\n"
+
+    def test_empty_bytes_returns_empty_string(self) -> None:
+        assert _decode_bash_output(b"", 0) == ""
+
+    def test_nul_byte_substituted_with_stub(self) -> None:
+        raw = b"some text\x00more bytes after"
+        out = _decode_bash_output(raw, 0)
+        assert out.startswith("[binary output:")
+        assert f"{len(raw)} bytes" in out
+        assert "exit=0" in out
+
+    def test_invalid_utf8_partial_substitutes_replacement_char(self) -> None:
+        # ~25% replacement chars after decode — well above the 5% binary
+        # threshold, should swap to the stub. The exact byte sequence
+        # 0xe0 is the one observed in the astropy__astropy-14539 row.
+        raw = b"abc\xe0\xe0\xe0def"
+        out = _decode_bash_output(raw, 1)
+        assert out.startswith("[binary output:")
+        assert "exit=1" in out
+
+    def test_mostly_valid_utf8_keeps_decoded_text(self) -> None:
+        # One stray bad byte in a long string → below 5% replacement,
+        # keep the (mostly intact) decoded text rather than stubbing.
+        raw = ("readable text " * 200).encode("utf-8") + b"\xe0"
+        out = _decode_bash_output(raw, 0)
+        assert "readable text" in out
+        assert not out.startswith("[binary output:")
+
+    def test_real_bash_run_on_binary_does_not_raise(
+        self, tmp_path: Path
+    ) -> None:
+        # End-to-end: a model-issued ``head`` on a binary file. Pre-fix
+        # this raised UnicodeDecodeError out of ``_run_bash`` →
+        # propagated all the way up to the runner. Post-fix it returns
+        # a normal observation dict with the binary-output stub.
+        bin_path = tmp_path / "blob.bin"
+        bin_path.write_bytes(bytes(range(256)) * 4)
+        result = _run_bash(
+            f"head -c 1024 {bin_path}", tmp_path,
+            timeout=10, output_cap=10_000,
+        )
+        assert result["exit_code"] == 0
+        assert result["timed_out"] is False
+        assert "[binary output:" in result["stdout"]
+
+
+# ---------------------------------------------------------------------------
+# Bug 1 — null content on tool-only assistant turn
+# ---------------------------------------------------------------------------
+
+
+class _FakeFunction:
+    def __init__(self, name: str, arguments: str) -> None:
+        self.name = name
+        self.arguments = arguments
+
+
+class _FakeToolCall:
+    def __init__(self, tc_id: str, name: str, arguments: str) -> None:
+        self.id = tc_id
+        self.type = "function"
+        self.function = _FakeFunction(name, arguments)
+
+
+class _FakeMessage:
+    def __init__(self, content: Any, tool_calls: List[Any]) -> None:
+        self.content = content
+        self.tool_calls = tool_calls
+
+
+class _FakeChoice:
+    def __init__(
+        self, message: _FakeMessage, finish_reason: str = "tool_calls"
+    ) -> None:
+        self.message = message
+        self.finish_reason = finish_reason
+
+
+class _FakeUsage:
+    prompt_tokens = 10
+    completion_tokens = 5
+
+
+class _FakeResp:
+    def __init__(self, choice: _FakeChoice) -> None:
+        self.choices = [choice]
+        self.usage = _FakeUsage()
+
+
+class _FakeCompletions:
+    """Records every (messages=...) the model is asked to score against.
+
+    Returns a scripted sequence: turn 1 → tool_only (no text), turn 2 →
+    final summary text, no tool_calls. We then assert that turn 2's
+    inbound messages contain the turn-1 assistant message with
+    ``content == ""`` (or omitted) — never ``None``, which is the bug.
+    """
+
+    def __init__(self, scripted: List[_FakeResp]) -> None:
+        self._scripted = list(scripted)
+        self.calls: List[List[Dict[str, Any]]] = []
+
+    def create(self, **kwargs: Any) -> _FakeResp:
+        # Deep enough copy so the agent appending to its messages list
+        # post-call doesn't mutate what we recorded here.
+        self.calls.append([dict(m) for m in kwargs["messages"]])
+        return self._scripted.pop(0)
+
+
+class _FakeChat:
+    def __init__(self, completions: _FakeCompletions) -> None:
+        self.completions = completions
+
+
+class _FakeClient:
+    def __init__(self, completions: _FakeCompletions) -> None:
+        self.chat = _FakeChat(completions)
+
+
+def test_assistant_message_content_never_none_on_tool_only_turn(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The OpenAI SDK 400s with ``content: null`` on replay. We must
+    serialize tool-only assistant turns as ``content == ""`` (or omit
+    the field) — never ``None``. Pre-fix this test failed because turn
+    2's recorded messages had ``messages[2]["content"] is None``.
+    """
+
+    tool_turn = _FakeResp(_FakeChoice(
+        _FakeMessage(
+            content=None,
+            tool_calls=[_FakeToolCall("c1", "bash", '{"command": "echo hi"}')],
+        ),
+    ))
+    done_turn = _FakeResp(_FakeChoice(
+        _FakeMessage(content="all done", tool_calls=[]),
+        finish_reason="stop",
+    ))
+    fake = _FakeCompletions([tool_turn, done_turn])
+    fake_client = _FakeClient(fake)
+
+    def _fake_openai_ctor(**kwargs: Any) -> _FakeClient:
+        return fake_client
+
+    # Swap in our fake at the import site inside _loop_cloud_openai.
+    import openai
+    monkeypatch.setattr(openai, "OpenAI", _fake_openai_ctor)
+
+    out = _loop_cloud_openai(
+        "fake problem", tmp_path,
+        model="gpt-5-mini-2025-08-07",
+        max_turns=4, bash_timeout=10, output_cap=10_000,
+        turn_max_tokens=64, trace_prefix="test",
+    )
+
+    # Two real model calls: tool-issuing turn + final turn.
+    assert len(fake.calls) == 2
+
+    second_call_messages = fake.calls[1]
+    # First two messages are system + user. The assistant turn from turn
+    # 1 should be index 2; its content must be a string (empty is fine),
+    # never None — that's the bug we're regressing.
+    assistant_msg = second_call_messages[2]
+    assert assistant_msg["role"] == "assistant"
+    assert "content" in assistant_msg
+    assert assistant_msg["content"] is not None
+    assert isinstance(assistant_msg["content"], str)
+    # Tool calls must still be present (we only fixed the content shape).
+    assert assistant_msg.get("tool_calls")
+
+    # Sanity: the loop terminated normally on the no-tool turn.
+    assert out["final_summary"] == "all done"
+    assert out["turns"] == 2

tests/agents/hybrid/test_openai_retry.py

@@ -0,0 +1,312 @@
+"""Smoke tests for the OpenAI SDK retry + per-org concurrency hardening.
+
+We deliberately don't hit the real OpenAI API. Instead we monkey-patch
+the underlying call to simulate the failure modes we care about:
+
+- Sustained ``RateLimitError`` walls.
+- ``APITimeoutError`` blips.
+- ``APIConnectionError`` blips.
+- ``InternalServerError`` 5xx blips.
+
+Then we hammer ``openai.OpenAI().chat.completions.create`` from a thread
+pool with a deliberately tight semaphore (concurrency=2, retries=4) and
+confirm:
+
+(a) no exceptions escape when the failure clears within the retry budget,
+(b) backoff actually fires (call count > attempted call count),
+(c) all requests eventually complete,
+(d) when the failure exceeds the retry budget, the final exception
+    propagates so the runner records ``error=...`` (no silent drop),
+(e) local vLLM-style clients (api_key="EMPTY" or localhost base_url)
+    bypass both the throttle and the retry loop.
+
+Run with:
+
+    .venv/bin/python -m pytest tests/agents/hybrid/test_openai_retry.py -v
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, List
+from unittest.mock import MagicMock
+
+import pytest
+
+# Make sure the patch picks up tight test settings, not the prod defaults.
+# Must be set BEFORE _openai_retry is imported.
+os.environ["OPENJARVIS_OPENAI_MAX_CONCURRENCY"] = "2"
+os.environ["OPENJARVIS_OPENAI_MAX_RETRIES"] = "4"
+os.environ["OPENJARVIS_OPENAI_RETRY_BASE"] = "0.05"   # fast tests
+os.environ["OPENJARVIS_OPENAI_RETRY_CAP"] = "0.2"
+
+
+# Reload the module fresh so env vars take effect (imports earlier in
+# the process may have frozen the defaults).
+import importlib
+
+from openjarvis.agents.hybrid import _openai_retry as _retry_mod
+
+importlib.reload(_retry_mod)
+_retry_mod.patch_openai_globally()
+
+
+def _make_fake_response() -> Any:
+    """Minimal stand-in for an OpenAI ChatCompletion response."""
+    r = MagicMock()
+    r.choices = [MagicMock(message=MagicMock(content="ok", tool_calls=None))]
+    r.usage = MagicMock(prompt_tokens=1, completion_tokens=1)
+    return r
+
+
+def _swap_underlying_create(
+    monkeypatch: pytest.MonkeyPatch, side_effect: Any,
+) -> List[int]:
+    """Replace the wrapped-underlying-orig with one that fires ``side_effect``.
+
+    Returns a mutable call-counter list so tests can assert on attempts.
+    """
+    from openai.resources.chat import completions as _comp_mod
+
+    counter: List[int] = [0]
+    wrapped = _comp_mod.Completions.create
+    # The patcher stashed the original at ``__wrapped__``.
+    orig = getattr(wrapped, "__wrapped__", None)
+    assert orig is not None, "patch_openai_globally didn't stash __wrapped__"
+
+    def fake(self: Any, *args: Any, **kwargs: Any) -> Any:
+        counter[0] += 1
+        if callable(side_effect):
+            return side_effect(counter[0])
+        if isinstance(side_effect, list):
+            i = min(counter[0] - 1, len(side_effect) - 1)
+            v = side_effect[i]
+            if isinstance(v, BaseException):
+                raise v
+            return v
+        if isinstance(side_effect, BaseException):
+            raise side_effect
+        return side_effect
+
+    # Replace the wrapped underlying directly.
+    new_wrap = _retry_mod._wrap_create(fake)
+    monkeypatch.setattr(_comp_mod.Completions, "create", new_wrap)
+    return counter
+
+
+def _rate_limit_error() -> BaseException:
+    """Build an ``openai.RateLimitError`` that the SDK would normally raise."""
+    import openai
+
+    # The SDK's RateLimitError wants (message, response, body) — we use a
+    # MagicMock for response so ``response.headers.get("retry-after")``
+    # returns None.
+    resp = MagicMock()
+    resp.headers = {}
+    resp.status_code = 429
+    return openai.RateLimitError("rate limit", response=resp, body=None)
+
+
+def _api_timeout_error() -> BaseException:
+    import openai
+
+    return openai.APITimeoutError(request=MagicMock())
+
+
+def _api_conn_error() -> BaseException:
+    import openai
+
+    return openai.APIConnectionError(request=MagicMock())
+
+
+def _internal_500_error() -> BaseException:
+    import openai
+
+    resp = MagicMock()
+    resp.headers = {}
+    resp.status_code = 500
+    return openai.InternalServerError("server error", response=resp, body=None)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_patches_installed() -> None:
+    import openai
+    from openai.resources.chat import completions
+
+    assert getattr(completions.Completions.create, "_hybrid_patched", False)
+    assert getattr(openai.OpenAI.__init__, "_hybrid_patched", False)
+
+
+def test_rate_limit_then_success(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Two 429s then a real response — retry path should swallow both."""
+    import openai
+
+    counter = _swap_underlying_create(
+        monkeypatch,
+        [_rate_limit_error(), _rate_limit_error(), _make_fake_response()],
+    )
+    client = openai.OpenAI(api_key="sk-fake")
+    resp = client.chat.completions.create(
+        model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+    )
+    assert resp.choices[0].message.content == "ok"
+    assert counter[0] == 3  # backoff fired twice, then succeeded
+
+
+def test_timeout_then_success(monkeypatch: pytest.MonkeyPatch) -> None:
+    import openai
+
+    counter = _swap_underlying_create(
+        monkeypatch, [_api_timeout_error(), _make_fake_response()]
+    )
+    client = openai.OpenAI(api_key="sk-fake")
+    resp = client.chat.completions.create(
+        model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+    )
+    assert resp.choices[0].message.content == "ok"
+    assert counter[0] == 2
+
+
+def test_500_then_success(monkeypatch: pytest.MonkeyPatch) -> None:
+    import openai
+
+    counter = _swap_underlying_create(
+        monkeypatch, [_internal_500_error(), _make_fake_response()]
+    )
+    client = openai.OpenAI(api_key="sk-fake")
+    resp = client.chat.completions.create(
+        model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+    )
+    assert resp.choices[0].message.content == "ok"
+    assert counter[0] == 2
+
+
+def test_retry_exhaustion_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Sustained 429 wall beyond retry budget → exception propagates."""
+    import openai
+
+    # max_retries=4 → 5 total attempts; feed 6 errors so the loop runs out.
+    errors = [_rate_limit_error() for _ in range(6)]
+    counter = _swap_underlying_create(monkeypatch, errors)
+    client = openai.OpenAI(api_key="sk-fake")
+    with pytest.raises(openai.RateLimitError):
+        client.chat.completions.create(
+            model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+        )
+    # 5 attempts: 1 initial + 4 retries.
+    assert counter[0] == 5
+
+
+def test_non_retryable_propagates_immediately(monkeypatch: pytest.MonkeyPatch) -> None:
+    """``BadRequestError`` is NOT retryable — should raise on attempt 1."""
+    import openai
+
+    resp = MagicMock()
+    resp.headers = {}
+    resp.status_code = 400
+    bad = openai.BadRequestError("nope", response=resp, body=None)
+    counter = _swap_underlying_create(monkeypatch, [bad])
+    client = openai.OpenAI(api_key="sk-fake")
+    with pytest.raises(openai.BadRequestError):
+        client.chat.completions.create(
+            model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+        )
+    assert counter[0] == 1  # no retries
+
+
+def test_local_vllm_bypasses_throttle_and_retry(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Local vLLM clients (api_key=EMPTY) must NOT pay the per-org throttle.
+
+    Verifies (a) detection works, (b) a non-retryable error fires only
+    once (no retry attempts for local endpoints — they're either up or
+    down). Also (c) the semaphore isn't acquired (which we can't directly
+    observe, but we can confirm the wrapper short-circuits).
+    """
+    import openai
+
+    counter = _swap_underlying_create(monkeypatch, _rate_limit_error())
+    client = openai.OpenAI(base_url="http://localhost:8001/v1", api_key="EMPTY")
+    # Should raise on attempt 1 (no retries for local endpoints).
+    with pytest.raises(openai.RateLimitError):
+        client.chat.completions.create(
+            model="qwen", messages=[{"role": "user", "content": "hi"}]
+        )
+    assert counter[0] == 1
+
+
+def test_concurrent_hammering_no_escape(monkeypatch: pytest.MonkeyPatch) -> None:
+    """16 concurrent calls, each fails twice with 429 then succeeds.
+
+    With concurrency=2 the semaphore queues, with retry budget=4 every
+    call eventually gets through. We confirm no exception escapes and
+    all 16 calls complete with the fake response.
+    """
+    import openai
+
+    # Per-call counter for failure injection.
+    call_state: dict = {}
+    state_lock = threading.Lock()
+
+    def side_effect(call_idx: int) -> Any:
+        # Tag by thread so each "logical call" fails its first 2 attempts.
+        tid = threading.get_ident()
+        with state_lock:
+            n = call_state.get(tid, 0) + 1
+            call_state[tid] = n
+        if n <= 2:
+            raise _rate_limit_error()
+        # Reset for the next logical call from this thread.
+        with state_lock:
+            call_state[tid] = 0
+        return _make_fake_response()
+
+    _swap_underlying_create(monkeypatch, side_effect)
+    client = openai.OpenAI(api_key="sk-fake")
+
+    def one() -> str:
+        r = client.chat.completions.create(
+            model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+        )
+        return r.choices[0].message.content
+
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=16) as ex:
+        futures = [ex.submit(one) for _ in range(16)]
+        results = [f.result() for f in as_completed(futures)]
+    elapsed = time.time() - t0
+    assert all(r == "ok" for r in results)
+    assert len(results) == 16
+    # Sanity: with concurrency=2 + backoff>=0.05s * 2 retries per call,
+    # 16 calls can't possibly finish instantly. Just confirm we spent
+    # *some* time in backoff, not that we measured it precisely.
+    assert elapsed > 0.1
+
+
+def test_retry_after_honored(monkeypatch: pytest.MonkeyPatch) -> None:
+    """If the SDK exposes a Retry-After header, we sleep at least that long."""
+    import openai
+
+    resp = MagicMock()
+    resp.headers = {"retry-after": "0.15"}
+    resp.status_code = 429
+    err = openai.RateLimitError("rate limit", response=resp, body=None)
+
+    counter = _swap_underlying_create(monkeypatch, [err, _make_fake_response()])
+    client = openai.OpenAI(api_key="sk-fake")
+    t0 = time.time()
+    client.chat.completions.create(
+        model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}]
+    )
+    elapsed = time.time() - t0
+    assert counter[0] == 2
+    # We sleep at least 0.15s (Retry-After) — jitter is additive ≤ 0.5.
+    assert elapsed >= 0.15

tests/agents/hybrid/test_registry_validation.py

@@ -0,0 +1,119 @@
+"""Regression tests for hybrid registry validation (Bug 5, 2026-05-15).
+
+Skillorchestra's SWE codepath gates on `method_cfg.swe_use_agent_loop`.
+Without that flag the cell silently falls back to a one-shot cloud call
+on SWE-bench tasks, which is almost never the intent. The registry
+loader (`load_registry`) now validates every skillorchestra SWE cell has
+the flag set and raises ValueError at load time if any are missing.
+
+Tests:
+1. Real registries on disk pass validation (no missing flags today).
+2. A synthetic registry with the flag missing fails with a clear error
+   naming the offending cell.
+3. The same synthetic cell with the flag set loads cleanly.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openjarvis.agents.hybrid.runner import (
+    DEFAULT_REGISTRY_DIR,
+    _SWE_BENCHES,
+    load_registry,
+)
+
+
+# ---------------------------------------------------------------------------
+# 1. Real on-disk registries: every skillorchestra SWE cell must already
+#    carry `swe_use_agent_loop = true`. This guards against future edits
+#    that drop the flag (the exact failure mode of Bug 5).
+# ---------------------------------------------------------------------------
+
+def test_skillorchestra_swe_has_loop_flag() -> None:
+    cells = load_registry(DEFAULT_REGISTRY_DIR)
+    swe_cells = {
+        name: cell
+        for name, cell in cells.items()
+        if cell.get("method") == "skillorchestra"
+        and cell.get("bench") in _SWE_BENCHES
+    }
+    assert swe_cells, (
+        "expected at least one skillorchestra SWE cell in the bundled "
+        "registry; if you deleted them all, drop this test too."
+    )
+    missing = [
+        name
+        for name, cell in swe_cells.items()
+        if not bool((cell.get("method_cfg") or {}).get("swe_use_agent_loop"))
+    ]
+    assert not missing, (
+        f"skillorchestra SWE cells missing swe_use_agent_loop: {missing}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# 2/3. Synthetic registry: round-trip a single SWE cell with and without
+#      the flag to confirm the validator fires (and only fires) at the
+#      right time.
+# ---------------------------------------------------------------------------
+
+_BAD_TOML = """\
+[cells.skillorchestra-fake-swe-3]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 3
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
+method_cfg = {}
+"""
+
+_GOOD_TOML = """\
+[cells.skillorchestra-fake-swe-3]
+method = "skillorchestra"
+bench  = "swebench-verified"
+n      = 3
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
+method_cfg = { swe_use_agent_loop = true, swe_max_turns = 10 }
+"""
+
+
+def test_registry_loader_rejects_missing_flag(tmp_path: Path) -> None:
+    (tmp_path / "fake.toml").write_text(_BAD_TOML)
+    with pytest.raises(ValueError) as ei:
+        load_registry(tmp_path)
+    msg = str(ei.value)
+    assert "skillorchestra-fake-swe-3" in msg, msg
+    assert "swe_use_agent_loop" in msg, msg
+
+
+def test_registry_loader_accepts_with_flag(tmp_path: Path) -> None:
+    (tmp_path / "fake.toml").write_text(_GOOD_TOML)
+    cells = load_registry(tmp_path)
+    assert "skillorchestra-fake-swe-3" in cells
+    mcfg = cells["skillorchestra-fake-swe-3"]["method_cfg"]
+    assert mcfg["swe_use_agent_loop"] is True
+
+
+# ---------------------------------------------------------------------------
+# Sanity: non-SWE benches and non-skillorchestra methods are unaffected.
+# ---------------------------------------------------------------------------
+
+_GAIA_NO_FLAG = """\
+[cells.skillorchestra-fake-gaia-3]
+method = "skillorchestra"
+bench  = "gaia"
+n      = 3
+local  = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
+cloud  = { model = "claude-opus-4-7",      endpoint = "anthropic" }
+method_cfg = {}
+"""
+
+
+def test_validator_ignores_non_swe_skillorchestra(tmp_path: Path) -> None:
+    (tmp_path / "fake.toml").write_text(_GAIA_NO_FLAG)
+    cells = load_registry(tmp_path)
+    assert "skillorchestra-fake-gaia-3" in cells

tests/agents/hybrid/test_swebench_dataset_variant.py

@@ -0,0 +1,68 @@
+"""Regression test for the SWE-bench variant bug.
+
+`SWEBenchDataset()` defaulted to the 50-task `verified_mini` variant, but
+hybrid n=100 subsets reference task_ids from the full 500-task
+`princeton-nlp/SWE-bench_Verified`. The runner now passes
+``variant="verified"`` explicitly; this test pins that behavior so the
+bug can't silently regress.
+
+Requires HF_TOKEN and a populated HF cache (or network). Cap each test
+at 60s; HF download can be slow on first run but the cache should be
+warm on the mkt cluster.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from openjarvis.agents.hybrid.runner import DEFAULT_SUBSETS_DIR, _load_swebench_tasks
+
+
+SUBSET_PATH = Path(
+    os.environ.get(
+        "HYBRID_SWEBENCH_N100_SUBSET",
+        DEFAULT_SUBSETS_DIR / "swebench_verified_n100_seed42.json",
+    )
+)
+KNOWN_FULL_TASK_ID = "pytest-dev__pytest-10081"
+
+
+def test_loads_full_verified_not_mini() -> None:
+    """No max_samples → at least 500 records (full Verified set, not 50)."""
+    tasks = _load_swebench_tasks(n=None)
+    assert len(tasks) >= 500, (
+        f"expected >=500 from full SWE-bench_Verified, got {len(tasks)}; "
+        "runner may have regressed to the 50-task verified_mini variant"
+    )
+
+
+def test_n100_returns_100_records() -> None:
+    """n=100 against the full variant → 100 records (not capped at 50)."""
+    tasks = _load_swebench_tasks(n=100)
+    assert len(tasks) == 100, (
+        f"expected 100 records, got {len(tasks)}; if 50 the runner is "
+        "loading verified_mini again"
+    )
+
+
+def test_known_full_task_id_present() -> None:
+    """`pytest-dev__pytest-10081` (from subset n=100 seed=42) must load.
+
+    This is the exact bug we hit: 94/100 subset ids were missing because
+    the dataset only had 50 mini-variant rows.
+    """
+    if SUBSET_PATH.exists():
+        data = json.loads(SUBSET_PATH.read_text())
+        ids = data["task_ids"] if isinstance(data, dict) else list(data)
+        assert KNOWN_FULL_TASK_ID in ids, (
+            f"sanity: {KNOWN_FULL_TASK_ID} should be in {SUBSET_PATH.name}"
+        )
+
+    tasks = _load_swebench_tasks(n=None)
+    loaded_ids = {t["task_id"] for t in tasks}
+    assert KNOWN_FULL_TASK_ID in loaded_ids, (
+        f"{KNOWN_FULL_TASK_ID} missing from loaded SWE-bench records; "
+        "runner is probably back on verified_mini"
+    )

tests/agents/hybrid/test_swebench_run_id_isolation.py

@@ -0,0 +1,143 @@
+"""Regression test for SWE-bench ``run_id`` collisions across concurrent cells.
+
+Failure mode (observed 2026-05-18, twice):
+    Multiple hybrid SWE cells running concurrently against the Modal
+    ``swebench-harness`` shared its per-instance cache via a ``run_id``
+    keyed only on ``instance_id``. Two cells scoring the same task hit
+    "1 instances already run, skipping..." on the second call; the
+    runner saw ``reason: no_report`` and scored that row 0 even when
+    the patch was correct. First seen with
+    ``minions-qwen27b-opus47-swe-n100`` vs the advisors cell; recurred
+    on 2026-05-18 across three of four ``qwen36`` SWE cells.
+
+Fix: ``_build_run_id(instance_id, cell_name)`` namespaces the run_id by
+cell so concurrent cells can't collide while same-cell resumes still
+hit the harness cache.
+
+This test pins the contract:
+
+1. Different cells + same instance → different run_ids (the bug).
+2. Same cell + same instance → identical run_id (resume still works).
+3. Both forms are filesystem-safe (no chars that would break the
+   ``logs/run_evaluation/<run_id>/...`` subtree or the
+   ``<model>.<run_id>.json`` summary glob).
+4. Cell name flows from :class:`SWEBenchHarnessScorer` into the run_id —
+   wiring regression guard, in case someone refactors and drops the
+   ``cell_name`` kwarg.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pytest
+
+from openjarvis.evals.scorers.swebench_harness import (
+    SWEBenchHarnessScorer,
+    _build_run_id,
+    _sanitize_run_id_part,
+)
+
+
+# Filenames + harness log paths must match this. Slashes, spaces, colons,
+# and other shell-hostile chars would silently corrupt scoring.
+_SAFE_RE = re.compile(r"^[A-Za-z0-9._-]+$")
+
+
+INSTANCE = "astropy__astropy-12907"
+CELL_A = "advisors-qwen36-opus47-swe-n100"
+CELL_B = "skillorchestra-qwen36-opus47-swe-n100"
+
+
+def test_different_cells_produce_different_run_ids():
+    """The core bug: two cells, same instance → must not collide."""
+    a = _build_run_id(INSTANCE, CELL_A)
+    b = _build_run_id(INSTANCE, CELL_B)
+    assert a != b, (
+        f"run_id collision between cells {CELL_A!r} and {CELL_B!r} "
+        f"on instance {INSTANCE!r}: both produced {a!r}"
+    )
+    # Both must still contain the instance id so the harness logs are
+    # greppable by instance.
+    assert INSTANCE in a
+    assert INSTANCE in b
+
+
+def test_same_cell_same_instance_is_stable():
+    """Resume semantics: re-running the same cell must hit the cache."""
+    assert _build_run_id(INSTANCE, CELL_A) == _build_run_id(INSTANCE, CELL_A)
+
+
+def test_legacy_no_cell_name_preserves_old_format():
+    """Single-cell callers (no ``cell_name``) keep the legacy ``oj-<id>``."""
+    assert _build_run_id(INSTANCE, None) == f"oj-{INSTANCE}"
+    assert _build_run_id(INSTANCE, "") == f"oj-{INSTANCE}"
+
+
+@pytest.mark.parametrize(
+    "cell_name",
+    [
+        CELL_A,
+        CELL_B,
+        "minions-qwen27b-opus47-swe-n100",
+        # Hostile inputs — must still produce something path-safe.
+        "weird/cell name with spaces",
+        "cell:with:colons",
+        "---leading-and-trailing---",
+    ],
+)
+def test_run_id_is_filesystem_safe(cell_name: str):
+    """Whatever we feed in, the run_id must be a clean path component."""
+    rid = _build_run_id(INSTANCE, cell_name)
+    assert _SAFE_RE.match(rid), f"run_id {rid!r} contains unsafe chars"
+
+
+def test_sanitizer_strips_unsafe_chars():
+    assert _sanitize_run_id_part("a/b c:d") == "a-b-c-d"
+    assert _sanitize_run_id_part("---x---") == "x"
+    assert _sanitize_run_id_part("ok-cell.1_2") == "ok-cell.1_2"
+
+
+def test_scorer_threads_cell_name_into_run_id(monkeypatch):
+    """End-to-end wiring: scorer constructor → ``_run_harness`` → run_id.
+
+    Guards against a refactor that quietly drops the ``cell_name`` kwarg
+    on the scorer or stops forwarding it to ``_run_harness``.
+    """
+    from openjarvis.evals.core.types import EvalRecord
+    from openjarvis.evals.scorers import swebench_harness as mod
+
+    captured: dict = {}
+
+    def fake_run_harness(instance_id, patch, timeout_s, cell_name=None):
+        captured["instance_id"] = instance_id
+        captured["cell_name"] = cell_name
+        captured["run_id"] = mod._build_run_id(instance_id, cell_name)
+        return {"success": True, "score": 1.0, "details": {}}
+
+    monkeypatch.setattr(mod, "_run_harness", fake_run_harness)
+
+    scorer = SWEBenchHarnessScorer(timeout_s=60, cell_name=CELL_A)
+    record = EvalRecord(
+        record_id=INSTANCE,
+        problem="",
+        reference="",
+        category="agentic",
+        metadata={"instance_id": INSTANCE},
+    )
+    # Minimal patch text that ``extract_patch`` will accept.
+    answer = "```diff\ndiff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -0,0 +1 @@\n+x\n```"
+    scorer.score(record, answer)
+
+    assert captured["cell_name"] == CELL_A
+    assert captured["instance_id"] == INSTANCE
+    assert CELL_A in captured["run_id"]
+    assert INSTANCE in captured["run_id"]
+
+    # And the other cell name produces a *different* run_id end-to-end.
+    captured.clear()
+    scorer_b = SWEBenchHarnessScorer(timeout_s=60, cell_name=CELL_B)
+    scorer_b.score(record, answer)
+    assert captured["cell_name"] == CELL_B
+    assert CELL_B in captured["run_id"]
+    assert CELL_A not in captured["run_id"]

tests/agents/hybrid/test_tool_calls_tracking.py

@@ -0,0 +1,452 @@
+"""Per-row ``tool_calls`` tracking across hybrid paradigms (2026-05-18).
+
+Confirms each paradigm surfaces a top-level ``tool_calls: int`` in the
+``AgentResult.metadata`` so the runner can write it into ``results.jsonl``.
+Definition per paradigm:
+
+  - SWE-bench cells: bash turns from ``run_swe_agent_loop``. One tool call
+    per bash command the agent ran.
+  - GAIA cells: number of native ``web_search`` invocations the cloud
+    backbone made. Zero for one-shot GAIA paths and zero for paradigms
+    whose GAIA path is just text-passing (e.g. Minions w/o prefetch).
+
+The legacy ``skillorchestra`` tool_calls numbers in
+``docs/results-table.md`` came from a defunct telemetry path; this
+re-establishes the contract on the actual row-write path.
+
+Companion follow-up (same day): the same row also carries
+``n_cloud_calls`` and ``n_local_calls`` — full LLM round-trip count per
+task, alongside ``tool_calls``. Counters live in ``_base._CALL_COUNTS``
+(thread-local, parallel to the trace buffer) and are bumped inside every
+SDK helper. Each paradigm test below asserts both are ``int`` and ``>=0``.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from openjarvis.agents._stubs import AgentContext
+
+
+def _assert_call_counts(meta: dict) -> None:
+    """Shared shape check for n_cloud_calls / n_local_calls in metadata.
+
+    Every paradigm result must expose both as ``int >= 0``; ``None`` is
+    not allowed (the runner casts to int unconditionally).
+    """
+    n_cloud = meta.get("n_cloud_calls")
+    n_local = meta.get("n_local_calls")
+    assert isinstance(n_cloud, int), f"n_cloud_calls not int: {n_cloud!r}"
+    assert isinstance(n_local, int), f"n_local_calls not int: {n_local!r}"
+    assert n_cloud >= 0
+    assert n_local >= 0
+
+
+# ---------------------------------------------------------------------------
+# Anthropic stub (web_search agent loop emits N web_search_requests).
+# ---------------------------------------------------------------------------
+
+def _fake_anthropic_response(text: str = "FINAL ANSWER: 7", n_searches: int = 3):
+    return SimpleNamespace(
+        content=[SimpleNamespace(type="text", text=text)],
+        usage=SimpleNamespace(
+            input_tokens=100,
+            output_tokens=20,
+            server_tool_use=SimpleNamespace(web_search_requests=n_searches),
+        ),
+        stop_reason="end_turn",
+    )
+
+
+class _FakeMessages:
+    def __init__(self, n_searches: int = 3):
+        self.n_searches = n_searches
+        self.calls = 0
+
+    def create(self, **kwargs):
+        self.calls += 1
+        # Each anthropic call returns the same shape; total searches
+        # accumulate via repeated calls (advisors does 2 executor passes,
+        # each reporting its own n_searches).
+        return _fake_anthropic_response(n_searches=self.n_searches)
+
+
+class _FakeAnthropic:
+    last_messages = None
+
+    def __init__(self, *args, **kwargs):
+        self.messages = _FakeMessages(n_searches=3)
+        type(self).last_messages = self.messages
+
+
+@pytest.fixture
+def fake_anthropic(monkeypatch):
+    import anthropic
+    monkeypatch.setattr(anthropic, "Anthropic", _FakeAnthropic)
+    yield _FakeAnthropic
+
+
+# ---------------------------------------------------------------------------
+# 1. baseline_cloud GAIA — tool_calls == n_searches on the agent-loop path.
+# ---------------------------------------------------------------------------
+
+def test_baseline_cloud_gaia_tool_calls_eq_web_searches(fake_anthropic):
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        cfg={
+            "cloud_max_tokens": 1024,
+            "web_search": {"enabled": True, "max_uses": 4},
+            "gaia_max_turns": 2,
+        },
+    )
+    ctx = AgentContext(metadata={"task": {"task_id": "t1", "question": "X?"}, "task_id": "t1"})
+    result = agent.run("X?", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    assert tc == result.metadata["web_search_uses"] == 3
+    _assert_call_counts(result.metadata)
+    # One turn of _call_anthropic_agent (fake returns end_turn immediately).
+    assert result.metadata["n_cloud_calls"] == 1
+    assert result.metadata["n_local_calls"] == 0
+
+
+def test_baseline_cloud_gaia_oneshot_tool_calls_zero(monkeypatch):
+    """The one-shot GAIA path makes zero countable tool calls."""
+    from openjarvis.agents.hybrid import baseline_cloud as bc_mod
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    def fake_call_cloud(self, *, user, system=None, max_tokens=4096,
+                       temperature=0.0, **kwargs):
+        return "FINAL ANSWER: x", 10, 5
+
+    monkeypatch.setattr(bc_mod.BaselineCloudAgent, "_call_cloud", fake_call_cloud)
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="gpt-5",
+        cloud_endpoint="openai",
+        cfg={"cloud_max_tokens": 1024},
+    )
+    ctx = AgentContext(metadata={"task": {"task_id": "t2", "question": "X?"}, "task_id": "t2"})
+    result = agent.run("X?", ctx)
+    assert isinstance(result.metadata.get("tool_calls"), int)
+    assert result.metadata["tool_calls"] == 0
+    _assert_call_counts(result.metadata)
+    # _call_cloud was stubbed at the method level, so the SDK helper that
+    # would bump the cloud counter never runs. Both counters stay at 0.
+    assert result.metadata["n_cloud_calls"] == 0
+    assert result.metadata["n_local_calls"] == 0
+
+
+# ---------------------------------------------------------------------------
+# 2. advisors GAIA — tool_calls == n_searches summed across executor passes.
+# ---------------------------------------------------------------------------
+
+def test_advisors_gaia_tool_calls_sums_search_counts(fake_anthropic, monkeypatch):
+    from openjarvis.agents.hybrid.advisors import AdvisorsAgent
+
+    # Stub the local vLLM advisor call so we don't need a server.
+    def fake_vllm(model, endpoint, *, user, max_tokens, temperature,
+                  enable_thinking=False):
+        return "be more careful", 50, 10
+
+    monkeypatch.setattr(
+        "openjarvis.agents.hybrid.advisors.AdvisorsAgent._call_vllm",
+        staticmethod(fake_vllm),
+    )
+
+    agent = AdvisorsAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        local_model="qwen",
+        local_endpoint="http://x",
+        cfg={
+            "executor_max_tokens": 1024,
+            "advisor_max_tokens": 512,
+            "web_search": {"enabled": True, "max_uses": 4},
+            "gaia_max_turns": 2,
+        },
+    )
+    ctx = AgentContext(metadata={"task": {"task_id": "t3", "question": "X?"}, "task_id": "t3"})
+    result = agent.run("X?", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    # Two executor passes, each reports 3 searches → 6 total.
+    assert tc == result.metadata["web_search_uses"] == 6
+    _assert_call_counts(result.metadata)
+    # Two cloud executor passes, each one turn → n_cloud_calls == 2.
+    # The local advisor call is stubbed (doesn't hit _call_vllm), so
+    # n_local_calls stays 0.
+    assert result.metadata["n_cloud_calls"] == 2
+    assert result.metadata["n_local_calls"] == 0
+
+
+# ---------------------------------------------------------------------------
+# 3. minions GAIA — tool_calls == prefetch n_searches.
+# ---------------------------------------------------------------------------
+
+def test_minions_gaia_tool_calls_eq_prefetch_searches(monkeypatch):
+    from openjarvis.agents.hybrid import minions as minions_mod
+    from openjarvis.agents.hybrid.minions import MinionsAgent
+
+    # Skip the Minions import / patch dance entirely by stubbing _run_paradigm's
+    # protocol layer. Easiest: stub _prefetch_context + replace Minions class.
+    def fake_prefetch(question, endpoint, model, max_uses):
+        return {
+            "text": "stub digest",
+            "tokens": 100,
+            "cost_usd": 0.01,
+            "n_searches": 2,
+        }
+
+    class _FakeProtocol:
+        def __init__(self, **kw):
+            pass
+
+        def __call__(self, *args, **kw):
+            return {
+                "final_answer": "FINAL ANSWER: 42",
+                "supervisor_messages": [],
+                "worker_messages": [],
+                "timing": {},
+                "log_file": "/dev/null",
+                "local_usage": SimpleNamespace(prompt_tokens=10, completion_tokens=5),
+                "remote_usage": SimpleNamespace(prompt_tokens=20, completion_tokens=10),
+            }
+
+    fake_minions_module = SimpleNamespace(
+        clients=SimpleNamespace(
+            anthropic=SimpleNamespace(AnthropicClient=lambda **kw: None),
+            openai=SimpleNamespace(OpenAIClient=lambda **kw: None),
+        ),
+        minion=SimpleNamespace(Minion=_FakeProtocol),
+        minions=SimpleNamespace(Minions=_FakeProtocol),
+    )
+
+    import sys
+    monkeypatch.setitem(sys.modules, "minions", fake_minions_module)
+    monkeypatch.setitem(sys.modules, "minions.clients", fake_minions_module.clients)
+    monkeypatch.setitem(sys.modules, "minions.clients.anthropic", fake_minions_module.clients.anthropic)
+    monkeypatch.setitem(sys.modules, "minions.clients.openai", fake_minions_module.clients.openai)
+    monkeypatch.setitem(sys.modules, "minions.minion", fake_minions_module.minion)
+    monkeypatch.setitem(sys.modules, "minions.minions", fake_minions_module.minions)
+    monkeypatch.setattr(minions_mod, "_apply_patches_once", lambda: None)
+    monkeypatch.setattr(minions_mod, "_prefetch_context", fake_prefetch)
+
+    agent = MinionsAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        local_model="qwen",
+        local_endpoint="http://x",
+        cfg={
+            "mode": "minion",
+            "max_rounds": 2,
+            "web_search": {"enabled": True, "max_uses": 4},
+        },
+    )
+    ctx = AgentContext(metadata={
+        "task": {"task_id": "tm", "question": "What is X?"},
+        "task_id": "tm",
+    })
+    result = agent.run("What is X?", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    assert tc == 2  # matches the stubbed prefetch n_searches
+    _assert_call_counts(result.metadata)
+
+
+# ---------------------------------------------------------------------------
+# 4. SWE path — tool_calls equals bash turns from run_swe_agent_loop.
+#    We stub run_swe_agent_loop directly so we don't need a repo / vLLM /
+#    Anthropic.
+# ---------------------------------------------------------------------------
+
+def _stub_swe_loop(turns: int = 7):
+    """Return a stub matching ``run_swe_agent_loop``'s output shape."""
+    def _stub(task, **kwargs):
+        return {
+            "answer": "stub\n\n```diff\n--- a\n+++ b\n```",
+            "patch": "--- a\n+++ b\n",
+            "final_summary": "stub",
+            "tokens_in": 200,
+            "tokens_out": 100,
+            "tokens_local": 0,
+            "tokens_cloud": 300,
+            "cost_usd": 0.05,
+            "turns": turns,
+            "max_turns_hit": False,
+            "workdir": "/tmp/stub",
+        }
+    return _stub
+
+
+def test_baseline_cloud_swe_tool_calls_eq_bash_turns(monkeypatch):
+    from openjarvis.agents.hybrid import baseline_cloud as bc_mod
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    monkeypatch.setattr(bc_mod, "run_swe_agent_loop", _stub_swe_loop(turns=11))
+
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        cfg={"swe_max_turns": 30, "cloud_max_tokens": 4096},
+    )
+    task = {
+        "task_id": "swe1",
+        "problem_statement": "fix it",
+        "repo": "a/b",
+        "base_commit": "deadbeef",
+    }
+    ctx = AgentContext(metadata={"task": task, "task_id": "swe1"})
+    result = agent.run("fix it", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    assert tc == 11
+    _assert_call_counts(result.metadata)
+
+
+def test_minions_swe_tool_calls_eq_worker_bash_turns(monkeypatch):
+    from openjarvis.agents.hybrid import minions as minions_mod
+    from openjarvis.agents.hybrid.minions import MinionsAgent
+
+    monkeypatch.setattr(minions_mod, "run_swe_agent_loop", _stub_swe_loop(turns=9))
+
+    def fake_call_cloud(self, *, user, system=None, max_tokens=4096,
+                       temperature=0.0, **kwargs):
+        return "plan: do X", 50, 25
+
+    monkeypatch.setattr(minions_mod.MinionsAgent, "_call_cloud", fake_call_cloud)
+
+    agent = MinionsAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        local_model="qwen",
+        local_endpoint="http://x",
+        cfg={"swe_use_agent_loop": True, "supervisor_max_tokens": 512},
+    )
+    task = {
+        "task_id": "swe2",
+        "problem_statement": "fix it",
+        "repo": "a/b",
+        "base_commit": "deadbeef",
+    }
+    ctx = AgentContext(metadata={"task": task, "task_id": "swe2"})
+    result = agent.run("fix it", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    assert tc == 9  # only the worker subloop counts; supervisor is text
+    _assert_call_counts(result.metadata)
+
+
+def test_advisors_swe_tool_calls_sums_both_executor_passes(monkeypatch):
+    from openjarvis.agents.hybrid import advisors as adv_mod
+    from openjarvis.agents.hybrid.advisors import AdvisorsAgent
+
+    # Both calls to run_swe_agent_loop return turns=4; sum should be 8.
+    monkeypatch.setattr(adv_mod, "run_swe_agent_loop", _stub_swe_loop(turns=4))
+
+    def fake_vllm(model, endpoint, *, user, max_tokens, temperature,
+                  enable_thinking=False):
+        return "advise", 30, 10
+
+    monkeypatch.setattr(
+        adv_mod.AdvisorsAgent, "_call_vllm", staticmethod(fake_vllm),
+    )
+
+    agent = AdvisorsAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        local_model="qwen",
+        local_endpoint="http://x",
+        cfg={"swe_use_agent_loop": True},
+    )
+    task = {
+        "task_id": "swe3",
+        "problem_statement": "fix it",
+        "repo": "a/b",
+        "base_commit": "deadbeef",
+    }
+    ctx = AgentContext(metadata={"task": task, "task_id": "swe3"})
+    result = agent.run("fix it", ctx)
+    tc = result.metadata.get("tool_calls")
+    assert isinstance(tc, int)
+    assert tc == 8  # initial + final executor passes, each 4 bash turns
+    _assert_call_counts(result.metadata)
+
+
+# ---------------------------------------------------------------------------
+# 5. Runner row contract — _run_one writes tool_calls into the row.
+# ---------------------------------------------------------------------------
+
+def test_runner_row_includes_tool_calls(monkeypatch):
+    """The runner's ``_run_one`` must surface ``tool_calls`` from meta
+    into the top-level row that ends up in ``results.jsonl``."""
+    from openjarvis.agents._stubs import AgentResult
+    from openjarvis.agents.hybrid import runner
+
+    class _StubAgent:
+        def run(self, prompt, ctx):
+            return AgentResult(
+                content="FINAL ANSWER: x",
+                metadata={
+                    "tokens_local": 0,
+                    "tokens_cloud": 100,
+                    "cost_usd": 0.01,
+                    "latency_s": 0.1,
+                    "web_search_uses": 2,
+                    "tool_calls": 5,
+                    "n_cloud_calls": 3,
+                    "n_local_calls": 0,
+                    "traces": {},
+                },
+                turns=1,
+            )
+
+    task = {"task_id": "row1", "question": "x", "reference": "x", "metadata": {}}
+    row = runner._run_one(_StubAgent(), "gaia", task, "/tmp/log")
+    assert "tool_calls" in row
+    assert isinstance(row["tool_calls"], int)
+    assert row["tool_calls"] == 5
+    # Companion follow-up: n_cloud_calls / n_local_calls likewise surface.
+    assert isinstance(row.get("n_cloud_calls"), int)
+    assert isinstance(row.get("n_local_calls"), int)
+    assert row["n_cloud_calls"] == 3
+    assert row["n_local_calls"] == 0
+
+
+# ---------------------------------------------------------------------------
+# 6. Runner row contract — missing fields default to 0 (don't crash).
+# ---------------------------------------------------------------------------
+
+def test_runner_row_defaults_call_counts_when_missing(monkeypatch):
+    """If an agent forgets to set ``n_cloud_calls`` / ``n_local_calls`` (e.g.
+    a third-party paradigm that doesn't use ``LocalCloudAgent.run``), the
+    runner must still emit the fields as ``int`` zero — never ``None``."""
+    from openjarvis.agents._stubs import AgentResult
+    from openjarvis.agents.hybrid import runner
+
+    class _BareAgent:
+        def run(self, prompt, ctx):
+            return AgentResult(
+                content="x",
+                metadata={"tokens_cloud": 10, "cost_usd": 0.0},
+                turns=1,
+            )
+
+    task = {"task_id": "rowZ", "question": "x", "reference": "x", "metadata": {}}
+    row = runner._run_one(_BareAgent(), "gaia", task, "/tmp/log")
+    assert isinstance(row["n_cloud_calls"], int)
+    assert isinstance(row["n_local_calls"], int)
+    assert row["n_cloud_calls"] == 0
+    assert row["n_local_calls"] == 0

tests/agents/hybrid/test_web_search_wiring.py

@@ -0,0 +1,196 @@
+"""Tests for the GAIA web_search wiring (2026-05-17).
+
+Confirms that when a GAIA cell sets ``method_cfg.web_search.enabled = true``
+and the cloud endpoint is Anthropic, every paradigm declares the native
+``web_search_20250305`` server-side tool on its Anthropic call. Uses mocks
+— no real API calls.
+
+Also confirms the default (``web_search`` absent / disabled) stays
+one-shot and does NOT declare the tool, preserving back-compat with the
+currently running n=100 cells.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.agents.hybrid._base import (
+    LocalCloudAgent,
+    build_web_search_tool,
+    web_search_cfg,
+)
+
+
+# ---------------------------------------------------------------------------
+# 1. Schema parser
+# ---------------------------------------------------------------------------
+
+def test_web_search_cfg_default_off() -> None:
+    assert web_search_cfg(None) == (False, 8)
+    assert web_search_cfg({}) == (False, 8)
+    assert web_search_cfg({"web_search": None}) == (False, 8)
+
+
+def test_web_search_cfg_enabled() -> None:
+    assert web_search_cfg({"web_search": {"enabled": True}}) == (True, 8)
+    assert web_search_cfg({"web_search": {"enabled": True, "max_uses": 3}}) == (True, 3)
+    assert web_search_cfg({"web_search": {"enabled": False, "max_uses": 5}}) == (False, 5)
+
+
+def test_build_web_search_tool_shape() -> None:
+    tool = build_web_search_tool(5)
+    assert tool == {
+        "type": "web_search_20250305",
+        "name": "web_search",
+        "max_uses": 5,
+    }
+
+
+# ---------------------------------------------------------------------------
+# 2. Baseline cloud GAIA cell declares the tool when opted in
+# ---------------------------------------------------------------------------
+
+def _fake_anthropic_response(
+    text: str = "FINAL ANSWER: 42",
+    n_searches: int = 2,
+    input_tokens: int = 100,
+    output_tokens: int = 50,
+):
+    """Build a minimal Anthropic-shaped response object."""
+    return SimpleNamespace(
+        content=[SimpleNamespace(
+            type="text", text=text,
+            # the serializer probes a fixed attr list; only `text` is
+            # needed for the text block path.
+        )],
+        usage=SimpleNamespace(
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            server_tool_use=SimpleNamespace(web_search_requests=n_searches),
+        ),
+        stop_reason="end_turn",
+    )
+
+
+class _FakeMessages:
+    """Captures the create() kwargs so the test can assert what was sent."""
+
+    def __init__(self):
+        self.last_kwargs = None
+        self.responses = [_fake_anthropic_response()]
+        self.calls = 0
+
+    def create(self, **kwargs):
+        self.last_kwargs = kwargs
+        idx = min(self.calls, len(self.responses) - 1)
+        self.calls += 1
+        return self.responses[idx]
+
+
+class _FakeAnthropic:
+    """anthropic.Anthropic() stub. Stores messages on a class attr so the
+    test can pull it out after the agent's call."""
+
+    last_messages = None
+
+    def __init__(self, *args, **kwargs):
+        self.messages = _FakeMessages()
+        type(self).last_messages = self.messages
+
+
+@pytest.fixture
+def fake_anthropic(monkeypatch):
+    """Patch anthropic.Anthropic at the SDK level so every paradigm's
+    raw client construction picks up the fake. Importing the real
+    library is fine — the agent uses ``anthropic.Anthropic()`` only.
+    """
+    import anthropic
+    monkeypatch.setattr(anthropic, "Anthropic", _FakeAnthropic)
+    yield _FakeAnthropic
+
+
+def test_baseline_cloud_declares_web_search_on_gaia_when_enabled(fake_anthropic):
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        cfg={
+            "cloud_max_tokens": 1024,
+            "web_search": {"enabled": True, "max_uses": 4},
+            "gaia_max_turns": 2,
+        },
+    )
+    ctx = AgentContext(metadata={
+        "task": {"task_id": "t1", "question": "What is X?"},
+        "task_id": "t1",
+    })
+    result = agent.run("What is X?", ctx)
+
+    sent = fake_anthropic.last_messages.last_kwargs
+    assert sent is not None, "Anthropic client was never called"
+    assert "tools" in sent, "web_search tool was not declared"
+    assert sent["tools"] == [
+        {"type": "web_search_20250305", "name": "web_search", "max_uses": 4}
+    ]
+    # Per-row web_search_uses surfaced through meta.
+    assert result.metadata["web_search_uses"] == 2
+
+
+def test_baseline_cloud_does_not_declare_tool_when_disabled(fake_anthropic):
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        cloud_endpoint="anthropic",
+        cfg={"cloud_max_tokens": 1024},  # no web_search block — default OFF
+    )
+    ctx = AgentContext(metadata={
+        "task": {"task_id": "t2", "question": "What is X?"},
+        "task_id": "t2",
+    })
+    agent.run("What is X?", ctx)
+
+    sent = fake_anthropic.last_messages.last_kwargs
+    assert sent is not None
+    assert "tools" not in sent, (
+        "Default behavior MUST be one-shot with no tools declared — "
+        "tools must only appear when method_cfg.web_search.enabled = true."
+    )
+
+
+def test_baseline_cloud_web_search_skipped_on_non_anthropic(monkeypatch):
+    """Non-Anthropic endpoint with web_search.enabled=true must not crash
+    and must not invoke a fake local web_search — it falls back to the
+    one-shot path (logged as web_search_skipped)."""
+    from openjarvis.agents.hybrid import baseline_cloud as bc_mod
+    from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+
+    # Stub the cloud call so we don't hit OpenAI.
+    def fake_call_cloud(self, *, user, system=None, max_tokens=4096,
+                       temperature=0.0, **kwargs):
+        return "FINAL ANSWER: x", 10, 5
+
+    monkeypatch.setattr(
+        bc_mod.BaselineCloudAgent, "_call_cloud", fake_call_cloud, raising=True,
+    )
+
+    agent = BaselineCloudAgent(
+        engine=None,
+        model="gpt-5",
+        cloud_endpoint="openai",
+        cfg={"cloud_max_tokens": 1024, "web_search": {"enabled": True}},
+    )
+    ctx = AgentContext(metadata={
+        "task": {"task_id": "t3", "question": "X?"},
+        "task_id": "t3",
+    })
+    res = agent.run("X?", ctx)
+    # Falls through to one-shot path; no crash, no fake web_search.
+    assert res.metadata["web_search_uses"] == 0

tests/agents/hybrid/test_worker_pool_override.py

@@ -0,0 +1,269 @@
+"""Tests for `method_cfg.worker_pool` cell-config override.
+
+Conductor and ToolOrchestra previously hardcoded a heterogeneous worker
+pool (Opus + gpt-5-mini + optional local Qwen + optional web-search). The
+override lets cells swap the pool composition without code changes.
+Strict replace (not merge) by design — simpler reasoning at the cell-config
+layer.
+
+These tests touch only the construction site:
+- override accepted + default replaced
+- invalid entries raise at init, with the right error message format
+- `$local` / `$cloud` substitution works
+- absent override = legacy behavior (default pool intact)
+
+We never exercise `_call_worker` here — that's the consumer logic and is
+out of scope for this change.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from openjarvis.agents.hybrid.conductor import (
+    ConductorAgent,
+    _resolve_worker_pool as _resolve_conductor_pool,
+)
+from openjarvis.agents.hybrid.toolorchestra import (
+    ToolOrchestraAgent,
+    _resolve_worker_pool as _resolve_toolorch_pool,
+)
+
+
+# ---------------------------------------------------------------------------
+# Common construction helpers
+# ---------------------------------------------------------------------------
+
+
+def _conductor(cfg: dict, *, local_model: str | None = None) -> ConductorAgent:
+    return ConductorAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        local_model=local_model,
+        local_endpoint="http://localhost:8001/v1" if local_model else None,
+        cloud_endpoint="anthropic",
+        cfg=cfg,
+    )
+
+
+def _toolorch(cfg: dict, *, local_model: str | None = None) -> ToolOrchestraAgent:
+    return ToolOrchestraAgent(
+        engine=None,
+        model="claude-opus-4-7",
+        local_model=local_model,
+        local_endpoint="http://localhost:8001/v1" if local_model else None,
+        cloud_endpoint="anthropic",
+        cfg=cfg,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Valid override replaces the default pool
+# ---------------------------------------------------------------------------
+
+
+def test_conductor_worker_pool_override_replaces_default() -> None:
+    pool = [
+        {
+            "id": 0,
+            "name": "cheap",
+            "endpoint": "openai",
+            "model": "gpt-5-mini",
+            "description": "cheap one",
+        },
+        {
+            "id": 1,
+            "name": "strong",
+            "endpoint": "anthropic",
+            "model": "claude-opus-4-7",
+            "description": "strong one",
+        },
+    ]
+    agent = _conductor({"worker_pool": pool})
+    resolved = _resolve_conductor_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    assert [w["name"] for w in resolved] == ["cheap", "strong"]
+    assert [w["model"] for w in resolved] == ["gpt-5-mini", "claude-opus-4-7"]
+    # Default pool would have included "frontier-anthropic" / "frontier-openai-mini"
+    # by name; the override must NOT carry those over.
+    assert "frontier-anthropic" not in {w["name"] for w in resolved}
+
+
+def test_toolorch_worker_pool_override_replaces_default() -> None:
+    pool = [
+        {
+            "id": 0,
+            "name": "search",
+            "type": "anthropic-web-search",
+            # model omitted on purpose — search type allows it
+        },
+        {
+            "id": 1,
+            "name": "solver",
+            "type": "openai",
+            "model": "gpt-5-mini",
+        },
+    ]
+    agent = _toolorch(pool=None, cfg=None) if False else _toolorch({"worker_pool": pool})
+    resolved = _resolve_toolorch_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    assert [w["name"] for w in resolved] == ["search", "solver"]
+    # Search entry got the default web-search model filled in.
+    assert resolved[0]["model"] == "claude-haiku-4-5"
+    # Default pool's "frontier-anthropic" / "frontier-openai-mini" should be gone.
+    assert "frontier-anthropic" not in {w["name"] for w in resolved}
+
+
+# ---------------------------------------------------------------------------
+# Invalid entries raise at agent init with the right message
+# ---------------------------------------------------------------------------
+
+
+def test_conductor_invalid_entry_raises_at_init() -> None:
+    bad_pool = [
+        {
+            "id": 0,
+            "name": "broken",
+            "endpoint": "openai",
+            "model": "not-a-real-model",
+        },
+    ]
+    with pytest.raises(ValueError, match=r"Invalid worker_pool entry \[0\]: model 'not-a-real-model'"):
+        _conductor({"worker_pool": bad_pool})
+
+
+def test_toolorch_invalid_type_raises_at_init() -> None:
+    bad_pool = [
+        {
+            "id": 0,
+            "name": "broken",
+            "type": "imaginary-endpoint",
+            "model": "gpt-5-mini",
+        },
+    ]
+    with pytest.raises(ValueError, match=r"Invalid worker_pool entry \[0\]: 'type' must be one of"):
+        _toolorch({"worker_pool": bad_pool})
+
+
+def test_only_search_workers_rejected() -> None:
+    # Toolorchestra: pool with ONLY web-search must fail — no solver.
+    bad_pool = [
+        {"id": 0, "name": "only-search", "type": "anthropic-web-search"},
+    ]
+    with pytest.raises(
+        ValueError, match=r"at least one non-search worker"
+    ):
+        _toolorch({"worker_pool": bad_pool})
+
+
+def test_empty_pool_rejected() -> None:
+    with pytest.raises(ValueError, match=r"non-empty list"):
+        _conductor({"worker_pool": []})
+    with pytest.raises(ValueError, match=r"non-empty list"):
+        _toolorch({"worker_pool": []})
+
+
+def test_duplicate_ids_rejected() -> None:
+    pool = [
+        {"id": 0, "name": "a", "endpoint": "openai", "model": "gpt-5-mini"},
+        {"id": 0, "name": "b", "endpoint": "anthropic", "model": "claude-opus-4-7"},
+    ]
+    with pytest.raises(ValueError, match=r"Invalid worker_pool entry \[0\]: duplicate id"):
+        _conductor({"worker_pool": pool})
+
+
+# ---------------------------------------------------------------------------
+# $local / $cloud substitution
+# ---------------------------------------------------------------------------
+
+
+def test_conductor_local_cloud_substitution() -> None:
+    pool = [
+        {
+            "id": 0,
+            "name": "local",
+            "endpoint": "vllm",
+            "model": "$local",
+        },
+        {
+            "id": 1,
+            "name": "cloud",
+            "endpoint": "anthropic",
+            "model": "$cloud",
+        },
+    ]
+    agent = _conductor(
+        {"worker_pool": pool},
+        local_model="Qwen/Qwen3.5-9B",
+    )
+    resolved = _resolve_conductor_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    assert resolved[0]["model"] == "Qwen/Qwen3.5-9B"
+    assert resolved[1]["model"] == "claude-opus-4-7"
+    # vllm worker got the local endpoint inherited as base_url default.
+    assert resolved[0]["base_url"] == "http://localhost:8001/v1"
+
+
+def test_toolorch_local_cloud_substitution_with_angle_brackets() -> None:
+    # Accept both `$local` / `$cloud` and `<local>` / `<cloud>` syntaxes.
+    pool = [
+        {"id": 0, "name": "lo", "type": "vllm", "model": "<local>"},
+        {"id": 1, "name": "hi", "type": "anthropic", "model": "<cloud>"},
+    ]
+    agent = _toolorch(
+        {"worker_pool": pool},
+        local_model="google/gemma-4-26B-A4B-it",
+    )
+    resolved = _resolve_toolorch_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    assert resolved[0]["model"] == "google/gemma-4-26B-A4B-it"
+    assert resolved[1]["model"] == "claude-opus-4-7"
+
+
+def test_local_substitution_without_local_model_raises() -> None:
+    pool = [
+        {"id": 0, "name": "lo", "endpoint": "vllm", "model": "$local"},
+    ]
+    with pytest.raises(ValueError, match=r"requires a local_model"):
+        _conductor({"worker_pool": pool})
+
+
+# ---------------------------------------------------------------------------
+# Absent override = default behavior unchanged
+# ---------------------------------------------------------------------------
+
+
+def test_no_override_uses_default_pool_conductor() -> None:
+    # No `worker_pool` key — resolver returns the default pool, which is the
+    # paper-faithful 7-worker composition (arXiv 2512.04388): Gemini-2.5-Pro,
+    # Claude Sonnet-4, GPT-5, plus four OpenRouter-routed open-weights
+    # workers (DeepSeek-R1-Distill-Qwen-32B, Gemma3-27B-it, Qwen3-32B,
+    # Qwen3-32B-thinking).
+    agent = _conductor({})
+    resolved = _resolve_conductor_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    names = {w["name"] for w in resolved}
+    assert names == {
+        "gemini-pro",
+        "claude-sonnet-4",
+        "gpt-5",
+        "deepseek-r1-distill-qwen-32b",
+        "gemma3-27b-it",
+        "qwen3-32b",
+        "qwen3-32b-thinking",
+    }
+
+
+def test_no_override_uses_default_pool_toolorch() -> None:
+    agent = _toolorch({})
+    resolved = _resolve_toolorch_pool(
+        agent._cfg, agent._local_model, agent._local_endpoint, agent._cloud_model,
+    )
+    names = {w["name"] for w in resolved}
+    # Default toolorch pool: web-search + frontier-anthropic + frontier-openai-mini.
+    assert names == {"web-search", "frontier-anthropic", "frontier-openai-mini"}

tests/test_modal_grading.py

@@ -0,0 +1,110 @@
+"""Regression tests for the swebench Modal cgroup-v2 patch.
+
+See ``src/openjarvis/evals/scorers/swebench_harness.py`` for context. The
+upstream swebench ``ModalSandboxRuntime.__init__`` writes to a cgroup-v1
+path (``/sys/fs/cgroup/cpu/cpu.shares``) that doesn't exist in Modal v1.4+
+sandboxes (cgroup-v2). Without the patch every grade silently scores 0.
+
+Cheap tests (idempotency, garbage-patch handling) run by default. The
+real Modal grade is marked ``@pytest.mark.modal`` and skipped unless
+``SWEBENCH_RUN_MODAL_TESTS=1`` is set and a Modal token is configured.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+
+from openjarvis.evals.core.types import EvalRecord
+from openjarvis.evals.scorers.swebench_harness import (
+    _CGROUP_SOURCE_SENTINEL,
+    _apply_patches_once,
+    _run_harness,
+    SWEBenchHarnessScorer,
+)
+
+
+# Known-good patch from cloud-only-opus47-swe-n100/results.jsonl row 0
+# (astropy__astropy-14539, scored 1.0 with the harness).
+ASTROPY_14539_PATCH = (
+    'diff --git a/astropy/io/fits/diff.py b/astropy/io/fits/diff.py\n'
+    '--- a/astropy/io/fits/diff.py\n'
+    '+++ b/astropy/io/fits/diff.py\n'
+    '@@ -1449,7 +1449,7 @@ class TableDataDiff(_BaseDiff):\n'
+    '                 arrb.dtype, np.floating\n'
+    '             ):\n'
+    '                 diffs = where_not_allclose(arra, arrb, rtol=self.rtol, atol=self.atol)\n'
+    '-            elif "P" in col.format:\n'
+    '+            elif "P" in col.format or "Q" in col.format:\n'
+    '                 diffs = (\n'
+    '                     [\n'
+    '                         idx\n'
+)
+
+
+def _swebench_modal_src() -> Path:
+    from swebench.harness.modal_eval import run_evaluation_modal as _m
+
+    return Path(_m.__file__)
+
+
+def test_patches_idempotent():
+    """Applying patches twice must leave exactly one sentinel in the source."""
+    _apply_patches_once()
+    _apply_patches_once()  # second call is a no-op when sentinel is present
+    src = _swebench_modal_src().read_text()
+    count = src.count(_CGROUP_SOURCE_SENTINEL)
+    assert count == 1, f"Expected sentinel exactly once, got {count}"
+    # Confirm the broken bare write is gone and the try/except is in place.
+    assert "try:" in src and "except FileNotFoundError:" in src
+
+
+def _modal_creds_present() -> bool:
+    if os.environ.get("SWEBENCH_RUN_MODAL_TESTS", "0") != "1":
+        return False
+    if os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET"):
+        return True
+    # Modal also stores creds in ~/.modal.toml.
+    return Path("~/.modal.toml").expanduser().exists()
+
+
+@pytest.mark.modal
+@pytest.mark.slow
+@pytest.mark.skipif(
+    not _modal_creds_present(),
+    reason="set SWEBENCH_RUN_MODAL_TESTS=1 and configure Modal token to run",
+)
+def test_grade_known_good_patch():
+    """End-to-end: known-good astropy fix must grade as resolved on Modal."""
+    result = _run_harness("astropy__astropy-14539", ASTROPY_14539_PATCH, 1800)
+    assert result["success"] is True, result.get("details", {})
+    assert result["score"] == 1.0
+
+
+@pytest.mark.modal
+@pytest.mark.slow
+@pytest.mark.skipif(
+    not _modal_creds_present(),
+    reason="set SWEBENCH_RUN_MODAL_TESTS=1 and configure Modal token to run",
+)
+def test_grade_bad_patch():
+    """Garbage patch must score 0.0 cleanly — no harness crash, no exception."""
+    scorer = SWEBenchHarnessScorer(timeout_s=600)
+    record = EvalRecord(
+        record_id="astropy__astropy-14539",
+        problem="",
+        reference="",
+        category="swebench",
+        metadata={"instance_id": "astropy__astropy-14539"},
+    )
+    # Wrap in a code fence so extract_patch returns something, exercising
+    # the harness end-to-end rather than the early "no_patch_extracted" path.
+    answer = "```diff\nnot a real diff\n```"
+    is_correct, details = scorer.score(record, answer)
+    assert is_correct is False
+    # Either the harness rejected the patch (report present, instance not in
+    # resolved_ids) or it bailed early with reason="no_report". Both are fine
+    # — what we're checking is that we didn't raise.
+    assert "patch" in details

tests/test_swe_compaction.py

@@ -0,0 +1,153 @@
+"""Tests for trajectory compaction in mini_swe_agent._compact_local_messages."""
+from __future__ import annotations
+
+import copy
+from typing import Any, Dict, List
+
+import pytest
+
+from openjarvis.agents.hybrid.mini_swe_agent import (
+    _compact_local_messages,
+    _estimate_prompt_tokens,
+    _get_tiktoken_enc,
+)
+
+
+TIKTOKEN_OK = bool(_get_tiktoken_enc())
+
+
+def _validate_openai_message_shape(messages: List[Dict[str, Any]]) -> None:
+    seen_tool_call_ids: set[str] = set()
+    for i, m in enumerate(messages):
+        role = m.get("role")
+        assert role in {"system", "user", "assistant", "tool"}, f"bad role at {i}: {role}"
+        if role == "assistant":
+            for tc in (m.get("tool_calls") or []):
+                tid = tc.get("id")
+                assert tid, f"assistant tool_call missing id at msg {i}"
+                seen_tool_call_ids.add(tid)
+        elif role == "tool":
+            tid = m.get("tool_call_id")
+            assert tid, f"tool message at {i} missing tool_call_id"
+            assert tid in seen_tool_call_ids, (
+                f"tool message at {i} references unknown tool_call_id={tid!r}"
+            )
+            assert m.get("content"), f"tool message at {i} has empty content"
+
+
+def _make_synthetic_messages() -> List[Dict[str, Any]]:
+    messages: List[Dict[str, Any]] = [
+        {"role": "system", "content": "You are a coding agent."},
+        {"role": "user", "content": "Fix the bug in foo.py."},
+    ]
+    big_idx = 7  # one of the assistant+tool pairs gets a giant blob
+    for t in range(19):
+        tc_id = f"call_{t:03d}"
+        messages.append({
+            "role": "assistant",
+            "content": f"Turn {t}: I'll run a command.",
+            "tool_calls": [{
+                "id": tc_id,
+                "type": "function",
+                "function": {
+                    "name": "bash",
+                    "arguments": '{"command": "ls -la dir_' + str(t) + '"}',
+                },
+            }],
+        })
+        if t == big_idx:
+            body = "OUTPUT " + ("x" * 29950)
+            obs = f"$ ls\n{body}\nexit_code=0"
+        else:
+            body = ("line " + str(t) + " ") * 800  # ~8000 chars
+            obs = f"$ ls\n{body}\nexit_code=0"
+        messages.append({
+            "role": "tool",
+            "tool_call_id": tc_id,
+            "content": obs[:8000] if t != big_idx else obs[:30000],
+        })
+    return messages
+
+
+def test_compaction_token_budget_and_shape():
+    messages = _make_synthetic_messages()
+    orig = copy.deepcopy(messages)
+    before = _estimate_prompt_tokens(messages)
+    assert before > 24_000, f"sanity: synthetic should exceed budget, got {before}"
+
+    compacted = _compact_local_messages(
+        messages,
+        client=None,  # forces stage-2 summary fallback (no API call)
+        model="dummy",
+        keep_last=4,
+        trace_prefix="test",
+        compact_at_tokens=24_000,
+    )
+
+    _validate_openai_message_shape(compacted)
+
+    if TIKTOKEN_OK:
+        after = _estimate_prompt_tokens(compacted)
+        assert after <= 24_000, f"after compaction tokens={after} > 24_000"
+
+    # System + initial user intact.
+    assert compacted[0] == orig[0]
+    assert compacted[1] == orig[1]
+
+    # Recent 4 turns (assistant + its tool replies) intact and deep-equal.
+    # In synthetic input, every turn is exactly 1 assistant + 1 tool, so the
+    # last 4 turns = last 8 messages.
+    assert compacted[-8:] == orig[-8:], "recent 4 turns must be intact"
+
+
+def test_stage1_only_when_sufficient():
+    """If stage 1 alone drops us below the budget, stage 2 must not run."""
+    # Build a smaller input that exceeds budget only because of giant tool outputs.
+    messages: List[Dict[str, Any]] = [
+        {"role": "system", "content": "sys"},
+        {"role": "user", "content": "task"},
+    ]
+    for t in range(8):
+        tc_id = f"call_{t}"
+        messages.append({
+            "role": "assistant",
+            "content": f"t{t}",
+            "tool_calls": [{
+                "id": tc_id, "type": "function",
+                "function": {"name": "bash", "arguments": "{}"},
+            }],
+        })
+        # Use random-ish text to avoid BPE collapse from repetition.
+        import random
+        rng = random.Random(t)
+        body = " ".join(rng.choice(["alpha", "beta", "gamma", "delta", "lambda",
+                                     "foo", "bar", "baz", "qux", "zeta"])
+                         for _ in range(3500))
+        messages.append({
+            "role": "tool",
+            "tool_call_id": tc_id,
+            "content": body + " exit_code=0",
+        })
+
+    before = _estimate_prompt_tokens(messages)
+    if not TIKTOKEN_OK:
+        pytest.skip("tiktoken unavailable; stage-1-only ordering is timing-sensitive without it")
+    assert before > 24_000
+
+    compacted = _compact_local_messages(
+        messages, client=None, model="dummy",
+        keep_last=4, trace_prefix="test", compact_at_tokens=24_000,
+    )
+    _validate_openai_message_shape(compacted)
+    # No stage-2 fold → length identical (only tool contents shortened).
+    assert len(compacted) == len(messages)
+    # The synthetic stage-2 system stub should NOT be present.
+    for m in compacted:
+        if m.get("role") == "system":
+            assert not str(m.get("content", "")).startswith("[turns "), \
+                "stage 2 should not have run"
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(pytest.main([__file__, "-v"]))

tests/test_swe_loop_dispatch.py

@@ -0,0 +1,299 @@
+"""Regression tests for Bug 4: SWE-bench dispatch through the agent loop
+for all three cloud endpoints (Anthropic / OpenAI / Gemini).
+
+Before the fix, ``baseline_cloud`` and ``skillorchestra`` gated the SWE
+agent-loop call on ``cloud_endpoint == "anthropic"`` and silently fell
+back to a one-shot ``_call_cloud`` for OpenAI / Gemini SWE tasks — a
+"blind patch" that scored near zero. ``_loop_cloud`` in
+``mini_swe_agent`` now dispatches per endpoint, so all three should
+route through ``run_swe_agent_loop``.
+
+These tests pin the dispatch behavior without doing any real cloud
+calls: ``run_swe_agent_loop`` is patched in each agent module to record
+the kwargs it was invoked with.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.agents.hybrid.baseline_cloud import BaselineCloudAgent
+from openjarvis.agents.hybrid.skillorchestra import SkillOrchestraAgent
+
+
+# ---------- helpers ----------
+
+def _swe_task() -> Dict[str, Any]:
+    """SWE-bench-shaped task that triggers the is_swe / swe_mode branch."""
+    return {
+        "task_id": "test__swe-1",
+        "repo": "test/repo",
+        "base_commit": "deadbeef",
+        "problem_statement": "Fix the off-by-one in the parser.",
+    }
+
+
+def _gaia_task() -> Dict[str, Any]:
+    """Non-SWE task (no repo / base_commit) — should hit the one-shot path."""
+    return {"task_id": "gaia-1"}
+
+
+def _loop_return(answer: str = "ok diff") -> Dict[str, Any]:
+    """Shape that run_swe_agent_loop returns."""
+    return {
+        "answer": answer,
+        "patch": "diff --git a/x b/x\n",
+        "final_summary": "fixed",
+        "tokens_in": 100,
+        "tokens_out": 50,
+        "tokens_local": 0,
+        "tokens_cloud": 150,
+        "cost_usd": 0.001,
+        "turns": 3,
+        "max_turns_hit": False,
+        "workdir": "/tmp/x",
+    }
+
+
+def _make_baseline(cloud_endpoint: str) -> BaselineCloudAgent:
+    return BaselineCloudAgent(
+        engine=MagicMock(name="engine"),
+        model="some-cloud-model",
+        cloud_endpoint=cloud_endpoint,
+        cfg={"cloud_max_tokens": 1024, "swe_max_turns": 5},
+    )
+
+
+def _make_skillorch(cloud_endpoint: str) -> SkillOrchestraAgent:
+    # swe_use_agent_loop must be on for skillorchestra to enter the
+    # swe_mode branch.
+    return SkillOrchestraAgent(
+        engine=MagicMock(name="engine"),
+        model="some-cloud-model",
+        cloud_endpoint=cloud_endpoint,
+        cfg={
+            "swe_use_agent_loop": True,
+            "swe_max_turns": 5,
+            "swe_turn_max_tokens": 1024,
+        },
+    )
+
+
+def _ctx(task: Dict[str, Any]) -> AgentContext:
+    ctx = AgentContext()
+    ctx.metadata["task"] = task
+    ctx.metadata["task_id"] = task.get("task_id", "")
+    return ctx
+
+
+# Router mock returns a JSON blob picking cloud-<cloud_model> so
+# skillorchestra routes to the cloud worker (not local). The agent key now
+# embeds the cell-configured cloud model — see skillorchestra._run_paradigm.
+_ROUTER_JSON = (
+    '{"chosen_agent": "cloud-some-cloud-model", '
+    '"skill_weights": {"factual_recall": 0.0, "multi_step_reasoning": 0.0, '
+    '"arithmetic": 0.0, "web_grounding": 0.0, "long_text_extraction": 0.0, '
+    '"format_compliance": 0.0, "code_or_logic": 1.0}, '
+    '"reasoning": "test"}'
+)
+
+
+# ---------- baseline_cloud ----------
+
+@pytest.mark.parametrize("endpoint", ["anthropic", "openai", "gemini"])
+def test_baseline_cloud_swe_uses_loop_for_all_endpoints(endpoint: str) -> None:
+    """SWE task + any cloud endpoint → run_swe_agent_loop called with that
+    endpoint. Pre-fix this only fired for anthropic; openai / gemini silently
+    fell back to a one-shot ``_call_cloud`` blind-patch."""
+    agent = _make_baseline(endpoint)
+    task = _swe_task()
+    ctx = _ctx(task)
+
+    with patch(
+        "openjarvis.agents.hybrid.baseline_cloud.run_swe_agent_loop",
+        return_value=_loop_return(),
+    ) as mock_loop, patch.object(
+        BaselineCloudAgent, "_call_cloud"
+    ) as mock_oneshot:
+        answer, meta = agent._run_paradigm("Fix the bug.", ctx)
+
+    assert mock_loop.call_count == 1, (
+        f"SWE task with cloud_endpoint={endpoint!r} must dispatch to "
+        f"run_swe_agent_loop, not the one-shot fallback"
+    )
+    assert mock_oneshot.call_count == 0, (
+        f"_call_cloud one-shot should NOT be invoked on SWE for {endpoint!r}"
+    )
+    call_kwargs = mock_loop.call_args.kwargs
+    assert call_kwargs["cloud_endpoint"] == endpoint
+    assert call_kwargs["backbone"] == "cloud"
+    assert call_kwargs["backbone_model"] == "some-cloud-model"
+    assert meta["tokens_cloud"] == 150
+
+
+# Per the task spec — explicit per-endpoint variants for clarity in failure
+# reports (parametrize above already covers them but these read as the
+# named tests in the task description).
+
+def test_baseline_cloud_anthropic_uses_loop() -> None:
+    test_baseline_cloud_swe_uses_loop_for_all_endpoints("anthropic")
+
+
+def test_baseline_cloud_openai_uses_loop() -> None:
+    test_baseline_cloud_swe_uses_loop_for_all_endpoints("openai")
+
+
+def test_baseline_cloud_gemini_uses_loop() -> None:
+    test_baseline_cloud_swe_uses_loop_for_all_endpoints("gemini")
+
+
+def test_baseline_cloud_gaia_uses_oneshot() -> None:
+    """Non-SWE task → one-shot ``_call_cloud``, NOT the SWE agent loop.
+
+    Regression guard against over-correcting Bug 4 by routing every task
+    through the SWE loop (which would crash on GAIA without repo / commit).
+    """
+    agent = _make_baseline("openai")
+    ctx = _ctx(_gaia_task())
+
+    with patch(
+        "openjarvis.agents.hybrid.baseline_cloud.run_swe_agent_loop",
+    ) as mock_loop, patch.object(
+        BaselineCloudAgent,
+        "_call_cloud",
+        return_value=("the answer", 10, 20),
+    ) as mock_oneshot:
+        answer, meta = agent._run_paradigm("What is 2+2?", ctx)
+
+    assert mock_loop.call_count == 0, (
+        "GAIA-shaped task must NOT enter the SWE agent loop"
+    )
+    assert mock_oneshot.call_count == 1
+    assert answer == "the answer"
+    assert meta["turns"] == 1
+    assert meta["traces"]["mode"] == "one_shot"
+
+
+# ---------- skillorchestra ----------
+
+@pytest.mark.parametrize("endpoint", ["anthropic", "openai", "gemini"])
+def test_skillorchestra_cloud_swe_uses_loop(endpoint: str) -> None:
+    """For each cloud endpoint, when skillorchestra routes to the cloud
+    worker on a SWE task it must call run_swe_agent_loop, not the one-shot
+    ``_call_cloud`` fallback (the Bug-4 regression path)."""
+    agent = _make_skillorch(endpoint)
+    ctx = _ctx(_swe_task())
+
+    # Router patches: anthropic path uses _call_anthropic with output_config;
+    # openai / gemini routers use _call_openai / _call_gemini with prompt-only
+    # JSON. Patch all three to be safe regardless of endpoint.
+    with patch.object(
+        SkillOrchestraAgent,
+        "_call_anthropic",
+        return_value=(_ROUTER_JSON, 5, 5, 0),
+    ), patch.object(
+        SkillOrchestraAgent,
+        "_call_openai",
+        return_value=(_ROUTER_JSON, 5, 5),
+    ), patch.object(
+        SkillOrchestraAgent,
+        "_call_gemini",
+        return_value=(_ROUTER_JSON, 5, 5),
+    ), patch.object(
+        SkillOrchestraAgent,
+        "_call_cloud",
+        return_value=("oneshot-answer", 99, 99),
+    ) as mock_oneshot, patch(
+        "openjarvis.agents.hybrid.skillorchestra.run_swe_agent_loop",
+        return_value=_loop_return(),
+    ) as mock_loop:
+        answer, meta = agent._run_paradigm("Fix the bug.", ctx)
+
+    assert mock_loop.call_count == 1, (
+        f"skillorchestra cloud-worker SWE with endpoint={endpoint!r} must "
+        f"dispatch through run_swe_agent_loop, not _call_cloud one-shot"
+    )
+    assert mock_oneshot.call_count == 0, (
+        f"_call_cloud one-shot should NOT be invoked on SWE for {endpoint!r}"
+    )
+    call_kwargs = mock_loop.call_args.kwargs
+    assert call_kwargs["cloud_endpoint"] == endpoint
+    assert call_kwargs["backbone"] == "cloud"
+    # chosen_agent must reflect the cell-configured cloud model (Bug A fix:
+    # 2026-05-17), not a hardcoded "cloud-opus-4-7" string.
+    assert meta["traces"]["chosen_agent"] == "cloud-some-cloud-model"
+
+
+def test_skillorchestra_cloud_anthropic_uses_loop() -> None:
+    test_skillorchestra_cloud_swe_uses_loop("anthropic")
+
+
+def test_skillorchestra_cloud_openai_uses_loop() -> None:
+    test_skillorchestra_cloud_swe_uses_loop("openai")
+
+
+def test_skillorchestra_cloud_gemini_uses_loop() -> None:
+    test_skillorchestra_cloud_swe_uses_loop("gemini")
+
+
+# ---------- skillorchestra cloud-agent-key contract ----------
+#
+# Bug A (2026-05-17): the router used to advertise a hardcoded
+# `cloud-opus-4-7` agent regardless of the cell-configured cloud model, so
+# Haiku / Gemini cells saw Opus's competence + cost in the prompt. The fix
+# plumbs `self._cloud_model` into the agent registry key. These assertions
+# pin that contract.
+
+def test_skillorchestra_cloud_key_matches_cloud_model() -> None:
+    """Default competence / cost tables for a cell are keyed by the
+    cell-configured cloud_model, not a hardcoded Opus string."""
+    from openjarvis.agents.hybrid.skillorchestra import (
+        _default_agent_competence,
+        _default_agent_cost,
+    )
+
+    comp = _default_agent_competence("qwen-27b", "claude-haiku-4-5")
+    cost = _default_agent_cost("qwen-27b", "claude-haiku-4-5")
+    assert "cloud-claude-haiku-4-5" in comp
+    assert "cloud-claude-haiku-4-5" in cost
+    # And the hardcoded Opus key must NOT appear for a non-Opus cell.
+    assert "cloud-claude-opus-4-7" not in comp
+    assert "cloud-claude-opus-4-7" not in cost
+    # Cost must come from the empirical per-task table — Haiku is ~$0.002,
+    # NOT Opus's $0.014, so price ratio actually bites for the router.
+    assert cost["cloud-claude-haiku-4-5"] < 0.005
+
+
+def test_skillorchestra_chosen_agent_uses_cloud_model_name() -> None:
+    """End-to-end: when a cell is configured with a non-Opus cloud, the
+    trace's `chosen_agent` field must name that cloud (not 'cloud-opus-4-7')."""
+    agent = SkillOrchestraAgent(
+        engine=MagicMock(name="engine"),
+        model="claude-haiku-4-5",
+        cloud_endpoint="anthropic",
+        cfg={"cloud_max_tokens": 256},
+    )
+    # Router emits the right haiku key; weight loads onto code_or_logic.
+    router_json = (
+        '{"chosen_agent": "cloud-claude-haiku-4-5", '
+        '"skill_weights": {"factual_recall": 0.0, "multi_step_reasoning": 0.0, '
+        '"arithmetic": 0.0, "web_grounding": 0.0, "long_text_extraction": 0.0, '
+        '"format_compliance": 0.0, "code_or_logic": 1.0}, '
+        '"reasoning": "test"}'
+    )
+    ctx = _ctx({"task_id": "gaia-1"})
+    with patch.object(
+        SkillOrchestraAgent, "_call_anthropic",
+        return_value=(router_json, 5, 5, 0),
+    ), patch.object(
+        SkillOrchestraAgent, "_call_cloud",
+        return_value=("the answer", 10, 20),
+    ):
+        _, meta = agent._run_paradigm("hello", ctx)
+
+    assert meta["traces"]["chosen_agent"] == "cloud-claude-haiku-4-5"
+    assert meta["traces"]["worker_model"] == "claude-haiku-4-5"

tests/test_swe_loop_silent_termination.py

@@ -0,0 +1,249 @@
+"""Regression tests for silent-termination bugs in the cloud SWE-agent loop.
+
+Pre-fix (observed 2026-05-15 on the n=100 skillorchestra-qwen-gpt5mini-swe
+and skillorchestra-qwen-gemini25flash-swe cells):
+
+- OpenAI loop: when ``finish_reason='length'`` truncated the model
+  mid-response, ``tool_calls`` was empty and ``text`` was empty/short.
+  The loop's ``if not tool_calls: break`` rule treated that as "model
+  done" and exited with ``final_summary=""`` → answer became the
+  ``[mini-swe-agent produced no summary text]`` placeholder → score 0.
+
+- Gemini loop: when ``finish_reason=MALFORMED_FUNCTION_CALL`` (model
+  tried to call ``bash`` but produced unparseable args), the response
+  had no ``function_call`` parts and no text → same silent exit. This
+  hit 24/100 tasks in the gemini-flash cell.
+
+Both loops now inject a one-shot recovery nudge and continue the loop
+instead of exiting, only treating natural ``stop`` / text-only ``STOP``
+as termination.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any, Dict, List
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+# ---------- OpenAI loop ----------
+
+def _openai_message(
+    *,
+    content: str = "",
+    tool_calls: List[Any] = None,
+    finish_reason: str = "stop",
+    prompt_tokens: int = 10,
+    completion_tokens: int = 5,
+) -> Any:
+    """Build a mock OpenAI ChatCompletion response."""
+    msg = SimpleNamespace(content=content, tool_calls=tool_calls or None)
+    choice = SimpleNamespace(message=msg, finish_reason=finish_reason)
+    usage = SimpleNamespace(
+        prompt_tokens=prompt_tokens, completion_tokens=completion_tokens,
+    )
+    return SimpleNamespace(choices=[choice], usage=usage)
+
+
+def _openai_tool_call(call_id: str, command: str) -> Any:
+    """Build a mock tool_calls entry — the loop reads .id, .function.name,
+    .function.arguments."""
+    return SimpleNamespace(
+        id=call_id,
+        function=SimpleNamespace(name="bash", arguments=f'{{"command":"{command}"}}'),
+    )
+
+
+def test_openai_loop_recovers_from_length_truncation(tmp_path: Any) -> None:
+    """When finish_reason='length' AND text is empty AND no tool_calls,
+    the loop must NOT terminate — it should inject a recovery nudge and
+    let the model retry. Pre-fix the loop exited silently with empty
+    final_summary (silent failure on gpt-5-mini SWE cells)."""
+    from openjarvis.agents.hybrid.mini_swe_agent import _loop_cloud_openai
+
+    # Sequence: turn 1 truncated (length, empty), turn 2 normal stop with summary
+    responses = [
+        _openai_message(content="", tool_calls=None, finish_reason="length"),
+        _openai_message(content="Done.", tool_calls=None, finish_reason="stop"),
+    ]
+    mock_client = MagicMock()
+    mock_client.chat.completions.create.side_effect = responses
+
+    with patch(
+        "openjarvis.agents.hybrid.mini_swe_agent.OpenAI", create=True,
+        return_value=mock_client,
+    ), patch("openai.OpenAI", return_value=mock_client):
+        result = _loop_cloud_openai(
+            "fix the bug", tmp_path,
+            model="gpt-5-mini", max_turns=5,
+            bash_timeout=10, output_cap=1000, turn_max_tokens=100,
+            trace_prefix="test",
+        )
+
+    # Must NOT have exited at turn 1 — should have retried.
+    assert result["turns"] == 2, (
+        f"Loop must retry after length-truncation with empty text; "
+        f"got turns={result['turns']}"
+    )
+    assert result["final_summary"] == "Done."
+    assert result["max_turns_hit"] is False
+
+
+def test_openai_loop_terminates_normally_on_stop(tmp_path: Any) -> None:
+    """Sanity guard: natural ``finish_reason='stop'`` with non-empty text
+    still terminates the loop on turn 1."""
+    from openjarvis.agents.hybrid.mini_swe_agent import _loop_cloud_openai
+
+    responses = [
+        _openai_message(content="Done.", tool_calls=None, finish_reason="stop"),
+    ]
+    mock_client = MagicMock()
+    mock_client.chat.completions.create.side_effect = responses
+
+    with patch("openai.OpenAI", return_value=mock_client):
+        result = _loop_cloud_openai(
+            "fix the bug", tmp_path,
+            model="gpt-5-mini", max_turns=5,
+            bash_timeout=10, output_cap=1000, turn_max_tokens=100,
+            trace_prefix="test",
+        )
+
+    assert result["turns"] == 1
+    assert result["final_summary"] == "Done."
+
+
+# ---------- Gemini loop ----------
+
+def _gemini_response(
+    *,
+    text: str = "",
+    function_calls: List[Dict[str, Any]] = None,
+    finish_reason: str = "FinishReason.STOP",
+    prompt_tokens: int = 10,
+    candidates_tokens: int = 5,
+) -> Any:
+    """Build a mock google-genai GenerateContentResponse."""
+    parts = []
+    if text:
+        parts.append(SimpleNamespace(text=text, function_call=None))
+    for fc in function_calls or []:
+        parts.append(SimpleNamespace(
+            text=None,
+            function_call=SimpleNamespace(name=fc["name"], args=fc["args"]),
+        ))
+    content = SimpleNamespace(parts=parts, role="model")
+    candidate = SimpleNamespace(content=content, finish_reason=finish_reason)
+    usage = SimpleNamespace(
+        prompt_token_count=prompt_tokens,
+        candidates_token_count=candidates_tokens,
+    )
+    return SimpleNamespace(candidates=[candidate], usage_metadata=usage)
+
+
+def _gemini_mocks(responses: List[Any]) -> Dict[str, Any]:
+    """Wire up sys.modules mocks for google.genai so the Gemini loop
+    sees our scripted ``generate_content`` responses. Returns the dict
+    suitable for ``patch.dict('sys.modules', ...)``.
+
+    The loop's ``from google import genai`` resolves via the
+    ``google`` parent module's ``genai`` attribute (NOT via
+    sys.modules['google.genai']), so we also bind ``fake_genai`` onto
+    the parent mock's attribute table — otherwise the import returns a
+    fresh unconfigured MagicMock and our scripted responses are bypassed.
+    """
+    fake_client = MagicMock()
+    fake_models = MagicMock()
+    fake_models.generate_content = MagicMock(side_effect=responses)
+    fake_client.models = fake_models
+    fake_genai = MagicMock()
+    fake_genai.Client = MagicMock(return_value=fake_client)
+    fake_types = MagicMock()
+    fake_genai.types = fake_types
+    fake_google = MagicMock()
+    fake_google.genai = fake_genai
+    return {
+        "google": fake_google,
+        "google.genai": fake_genai,
+        "google.genai.types": fake_types,
+    }
+
+
+def test_gemini_loop_recovers_from_malformed_function_call(tmp_path: Any) -> None:
+    """When finish_reason includes MALFORMED_FUNCTION_CALL AND no text AND
+    no function_calls, the loop must inject a recovery nudge and retry —
+    NOT exit silently. Pre-fix this hit 24/100 tasks on gemini-flash."""
+    from openjarvis.agents.hybrid import mini_swe_agent
+
+    responses = [
+        _gemini_response(
+            text="", function_calls=None,
+            finish_reason="FinishReason.MALFORMED_FUNCTION_CALL",
+        ),
+        _gemini_response(
+            text="Fixed.", function_calls=None,
+            finish_reason="FinishReason.STOP",
+        ),
+    ]
+    with patch.dict("sys.modules", _gemini_mocks(responses)):
+        result = mini_swe_agent._loop_cloud_gemini(
+            "fix the bug", tmp_path,
+            model="gemini-2.5-flash", max_turns=5,
+            bash_timeout=10, output_cap=1000, turn_max_tokens=100,
+            trace_prefix="test",
+        )
+
+    assert result["turns"] == 2, (
+        f"Gemini loop must retry on MALFORMED_FUNCTION_CALL with empty text; "
+        f"got turns={result['turns']}"
+    )
+    assert result["final_summary"] == "Fixed."
+
+
+def test_gemini_loop_recovers_from_max_tokens(tmp_path: Any) -> None:
+    """MAX_TOKENS truncation parallel of the OpenAI ``length`` recovery."""
+    from openjarvis.agents.hybrid import mini_swe_agent
+
+    responses = [
+        _gemini_response(
+            text="", function_calls=None,
+            finish_reason="FinishReason.MAX_TOKENS",
+        ),
+        _gemini_response(
+            text="Done.", function_calls=None,
+            finish_reason="FinishReason.STOP",
+        ),
+    ]
+    with patch.dict("sys.modules", _gemini_mocks(responses)):
+        result = mini_swe_agent._loop_cloud_gemini(
+            "fix the bug", tmp_path,
+            model="gemini-2.5-flash", max_turns=5,
+            bash_timeout=10, output_cap=1000, turn_max_tokens=100,
+            trace_prefix="test",
+        )
+
+    assert result["turns"] == 2
+    assert result["final_summary"] == "Done."
+
+
+def test_gemini_loop_terminates_normally_on_stop_with_text(tmp_path: Any) -> None:
+    """Sanity guard: text-only STOP still ends the loop (no recovery)."""
+    from openjarvis.agents.hybrid import mini_swe_agent
+
+    responses = [
+        _gemini_response(
+            text="All done.", function_calls=None,
+            finish_reason="FinishReason.STOP",
+        ),
+    ]
+    with patch.dict("sys.modules", _gemini_mocks(responses)):
+        result = mini_swe_agent._loop_cloud_gemini(
+            "fix the bug", tmp_path,
+            model="gemini-2.5-flash", max_turns=5,
+            bash_timeout=10, output_cap=1000, turn_max_tokens=100,
+            trace_prefix="test",
+        )
+
+    assert result["turns"] == 1
+    assert result["final_summary"] == "All done."