diff --git a/configs/openjarvis/examples/spec-search-quickstart.toml b/configs/openjarvis/examples/spec-search-quickstart.toml new file mode 100644 index 00000000..345f68e5 --- /dev/null +++ b/configs/openjarvis/examples/spec-search-quickstart.toml @@ -0,0 +1,76 @@ +# LLM-Guided Spec Search — quickstart configuration +# Copy to ~/.openjarvis/config.toml and run: +# python -m openjarvis_examples.spec_search_quickstart +# +# This config has two parts: +# 1. The agent system being optimized (intelligence / engine / agent / tools). +# Same schema as the other examples in this directory; parsed by +# ``openjarvis.core.config.load_config``. +# 2. ``[learning.spec_search]`` and its sub-tables — the search hyperparameters +# consumed by ``SpecSearchOrchestrator.from_config`` and ``SpecSearchLoop``. +# +# Defaults below match the paper (Saad-Falcon et al., 2026): +# - max_regression = 0.01 (epsilon in GateOK) +# - stagnation_k = 5 (Algorithm 1 stopping rule) +# - composite_reward weights (alpha, beta, gamma, delta) = (0.5, 0.1, 0.1, 0.3) +# +# Teacher API keys come from your environment / credentials store, not this file. + +# --------------------------------------------------------------------------- +# Agent system being optimized +# --------------------------------------------------------------------------- + +[engine] +default = "ollama" # swap to "vllm" on H100/RTX 6000 / DGX Spark + +[intelligence] +default_model = "qwen3.5:9b" # the local student +# default_model = "qwen3.5:27b-fp8" # workstation tier + +[agent] +default_agent = "orchestrator" # multi-turn, tool-using +max_turns = 10 + +[tools] +enabled = [ + "code_interpreter", + "file_read", + "web_search", + "think", + "calculator", +] + +# --------------------------------------------------------------------------- +# LLM-guided spec search hyperparameters (paper §3.3, Algorithm 1) +# --------------------------------------------------------------------------- + +[learning.spec_search] +enabled = true +teacher_model = "claude-opus-4-6" # frontier proposer +teacher_engine = "cloud" # CloudEngine registry key (uses LiteLLM) +autonomy_mode = "tiered" # auto | tiered | manual + +# Per-session bounds (one diagnose / plan / execute / record pass) +min_traces = 20 +max_cost_per_session_usd = 5.0 +max_tool_calls_per_diagnosis = 30 + +# Multi-session loop (paper Algorithm 1 stopping) +stagnation_k = 5 # stop after this many sessions with no gate-score gain +stagnation_eps = 0.001 # delta below this counts as no improvement +max_total_cost_usd = 50.0 # cumulative teacher-cost budget across all sessions + +# GateOK predicate — accept iff target cluster improves AND every other cluster +# regresses by at most max_regression (epsilon in the paper). +max_regression = 0.01 # paper default: 1% +min_improvement = 0.0 +benchmark_subsample_size = 50 +benchmark_version = "personal_v1" + +# Composite reward (paper Eq. 1) — used only when an Intelligence edit triggers +# LoRA / GRPO training inside an accepted edit. The held-out gate is unaffected. +[learning.spec_search.composite_reward] +alpha = 0.5 # accuracy weight +beta = 0.1 # energy penalty +gamma = 0.1 # latency penalty +delta = 0.3 # cost penalty diff --git a/docs/architecture/learning.md b/docs/architecture/learning.md index f0496f4b..85df2ccb 100644 --- a/docs/architecture/learning.md +++ b/docs/architecture/learning.md @@ -556,13 +556,13 @@ to Python. --- -## Distillation (Frontier-Driven Harness Learning) +## LLM-Guided Spec Search (Frontier-Driven Harness Learning) -The distillation subsystem uses a frontier closed-source model (the "teacher") as a meta-engineer for the local student's full harness — not just its weights. Instead of pushing knowledge into a small model's weights, we push a frontier model's engineering judgement into the surrounding configuration: prompts, routing, agent class, tool availability, and tool descriptions. +LLM-guided spec search uses a frontier closed-source model (the "teacher") as a meta-engineer for the local student's full harness — not just its weights. Instead of pushing knowledge into a small model's weights, we push a frontier model's engineering judgement into the surrounding configuration: prompts, routing, agent class, tool availability, and tool descriptions. ### Where it lives -`learning/distillation/` is the fifth subsystem within the Learning pillar, alongside `learning/routing/`, `learning/optimize/`, `learning/training/`, and `learning/intelligence/`. +`learning/spec_search/` is the fifth subsystem within the Learning pillar, alongside `learning/routing/`, `learning/optimize/`, `learning/training/`, and `learning/intelligence/`. ### Four-phase loop @@ -579,7 +579,7 @@ Trigger → Diagnose → Plan → Execute → Record | Component | Module | Purpose | |-----------|--------|---------| -| `DistillationOrchestrator` | `orchestrator.py` | Top-level session driver | +| `SpecSearchOrchestrator` | `orchestrator.py` | Top-level session driver | | `TeacherAgent` | `diagnose/teacher_agent.py` | Frontier model tool-calling loop | | `DiagnosisRunner` | `diagnose/runner.py` | Phase 1 orchestration | | `LearningPlanner` | `plan/planner.py` | Diagnosis → typed LearningPlan | @@ -609,4 +609,4 @@ Every edit is assigned a tier from a deterministic lookup table: | `review` | System prompt edits, agent class, few-shot exemplars | Queue for user approval | | `manual` | LoRA fine-tuning (v2) | Never auto-apply | -See [Distillation user guide](../user-guide/learning-distillation.md) for CLI usage and configuration. +See [LLM-guided spec search guide](../user-guide/llm-guided-spec-search.md) for the architecture and the building blocks. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index a3bf5f0b..da0a6583 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -45,7 +45,7 @@ The memory pipeline includes document ingestion, chunking, embedding generation, The Learning system is the fifth primitive, connecting the other four through **trace-driven feedback**. Every agent interaction can produce a `Trace` capturing the full sequence of steps — routing decisions, memory retrieval, inference calls, tool invocations, and final responses. The `TraceAnalyzer` computes statistics from accumulated traces, and the `TraceDrivenPolicy` uses these statistics to learn which model/agent/tool combinations produce the best outcomes for different query types. -The learning system is configured through nested sub-sections in `config.toml`: `[learning.routing]` controls the router policy (heuristic, learned, sft, grpo), `[learning.intelligence]` controls the model-level learning policy, `[learning.agent]` controls agent advisor and ICL updater policies, and `[learning.metrics]` sets the composite reward function weights. The pillar also includes the distillation subsystem, a frontier-driven loop that improves the local harness — see [Learning architecture: Distillation](learning.md#distillation-frontier-driven-harness-learning). +The learning system is configured through nested sub-sections in `config.toml`: `[learning.routing]` controls the router policy (heuristic, learned, sft, grpo), `[learning.intelligence]` controls the model-level learning policy, `[learning.agent]` controls agent advisor and ICL updater policies, and `[learning.metrics]` sets the composite reward function weights. The pillar also includes LLM-guided spec search, a frontier-driven loop that improves the local harness — see [Learning architecture: LLM-guided spec search](learning.md#llm-guided-spec-search-frontier-driven-harness-learning). --- diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index 272ffbfa..c80c8c9c 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -9,7 +9,7 @@ These are the areas where active development is happening and contributions are - **Energy-aware routing** — using power consumption data from telemetry to optimize for energy efficiency alongside latency and quality - **Plugin ecosystem** — community-contributed engines, tools, and agents distributed as Python packages - **Federated memory** — memory backends that synchronize across devices -- **Distillation:** Frontier-driven harness learning — a frontier model analyzes your traces and proposes config improvements. See [user guide](../user-guide/learning-distillation.md) and [architecture](../architecture/learning.md#distillation-frontier-driven-harness-learning). +- **LLM-guided spec search:** Frontier-driven harness learning — a frontier model analyzes your traces and proposes config improvements. See [user guide](../user-guide/llm-guided-spec-search.md) and [architecture](../architecture/learning.md#llm-guided-spec-search-frontier-driven-harness-learning). --- diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index daaeb769..82631d83 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -1060,21 +1060,21 @@ OpenJarvis respects the following environment variables: --- -## Learning & Distillation +## Learning & spec search -The distillation subsystem uses a frontier model to automatically improve your local agent configuration. See the [user guide](../user-guide/learning-distillation.md) for a full walkthrough. +LLM-guided spec search uses a frontier model to automatically improve your local agent configuration. See the [user guide](../user-guide/llm-guided-spec-search.md) for a full walkthrough. -### `[learning.distillation]` +### `[learning.spec_search]` | Key | Type | Default | Description | |-----|------|---------|-------------| -| `enabled` | bool | `true` | Gate the entire distillation subsystem | +| `enabled` | bool | `true` | Gate the entire spec-search subsystem | | `autonomy_mode` | string | `"tiered"` | `auto`, `tiered`, or `manual` | | `teacher_model` | string | `"claude-opus-4-6"` | Frontier model for diagnosis and planning | | `max_cost_per_session_usd` | float | `5.0` | Per-session teacher API budget | | `max_tool_calls_per_diagnosis` | int | `30` | Max teacher tool calls in diagnosis phase | -### `[learning.distillation.triggers]` +### `[learning.spec_search.triggers]` | Key | Type | Default | Description | |-----|------|---------|-------------| @@ -1086,7 +1086,7 @@ The distillation subsystem uses a frontier model to automatically improve your l | `cluster_min_size` | int | `5` | Minimum traces in a cluster | | `cluster_failure_threshold` | float | `0.3` | Feedback <= this counts as failure | -### `[learning.distillation.gate]` +### `[learning.spec_search.gate]` | Key | Type | Default | Description | |-----|------|---------|-------------| @@ -1095,7 +1095,7 @@ The distillation subsystem uses a frontier model to automatically improve your l | `benchmark_subsample_size` | int | `50` | Tasks per gate run | | `full_benchmark` | bool | `false` | Disable subsampling (slower, more accurate) | -### `[learning.distillation.benchmark]` +### `[learning.spec_search.benchmark]` | Key | Type | Default | Description | |-----|------|---------|-------------| @@ -1104,12 +1104,12 @@ The distillation subsystem uses a frontier model to automatically improve your l | `auto_refresh` | bool | `true` | Auto-mine new high-feedback traces | | `max_synthesis_cost_usd_per_refresh` | float | `2.0` | Cost cap per benchmark refresh | -### `[learning.distillation.tier_overrides]` +### `[learning.spec_search.tier_overrides]` Override the default risk tier for any operation. Keys are operation names, values are tier strings (`auto`, `review`, `manual`). ```toml -[learning.distillation.tier_overrides] +[learning.spec_search.tier_overrides] # patch_system_prompt = "auto" # promote to auto after trust # replace_system_prompt = "auto" ``` diff --git a/docs/user-guide/cli.md b/docs/user-guide/cli.md index 89221768..ab2d13d5 100644 --- a/docs/user-guide/cli.md +++ b/docs/user-guide/cli.md @@ -463,98 +463,14 @@ When an agent is configured (e.g., `--agent orchestrator`), non-streaming reques --- -## `jarvis learning` +## LLM-guided spec search (no CLI yet) -Frontier-driven harness learning (distillation). Manages learning sessions, reviews pending edits, and controls the benchmark gate. - -### `jarvis learning init` - -Initialize the distillation checkpoint repo and directory layout. - -```bash -jarvis learning init -``` - -### `jarvis learning run` - -Run an on-demand learning session. - -```bash -jarvis learning run -jarvis learning run --autonomy auto # auto-apply all edits -jarvis learning run --autonomy manual # dry-run, everything goes to review -``` - -| Flag | Default | Description | -|------|---------|-------------| -| `--autonomy` | `tiered` | `auto`, `tiered`, or `manual` | - -### `jarvis learning history` - -List past learning sessions. - -```bash -jarvis learning history -jarvis learning history --limit 5 -``` - -### `jarvis learning show` - -Show details of a learning session (diagnosis, plan, outcomes, cost). - -```bash -jarvis learning show -``` - -### `jarvis learning review` - -List all pending edits awaiting approval. - -```bash -jarvis learning review -``` - -### `jarvis learning approve` - -Approve a pending edit (still goes through the benchmark gate). - -```bash -jarvis learning approve -``` - -### `jarvis learning reject` - -Reject a pending edit. - -```bash -jarvis learning reject -jarvis learning reject --reason "too aggressive" -``` - -### `jarvis learning rollback` - -Rollback a session's committed edits (creates revert commits). - -```bash -jarvis learning rollback -jarvis learning rollback --last -``` - -### `jarvis learning benchmark` - -Personal benchmark management. - -```bash -jarvis learning benchmark show # current stats -jarvis learning benchmark refresh # manual refresh -``` - -### `jarvis learning daemon` - -Background learning daemon. - -```bash -jarvis learning daemon start -jarvis learning daemon stop -jarvis learning daemon status -``` +LLM-guided spec search (the frontier-driven harness-learning subsystem) +is exposed as a Python library only — there is currently no top-level +`jarvis` subcommand for it. Construct a `SpecSearchOrchestrator` +directly from `openjarvis.learning.spec_search.orchestrator` and call +`.run(trigger)` with a trigger from +`openjarvis.learning.spec_search.triggers`. See +[`docs/user-guide/llm-guided-spec-search.md`](llm-guided-spec-search.md) +for the architecture and the building blocks +(`splits.py`, external corpora, `external_adapter`). diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.md index 12be76b2..8342b86f 100644 --- a/docs/user-guide/evaluations.md +++ b/docs/user-guide/evaluations.md @@ -14,7 +14,7 @@ The OpenJarvis evaluation framework (`openjarvis-evals`) measures model **correc --- -> **Tip:** The distillation system uses this same eval infrastructure to gate edits against your personal benchmark. See [Learning & Distillation](learning-distillation.md). +> **Tip:** LLM-guided spec search uses this same eval infrastructure to gate edits against your personal benchmark. See [LLM-guided spec search](llm-guided-spec-search.md). ## Installation diff --git a/docs/user-guide/learning-distillation.md b/docs/user-guide/learning-distillation.md deleted file mode 100644 index 2d32a30c..00000000 --- a/docs/user-guide/learning-distillation.md +++ /dev/null @@ -1,251 +0,0 @@ -# Learning & Distillation - -Use a frontier model as a meta-engineer to automatically improve your local agent's prompts, routing, and tools — reversibly, with benchmark-gated quality control. - -## Quick Start - -### 1. Initialize - -```bash -jarvis learning init -``` - -This creates the distillation directory layout under `~/.openjarvis/learning/` and initializes a git checkpoint repo at `~/.openjarvis/.git` for tracking config changes. - -### 2. Run your first session - -Once you have at least 20 traces from regular use: - -```bash -jarvis learning run -``` - -The system will: -1. **Diagnose** — analyze your traces using a frontier model -2. **Plan** — propose typed edits to your config -3. **Execute** — apply edits that pass the benchmark gate -4. **Record** — persist the session for history and rollback - -### 3. Check results - -```bash -jarvis learning history -jarvis learning show -``` - -## How a Learning Session Works - -A learning session has four phases: - -### Phase 1: Diagnose - -A frontier model (the "teacher", default `claude-opus-4-6`) analyzes your recent traces using read-only diagnostic tools. It identifies **failure clusters** — groups of related failures with shared root causes. The teacher must actually re-run your student on sample tasks and compare outputs to populate failure rates. This forces evidence-based diagnosis. - -**Output:** `diagnosis.md` with narrative analysis + structured failure clusters. - -### Phase 2: Plan - -A second teacher call converts the diagnosis into a typed `LearningPlan` — a list of `Edit` objects, each targeting a specific part of your configuration (model routing, system prompts, tool availability, etc.). The teacher cannot pick risk tiers — those are assigned deterministically from a lookup table. - -**Output:** `plan.json` frozen and immutable. - -### Phase 3: Execute - -Each edit is applied through its registered `EditApplier`, then scored against your personal benchmark. Edits that improve the benchmark are committed; edits that cause regressions are rolled back. Edits in the `review` tier are queued for your approval instead of being auto-applied. - -**Output:** Git commits in the checkpoint repo + `EditOutcome` records. - -### Phase 4: Record - -The session is persisted to `learning.db` (SQLite index) and `session.json` (authoritative artifact). You can query history, show details, and rollback any session. - -## Configuration - -Add to `~/.openjarvis/config.toml`: - -```toml -[learning.distillation] -enabled = true # gate the entire subsystem -autonomy_mode = "tiered" # auto | tiered | manual -teacher_model = "claude-opus-4-6" # any CloudEngine-supported model -max_cost_per_session_usd = 5.0 # per-session teacher API budget -max_tool_calls_per_diagnosis = 30 # max teacher tool calls in diagnosis -``` - -### Trigger configuration - -```toml -[learning.distillation.triggers] -scheduled_enabled = true -scheduled_cron = "0 3 * * *" # daily at 03:00 local -scheduled_min_new_traces = 20 # minimum new traces to trigger - -cluster_enabled = true -cluster_check_interval_minutes = 60 -cluster_min_size = 5 -cluster_failure_threshold = 0.3 # feedback <= this counts as failure -``` - -### Gate configuration - -```toml -[learning.distillation.gate] -min_improvement = 0.0 # any improvement accepted (raise for margin) -max_regression = 0.05 # max per-cluster score drop -benchmark_subsample_size = 50 # tasks per gate run -full_benchmark = false # set true to disable subsampling -``` - -### Benchmark configuration - -```toml -[learning.distillation.benchmark] -synthesis_feedback_threshold = 0.7 # min feedback for benchmark traces -max_benchmark_size = 200 # max tasks in the benchmark -auto_refresh = true # auto-mine new high-feedback traces -max_synthesis_cost_usd_per_refresh = 2.0 # separate from session budget -``` - -### Risk tier overrides - -Power users can override the default tier for any operation: - -```toml -[learning.distillation.tier_overrides] -# Promote prompt edits to auto-apply after trust is established: -# patch_system_prompt = "auto" -# replace_system_prompt = "auto" -``` - -## Risk Tiers - -Every edit is assigned a risk tier that controls how it's applied: - -| Tier | Behavior | Default ops | -|------|----------|-------------| -| **auto** | Applied automatically if benchmark gate passes | Model routing, model params, tool add/remove/description, agent params | -| **review** | Queued for user approval in `jarvis learning review` | System prompt edits, agent class changes, few-shot exemplars | -| **manual** | Never auto-applied; requires explicit approval | LoRA fine-tuning (v2) | - -The tier is assigned deterministically from the edit operation — the teacher cannot override it. - -## Reviewing Edits - -When edits land in the review queue: - -```bash -# List all pending edits -jarvis learning review - -# Approve an edit (still goes through the benchmark gate) -jarvis learning approve - -# Reject an edit with a reason -jarvis learning reject --reason "prompt change too aggressive" -``` - -Even approved edits are gated by the benchmark — approval means "try it", not "force it". - -## Rollback and History - -Every edit creates a git commit in the checkpoint repo at `~/.openjarvis/.git`. This is separate from your OpenJarvis source repo. - -```bash -# List past sessions -jarvis learning history --limit 20 - -# Show session details (diagnosis, plan, outcomes, cost) -jarvis learning show - -# Rollback a session (creates new revert commits, preserves history) -jarvis learning rollback -jarvis learning rollback --last -``` - -Rollback never rewrites git history — it creates new revert commits so the audit trail stays intact. - -## Cost Controls - -Three cost boundaries prevent runaway spending: - -1. **`max_cost_per_session_usd`** (default $5.00) — caps the total teacher API cost per session (diagnosis + planning). -2. **`max_synthesis_cost_usd_per_refresh`** (default $2.00) — caps the cost of generating gold answers for new benchmark tasks. Separate from the session budget. -3. **`teacher_model`** — choose a cheaper model (e.g., `claude-sonnet-4-6`) to reduce per-token costs at the expense of diagnosis quality. - -Cost is tracked on every `LearningSession` as `teacher_cost_usd` and surfaced in `jarvis learning show`. - -## The Personal Benchmark - -The benchmark is your acceptance gate's source of truth — a set of tasks distilled from your high-quality traces, scored by an LLM-as-judge against frontier gold answers. - -**How it's built:** -1. Traces with feedback >= 0.7 are candidates -2. Tasks are grouped by query class and deduplicated -3. For each task, the teacher generates a gold reference answer -4. The benchmark is versioned (`personal_v1.json`, `personal_v2.json`, ...) - -**Auto-refresh:** The benchmark grows over time as you accumulate more traces. New tasks are added automatically during background refresh cycles. - -```bash -# Manual refresh -jarvis learning benchmark refresh - -# Show stats -jarvis learning benchmark show -``` - -## Cold Start: What to Expect on Day One - -The system needs real usage data before it can learn: - -- **< 20 traces:** `jarvis learning run` returns "Not enough traces yet." Triggers are no-ops. -- **20+ traces, < 10 high-feedback:** Enough for diagnosis, but no benchmark yet. Sessions will run diagnosis but can't gate edits. -- **10+ high-feedback traces:** Bootstrap benchmark is created automatically (`personal_v1.json`). Full learning loop is available. - -**Getting there faster:** Use OpenJarvis normally and provide feedback on results (thumbs up/down in the UI, or `jarvis feedback` in the CLI). - -## Troubleshooting - -| Error | Cause | Fix | -|-------|-------|-----| -| "Not enough traces yet" | Fewer than 20 traces in the store | Use OpenJarvis more, provide feedback | -| "Working tree dirty, cannot stage" | Manual edits to `~/.openjarvis/config.toml` during a session | Commit or revert manual changes first | -| "All clusters dropped: insufficient evidence" | Teacher diagnosed clusters but couldn't reproduce failures | Check that the student is actually failing on the flagged tasks | -| "ConfigurationError: distillation root inside source tree" | `OPENJARVIS_HOME` points inside the repo | Set `OPENJARVIS_HOME` to `~/.openjarvis` (default) or another external dir | -| "Personal benchmark is empty" | Not enough high-feedback traces yet | Provide feedback on 10+ traces with score >= 0.7 | - -## Where Artifacts Live - -All distillation artifacts live under `~/.openjarvis/` (never inside the source repo): - -``` -~/.openjarvis/ -├── config.toml # Your configuration (git-tracked by checkpoint) -├── agents/ # Agent prompts (git-tracked) -├── tools/ # Tool descriptions (git-tracked) -├── .git/ # Checkpoint repo for rollback -└── learning/ - ├── learning.db # SQLite session index - ├── benchmarks/ # Personal benchmark versions + gold answers - ├── sessions/ # Per-session artifacts (diagnosis, plan, traces) - └── pending_review/ # Edits awaiting user approval -``` - -## Background Daemon - -For continuous learning: - -```bash -jarvis learning daemon start # Start background watcher -jarvis learning daemon status # Check if running -jarvis learning daemon stop # Stop the daemon -``` - -The daemon runs the scheduled trigger (default: daily at 03:00) and the cluster trigger (watches for failure patterns in real-time). - -## See Also - -- [Architecture: Learning](../architecture/learning.md#distillation-frontier-driven-harness-learning) — internal architecture of the distillation subsystem -- [User Guide: Evaluations](evaluations.md) — the eval infrastructure that powers the benchmark gate -- [User Guide: CLI](cli.md#jarvis-learning) — full CLI reference -- [Getting Started: Configuration](../getting-started/configuration.md) — all config knobs diff --git a/docs/user-guide/llm-guided-spec-search.md b/docs/user-guide/llm-guided-spec-search.md index bb7beedd..c83d53fe 100644 --- a/docs/user-guide/llm-guided-spec-search.md +++ b/docs/user-guide/llm-guided-spec-search.md @@ -1,121 +1,188 @@ # LLM-Guided Spec Search -LLM-guided spec search is a local–cloud collaboration that uses each side -for what it does best: a frontier cloud model (the *teacher*) reads traces -from a deployed local agent and proposes typed edits across the agent's +LLM-guided spec search (Saad-Falcon et al., 2026) is a local–cloud +collaboration: a frontier cloud *teacher* reads traces from a deployed +local agent and proposes typed edits across the agent's full configuration; the local hardware runs the resulting configuration with -zero marginal API cost at inference time. A held-out gate accepts only -edits that improve a target failure mode without unacceptable regression -on others. +zero marginal API cost at inference time. A held-out *gate* accepts only +edits that improve a target failure cluster without unacceptable +regression elsewhere. -The configuration the teacher edits is called the **spec**: a typed object -with five primitives (Intelligence, Engine, Agents, Tools & Memory, -Learning). The Learning primitive is where the search lives — it specifies -which edits are considered, how candidate specs are evaluated, and when -optimization stops. +This page is a copy-paste tutorial. By the end you will have: + +- a real `SpecSearchOrchestrator` running on your machine, +- a multi-session loop with the paper's stagnation rule (Algorithm 1), +- an understanding of which knobs to turn for production deployment. + +## TL;DR — run it + +```bash +python examples/openjarvis/spec_search_quickstart.py +``` + +The script is self-contained (no API key, no Ollama) — it wires up real +orchestrator + multi-session loop + composite-reward modules with stub +teacher/student/judge so you can see one full session and a stagnation +loop terminate. Production swap-points are commented inline; see +[Going to production](#going-to-production) below. ## How it works -A search session repeats four phases: +A search session repeats four phases (paper §3.3): | Phase | What happens | |---|---| -| **Diagnose** | The teacher reads eligible traces and groups failures into clusters, each annotated with a natural-language characterization of the skill gap. | -| **Plan** | The teacher proposes typed edits across the editable primitives (Intelligence, Engine, Agents, Tools & Memory). A single proposal can edit multiple slots at once. | -| **Execute** | The candidate spec is evaluated on a held-out gate: the targeted cluster must improve, and every other cluster must regress by no more than a per-cluster tolerance. | -| **Record** | Accepted edits are committed; rejected edits are rolled back. | +| **Diagnose** | Teacher reads eligible traces and groups failures into clusters, each annotated with `(student_failure_rate, teacher_success_rate, skill_gap)`. | +| **Plan** | Teacher proposes typed edits across the four editable primitives (Intelligence, Engine, Agents, Tools & Memory). One proposal can edit multiple slots at once. | +| **Execute** | Each candidate edit is applied; the gate scores the resulting spec on a held-out subsample. Accepted iff `GateOK` holds (see below). | +| **Record** | Accepted edits commit to the checkpoint store; rejected edits roll back. The session is persisted to `SessionStore`. | -The four-phase loop is implemented by the existing `jarvis learning` -command — see [Learning & Distillation](learning-distillation.md) for the -end-to-end runnable workflow, edit applier registry, and configuration -schema. This document covers the new building blocks added on top. +`SpecSearchOrchestrator.run(trigger)` runs **one** session end-to-end. +`SpecSearchLoop` (paper Algorithm 1) wraps the orchestrator and repeats +sessions until either gate-score stagnation (default *k* = 5 sessions) +or budget exhaustion. -### Alignment with the paper +### `GateOK` — the acceptance predicate -The paper's Algorithm 1 specifies a **multi-session loop** that runs -diagnose → plan → execute → record repeatedly until gate-score stagnation -(default *k* = 5 sessions) or budget exhaustion, with default per-cluster -tolerance ε = 1 %. The current `DistillationOrchestrator` runs -**one session per `jarvis learning run` invocation** (one diagnose, one -plan, one round of edits + gate decisions, one record); the per-cluster -regression check uses a default `max_regression = 0.05` (5 %). +Let `G_c(S)` be the held-out gate score of spec `S` restricted to +failure cluster `c`. For an edit `e` targeting cluster `c`, with +`S' = apply(S, e)`: -To match the paper's defaults today: - -- pass `max_regression=0.01` when constructing the orchestrator or the - `BenchmarkGate` directly, and -- run `jarvis learning run` from a wrapper that re-invokes it until - per-session gate-score deltas stagnate for *k* sessions. - -Wiring the multi-session stagnation loop into the orchestrator natively, -plus changing the default `max_regression` to 1 %, are tracked as -follow-ups — they're not part of this PR. - -## What this PR adds - -### `splits.py` — deterministic train / test splits - -`src/openjarvis/evals/core/splits.py` adds a small helper that takes a -list of records and a benchmark name and returns a deterministic -train / test partition. The split is keyed off a stable hash of each -record's id, so the same `(records, train_frac, seed)` always yields the -same partition. This is the substrate that makes "evaluate on -`split=test`" and "search over `split=train`" reproducible. - -```python -from openjarvis.evals.core.splits import apply_split - -train = apply_split(records, split="train", train_frac=0.2, seed=42) -test = apply_split(records, split="test", train_frac=0.2, seed=42) -all_ = apply_split(records, split="all", train_frac=0.2, seed=42) # passthrough +``` +GateOK(S', S, c, eps) ⟺ + G_c(S') > G_c(S) # target cluster improves, AND + G_c'(S') >= G_c'(S) − eps # every other cluster regresses by ≤ eps ``` -The `split` kwarg is now wired through every dataset provider in -`openjarvis.evals.datasets` (gaia, livecodebench, liveresearch, -liveresearchbench, pinchbench, taubench, toolcall15) so any caller that -constructs a `BenchmarkConfig` can request a particular split. +Default `eps = 0.01` (1 %) per the paper. The `BenchmarkGate` class +implements this; the `max_regression` knob is `eps`. -### External-corpus dataset providers +### Composite reward (Intelligence-edit training only) -When the diagnose phase wants the teacher to reason over a broader -agent-trace corpus instead of just the local student's own traces, it -can ingest records from a HuggingFace-backed external corpus. Three -providers are included: +When an Intelligence edit triggers LoRA / GRPO training inside the +execute phase, candidate responses `y` to query `q` are scored by +(paper Eq. 1): -| Corpus | HuggingFace dataset | What it surfaces | +``` +R(q, y) = α · R_acc(q, y) + − β · Ê(q, y) # energy + − γ · L̂(q, y) # latency + − δ · Ĉ(q, y) # cost +``` + +Defaults `(α, β, γ, δ) = (0.5, 0.1, 0.1, 0.3)`. The efficiency +quantities (E, L, C) are z-scored *within batch* before weighting, so +the reward trades dimensionless deviations rather than raw joules / +seconds / dollars (paper Appendix C.6). Implementation: +`openjarvis.learning.spec_search.composite_reward.score_batch`. + +The held-out gate evaluates the resulting spec end-to-end; it is +unaffected by these weights. + +## Configuration + +The prebuilt config lives at +`configs/openjarvis/examples/spec-search-quickstart.toml`. Copy it to +`~/.openjarvis/config.toml` (or set `OPENJARVIS_CONFIG` to it) and the +regular loader picks it up: + +```python +from openjarvis.core.config import load_config +cfg = load_config().learning.spec_search # SpecSearchLearningConfig +``` + +The `[learning.spec_search]` table maps 1:1 onto the +`SpecSearchLearningConfig` dataclass and is read by both +`SpecSearchOrchestrator.from_config` and `SpecSearchLoop`: + +```toml +[learning.spec_search] +enabled = true +teacher_model = "claude-opus-4-6" +teacher_engine = "cloud" +autonomy_mode = "tiered" # auto | tiered | manual + +# Per-session bounds +min_traces = 20 +max_cost_per_session_usd = 5.0 +max_tool_calls_per_diagnosis = 30 + +# Multi-session loop (paper Algorithm 1) +stagnation_k = 5 # paper default +stagnation_eps = 0.001 +max_total_cost_usd = 50.0 + +# Gate (GateOK) +max_regression = 0.01 # paper default: epsilon = 1% +min_improvement = 0.0 +benchmark_subsample_size = 50 +benchmark_version = "personal_v1" + +[learning.spec_search.composite_reward] +alpha = 0.5 # accuracy +beta = 0.1 # energy +gamma = 0.1 # latency +delta = 0.3 # cost +``` + +## Going to production + +The quickstart uses fakes for the teacher engine, student runner, and +judge so it runs without external services. To run a real session, +swap each fake for the corresponding production component: + +| Slot | Quickstart | Production | |---|---|---| -| `adp` | `neulab/agent-data-collection` | Multi-turn agent trajectories from AgentTuning, CodeAct, OpenHands, and others | -| `toolorchestra` | `nvidia/ToolScale` | Tool-use trajectories (the dataset underlying the ToolOrchestra paper) | -| `generalthoughts` | `natolambert/GeneralThought-430K-filtered` | A filtered reasoning-trace pool | +| `teacher_engine` | `FakeTeacherEngine` | `EngineRegistry.get(cfg.teacher_engine)(model=cfg.teacher_model)` — set `ANTHROPIC_API_KEY` etc. | +| `trace_store` | `MagicMock` | `openjarvis.traces.store.TraceStore(home / "traces.db")` | +| `student_runner` | `MagicMock` | `openjarvis.learning.spec_search.student_runner.VLLMStudentRunner(host=..., model=...)` | +| `judge` | `MagicMock` | `openjarvis.evals.core.scorer.LLMJudgeScorer(...)` (or a deterministic scorer if your benchmark provides one) | +| `session_store` | `MagicMock` | `openjarvis.learning.spec_search.storage.session_store.SessionStore(home / "learning" / "sessions.db")` | +| `checkpoint_store` | `MagicMock` | `openjarvis.learning.spec_search.checkpoint.store.CheckpointStore(home / "learning" / "checkpoints")` | +| `scorer` | climbing-plateau fake | a real `Scorer` callable (typically a `BenchmarkGate.score` adapter) | -Each provider implements the standard `DatasetProvider.load(...)` -interface so corpus records can be loaded the same way benchmark records -are. They're labeled "NOT USED FOR EVALUATION" in their docstrings — -they exist purely to feed the proposer's diagnose phase. +The orchestrator only depends on the *interface* of each slot, not the +concrete class — anything implementing the corresponding protocol works. -### `external_adapter.py` — corpus records as synthetic traces +## Adding a new external corpus -`src/openjarvis/learning/distillation/external_adapter.py` adapts records -from any external corpus into rows the proposer's existing trace tools -understand. The proposer reads from the SQLite TraceStore via search / -get tools; this adapter writes each `EvalRecord` as a synthetic `Trace` -with `feedback=0.5` (no ground truth) and a `source_name` tag in -metadata so multi-source diagnose runs can filter downstream. +Diagnose phase can ingest records from a HuggingFace-backed external +corpus. Three providers ship in-tree (`adp`, `toolorchestra`, +`generalthoughts`); to add a new one: + +1. Create `src/openjarvis/evals/datasets/.py` implementing + `DatasetProvider` (`adp.py` is a small reference). The provider's + `load(max_samples, seed, split)` must respect `split` via + `apply_split` from `openjarvis.evals.core.splits`. +2. Register: `@DatasetRegistry.register("")`. +3. Feed it to the proposer via the trace store: ```python from openjarvis.evals.datasets.adp import ADPDataset -from openjarvis.learning.distillation.external_adapter import ( +from openjarvis.learning.spec_search.external_adapter import ( write_external_records_as_traces, ) from openjarvis.traces.store import TraceStore -records = ADPDataset().load(max_samples=200, seed=42, split="all") +records = list(ADPDataset().load(max_samples=200, seed=42, split="all")) store = TraceStore("~/.openjarvis/traces.db") n = write_external_records_as_traces(store, records, source_name="adp") -# proposer's diagnose phase can now search/filter these traces by source="adp" +# proposer can now filter on metadata["source"] == "adp" ``` -### Bug fix: agent-backend trace toggle +## What runs where + +At inference time, the resulting spec runs entirely on-device — model +inference, agent execution, tool invocation. Teacher API calls happen +only at search time (diagnose + plan), and only **eligible scrubbed +traces** are transmitted (per the trace-eligibility rules in your +config). + +Users requiring strict local-only operation can swap a larger local +model in as the teacher; this trades search quality for zero cloud +exposure. + +## Bug fix bundled with this release `src/openjarvis/evals/backends/jarvis_agent.py` previously hardcoded `builder.telemetry(telemetry).traces(True).build()`, ignoring the @@ -125,7 +192,7 @@ intent. A corrupt traces.db then turned every agent eval into "database disk image is malformed" errors that the eval scorer dropped, producing fake high accuracies from a handful of successful samples. -The fix is one line — match the `JarvisDirectBackend` pattern: +The one-line fix: ```python self._system = builder.telemetry(telemetry).traces(telemetry).build() @@ -134,75 +201,14 @@ self._system = builder.telemetry(telemetry).traces(telemetry).build() Callers that previously expected traces to always be written should pass `telemetry=True` explicitly. -## Configuration +## See also -`jarvis learning` reads its configuration from `~/.openjarvis/config.toml`: - -```toml -[learning.distillation] -enabled = true # gate the entire subsystem -autonomy_mode = "tiered" # auto | tiered | manual -teacher_model = "claude-opus-4-6" # any CloudEngine-supported model -max_cost_per_session_usd = 5.0 # per-session teacher API budget -max_tool_calls_per_diagnosis = 30 # max teacher tool calls in diagnosis -``` - -Gate / acceptance knobs (constructor arguments to `BenchmarkGate` and -`DistillationOrchestrator`): - -| Argument | Meaning | Current default | Paper default | -|---|---|---|---| -| `max_regression` | Maximum per-cluster score drop tolerated before rejecting an edit (the ε in the paper's `GateOK`). | `0.05` (5 %) | `0.01` (1 %) | -| `min_improvement` | Minimum overall score improvement required to accept an edit. | `0.0` | (paper does not specify; uses `> 0` per `Gc(S') > Gc(S)`) | -| `n_tasks` | Number of tasks scored per gate run. | `50` | n/a (gate is per-cluster) | - -Pass `max_regression=0.01` explicitly to match the paper's default. See -the *Alignment with the paper* note above for the gap between the -single-session `DistillationOrchestrator` and the paper's multi-session -loop with stagnation criterion. - -Each `EditApplier` registers an autonomy tier (`auto`, `review`, -`manual`); see the -[Learning & Distillation](learning-distillation.md) doc for the per-edit -configuration schema and the registered appliers. - -## What runs where - -At inference time, the resulting spec runs entirely on-device — model -inference, agent execution, and tool invocation. Teacher API calls are -made only at search time, for diagnosis and edit proposal. The local -spec makes zero teacher calls at inference time. - -When a frontier teacher is used, only **eligible** scrubbed traces are -transmitted (per the trace-eligibility rules in the configuration). -Users requiring strict local-only operation can swap a larger local -model in as the teacher; this trades some search quality for zero cloud -exposure. - -## Why decouple primitives at all - -Existing personal AI frameworks bundle agent prompts, tool descriptions, -memory configuration, and runtime settings around a specific cloud -model. Naive substitution of a local model for the cloud model collapses -accuracy because none of the surrounding configuration was tuned for the -new model. Optimizing across primitives jointly — instead of -single-primitive optimization (LoRA-only, prompt-only) — is what allows -LLM-guided spec search to recover that lost accuracy. The decomposition -into Intelligence / Engine / Agents / Tools & Memory / Learning, with -the spec as the typed configuration object, is what makes the search -space well-defined. - -## Adding a new external corpus - -1. Create `src/openjarvis/evals/datasets/.py` implementing - `DatasetProvider` (look at `adp.py` for a small reference). The - provider's `load(max_samples, seed, split)` should respect the - `split` kwarg via `apply_split` from `openjarvis.evals.core.splits`. -2. Register the new dataset in your local `DatasetRegistry` (typically - via `@DatasetRegistry.register("")` decorator on the class). -3. Use it the same way as the bundled corpora: load records, then call - `write_external_records_as_traces(store, records, source_name="")` - to make them visible to the proposer's diagnose phase. - -The proposer can then filter on `metadata["source"] == ""` if -you're feeding multiple corpora into the same search session. +- `examples/openjarvis/spec_search_quickstart.py` — runnable end-to-end demo. +- `configs/openjarvis/examples/spec-search-quickstart.toml` — prebuilt config. +- `src/openjarvis/learning/spec_search/orchestrator.py` — `SpecSearchOrchestrator` (single session). +- `src/openjarvis/learning/spec_search/multi_session.py` — `SpecSearchLoop` (Algorithm 1). +- `src/openjarvis/learning/spec_search/composite_reward.py` — paper Eq. 1. +- `src/openjarvis/learning/spec_search/gate/benchmark_gate.py` — `GateOK` predicate. +- `src/openjarvis/learning/spec_search/external_adapter.py` — corpus → trace adapter. +- `tests/learning/spec_search/test_multi_session.py`, `test_composite_reward.py` — unit tests. +- `tests/learning/spec_search/test_orchestrator.py` — full-session test with mocks. diff --git a/examples/openjarvis/spec_search_quickstart.py b/examples/openjarvis/spec_search_quickstart.py new file mode 100644 index 00000000..c3a21f76 --- /dev/null +++ b/examples/openjarvis/spec_search_quickstart.py @@ -0,0 +1,306 @@ +"""LLM-Guided Spec Search — runnable quickstart. + +This script wires up every primitive ``SpecSearchOrchestrator`` needs and runs +one full session, then a multi-session loop with the paper's stagnation rule +(Algorithm 1, k=5, epsilon=1%). + +It is *self-contained*: by default the teacher engine and the student runner +are local fakes, so you can ``python examples/openjarvis/spec_search_quickstart.py`` +with no API keys and no Ollama / vLLM running. Every "swap this for production" +hookpoint is called out inline. + +Configuration is read from ``configs/openjarvis/examples/spec-search-quickstart.toml`` +via the regular ``openjarvis.core.config.load_config`` machinery — you can copy +that TOML to ``~/.openjarvis/config.toml`` and tune the gate / stagnation / +reward knobs without editing this file. + +Run: + + OPENJARVIS_HOME=/tmp/openjarvis-spec-search-demo \\ + python examples/openjarvis/spec_search_quickstart.py +""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +from openjarvis.core.config import SpecSearchLearningConfig +from openjarvis.learning.spec_search.composite_reward import ( + RewardWeights, + TrainingSample, + score_batch, +) +from openjarvis.learning.spec_search.models import ( + BenchmarkSnapshot, + FailureCluster, +) +from openjarvis.learning.spec_search.multi_session import SpecSearchLoop +from openjarvis.learning.spec_search.orchestrator import SpecSearchOrchestrator +from openjarvis.learning.spec_search.triggers import OnDemandTrigger + +# --------------------------------------------------------------------------- +# Fakes — replace these with real production components. +# --------------------------------------------------------------------------- + + +@dataclass +class FakeTeacherEngine: + """Stand-in for ``CloudEngine``. + + For production, use the registry:: + + from openjarvis.core.registry import EngineRegistry + engine_cls = EngineRegistry.get(cfg.teacher_engine) # "cloud" + engine = engine_cls(model=cfg.teacher_model) # set ANTHROPIC_API_KEY + + The orchestrator only calls ``engine.generate(...)`` so any object with + that method will work. + """ + + call_count: int = 0 + + def generate(self, **_: Any) -> dict[str, Any]: + self.call_count += 1 + # Propose one auto-tier Tools edit so the gate has something to score. + return { + "content": json.dumps( + { + "edits": [ + { + "id": f"edit-{self.call_count:03d}", + "pillar": "tools", + "op": "edit_tool_description", + "target": "tools.web_search", + "payload": { + "tool_name": "web_search", + "new_description": ( + "Search the web for recent information. " + "Prefer this for time-sensitive queries." + ), + }, + "rationale": ( + "Student under-invokes web_search on multi-hop queries." + ), + "expected_improvement": "c1", + "risk_tier": "auto", + "references": ["t-001"], + } + ] + } + ), + "usage": {"total_tokens": 1200}, + "cost_usd": 0.04, + "finish_reason": "stop", + } + + +def _fake_diagnosis() -> Any: + """A canned DiagnosisResult so the demo doesn't need a real teacher loop.""" + from openjarvis.learning.spec_search.diagnose.runner import DiagnosisResult + + return DiagnosisResult( + diagnosis_md=( + "## Diagnosis\n\n" + "Cluster `c1` (multi-hop research): student fails to invoke " + "`web_search` on time-sensitive queries; teacher invokes it " + "consistently. Likely cause: tool description does not signal " + "freshness." + ), + clusters=[ + FailureCluster( + id="c1", + description="Multi-hop research; web_search under-invocation", + sample_trace_ids=["t-001", "t-002", "t-003"], + student_failure_rate=0.7, + teacher_success_rate=0.95, + skill_gap="Student does not invoke web_search on multi-hop research.", + ) + ], + cost_usd=0.05, + tool_call_records=[], + ) + + +def _fake_scorer_factory(start_score: float = 0.60, step: float = 0.06): + """Return a scorer whose overall score climbs by ``step`` on each call. + + Mimics the loop the paper describes: each accepted edit lifts the gate + score, and the multi-session loop stops once gains plateau. + """ + state = {"score": start_score - step, "calls": 0} + + def scorer(**_: Any) -> BenchmarkSnapshot: + state["calls"] += 1 + # Every other call (the "after" snapshot) bumps the score; gain + # tapers after 3 sessions so the stagnation rule fires. + if state["calls"] % 2 == 0: + bump = step if state["calls"] // 2 <= 3 else 0.0 + state["score"] = min(1.0, state["score"] + bump) + return BenchmarkSnapshot( + benchmark_version="personal_v1", + overall_score=state["score"], + cluster_scores={"c1": state["score"]}, + task_count=10, + elapsed_seconds=5.0, + ) + + return scorer + + +# --------------------------------------------------------------------------- +# Wire-up +# --------------------------------------------------------------------------- + + +def build_orchestrator( + config: SpecSearchLearningConfig, + home: Path, +) -> SpecSearchOrchestrator: + """Construct a SpecSearchOrchestrator from a SpecSearchLearningConfig. + + All five injected primitives below are demo fakes; the comments next to + each one show the production replacement. + """ + return SpecSearchOrchestrator.from_config( + config, + # Teacher: replace with EngineRegistry.get("cloud")(model=cfg.teacher_model) + teacher_engine=FakeTeacherEngine(), + # TraceStore: production = TraceStore(home / "traces.db") + trace_store=MagicMock(count=MagicMock(return_value=config.min_traces + 10)), + benchmark_samples=[], + # StudentRunner: production = VLLMStudentRunner(host=..., model=...) + student_runner=MagicMock(), + # Judge: production = openjarvis.evals.core.scorer.LLMJudgeScorer(...) + judge=MagicMock(), + # SessionStore + CheckpointStore: production = real on-disk stores + session_store=MagicMock(), + checkpoint_store=MagicMock( + current_sha=MagicMock(return_value="demo-sha"), + begin_stage=MagicMock( + return_value=MagicMock(pre_stage_sha="demo-sha"), + ), + ), + openjarvis_home=home, + scorer=_fake_scorer_factory(), + ) + + +def demo_composite_reward(weights: RewardWeights) -> None: + """Show how the paper Eq. 1 reward ranks Intelligence-edit candidates.""" + candidates = [ + TrainingSample( + accuracy=1.0, energy_joules=200, latency_seconds=5.0, cost_usd=0.0 + ), + TrainingSample( + accuracy=1.0, energy_joules=400, latency_seconds=8.0, cost_usd=0.0 + ), + TrainingSample( + accuracy=0.0, energy_joules=100, latency_seconds=2.0, cost_usd=0.0 + ), + ] + rewards = score_batch(candidates, weights=weights) + print("\nComposite reward (paper Eq. 1) — ranking 3 candidates:") + print( + f" weights = (alpha={weights.alpha}, beta={weights.beta}, " + f"gamma={weights.gamma}, delta={weights.delta})" + ) + for i, (c, r) in enumerate(zip(candidates, rewards)): + print( + f" candidate {i}: acc={c.accuracy} energy={c.energy_joules}J " + f"latency={c.latency_seconds}s -> reward={r:+.3f}" + ) + + +def main() -> None: + # In a real deployment, ``load_config()`` reads from ``~/.openjarvis/config.toml``. + # For a self-contained demo we synthesize the spec-search config inline so the + # script is runnable without copying any files. To use the prebuilt TOML: + # + # from openjarvis.core.config import load_config + # cfg = load_config().learning.spec_search + # + # (after copying ``configs/openjarvis/examples/spec-search-quickstart.toml`` + # to ``~/.openjarvis/config.toml``) + + cfg = SpecSearchLearningConfig( + enabled=True, + teacher_model="claude-opus-4-6", + teacher_engine="cloud", + autonomy_mode="auto", + min_traces=20, + max_cost_per_session_usd=5.0, + max_tool_calls_per_diagnosis=30, + stagnation_k=5, # paper default + stagnation_eps=0.001, + max_total_cost_usd=50.0, + max_regression=0.01, # paper default: epsilon = 1% + min_improvement=0.0, + benchmark_subsample_size=10, + ) + + home = Path( + os.environ.get("OPENJARVIS_HOME") + or tempfile.mkdtemp(prefix="openjarvis-spec-search-") + ) + print(f"OPENJARVIS_HOME = {home}") + + # ----- Single session --------------------------------------------------- + orch = build_orchestrator(cfg, home) + + # The orchestrator's diagnose phase calls a real DiagnosisRunner that + # invokes the teacher. We swap it out for the canned diagnosis above so + # the demo does not need an API key. + from unittest.mock import patch + + print("\n=== Single session (one diagnose / plan / execute / record) ===") + with patch( + "openjarvis.learning.spec_search.orchestrator.DiagnosisRunner" + ) as MockDiag: + MockDiag.return_value.run.return_value = _fake_diagnosis() + session = orch.run(OnDemandTrigger()) + + print(f" status = {session.status.value}") + print(f" teacher_cost_usd = ${session.teacher_cost_usd:.4f}") + print(f" edit_outcomes = {[(o.edit_id, o.status) for o in session.edit_outcomes]}") + if session.benchmark_after is not None: + print( + f" before -> after = " + f"{session.benchmark_before.overall_score:.3f} -> " + f"{session.benchmark_after.overall_score:.3f}" + ) + + # ----- Multi-session loop (paper Algorithm 1) --------------------------- + orch = build_orchestrator(cfg, home) # fresh fakes for clean state + loop = SpecSearchLoop( + orch, + stagnation_k=cfg.stagnation_k, + stagnation_eps=cfg.stagnation_eps, + max_total_cost_usd=cfg.max_total_cost_usd, + ) + + print( + "\n=== Multi-session loop (stagnation_k = " + f"{cfg.stagnation_k}, max_total_cost = ${cfg.max_total_cost_usd}) ===" + ) + with patch( + "openjarvis.learning.spec_search.orchestrator.DiagnosisRunner" + ) as MockDiag: + MockDiag.return_value.run.return_value = _fake_diagnosis() + result = loop.run() + + print(f" sessions = {len(result.sessions)}") + print(f" stop_reason = {result.stop_reason}") + print(f" total cost = ${result.total_cost_usd:.4f}") + print(f" best score = {result.best_overall_score:.3f}") + + demo_composite_reward(RewardWeights()) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 7da46602..25253956 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -183,7 +183,7 @@ nav: - Scheduler: user-guide/scheduler.md - Telemetry: user-guide/telemetry.md - Security: user-guide/security.md - - Learning & Distillation: user-guide/learning-distillation.md + - LLM-guided spec search: user-guide/llm-guided-spec-search.md - Leaderboard: leaderboard.md - Roadmap: development/roadmap.md - Development: diff --git a/src/openjarvis/agents/prompt_loader.py b/src/openjarvis/agents/prompt_loader.py index 429d8b17..4a508e66 100644 --- a/src/openjarvis/agents/prompt_loader.py +++ b/src/openjarvis/agents/prompt_loader.py @@ -1,6 +1,6 @@ """Load system prompt and few-shot overrides from $OPENJARVIS_HOME. -Distillation (M1) proposes edits that get written to disk by appliers. +LLM-guided spec search (M1) proposes edits that get written to disk by appliers. This module lets agents pick those overrides up at runtime: - System prompts: ``$OPENJARVIS_HOME/agents/{name}/system_prompt.md`` diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py index de29bcc7..658682f4 100644 --- a/src/openjarvis/cli/__init__.py +++ b/src/openjarvis/cli/__init__.py @@ -41,7 +41,6 @@ from openjarvis.cli.telemetry_cmd import telemetry from openjarvis.cli.tool_cmd import tool from openjarvis.cli.vault_cmd import vault from openjarvis.cli.workflow_cmd import workflow -from openjarvis.learning.distillation.cli import learning_group @click.group( @@ -113,7 +112,6 @@ cli.add_command(connect, "connect") cli.add_command(digest, "digest") cli.add_command(deep_research_setup, "deep-research-setup") cli.add_command(deep_research_setup, "research") -cli.add_command(learning_group, "learning") cli.add_command(bootstrap_cmd, "_bootstrap") # Gateway CLI commands (lazy import to avoid pulling starlette) diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 2d9237c0..855e752f 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -684,6 +684,54 @@ class MetricsConfig: efficiency_weight: float = 0.1 +@dataclass(slots=True) +class SpecSearchCompositeRewardConfig: + """Composite reward weights for Intelligence-edit training (paper Eq. 1). + + R(q, y) = alpha * R_acc - beta * E_hat - gamma * L_hat - delta * C_hat + """ + + alpha: float = 0.5 + beta: float = 0.1 + gamma: float = 0.1 + delta: float = 0.3 + + +@dataclass(slots=True) +class SpecSearchLearningConfig: + """LLM-guided spec search config (paper §3.3, Algorithm 1). + + Maps to ``[learning.spec_search]`` and is consumed by + ``SpecSearchOrchestrator.from_config`` and ``SpecSearchLoop``. + """ + + enabled: bool = False + teacher_model: str = "claude-opus-4-6" + teacher_engine: str = "cloud" # registry key for the cloud engine + autonomy_mode: str = "tiered" # auto | tiered | manual + + # Per-session bounds (one diagnose/plan/execute pass) + min_traces: int = 20 + max_cost_per_session_usd: float = 5.0 + max_tool_calls_per_diagnosis: int = 30 + + # Multi-session loop (paper Algorithm 1) + stagnation_k: int = 5 + max_total_cost_usd: float = 50.0 + stagnation_eps: float = 0.001 # gate-score delta below this counts as no progress + + # Gate (GateOK predicate) + max_regression: float = 0.01 # paper default: epsilon = 1% + min_improvement: float = 0.0 + benchmark_subsample_size: int = 50 + benchmark_version: str = "personal_v1" + + # Composite reward (only used when an Intelligence edit triggers training) + composite_reward: SpecSearchCompositeRewardConfig = field( + default_factory=SpecSearchCompositeRewardConfig, + ) + + @dataclass class LearningConfig: """Learning system settings with per-primitive sub-policies.""" @@ -697,6 +745,9 @@ class LearningConfig: ) agent: AgentLearningConfig = field(default_factory=AgentLearningConfig) skills: SkillsLearningConfig = field(default_factory=SkillsLearningConfig) + spec_search: SpecSearchLearningConfig = field( + default_factory=SpecSearchLearningConfig, + ) metrics: MetricsConfig = field(default_factory=MetricsConfig) # Training pipeline @@ -1578,7 +1629,7 @@ def _parse_mining_section(data: dict) -> Optional["MiningConfig"]: pearld_rpc_url=extra.get("pearld_rpc_url", "http://localhost:44107") ) elif isinstance(target_str, str) and target_str.startswith("pool:"): - submit_target = PoolTarget(url=target_str[len("pool:"):]) + submit_target = PoolTarget(url=target_str[len("pool:") :]) else: raise ValueError( f"[mining].submit_target must be 'solo' or 'pool:', got {target_str!r}" diff --git a/src/openjarvis/evals/comparison/table_gen.py b/src/openjarvis/evals/comparison/table_gen.py index 575a1660..4e703182 100644 --- a/src/openjarvis/evals/comparison/table_gen.py +++ b/src/openjarvis/evals/comparison/table_gen.py @@ -339,9 +339,9 @@ def _build_t6(frame: ResultsFrame) -> Tuple[str, str]: def _build_t7(frame: ResultsFrame) -> Tuple[str, str]: """T7: Edit category x framework — preliminary (raw deltas). - Spec §10 says richer edit-attribution data needs the spec-distillation - pipeline to tag accepted edits. For now, raw per-benchmark accuracy - deltas across frameworks. + Spec §10 says richer edit-attribution data needs the LLM-guided + spec-search pipeline to tag accepted edits. For now, raw per-benchmark + accuracy deltas across frameworks. """ df = frame.df.filter(pl.col("metric_name") == "accuracy") pivot = ( diff --git a/src/openjarvis/evals/datasets/adp.py b/src/openjarvis/evals/datasets/adp.py index ab9c665b..9394bf45 100644 --- a/src/openjarvis/evals/datasets/adp.py +++ b/src/openjarvis/evals/datasets/adp.py @@ -2,7 +2,7 @@ NOT USED FOR EVALUATION. Surfaces multi-turn agent trajectories to the LLM-guided spec search proposer via -``openjarvis.learning.distillation.external_adapter`` so the diagnose +``openjarvis.learning.spec_search.external_adapter`` so the diagnose phase can reason over a broad pool of agent traces without depending on the per-cell student's own trace history. diff --git a/src/openjarvis/evals/datasets/generalthoughts.py b/src/openjarvis/evals/datasets/generalthoughts.py index 08306310..1d3a8ea2 100644 --- a/src/openjarvis/evals/datasets/generalthoughts.py +++ b/src/openjarvis/evals/datasets/generalthoughts.py @@ -2,7 +2,7 @@ NOT USED FOR EVALUATION. Provides the LLM-guided spec search proposer with a large pool of reasoning trajectories to reason over (via -`openjarvis.learning.distillation.external_adapter`) when the diagnose +`openjarvis.learning.spec_search.external_adapter`) when the diagnose phase wants signal from a broad reasoning corpus rather than the per-cell student's own trace history. diff --git a/src/openjarvis/evals/datasets/toolorchestra.py b/src/openjarvis/evals/datasets/toolorchestra.py index 3f546a53..dbd4325a 100644 --- a/src/openjarvis/evals/datasets/toolorchestra.py +++ b/src/openjarvis/evals/datasets/toolorchestra.py @@ -3,7 +3,7 @@ NOT USED FOR EVALUATION. Surfaces tool-use trajectories from ``nvidia/ToolScale`` (the dataset underlying the ToolOrchestra paper) to the LLM-guided spec search proposer via -``openjarvis.learning.distillation.external_adapter`` so the diagnose +``openjarvis.learning.spec_search.external_adapter`` so the diagnose phase can reason over a broad pool of tool-use traces. The ``dataset_id`` is kept as ``"toolorchestra"`` (matching the published diff --git a/src/openjarvis/learning/distillation/__init__.py b/src/openjarvis/learning/distillation/__init__.py deleted file mode 100644 index 7e5f027b..00000000 --- a/src/openjarvis/learning/distillation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Harness distillation: frontier-driven learning subsystem.""" diff --git a/src/openjarvis/learning/distillation/cli.py b/src/openjarvis/learning/distillation/cli.py deleted file mode 100644 index 0daf61e4..00000000 --- a/src/openjarvis/learning/distillation/cli.py +++ /dev/null @@ -1,132 +0,0 @@ -"""``jarvis learning`` — distillation learning CLI subcommands. - -See spec §12. -""" - -from __future__ import annotations - -import click -from rich.console import Console - -console = Console() - - -@click.group("learning") -def learning_group() -> None: - """Frontier-driven harness learning (distillation).""" - - -@learning_group.command("init") -def learning_init() -> None: - """Initialize the distillation checkpoint repo and directory layout.""" - from openjarvis.learning.distillation.checkpoint.store import CheckpointStore - from openjarvis.learning.distillation.storage.paths import ( - ensure_distillation_dirs, - resolve_distillation_root, - ) - - root = resolve_distillation_root() - ensure_distillation_dirs() - home = root.parent # ~/.openjarvis - store = CheckpointStore(home) - store.init() - console.print(f"[green]Initialized distillation at {root}[/green]") - - -@learning_group.command("run") -@click.option( - "--autonomy", - type=click.Choice(["auto", "tiered", "manual"]), - default="tiered", -) -def learning_run(autonomy: str) -> None: - """Run an on-demand learning session.""" - console.print("[yellow]On-demand session started.[/yellow]") - console.print("[dim]Use 'jarvis learning history' to check results.[/dim]") - console.print(f"[dim]Autonomy mode: {autonomy}[/dim]") - # Full wiring deferred to integration — this registers the CLI surface. - console.print("[dim]Full orchestration requires configured teacher engine.[/dim]") - - -@learning_group.command("history") -@click.option("--limit", type=int, default=10, help="Max sessions to show.") -def learning_history(limit: int) -> None: - """List past learning sessions.""" - console.print(f"[dim]Showing last {limit} sessions (requires learning.db).[/dim]") - - -@learning_group.command("show") -@click.argument("session_id") -def learning_show(session_id: str) -> None: - """Show details of a learning session.""" - console.print(f"[dim]Session: {session_id}[/dim]") - - -@learning_group.command("review") -def learning_review() -> None: - """Review pending edits awaiting approval.""" - console.print("[dim]Pending review queue.[/dim]") - - -@learning_group.command("approve") -@click.argument("edit_id") -def learning_approve(edit_id: str) -> None: - """Approve a pending edit.""" - console.print(f"[dim]Approving edit: {edit_id}[/dim]") - - -@learning_group.command("reject") -@click.argument("edit_id") -@click.option("--reason", type=str, default="", help="Rejection reason.") -def learning_reject(edit_id: str, reason: str) -> None: - """Reject a pending edit.""" - console.print(f"[dim]Rejecting edit: {edit_id}[/dim]") - - -@learning_group.command("rollback") -@click.argument("session_id", required=False) -@click.option("--last", is_flag=True, help="Rollback the most recent session.") -def learning_rollback(session_id: str | None, last: bool) -> None: - """Rollback a learning session's commits.""" - target = session_id or ("last session" if last else "none") - console.print(f"[dim]Rolling back: {target}[/dim]") - - -@learning_group.group("benchmark") -def benchmark_group() -> None: - """Personal benchmark management.""" - - -@benchmark_group.command("refresh") -def benchmark_refresh() -> None: - """Manually refresh the personal benchmark.""" - console.print("[dim]Refreshing personal benchmark.[/dim]") - - -@benchmark_group.command("show") -def benchmark_show() -> None: - """Show current benchmark statistics.""" - console.print("[dim]Benchmark stats.[/dim]") - - -@learning_group.group("daemon") -def daemon_group() -> None: - """Background learning daemon.""" - - -@daemon_group.command("start") -def daemon_start() -> None: - """Start the learning daemon.""" - console.print("[dim]Starting daemon.[/dim]") - - -@daemon_group.command("stop") -def daemon_stop() -> None: - """Stop the learning daemon.""" - console.print("[dim]Stopping daemon.[/dim]") - - -@daemon_group.command("status") -def daemon_status() -> None: - """Check daemon status.""" - console.print("[dim]Daemon status.[/dim]") diff --git a/src/openjarvis/learning/spec_search/__init__.py b/src/openjarvis/learning/spec_search/__init__.py new file mode 100644 index 00000000..cadb4fe2 --- /dev/null +++ b/src/openjarvis/learning/spec_search/__init__.py @@ -0,0 +1 @@ +"""LLM-guided spec search: frontier-driven learning subsystem.""" diff --git a/src/openjarvis/learning/distillation/checkpoint/__init__.py b/src/openjarvis/learning/spec_search/checkpoint/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/checkpoint/__init__.py rename to src/openjarvis/learning/spec_search/checkpoint/__init__.py diff --git a/src/openjarvis/learning/distillation/checkpoint/store.py b/src/openjarvis/learning/spec_search/checkpoint/store.py similarity index 95% rename from src/openjarvis/learning/distillation/checkpoint/store.py rename to src/openjarvis/learning/spec_search/checkpoint/store.py index deda782a..bdb92d3f 100644 --- a/src/openjarvis/learning/distillation/checkpoint/store.py +++ b/src/openjarvis/learning/spec_search/checkpoint/store.py @@ -1,4 +1,4 @@ -"""Git-backed checkpoint store for distillation config rollback. +"""Git-backed checkpoint store for spec-search config rollback. A thin wrapper over a local git repository at ``/.git``. The repo tracks ``config.toml``, ``agents/``, and ``tools/`` so that the @@ -19,7 +19,7 @@ import subprocess from dataclasses import dataclass from pathlib import Path -from openjarvis.learning.distillation.storage.paths import ( +from openjarvis.learning.spec_search.storage.paths import ( ConfigurationError, _find_source_root, ) @@ -47,7 +47,7 @@ class StageHandle: class CheckpointStore: - """Thin git wrapper for the distillation checkpoint repo. + """Thin git wrapper for the spec-search checkpoint repo. Parameters ---------- @@ -74,7 +74,7 @@ class CheckpointStore: Refuses to initialize if ``self.root`` is inside the OpenJarvis source tree — this is the same defense-in-depth check as - ``resolve_distillation_root``: we never want a stray git repo writing + ``resolve_spec_search_root``: we never want a stray git repo writing config snapshots into the working copy. Idempotent: if ``.git`` already exists and contains a baseline commit, @@ -100,8 +100,8 @@ class CheckpointStore: if not (self._root / ".git").exists(): self._git("init", "-q") - self._git("config", "user.email", "distillation@openjarvis.local") - self._git("config", "user.name", "OpenJarvis Distillation") + self._git("config", "user.email", "spec-search@openjarvis.local") + self._git("config", "user.name", "OpenJarvis Spec Search") # Stage whatever tracked paths currently exist (it's OK if some # don't yet — the user may not have agents or tools dirs at first diff --git a/src/openjarvis/learning/spec_search/composite_reward.py b/src/openjarvis/learning/spec_search/composite_reward.py new file mode 100644 index 00000000..7d5ec4c7 --- /dev/null +++ b/src/openjarvis/learning/spec_search/composite_reward.py @@ -0,0 +1,97 @@ +"""Composite reward for Intelligence-edit training (paper §3.3, Eq. 1). + +R(q, y) = alpha * R_acc(q, y) - beta * E_hat(q, y) + - gamma * L_hat(q, y) + - delta * C_hat(q, y) + +The efficiency quantities (E, L, C) are normalised within the evaluated +benchmark before weighting (z-score), so the reward trades dimensionless +deviations rather than raw joules / seconds / dollars (paper Appendix C.6). + +Default weights (alpha, beta, gamma, delta) = (0.5, 0.1, 0.1, 0.3). + +This reward is consumed only inside an Intelligence edit that triggers +training (LoRA/GRPO). The held-out gate (BenchmarkGate) evaluates the +resulting spec end-to-end and is not affected by these weights. +""" + +from __future__ import annotations + +import statistics +from dataclasses import dataclass +from typing import Sequence + + +@dataclass(slots=True, frozen=True) +class RewardWeights: + """Composite-reward weights from paper Eq. 1.""" + + alpha: float = 0.5 + beta: float = 0.1 + gamma: float = 0.1 + delta: float = 0.3 + + +@dataclass(slots=True, frozen=True) +class TrainingSample: + """One (query, response) candidate scored during Intelligence training. + + All efficiency quantities are raw (un-normalised); normalisation is + applied across the batch by ``score_batch``. + """ + + accuracy: float # 0..1; for binary tasks, 0 or 1 + energy_joules: float + latency_seconds: float + cost_usd: float + + +def _zscore(values: Sequence[float]) -> list[float]: + """Per-batch z-score; returns zeros if the batch has zero variance. + + Within-batch normalisation is the paper's choice (Appendix C.6) so + the composite reward trades dimensionless deviations. + """ + if not values: + return [] + mean = statistics.fmean(values) + if len(values) < 2: + return [0.0] * len(values) + stdev = statistics.pstdev(values) + if stdev == 0.0: + return [0.0] * len(values) + return [(v - mean) / stdev for v in values] + + +def score_batch( + samples: Sequence[TrainingSample], + weights: RewardWeights | None = None, +) -> list[float]: + """Score a batch of candidates with the paper's composite reward. + + Energy / latency / cost are z-scored within the batch before being + weighted, so the reward magnitudes are comparable across benchmarks + and hardware platforms. + + Args: + samples: candidate (query, response) pairs to score. + weights: composite-reward weights; uses paper defaults if None. + + Returns: + One scalar reward per sample, in the same order. + """ + w = weights or RewardWeights() + if not samples: + return [] + + e_norm = _zscore([s.energy_joules for s in samples]) + l_norm = _zscore([s.latency_seconds for s in samples]) + c_norm = _zscore([s.cost_usd for s in samples]) + + return [ + w.alpha * s.accuracy - w.beta * e - w.gamma * lat - w.delta * cost + for s, e, lat, cost in zip(samples, e_norm, l_norm, c_norm) + ] + + +__all__ = ["RewardWeights", "TrainingSample", "score_batch"] diff --git a/src/openjarvis/learning/distillation/diagnose/__init__.py b/src/openjarvis/learning/spec_search/diagnose/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/diagnose/__init__.py rename to src/openjarvis/learning/spec_search/diagnose/__init__.py diff --git a/src/openjarvis/learning/distillation/diagnose/runner.py b/src/openjarvis/learning/spec_search/diagnose/runner.py similarity index 96% rename from src/openjarvis/learning/distillation/diagnose/runner.py rename to src/openjarvis/learning/spec_search/diagnose/runner.py index b0d34616..483a2cce 100644 --- a/src/openjarvis/learning/distillation/diagnose/runner.py +++ b/src/openjarvis/learning/spec_search/diagnose/runner.py @@ -1,4 +1,4 @@ -"""DiagnosisRunner: orchestrates phase 1 of the distillation loop. +"""DiagnosisRunner: orchestrates phase 1 of the spec-search loop. Builds diagnostic tools, runs the TeacherAgent, parses failure clusters from the teacher's output, and persists artifacts. @@ -15,14 +15,14 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any -from openjarvis.learning.distillation.diagnose.teacher_agent import ( +from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) -from openjarvis.learning.distillation.diagnose.tools import ( +from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) -from openjarvis.learning.distillation.diagnose.types import ToolCallRecord -from openjarvis.learning.distillation.models import FailureCluster +from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord +from openjarvis.learning.spec_search.models import FailureCluster logger = logging.getLogger(__name__) @@ -85,7 +85,7 @@ class DiagnosisResult: class DiagnosisRunner: - """Orchestrates phase 1 of the distillation loop. + """Orchestrates phase 1 of the spec-search loop. Parameters ---------- diff --git a/src/openjarvis/learning/distillation/diagnose/teacher_agent.py b/src/openjarvis/learning/spec_search/diagnose/teacher_agent.py similarity index 99% rename from src/openjarvis/learning/distillation/diagnose/teacher_agent.py rename to src/openjarvis/learning/spec_search/diagnose/teacher_agent.py index c919ced6..bfcde3f5 100644 --- a/src/openjarvis/learning/distillation/diagnose/teacher_agent.py +++ b/src/openjarvis/learning/spec_search/diagnose/teacher_agent.py @@ -24,7 +24,7 @@ from datetime import datetime, timezone from typing import Any from openjarvis.core.types import Message, Role, ToolCall -from openjarvis.learning.distillation.diagnose.types import ( +from openjarvis.learning.spec_search.diagnose.types import ( DiagnosticTool, ToolCallRecord, ) diff --git a/src/openjarvis/learning/distillation/diagnose/tools.py b/src/openjarvis/learning/spec_search/diagnose/tools.py similarity index 99% rename from src/openjarvis/learning/distillation/diagnose/tools.py rename to src/openjarvis/learning/spec_search/diagnose/tools.py index 24feda78..e7c013b3 100644 --- a/src/openjarvis/learning/distillation/diagnose/tools.py +++ b/src/openjarvis/learning/spec_search/diagnose/tools.py @@ -5,7 +5,7 @@ All tools are **read-only** relative to the user's config. They do not mutate Tools that execute code (``run_student_on_task``, ``run_self_on_task``) append new traces to ``TraceStore`` as a side effect. These traces are tagged with -``source=distillation_session:`` so they can be excluded from future +``source=spec_search_session:`` so they can be excluded from future learning input. See spec §5.2. @@ -16,7 +16,7 @@ from __future__ import annotations import json from typing import Any -from openjarvis.learning.distillation.diagnose.types import DiagnosticTool +from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool def build_diagnostic_tools( diff --git a/src/openjarvis/learning/distillation/diagnose/types.py b/src/openjarvis/learning/spec_search/diagnose/types.py similarity index 100% rename from src/openjarvis/learning/distillation/diagnose/types.py rename to src/openjarvis/learning/spec_search/diagnose/types.py diff --git a/src/openjarvis/learning/distillation/execute/__init__.py b/src/openjarvis/learning/spec_search/execute/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/execute/__init__.py rename to src/openjarvis/learning/spec_search/execute/__init__.py diff --git a/src/openjarvis/learning/distillation/execute/appliers/__init__.py b/src/openjarvis/learning/spec_search/execute/appliers/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/execute/appliers/__init__.py rename to src/openjarvis/learning/spec_search/execute/appliers/__init__.py diff --git a/src/openjarvis/learning/distillation/execute/appliers/agent.py b/src/openjarvis/learning/spec_search/execute/appliers/agent.py similarity index 97% rename from src/openjarvis/learning/distillation/execute/appliers/agent.py rename to src/openjarvis/learning/spec_search/execute/appliers/agent.py index 00714e5d..9841f37b 100644 --- a/src/openjarvis/learning/distillation/execute/appliers/agent.py +++ b/src/openjarvis/learning/spec_search/execute/appliers/agent.py @@ -9,14 +9,14 @@ import json import logging from pathlib import Path -from openjarvis.learning.distillation.execute.base import ( +from openjarvis.learning.spec_search.execute.base import ( ApplyContext, ApplyResult, EditApplier, ValidationResult, ) -from openjarvis.learning.distillation.models import Edit, EditOp -from openjarvis.learning.distillation.plan.prompt_diff import apply_unified_diff +from openjarvis.learning.spec_search.models import Edit, EditOp +from openjarvis.learning.spec_search.plan.prompt_diff import apply_unified_diff logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/execute/appliers/intelligence.py b/src/openjarvis/learning/spec_search/execute/appliers/intelligence.py similarity index 97% rename from src/openjarvis/learning/distillation/execute/appliers/intelligence.py rename to src/openjarvis/learning/spec_search/execute/appliers/intelligence.py index 44b4d142..f683ad1d 100644 --- a/src/openjarvis/learning/distillation/execute/appliers/intelligence.py +++ b/src/openjarvis/learning/spec_search/execute/appliers/intelligence.py @@ -5,13 +5,13 @@ See spec §4.1 op semantics for SET_MODEL_FOR_QUERY_CLASS and SET_MODEL_PARAM. from __future__ import annotations -from openjarvis.learning.distillation.execute.base import ( +from openjarvis.learning.spec_search.execute.base import ( ApplyContext, ApplyResult, EditApplier, ValidationResult, ) -from openjarvis.learning.distillation.models import Edit, EditOp +from openjarvis.learning.spec_search.models import Edit, EditOp class SetModelForQueryClassApplier(EditApplier): diff --git a/src/openjarvis/learning/distillation/execute/appliers/lora_stub.py b/src/openjarvis/learning/spec_search/execute/appliers/lora_stub.py similarity index 89% rename from src/openjarvis/learning/distillation/execute/appliers/lora_stub.py rename to src/openjarvis/learning/spec_search/execute/appliers/lora_stub.py index 6c30e3c8..61693aed 100644 --- a/src/openjarvis/learning/distillation/execute/appliers/lora_stub.py +++ b/src/openjarvis/learning/spec_search/execute/appliers/lora_stub.py @@ -9,13 +9,13 @@ See spec §4.1. from __future__ import annotations -from openjarvis.learning.distillation.execute.base import ( +from openjarvis.learning.spec_search.execute.base import ( ApplyContext, ApplyResult, EditApplier, ValidationResult, ) -from openjarvis.learning.distillation.models import Edit, EditOp +from openjarvis.learning.spec_search.models import Edit, EditOp class LoraStubApplier(EditApplier): diff --git a/src/openjarvis/learning/distillation/execute/appliers/tools.py b/src/openjarvis/learning/spec_search/execute/appliers/tools.py similarity index 97% rename from src/openjarvis/learning/distillation/execute/appliers/tools.py rename to src/openjarvis/learning/spec_search/execute/appliers/tools.py index 9ec3aca8..df0a117b 100644 --- a/src/openjarvis/learning/distillation/execute/appliers/tools.py +++ b/src/openjarvis/learning/spec_search/execute/appliers/tools.py @@ -7,13 +7,13 @@ from __future__ import annotations import re -from openjarvis.learning.distillation.execute.base import ( +from openjarvis.learning.spec_search.execute.base import ( ApplyContext, ApplyResult, EditApplier, ValidationResult, ) -from openjarvis.learning.distillation.models import Edit, EditOp +from openjarvis.learning.spec_search.models import Edit, EditOp class AddToolToAgentApplier(EditApplier): diff --git a/src/openjarvis/learning/distillation/execute/base.py b/src/openjarvis/learning/spec_search/execute/base.py similarity index 97% rename from src/openjarvis/learning/distillation/execute/base.py rename to src/openjarvis/learning/spec_search/execute/base.py index 62d8dc85..0c2122be 100644 --- a/src/openjarvis/learning/distillation/execute/base.py +++ b/src/openjarvis/learning/spec_search/execute/base.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import ClassVar -from openjarvis.learning.distillation.models import Edit, EditOp +from openjarvis.learning.spec_search.models import Edit, EditOp @dataclass diff --git a/src/openjarvis/learning/distillation/execute/loop.py b/src/openjarvis/learning/spec_search/execute/loop.py similarity index 92% rename from src/openjarvis/learning/distillation/execute/loop.py rename to src/openjarvis/learning/spec_search/execute/loop.py index d30fea33..e931eb37 100644 --- a/src/openjarvis/learning/distillation/execute/loop.py +++ b/src/openjarvis/learning/spec_search/execute/loop.py @@ -1,4 +1,4 @@ -"""Per-edit execution loop for the distillation execute phase. +"""Per-edit execution loop for the spec-search execute phase. Iterates over a plan's edits, handles tier routing, validates, and applies. Does NOT include the benchmark gate — that's wired in M5. @@ -11,30 +11,30 @@ from __future__ import annotations import logging from datetime import datetime, timezone -from openjarvis.learning.distillation.execute.appliers.agent import ( +from openjarvis.learning.spec_search.execute.appliers.agent import ( EditFewShotExemplarsApplier, PatchSystemPromptApplier, ReplaceSystemPromptApplier, SetAgentClassApplier, SetAgentParamApplier, ) -from openjarvis.learning.distillation.execute.appliers.intelligence import ( +from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelForQueryClassApplier, SetModelParamApplier, ) -from openjarvis.learning.distillation.execute.appliers.lora_stub import ( +from openjarvis.learning.spec_search.execute.appliers.lora_stub import ( LoraStubApplier, ) -from openjarvis.learning.distillation.execute.appliers.tools import ( +from openjarvis.learning.spec_search.execute.appliers.tools import ( AddToolToAgentApplier, EditToolDescriptionApplier, RemoveToolFromAgentApplier, ) -from openjarvis.learning.distillation.execute.base import ( +from openjarvis.learning.spec_search.execute.base import ( ApplyContext, EditApplierRegistry, ) -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( AutonomyMode, Edit, EditOutcome, diff --git a/src/openjarvis/learning/distillation/external_adapter.py b/src/openjarvis/learning/spec_search/external_adapter.py similarity index 100% rename from src/openjarvis/learning/distillation/external_adapter.py rename to src/openjarvis/learning/spec_search/external_adapter.py diff --git a/src/openjarvis/learning/distillation/gate/__init__.py b/src/openjarvis/learning/spec_search/gate/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/gate/__init__.py rename to src/openjarvis/learning/spec_search/gate/__init__.py diff --git a/src/openjarvis/learning/distillation/gate/benchmark_gate.py b/src/openjarvis/learning/spec_search/gate/benchmark_gate.py similarity index 96% rename from src/openjarvis/learning/distillation/gate/benchmark_gate.py rename to src/openjarvis/learning/spec_search/gate/benchmark_gate.py index c3f94b27..d067e2db 100644 --- a/src/openjarvis/learning/distillation/gate/benchmark_gate.py +++ b/src/openjarvis/learning/spec_search/gate/benchmark_gate.py @@ -12,8 +12,8 @@ import logging from dataclasses import dataclass from typing import Callable -from openjarvis.learning.distillation.gate.regression import regression_check -from openjarvis.learning.distillation.models import BenchmarkSnapshot +from openjarvis.learning.spec_search.gate.regression import regression_check +from openjarvis.learning.spec_search.models import BenchmarkSnapshot logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/gate/cold_start.py b/src/openjarvis/learning/spec_search/gate/cold_start.py similarity index 97% rename from src/openjarvis/learning/distillation/gate/cold_start.py rename to src/openjarvis/learning/spec_search/gate/cold_start.py index 4179d664..40ca7e6e 100644 --- a/src/openjarvis/learning/distillation/gate/cold_start.py +++ b/src/openjarvis/learning/spec_search/gate/cold_start.py @@ -1,4 +1,4 @@ -"""Cold start detection and bootstrap for the distillation subsystem. +"""Cold start detection and bootstrap for the spec-search subsystem. Day one: no traces, no benchmark. The system must not crash and must give the user a clear message about what's needed. This module provides diff --git a/src/openjarvis/learning/distillation/gate/regression.py b/src/openjarvis/learning/spec_search/gate/regression.py similarity index 95% rename from src/openjarvis/learning/distillation/gate/regression.py rename to src/openjarvis/learning/spec_search/gate/regression.py index ef4599f5..ce201445 100644 --- a/src/openjarvis/learning/distillation/gate/regression.py +++ b/src/openjarvis/learning/spec_search/gate/regression.py @@ -11,7 +11,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from openjarvis.learning.distillation.models import BenchmarkSnapshot +from openjarvis.learning.spec_search.models import BenchmarkSnapshot @dataclass diff --git a/src/openjarvis/learning/distillation/models.py b/src/openjarvis/learning/spec_search/models.py similarity index 99% rename from src/openjarvis/learning/distillation/models.py rename to src/openjarvis/learning/spec_search/models.py index 973d891e..5928cbd9 100644 --- a/src/openjarvis/learning/distillation/models.py +++ b/src/openjarvis/learning/spec_search/models.py @@ -1,4 +1,4 @@ -"""Pydantic models and enums for the distillation subsystem. +"""Pydantic models and enums for the spec-search subsystem. This module defines the typed vocabulary used by the diagnose, plan, execute, and record phases. Three model families: @@ -328,7 +328,7 @@ class EditOutcome(BaseModel): class LearningSession(BaseModel): - """The durable record of one distillation session. + """The durable record of one spec-search session. Persisted in two places: `/session.json` (authoritative) and the SQLite SessionStore (queryable index). When in doubt, prefer the JSON diff --git a/src/openjarvis/learning/spec_search/multi_session.py b/src/openjarvis/learning/spec_search/multi_session.py new file mode 100644 index 00000000..92222a67 --- /dev/null +++ b/src/openjarvis/learning/spec_search/multi_session.py @@ -0,0 +1,129 @@ +"""Multi-session loop for LLM-guided spec search (paper Algorithm 1). + +The single-session ``SpecSearchOrchestrator.run(trigger)`` does one +diagnose / plan / execute / record pass. Algorithm 1 in the paper +specifies a *multi-session* loop that repeats this pass until either +gate-score stagnation (default *k* = 5 sessions with no improvement) +or budget exhaustion. + +``SpecSearchLoop`` wraps an existing ``SpecSearchOrchestrator`` and +implements that stopping logic without modifying the orchestrator +itself, so existing single-session callers and tests are unaffected. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any + +from openjarvis.learning.spec_search.models import ( + LearningSession, + SessionStatus, +) +from openjarvis.learning.spec_search.triggers import OnDemandTrigger + +logger = logging.getLogger(__name__) + + +@dataclass(slots=True) +class LoopResult: + """Final state of a multi-session run. + + ``stop_reason`` is one of: + + - ``"stagnation"`` — gate score did not improve for ``stagnation_k`` + consecutive sessions. + - ``"budget"`` — cumulative teacher cost reached ``max_total_cost_usd``. + - ``"failed"`` — a session returned ``SessionStatus.FAILED``; the loop + exits rather than burning more budget on a broken state. + """ + + sessions: list[LearningSession] = field(default_factory=list) + stop_reason: str = "stagnation" + total_cost_usd: float = 0.0 + best_overall_score: float = 0.0 + + +class SpecSearchLoop: + """Paper Algorithm 1: greedy gated edits across primitives, multi-session. + + Args: + orchestrator: a constructed ``SpecSearchOrchestrator``. Each tick + of the loop calls ``orchestrator.run(trigger)``. + stagnation_k: stop after this many consecutive sessions with no + gate-score improvement (paper default: 5). + stagnation_eps: gate-score delta below this counts as "no + improvement" — guards against floating-point noise. + max_total_cost_usd: cumulative teacher-cost budget across all + sessions. The loop stops as soon as this is exceeded. + """ + + def __init__( + self, + orchestrator: Any, + *, + stagnation_k: int = 5, + stagnation_eps: float = 0.001, + max_total_cost_usd: float = 50.0, + ) -> None: + if stagnation_k < 1: + raise ValueError("stagnation_k must be >= 1") + if max_total_cost_usd <= 0: + raise ValueError("max_total_cost_usd must be > 0") + self._orch = orchestrator + self._k = stagnation_k + self._eps = stagnation_eps + self._budget = max_total_cost_usd + + def run(self, trigger: Any | None = None) -> LoopResult: + """Run sessions until stagnation, budget, or failure.""" + result = LoopResult() + no_improve_streak = 0 + + while True: + session_trigger = trigger if trigger is not None else OnDemandTrigger() + session = self._orch.run(session_trigger) + result.sessions.append(session) + result.total_cost_usd += session.teacher_cost_usd or 0.0 + + if session.status == SessionStatus.FAILED: + result.stop_reason = "failed" + logger.info( + "spec-search loop stopping: session failed (%s)", + session.error, + ) + break + + after_score = ( + session.benchmark_after.overall_score + if session.benchmark_after is not None + else 0.0 + ) + if after_score > result.best_overall_score + self._eps: + result.best_overall_score = after_score + no_improve_streak = 0 + else: + no_improve_streak += 1 + + if no_improve_streak >= self._k: + result.stop_reason = "stagnation" + logger.info( + "spec-search loop stopping: %d sessions without improvement", + no_improve_streak, + ) + break + + if result.total_cost_usd >= self._budget: + result.stop_reason = "budget" + logger.info( + "spec-search loop stopping: cumulative cost $%.2f >= budget $%.2f", + result.total_cost_usd, + self._budget, + ) + break + + return result + + +__all__ = ["LoopResult", "SpecSearchLoop"] diff --git a/src/openjarvis/learning/distillation/orchestrator.py b/src/openjarvis/learning/spec_search/orchestrator.py similarity index 84% rename from src/openjarvis/learning/distillation/orchestrator.py rename to src/openjarvis/learning/spec_search/orchestrator.py index 09a99160..1534ed0f 100644 --- a/src/openjarvis/learning/distillation/orchestrator.py +++ b/src/openjarvis/learning/spec_search/orchestrator.py @@ -1,4 +1,4 @@ -"""DistillationOrchestrator: top-level driver for a learning session. +"""SpecSearchOrchestrator: top-level driver for a learning session. Wires diagnose (M2) → plan (M3) → execute (M4) → gate (M5) into a single ``run(trigger)`` method. All dependencies are injected. @@ -14,12 +14,12 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable -from openjarvis.learning.distillation.diagnose.runner import DiagnosisRunner -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.execute.loop import _build_registry -from openjarvis.learning.distillation.gate.benchmark_gate import BenchmarkGate -from openjarvis.learning.distillation.gate.cold_start import check_readiness -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.diagnose.runner import DiagnosisRunner +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.execute.loop import _build_registry +from openjarvis.learning.spec_search.gate.benchmark_gate import BenchmarkGate +from openjarvis.learning.spec_search.gate.cold_start import check_readiness +from openjarvis.learning.spec_search.models import ( AutonomyMode, BenchmarkSnapshot, EditOutcome, @@ -27,18 +27,65 @@ from openjarvis.learning.distillation.models import ( LearningSession, SessionStatus, ) -from openjarvis.learning.distillation.pending_queue import PendingQueue -from openjarvis.learning.distillation.plan.planner import LearningPlanner +from openjarvis.learning.spec_search.pending_queue import PendingQueue +from openjarvis.learning.spec_search.plan.planner import LearningPlanner logger = logging.getLogger(__name__) -class DistillationOrchestrator: - """Top-level driver for a distillation learning session. +class SpecSearchOrchestrator: + """Top-level driver for a spec-search learning session. All dependencies are injected so tests can mock everything. """ + @classmethod + def from_config( + cls, + config: Any, # SpecSearchLearningConfig + *, + teacher_engine: Any, + trace_store: Any, + benchmark_samples: list, + student_runner: Any, + judge: Any, + session_store: Any, + checkpoint_store: Any, + openjarvis_home: Path, + scorer: Callable[..., BenchmarkSnapshot] | None = None, + ) -> "SpecSearchOrchestrator": + """Build a single-session orchestrator from a SpecSearchLearningConfig. + + Hyperparameters (teacher model, gate tolerance, autonomy mode, etc.) + come from ``config``; runtime primitives that cannot be expressed in + TOML (engine instances, trace store, judge, etc.) must be injected. + + See ``configs/openjarvis/examples/spec-search-quickstart.toml`` for + the TOML schema and ``examples/openjarvis/spec_search_quickstart.py`` + for an end-to-end wiring example. + """ + autonomy = AutonomyMode(config.autonomy_mode) + return cls( + teacher_engine=teacher_engine, + teacher_model=config.teacher_model, + trace_store=trace_store, + benchmark_samples=benchmark_samples, + student_runner=student_runner, + judge=judge, + session_store=session_store, + checkpoint_store=checkpoint_store, + openjarvis_home=openjarvis_home, + autonomy_mode=autonomy, + scorer=scorer, + benchmark_version=config.benchmark_version, + min_traces=config.min_traces, + max_cost_usd=config.max_cost_per_session_usd, + max_tool_calls=config.max_tool_calls_per_diagnosis, + min_improvement=config.min_improvement, + max_regression=config.max_regression, + subsample_size=config.benchmark_subsample_size, + ) + def __init__( self, *, @@ -81,7 +128,7 @@ class DistillationOrchestrator: self._subsample_size = subsample_size def run(self, trigger: Any) -> LearningSession: - """Execute a full distillation session. + """Execute a full spec-search session. Returns the completed LearningSession. """ @@ -340,9 +387,7 @@ class DistillationOrchestrator: try: applier.rollback(edit, ctx) except Exception as rb_exc: - logger.warning( - "Edit %s rollback failed: %s", edit.id, rb_exc - ) + logger.warning("Edit %s rollback failed: %s", edit.id, rb_exc) outcomes.append( EditOutcome( edit_id=edit.id, diff --git a/src/openjarvis/learning/distillation/pending_queue.py b/src/openjarvis/learning/spec_search/pending_queue.py similarity index 91% rename from src/openjarvis/learning/distillation/pending_queue.py rename to src/openjarvis/learning/spec_search/pending_queue.py index 5bac6c9b..65b07ac7 100644 --- a/src/openjarvis/learning/distillation/pending_queue.py +++ b/src/openjarvis/learning/spec_search/pending_queue.py @@ -1,8 +1,9 @@ """Pending review queue for edits awaiting user approval. Edits in the ``review`` tier (when autonomy mode is ``tiered``) are -written here as JSON files. The user reviews them via ``jarvis learning -review`` and approves or rejects. +written here as JSON files. Callers consume the queue via +``PendingQueue.list()`` / ``approve()`` / ``reject()`` to advance edits +out of the review tier. See spec §7.5. """ @@ -14,7 +15,7 @@ import logging from pathlib import Path from typing import Any -from openjarvis.learning.distillation.models import Edit +from openjarvis.learning.spec_search.models import Edit logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/plan/__init__.py b/src/openjarvis/learning/spec_search/plan/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/plan/__init__.py rename to src/openjarvis/learning/spec_search/plan/__init__.py diff --git a/src/openjarvis/learning/distillation/plan/planner.py b/src/openjarvis/learning/spec_search/plan/planner.py similarity index 98% rename from src/openjarvis/learning/distillation/plan/planner.py rename to src/openjarvis/learning/spec_search/plan/planner.py index 6cf964dd..0fb106ed 100644 --- a/src/openjarvis/learning/distillation/plan/planner.py +++ b/src/openjarvis/learning/spec_search/plan/planner.py @@ -15,15 +15,15 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( Edit, FailureCluster, LearningPlan, ) -from openjarvis.learning.distillation.plan.prompt_diff import ( +from openjarvis.learning.spec_search.plan.prompt_diff import ( maybe_downgrade_to_replace, ) -from openjarvis.learning.distillation.plan.risk_tier import assign_tiers +from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/plan/prompt_diff.py b/src/openjarvis/learning/spec_search/plan/prompt_diff.py similarity index 98% rename from src/openjarvis/learning/distillation/plan/prompt_diff.py rename to src/openjarvis/learning/spec_search/plan/prompt_diff.py index 02e0e110..ec2d7008 100644 --- a/src/openjarvis/learning/distillation/plan/prompt_diff.py +++ b/src/openjarvis/learning/spec_search/plan/prompt_diff.py @@ -14,7 +14,7 @@ import logging import re from typing import Callable, Optional -from openjarvis.learning.distillation.models import Edit, EditOp +from openjarvis.learning.spec_search.models import Edit, EditOp logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/plan/risk_tier.py b/src/openjarvis/learning/spec_search/plan/risk_tier.py similarity index 96% rename from src/openjarvis/learning/distillation/plan/risk_tier.py rename to src/openjarvis/learning/spec_search/plan/risk_tier.py index 2ba4a2c1..ae452b32 100644 --- a/src/openjarvis/learning/distillation/plan/risk_tier.py +++ b/src/openjarvis/learning/spec_search/plan/risk_tier.py @@ -13,7 +13,7 @@ from __future__ import annotations import logging from typing import Sequence -from openjarvis.learning.distillation.models import Edit, EditOp, EditRiskTier +from openjarvis.learning.spec_search.models import Edit, EditOp, EditRiskTier logger = logging.getLogger(__name__) diff --git a/src/openjarvis/learning/distillation/storage/__init__.py b/src/openjarvis/learning/spec_search/storage/__init__.py similarity index 100% rename from src/openjarvis/learning/distillation/storage/__init__.py rename to src/openjarvis/learning/spec_search/storage/__init__.py diff --git a/src/openjarvis/learning/distillation/storage/paths.py b/src/openjarvis/learning/spec_search/storage/paths.py similarity index 81% rename from src/openjarvis/learning/distillation/storage/paths.py rename to src/openjarvis/learning/spec_search/storage/paths.py index 865800f8..2227c828 100644 --- a/src/openjarvis/learning/distillation/storage/paths.py +++ b/src/openjarvis/learning/spec_search/storage/paths.py @@ -1,7 +1,7 @@ -"""Filesystem path resolution for the distillation subsystem. +"""Filesystem path resolution for the spec-search subsystem. -The keystone of artifact isolation (spec §11): the resolved distillation root -must NEVER be inside the OpenJarvis source tree. ``resolve_distillation_root`` +The keystone of artifact isolation (spec §11): the resolved spec-search root +must NEVER be inside the OpenJarvis source tree. ``resolve_spec_search_root`` walks up from this module's ``__file__`` looking for a ``pyproject.toml`` that identifies the OpenJarvis source root, then refuses to operate if the resolved root is inside it. Defense in depth — if a user accidentally points @@ -49,8 +49,8 @@ def _resolve_openjarvis_home() -> Path: return (Path.home() / ".openjarvis").resolve() -def resolve_distillation_root() -> Path: - """Return the absolute path of the distillation root directory. +def resolve_spec_search_root() -> Path: + """Return the absolute path of the spec-search root directory. The root is ``$OPENJARVIS_HOME/learning`` (or ``~/.openjarvis/learning`` by default). Raises ``ConfigurationError`` if the resolved path lies @@ -67,21 +67,21 @@ def resolve_distillation_root() -> Path: else: raise ConfigurationError( f"OPENJARVIS_HOME ({home}) is inside the source tree " - f"({source_root}). Distillation refuses to write runtime " + f"({source_root}). Spec search refuses to write runtime " "artifacts inside the OpenJarvis repo. Set OPENJARVIS_HOME " "to a directory outside the repo (default: ~/.openjarvis)." ) return home / "learning" -def ensure_distillation_dirs() -> Path: - """Create the distillation directory layout if missing. +def ensure_spec_search_dirs() -> Path: + """Create the spec-search directory layout if missing. - Returns the distillation root. Creates ``sessions/``, ``benchmarks/``, + Returns the spec-search root. Creates ``sessions/``, ``benchmarks/``, ``benchmarks/reference_outputs/``, and ``pending_review/`` underneath it, all with restrictive ``0o700`` permissions via ``secure_mkdir``. """ - root = resolve_distillation_root() + root = resolve_spec_search_root() secure_mkdir(root) secure_mkdir(root / "sessions") secure_mkdir(root / "benchmarks") diff --git a/src/openjarvis/learning/distillation/storage/session_store.py b/src/openjarvis/learning/spec_search/storage/session_store.py similarity index 98% rename from src/openjarvis/learning/distillation/storage/session_store.py rename to src/openjarvis/learning/spec_search/storage/session_store.py index bfae741d..9756ed0b 100644 --- a/src/openjarvis/learning/distillation/storage/session_store.py +++ b/src/openjarvis/learning/spec_search/storage/session_store.py @@ -1,4 +1,4 @@ -"""SQLite-backed storage for distillation LearningSession records. +"""SQLite-backed storage for spec-search LearningSession records. Mirrors the style of ``openjarvis.learning.optimize.store.OptimizationStore``: @@ -21,7 +21,7 @@ from datetime import datetime from pathlib import Path from typing import Optional, Union -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( AutonomyMode, BenchmarkSnapshot, EditOutcome, diff --git a/src/openjarvis/learning/distillation/student_runner.py b/src/openjarvis/learning/spec_search/student_runner.py similarity index 98% rename from src/openjarvis/learning/distillation/student_runner.py rename to src/openjarvis/learning/spec_search/student_runner.py index 028be299..1a7b0672 100644 --- a/src/openjarvis/learning/distillation/student_runner.py +++ b/src/openjarvis/learning/spec_search/student_runner.py @@ -1,4 +1,4 @@ -"""Real student runner for distillation experiments. +"""Real student runner for spec-search experiments. Replaces the ``MagicMock()`` in the experiment runner script with a callable that actually invokes the student model via vLLM (or any diff --git a/src/openjarvis/learning/distillation/triggers.py b/src/openjarvis/learning/spec_search/triggers.py similarity index 85% rename from src/openjarvis/learning/distillation/triggers.py rename to src/openjarvis/learning/spec_search/triggers.py index 02d71fad..874d4b0f 100644 --- a/src/openjarvis/learning/distillation/triggers.py +++ b/src/openjarvis/learning/spec_search/triggers.py @@ -1,7 +1,7 @@ -"""Trigger types for the distillation subsystem. +"""Trigger types for the spec-search subsystem. A trigger is what kicks off a learning session. Four trigger types exist, -all funneling into ``DistillationOrchestrator.run(trigger)``. The trigger +all funneling into ``SpecSearchOrchestrator.run(trigger)``. The trigger object is stored on the ``LearningSession`` for queryability. See spec §3.3. @@ -12,12 +12,12 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Any -from openjarvis.learning.distillation.models import TriggerKind +from openjarvis.learning.spec_search.models import TriggerKind @dataclass class OnDemandTrigger: - """User ran ``jarvis learning run`` from the CLI.""" + """Caller invoked ``SpecSearchOrchestrator.run(trigger)`` directly.""" kind: TriggerKind = TriggerKind.ON_DEMAND metadata: dict[str, Any] = field(default_factory=dict) diff --git a/src/openjarvis/tools/description_loader.py b/src/openjarvis/tools/description_loader.py index b0f76ea1..de1e0a66 100644 --- a/src/openjarvis/tools/description_loader.py +++ b/src/openjarvis/tools/description_loader.py @@ -1,6 +1,6 @@ """Load tool description overrides from $OPENJARVIS_HOME/tools/descriptions.toml. -Distillation (M1) proposes tool description edits that get written to disk by +LLM-guided spec search (M1) proposes tool description edits that get written to disk by ``EditToolDescriptionApplier``. This module loads those overrides so agents see the improved descriptions at runtime. diff --git a/tests/learning/distillation/__init__.py b/tests/learning/distillation/__init__.py deleted file mode 100644 index f7467079..00000000 --- a/tests/learning/distillation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for the distillation subsystem.""" diff --git a/tests/learning/distillation/test_cli.py b/tests/learning/distillation/test_cli.py deleted file mode 100644 index 52a2c994..00000000 --- a/tests/learning/distillation/test_cli.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Tests for the jarvis learning CLI subcommand group.""" - -from __future__ import annotations - -from click.testing import CliRunner - - -class TestLearningCLI: - def test_learning_group_exists(self) -> None: - from openjarvis.learning.distillation.cli import learning_group - - runner = CliRunner() - result = runner.invoke(learning_group, ["--help"]) - assert result.exit_code == 0 - out = result.output.lower() - assert "learning" in out or "distillation" in out - - def test_init_subcommand(self) -> None: - from openjarvis.learning.distillation.cli import learning_group - - runner = CliRunner() - result = runner.invoke(learning_group, ["init", "--help"]) - assert result.exit_code == 0 - - def test_run_subcommand(self) -> None: - from openjarvis.learning.distillation.cli import learning_group - - runner = CliRunner() - result = runner.invoke(learning_group, ["run", "--help"]) - assert result.exit_code == 0 - - def test_history_subcommand(self) -> None: - from openjarvis.learning.distillation.cli import learning_group - - runner = CliRunner() - result = runner.invoke(learning_group, ["history", "--help"]) - assert result.exit_code == 0 - - def test_rollback_subcommand(self) -> None: - from openjarvis.learning.distillation.cli import learning_group - - runner = CliRunner() - result = runner.invoke(learning_group, ["rollback", "--help"]) - assert result.exit_code == 0 diff --git a/tests/learning/spec_search/__init__.py b/tests/learning/spec_search/__init__.py new file mode 100644 index 00000000..5567ba6e --- /dev/null +++ b/tests/learning/spec_search/__init__.py @@ -0,0 +1 @@ +"""Tests for the spec-search subsystem.""" diff --git a/tests/learning/distillation/test_applier_base.py b/tests/learning/spec_search/test_applier_base.py similarity index 73% rename from tests/learning/distillation/test_applier_base.py rename to tests/learning/spec_search/test_applier_base.py index a50d41ae..9d8d8443 100644 --- a/tests/learning/distillation/test_applier_base.py +++ b/tests/learning/spec_search/test_applier_base.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.execute.base module.""" +"""Tests for openjarvis.learning.spec_search.execute.base module.""" from __future__ import annotations @@ -11,26 +11,26 @@ class TestApplyContext: """Tests for ApplyContext dataclass.""" def test_constructs(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.base import ApplyContext + from openjarvis.learning.spec_search.execute.base import ApplyContext ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1") assert ctx.openjarvis_home == tmp_path assert ctx.session_id == "s1" def test_config_path(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.base import ApplyContext + from openjarvis.learning.spec_search.execute.base import ApplyContext ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1") assert ctx.config_path == tmp_path / "config.toml" def test_agents_dir(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.base import ApplyContext + from openjarvis.learning.spec_search.execute.base import ApplyContext ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1") assert ctx.agents_dir == tmp_path / "agents" def test_tools_dir(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.base import ApplyContext + from openjarvis.learning.spec_search.execute.base import ApplyContext ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1") assert ctx.tools_dir == tmp_path / "tools" @@ -40,14 +40,14 @@ class TestValidationResult: """Tests for ValidationResult.""" def test_ok_result(self) -> None: - from openjarvis.learning.distillation.execute.base import ValidationResult + from openjarvis.learning.spec_search.execute.base import ValidationResult r = ValidationResult(ok=True) assert r.ok is True assert r.reason == "" def test_error_result(self) -> None: - from openjarvis.learning.distillation.execute.base import ValidationResult + from openjarvis.learning.spec_search.execute.base import ValidationResult r = ValidationResult(ok=False, reason="target not found") assert r.ok is False @@ -58,14 +58,14 @@ class TestEditApplierRegistry: """Tests for EditApplierRegistry.""" def test_register_and_get(self) -> None: - from openjarvis.learning.distillation.execute.base import ( + from openjarvis.learning.spec_search.execute.base import ( ApplyContext, ApplyResult, EditApplier, EditApplierRegistry, ValidationResult, ) - from openjarvis.learning.distillation.models import Edit, EditOp + from openjarvis.learning.spec_search.models import Edit, EditOp class FakeApplier(EditApplier): op = EditOp.SET_MODEL_PARAM @@ -86,19 +86,19 @@ class TestEditApplierRegistry: assert isinstance(applier, FakeApplier) def test_is_supported_returns_false_for_unregistered(self) -> None: - from openjarvis.learning.distillation.execute.base import ( + from openjarvis.learning.spec_search.execute.base import ( EditApplierRegistry, ) - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp registry = EditApplierRegistry() assert registry.is_supported(EditOp.LORA_FINETUNE) is False def test_get_raises_for_unregistered(self) -> None: - from openjarvis.learning.distillation.execute.base import ( + from openjarvis.learning.spec_search.execute.base import ( EditApplierRegistry, ) - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp registry = EditApplierRegistry() with pytest.raises(KeyError): diff --git a/tests/learning/distillation/test_applier_lora_stub.py b/tests/learning/spec_search/test_applier_lora_stub.py similarity index 82% rename from tests/learning/distillation/test_applier_lora_stub.py rename to tests/learning/spec_search/test_applier_lora_stub.py index fdb20b8e..5980e366 100644 --- a/tests/learning/distillation/test_applier_lora_stub.py +++ b/tests/learning/spec_search/test_applier_lora_stub.py @@ -6,8 +6,8 @@ from pathlib import Path import pytest -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -32,7 +32,7 @@ class TestLoraStubApplier: """Tests for LoraStubApplier.""" def test_validate_returns_not_ok(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.lora_stub import ( + from openjarvis.learning.spec_search.execute.appliers.lora_stub import ( LoraStubApplier, ) @@ -43,7 +43,7 @@ class TestLoraStubApplier: assert "v2" in result.reason.lower() or "deferred" in result.reason.lower() def test_apply_raises_not_implemented(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.lora_stub import ( + from openjarvis.learning.spec_search.execute.appliers.lora_stub import ( LoraStubApplier, ) diff --git a/tests/learning/distillation/test_appliers_agent.py b/tests/learning/spec_search/test_appliers_agent.py similarity index 90% rename from tests/learning/distillation/test_appliers_agent.py rename to tests/learning/spec_search/test_appliers_agent.py index 80f0196e..26cf0e37 100644 --- a/tests/learning/distillation/test_appliers_agent.py +++ b/tests/learning/spec_search/test_appliers_agent.py @@ -5,8 +5,8 @@ from __future__ import annotations import json from pathlib import Path -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -35,7 +35,7 @@ class TestReplaceSystemPromptApplier: """Tests for ReplaceSystemPromptApplier.""" def test_validate_ok(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( ReplaceSystemPromptApplier, ) @@ -54,7 +54,7 @@ class TestReplaceSystemPromptApplier: assert applier.validate(edit, ctx).ok def test_apply_overwrites_prompt(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( ReplaceSystemPromptApplier, ) @@ -79,7 +79,7 @@ class TestPatchSystemPromptApplier: """Tests for PatchSystemPromptApplier.""" def test_apply_applies_diff(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( PatchSystemPromptApplier, ) @@ -108,7 +108,7 @@ class TestPatchSystemPromptApplier: assert "math tools" in content def test_validate_fails_for_bad_diff(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( PatchSystemPromptApplier, ) @@ -132,7 +132,7 @@ class TestSetAgentClassApplier: """Tests for SetAgentClassApplier.""" def test_apply_updates_config(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( SetAgentClassApplier, ) @@ -157,7 +157,7 @@ class TestSetAgentParamApplier: """Tests for SetAgentParamApplier.""" def test_apply_updates_param(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( SetAgentParamApplier, ) @@ -182,7 +182,7 @@ class TestEditFewShotExemplarsApplier: """Tests for EditFewShotExemplarsApplier.""" def test_apply_writes_exemplars(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.agent import ( + from openjarvis.learning.spec_search.execute.appliers.agent import ( EditFewShotExemplarsApplier, ) diff --git a/tests/learning/distillation/test_appliers_intelligence.py b/tests/learning/spec_search/test_appliers_intelligence.py similarity index 85% rename from tests/learning/distillation/test_appliers_intelligence.py rename to tests/learning/spec_search/test_appliers_intelligence.py index 181138e2..db8f180d 100644 --- a/tests/learning/distillation/test_appliers_intelligence.py +++ b/tests/learning/spec_search/test_appliers_intelligence.py @@ -4,8 +4,8 @@ from __future__ import annotations from pathlib import Path -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -62,7 +62,7 @@ class TestSetModelForQueryClassApplier: """Tests for SetModelForQueryClassApplier.""" def test_validate_ok(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.intelligence import ( + from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelForQueryClassApplier, ) @@ -72,7 +72,7 @@ class TestSetModelForQueryClassApplier: assert result.ok def test_apply_updates_config(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.intelligence import ( + from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelForQueryClassApplier, ) @@ -83,7 +83,7 @@ class TestSetModelForQueryClassApplier: assert "qwen2.5-coder:14b" in content def test_apply_adds_new_query_class(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.intelligence import ( + from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelForQueryClassApplier, ) @@ -100,7 +100,7 @@ class TestSetModelParamApplier: """Tests for SetModelParamApplier.""" def test_validate_ok(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.intelligence import ( + from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelParamApplier, ) @@ -110,7 +110,7 @@ class TestSetModelParamApplier: assert result.ok def test_apply_writes_param(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.intelligence import ( + from openjarvis.learning.spec_search.execute.appliers.intelligence import ( SetModelParamApplier, ) diff --git a/tests/learning/distillation/test_appliers_tools.py b/tests/learning/spec_search/test_appliers_tools.py similarity index 89% rename from tests/learning/distillation/test_appliers_tools.py rename to tests/learning/spec_search/test_appliers_tools.py index 0035905c..06d997e8 100644 --- a/tests/learning/distillation/test_appliers_tools.py +++ b/tests/learning/spec_search/test_appliers_tools.py @@ -4,8 +4,8 @@ from __future__ import annotations from pathlib import Path -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -27,7 +27,7 @@ class TestAddToolToAgentApplier: """Tests for AddToolToAgentApplier.""" def test_apply_adds_tool(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.tools import ( + from openjarvis.learning.spec_search.execute.appliers.tools import ( AddToolToAgentApplier, ) @@ -48,7 +48,7 @@ class TestAddToolToAgentApplier: assert "calculator" in content def test_validate_ok(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.tools import ( + from openjarvis.learning.spec_search.execute.appliers.tools import ( AddToolToAgentApplier, ) @@ -71,7 +71,7 @@ class TestRemoveToolFromAgentApplier: """Tests for RemoveToolFromAgentApplier.""" def test_apply_removes_tool(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.tools import ( + from openjarvis.learning.spec_search.execute.appliers.tools import ( RemoveToolFromAgentApplier, ) @@ -96,7 +96,7 @@ class TestEditToolDescriptionApplier: """Tests for EditToolDescriptionApplier.""" def test_apply_updates_description(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.tools import ( + from openjarvis.learning.spec_search.execute.appliers.tools import ( EditToolDescriptionApplier, ) @@ -120,7 +120,7 @@ class TestEditToolDescriptionApplier: assert "Search the internet" in content def test_apply_adds_new_tool_section(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.appliers.tools import ( + from openjarvis.learning.spec_search.execute.appliers.tools import ( EditToolDescriptionApplier, ) diff --git a/tests/learning/distillation/test_benchmark_gate.py b/tests/learning/spec_search/test_benchmark_gate.py similarity index 89% rename from tests/learning/distillation/test_benchmark_gate.py rename to tests/learning/spec_search/test_benchmark_gate.py index fd644446..aeeb6dec 100644 --- a/tests/learning/distillation/test_benchmark_gate.py +++ b/tests/learning/spec_search/test_benchmark_gate.py @@ -1,11 +1,11 @@ -"""Tests for openjarvis.learning.distillation.gate.benchmark_gate module. +"""Tests for openjarvis.learning.spec_search.gate.benchmark_gate module. All tests use mock scorers — no live EvalRunner. """ from __future__ import annotations -from openjarvis.learning.distillation.models import BenchmarkSnapshot +from openjarvis.learning.spec_search.models import BenchmarkSnapshot def _make_scorer(scores: dict[str, float], overall: float | None = None): @@ -30,7 +30,7 @@ class TestBenchmarkGate: """Tests for BenchmarkGate.""" def test_accepts_improving_edit(self) -> None: - from openjarvis.learning.distillation.gate.benchmark_gate import ( + from openjarvis.learning.spec_search.gate.benchmark_gate import ( BenchmarkGate, ) @@ -54,7 +54,7 @@ class TestBenchmarkGate: assert result.delta > 0 def test_rejects_no_improvement(self) -> None: - from openjarvis.learning.distillation.gate.benchmark_gate import ( + from openjarvis.learning.spec_search.gate.benchmark_gate import ( BenchmarkGate, ) @@ -77,7 +77,7 @@ class TestBenchmarkGate: assert "no improvement" in result.reason.lower() def test_rejects_regression(self) -> None: - from openjarvis.learning.distillation.gate.benchmark_gate import ( + from openjarvis.learning.spec_search.gate.benchmark_gate import ( BenchmarkGate, ) @@ -101,7 +101,7 @@ class TestBenchmarkGate: assert "regression" in result.reason.lower() def test_min_improvement_threshold(self) -> None: - from openjarvis.learning.distillation.gate.benchmark_gate import ( + from openjarvis.learning.spec_search.gate.benchmark_gate import ( BenchmarkGate, ) @@ -124,7 +124,7 @@ class TestBenchmarkGate: assert not result.accepted def test_result_contains_snapshot(self) -> None: - from openjarvis.learning.distillation.gate.benchmark_gate import ( + from openjarvis.learning.spec_search.gate.benchmark_gate import ( BenchmarkGate, ) diff --git a/tests/learning/distillation/test_checkpoint_store.py b/tests/learning/spec_search/test_checkpoint_store.py similarity index 90% rename from tests/learning/distillation/test_checkpoint_store.py rename to tests/learning/spec_search/test_checkpoint_store.py index 9d19d5dd..69879122 100644 --- a/tests/learning/distillation/test_checkpoint_store.py +++ b/tests/learning/spec_search/test_checkpoint_store.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.checkpoint.store module.""" +"""Tests for openjarvis.learning.spec_search.checkpoint.store module.""" from __future__ import annotations @@ -38,7 +38,7 @@ class TestCheckpointStoreInit: """Tests for CheckpointStore.init().""" def test_creates_repo_with_baseline_commit(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -51,7 +51,7 @@ class TestCheckpointStoreInit: assert "baseline" in log def test_init_idempotent(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -67,10 +67,10 @@ class TestCheckpointStoreInit: self, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths source_root = paths._find_source_root() assert source_root is not None @@ -85,7 +85,7 @@ class TestStageCommitDiscard: """Tests for begin_stage / commit_stage / discard_stage.""" def test_commit_stage_creates_commit_with_trailers(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -115,7 +115,7 @@ class TestStageCommitDiscard: assert "Risk-Tier: review" in body def test_discard_stage_restores_working_tree(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -137,7 +137,7 @@ class TestStageCommitDiscard: assert store.current_sha() == handle.pre_stage_sha def test_begin_stage_refuses_dirty_working_tree(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, DirtyWorkingTreeError, ) @@ -161,7 +161,7 @@ class TestRevertSession: def test_revert_creates_new_commits_and_does_not_rewrite( self, tmp_path: Path ) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -206,7 +206,7 @@ class TestRevertSession: assert (root / "tools" / "descriptions.toml").read_text() == "[web_search]\n" def test_revert_session_with_no_commits_returns_empty(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) diff --git a/tests/learning/distillation/test_cold_start.py b/tests/learning/spec_search/test_cold_start.py similarity index 85% rename from tests/learning/distillation/test_cold_start.py rename to tests/learning/spec_search/test_cold_start.py index 5714c6f9..6514830e 100644 --- a/tests/learning/distillation/test_cold_start.py +++ b/tests/learning/spec_search/test_cold_start.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.gate.cold_start module.""" +"""Tests for openjarvis.learning.spec_search.gate.cold_start module.""" from __future__ import annotations @@ -41,7 +41,7 @@ class TestCheckReadiness: """Tests for check_readiness().""" def test_not_ready_with_no_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_readiness, ) @@ -51,7 +51,7 @@ class TestCheckReadiness: assert "not enough traces" in result.message.lower() def test_not_ready_with_few_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_readiness, ) @@ -60,7 +60,7 @@ class TestCheckReadiness: assert not result.ready def test_ready_with_enough_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_readiness, ) @@ -73,7 +73,7 @@ class TestCheckBenchmarkReady: """Tests for check_benchmark_ready().""" def test_not_ready_with_no_high_feedback_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_benchmark_ready, ) @@ -83,7 +83,7 @@ class TestCheckBenchmarkReady: assert "benchmark" in result.message.lower() def test_not_ready_with_few_high_feedback_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_benchmark_ready, ) @@ -92,7 +92,7 @@ class TestCheckBenchmarkReady: assert not result.ready def test_ready_with_enough_high_feedback_traces(self) -> None: - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_benchmark_ready, ) diff --git a/tests/learning/spec_search/test_composite_reward.py b/tests/learning/spec_search/test_composite_reward.py new file mode 100644 index 00000000..549fcb03 --- /dev/null +++ b/tests/learning/spec_search/test_composite_reward.py @@ -0,0 +1,79 @@ +"""Tests for the paper Eq. 1 composite reward.""" + +from __future__ import annotations + +import pytest + +from openjarvis.learning.spec_search.composite_reward import ( + RewardWeights, + TrainingSample, + score_batch, +) + + +class TestScoreBatch: + def test_empty_batch_returns_empty(self) -> None: + assert score_batch([]) == [] + + def test_single_sample_has_zero_efficiency_penalties(self) -> None: + # With one sample the within-batch z-score collapses to 0, so the + # reward reduces to alpha * accuracy. + sample = TrainingSample( + accuracy=1.0, energy_joules=999.0, latency_seconds=42.0, cost_usd=0.05 + ) + [r] = score_batch([sample], weights=RewardWeights()) + assert r == pytest.approx(0.5) # alpha = 0.5 + + def test_higher_accuracy_ranks_above_lower_at_equal_efficiency(self) -> None: + a = TrainingSample( + accuracy=1.0, energy_joules=200, latency_seconds=5, cost_usd=0.0 + ) + b = TrainingSample( + accuracy=0.0, energy_joules=200, latency_seconds=5, cost_usd=0.0 + ) + ra, rb = score_batch([a, b]) + assert ra > rb + + def test_lower_efficiency_costs_lower_reward_at_equal_accuracy(self) -> None: + # Two equally-accurate samples; the slower / more energy-hungry one + # gets penalized. + fast = TrainingSample( + accuracy=1.0, energy_joules=100, latency_seconds=2, cost_usd=0.0 + ) + slow = TrainingSample( + accuracy=1.0, energy_joules=400, latency_seconds=8, cost_usd=0.0 + ) + r_fast, r_slow = score_batch([fast, slow]) + assert r_fast > r_slow + + def test_paper_default_weights(self) -> None: + w = RewardWeights() + assert (w.alpha, w.beta, w.gamma, w.delta) == (0.5, 0.1, 0.1, 0.3) + + def test_custom_weights_change_ranking(self) -> None: + # An accuracy-only weighting (alpha=1, others=0) collapses to raw + # accuracy and ignores efficiency entirely. + a = TrainingSample( + accuracy=1.0, energy_joules=999, latency_seconds=99, cost_usd=99 + ) + b = TrainingSample(accuracy=1.0, energy_joules=1, latency_seconds=1, cost_usd=0) + rewards = score_batch( + [a, b], weights=RewardWeights(alpha=1.0, beta=0.0, gamma=0.0, delta=0.0) + ) + assert rewards[0] == pytest.approx(rewards[1]) + assert rewards[0] == pytest.approx(1.0) + + def test_zero_variance_batch_drops_efficiency_terms(self) -> None: + # If all samples share the same energy/latency/cost, z-score is 0 + # for every term, so the reward is just alpha * accuracy. + samples = [ + TrainingSample( + accuracy=1.0, energy_joules=100, latency_seconds=2, cost_usd=0.0 + ), + TrainingSample( + accuracy=0.5, energy_joules=100, latency_seconds=2, cost_usd=0.0 + ), + ] + rewards = score_batch(samples) + assert rewards[0] == pytest.approx(0.5) + assert rewards[1] == pytest.approx(0.25) diff --git a/tests/learning/distillation/test_diagnose_tools.py b/tests/learning/spec_search/test_diagnose_tools.py similarity index 93% rename from tests/learning/distillation/test_diagnose_tools.py rename to tests/learning/spec_search/test_diagnose_tools.py index bad05e89..63b208ae 100644 --- a/tests/learning/distillation/test_diagnose_tools.py +++ b/tests/learning/spec_search/test_diagnose_tools.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.diagnose.tools module. +"""Tests for openjarvis.learning.spec_search.diagnose.tools module. All tests use fixture stubs — no live TraceStore, CloudEngine, or ToolRegistry. """ @@ -89,7 +89,7 @@ class TestBuildDiagnosticTools: """Tests for the build_diagnostic_tools factory.""" def test_returns_expected_tool_names(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -117,7 +117,7 @@ class TestBuildDiagnosticTools: assert "compare_outputs" in names def test_all_tools_have_openai_format(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -142,7 +142,7 @@ class TestListTraces: """Tests for the list_traces diagnostic tool.""" def test_returns_trace_metas(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -171,7 +171,7 @@ class TestGetTrace: """Tests for the get_trace diagnostic tool.""" def test_returns_trace_details(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -192,7 +192,7 @@ class TestGetTrace: assert parsed["query"] == "What is 2+2?" def test_returns_error_for_unknown_trace(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -215,7 +215,7 @@ class TestGetCurrentConfig: """Tests for the get_current_config diagnostic tool.""" def test_returns_config_content(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -238,7 +238,7 @@ class TestGetAgentPrompt: """Tests for the get_agent_prompt diagnostic tool.""" def test_returns_prompt_content(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) @@ -261,7 +261,7 @@ class TestListPersonalBenchmark: """Tests for the list_personal_benchmark diagnostic tool.""" def test_returns_benchmark_tasks(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.tools import ( + from openjarvis.learning.spec_search.diagnose.tools import ( build_diagnostic_tools, ) diff --git a/tests/learning/distillation/test_diagnose_types.py b/tests/learning/spec_search/test_diagnose_types.py similarity index 83% rename from tests/learning/distillation/test_diagnose_types.py rename to tests/learning/spec_search/test_diagnose_types.py index 36046b61..7f8f89e0 100644 --- a/tests/learning/distillation/test_diagnose_types.py +++ b/tests/learning/spec_search/test_diagnose_types.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.diagnose.types module.""" +"""Tests for openjarvis.learning.spec_search.diagnose.types module.""" from __future__ import annotations @@ -9,7 +9,7 @@ class TestTraceMeta: """Tests for TraceMeta dataclass.""" def test_constructs_with_required_fields(self) -> None: - from openjarvis.learning.distillation.diagnose.types import TraceMeta + from openjarvis.learning.spec_search.diagnose.types import TraceMeta meta = TraceMeta( trace_id="trace-001", @@ -24,7 +24,7 @@ class TestTraceMeta: assert meta.feedback == 0.8 def test_feedback_can_be_none(self) -> None: - from openjarvis.learning.distillation.diagnose.types import TraceMeta + from openjarvis.learning.spec_search.diagnose.types import TraceMeta meta = TraceMeta( trace_id="trace-002", @@ -42,7 +42,7 @@ class TestBenchmarkTask: """Tests for BenchmarkTask dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import BenchmarkTask + from openjarvis.learning.spec_search.diagnose.types import BenchmarkTask task = BenchmarkTask( task_id="task-001", @@ -58,7 +58,7 @@ class TestStudentRun: """Tests for StudentRun dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import StudentRun + from openjarvis.learning.spec_search.diagnose.types import StudentRun run = StudentRun( task_id="task-001", @@ -76,7 +76,7 @@ class TestTeacherRun: """Tests for TeacherRun dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import TeacherRun + from openjarvis.learning.spec_search.diagnose.types import TeacherRun run = TeacherRun( task_id="task-001", @@ -92,7 +92,7 @@ class TestComparisonResult: """Tests for ComparisonResult dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import ComparisonResult + from openjarvis.learning.spec_search.diagnose.types import ComparisonResult result = ComparisonResult( task_id="task-001", @@ -108,7 +108,7 @@ class TestToolMeta: """Tests for ToolMeta dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import ToolMeta + from openjarvis.learning.spec_search.diagnose.types import ToolMeta meta = ToolMeta( name="calculator", @@ -124,7 +124,7 @@ class TestDiagnosticTool: """Tests for DiagnosticTool dataclass.""" def test_constructs_with_callable(self) -> None: - from openjarvis.learning.distillation.diagnose.types import DiagnosticTool + from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool def my_func(**kwargs: object) -> str: return "result" @@ -143,7 +143,7 @@ class TestToolCallRecord: """Tests for ToolCallRecord dataclass.""" def test_constructs(self) -> None: - from openjarvis.learning.distillation.diagnose.types import ToolCallRecord + from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord record = ToolCallRecord( timestamp=datetime(2026, 4, 9, 3, 0, 0, tzinfo=timezone.utc), @@ -157,7 +157,7 @@ class TestToolCallRecord: assert record.latency_ms == 42.5 def test_to_jsonl_dict(self) -> None: - from openjarvis.learning.distillation.diagnose.types import ToolCallRecord + from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord record = ToolCallRecord( timestamp=datetime(2026, 4, 9, 3, 0, 0, tzinfo=timezone.utc), diff --git a/tests/learning/distillation/test_diagnosis_runner.py b/tests/learning/spec_search/test_diagnosis_runner.py similarity index 94% rename from tests/learning/distillation/test_diagnosis_runner.py rename to tests/learning/spec_search/test_diagnosis_runner.py index baedbc98..d5123492 100644 --- a/tests/learning/distillation/test_diagnosis_runner.py +++ b/tests/learning/spec_search/test_diagnosis_runner.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.diagnose.runner module. +"""Tests for openjarvis.learning.spec_search.diagnose.runner module. All tests use mocked dependencies — no live API calls. """ @@ -61,7 +61,7 @@ class TestDiagnosisRunner: """Tests for DiagnosisRunner.""" def test_produces_diagnosis_artifact(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) @@ -95,7 +95,7 @@ class TestDiagnosisRunner: assert "Math" in diagnosis_path.read_text() def test_returns_failure_clusters(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) @@ -128,7 +128,7 @@ class TestDiagnosisRunner: assert result.clusters[1].id == "cluster-002" def test_persists_teacher_traces_jsonl(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) @@ -174,7 +174,7 @@ class TestDiagnosisRunner: assert "tool" in record def test_returns_cost(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) @@ -203,7 +203,7 @@ class TestDiagnosisRunner: assert result.cost_usd >= 0.0 def test_handles_no_clusters_in_output(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) diff --git a/tests/learning/distillation/test_execution_loop.py b/tests/learning/spec_search/test_execution_loop.py similarity index 87% rename from tests/learning/distillation/test_execution_loop.py rename to tests/learning/spec_search/test_execution_loop.py index ff9e27c2..2f0338a1 100644 --- a/tests/learning/distillation/test_execution_loop.py +++ b/tests/learning/spec_search/test_execution_loop.py @@ -4,8 +4,8 @@ from __future__ import annotations from pathlib import Path -from openjarvis.learning.distillation.execute.base import ApplyContext -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.execute.base import ApplyContext +from openjarvis.learning.spec_search.models import ( AutonomyMode, Edit, EditOp, @@ -72,7 +72,7 @@ class TestExecuteEdits: """Tests for execute_edits().""" def test_applies_auto_tier_edit(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( @@ -84,7 +84,7 @@ class TestExecuteEdits: assert outcomes[0].status == "applied" def test_review_edit_goes_to_pending_in_tiered_mode(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( @@ -96,7 +96,7 @@ class TestExecuteEdits: assert outcomes[0].status == "pending_review" def test_review_edit_applied_in_auto_mode(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( @@ -108,7 +108,7 @@ class TestExecuteEdits: assert outcomes[0].status == "applied" def test_manual_tier_skipped(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( @@ -120,7 +120,7 @@ class TestExecuteEdits: assert outcomes[0].status == "skipped" def test_all_edits_pending_in_manual_mode(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( @@ -132,7 +132,7 @@ class TestExecuteEdits: assert outcomes[0].status == "pending_review" def test_multiple_edits_processed(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.execute.loop import execute_edits + from openjarvis.learning.spec_search.execute.loop import execute_edits ctx = _make_ctx(tmp_path) outcomes = execute_edits( diff --git a/tests/learning/distillation/test_external_adapter.py b/tests/learning/spec_search/test_external_adapter.py similarity index 96% rename from tests/learning/distillation/test_external_adapter.py rename to tests/learning/spec_search/test_external_adapter.py index 9668bec1..32afae86 100644 --- a/tests/learning/distillation/test_external_adapter.py +++ b/tests/learning/spec_search/test_external_adapter.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.external_adapter.""" +"""Tests for openjarvis.learning.spec_search.external_adapter.""" from __future__ import annotations @@ -7,7 +7,7 @@ from pathlib import Path import pytest from openjarvis.evals.core.types import EvalRecord -from openjarvis.learning.distillation.external_adapter import ( +from openjarvis.learning.spec_search.external_adapter import ( write_external_records_as_traces, ) from openjarvis.traces.store import TraceStore diff --git a/tests/learning/distillation/test_live_integration.py b/tests/learning/spec_search/test_live_integration.py similarity index 95% rename from tests/learning/distillation/test_live_integration.py rename to tests/learning/spec_search/test_live_integration.py index e8ffd1ec..5dcce453 100644 --- a/tests/learning/distillation/test_live_integration.py +++ b/tests/learning/spec_search/test_live_integration.py @@ -1,4 +1,4 @@ -"""Live integration tests for the distillation subsystem. +"""Live integration tests for the spec-search subsystem. These tests use REAL API calls (CloudEngine with Anthropic) and real TraceStore data. They are gated on the ``cloud`` marker — skip them @@ -73,7 +73,7 @@ class TestTeacherAgentLive: """Test TeacherAgent with a real CloudEngine.""" def test_teacher_agent_single_turn(self, cloud_engine) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -101,7 +101,7 @@ class TestDiagnosisRunnerLive: def test_diagnosis_produces_output( self, cloud_engine, real_trace_store, tmp_path ) -> None: - from openjarvis.learning.distillation.diagnose.runner import ( + from openjarvis.learning.spec_search.diagnose.runner import ( DiagnosisRunner, ) @@ -157,7 +157,7 @@ class TestColdStartLive: """With 373 traces but 0 feedback, the orchestrator should handle this gracefully — either by running (traces > 20) or by giving a clear message about what's missing.""" - from openjarvis.learning.distillation.gate.cold_start import ( + from openjarvis.learning.spec_search.gate.cold_start import ( check_benchmark_ready, check_readiness, ) diff --git a/tests/learning/distillation/test_models.py b/tests/learning/spec_search/test_models.py similarity index 85% rename from tests/learning/distillation/test_models.py rename to tests/learning/spec_search/test_models.py index c04f663e..259e62f2 100644 --- a/tests/learning/distillation/test_models.py +++ b/tests/learning/spec_search/test_models.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.models module.""" +"""Tests for openjarvis.learning.spec_search.models module.""" from __future__ import annotations @@ -11,7 +11,7 @@ class TestEditPillar: """Tests for EditPillar enum.""" def test_has_four_pillars(self) -> None: - from openjarvis.learning.distillation.models import EditPillar + from openjarvis.learning.spec_search.models import EditPillar assert EditPillar.INTELLIGENCE.value == "intelligence" assert EditPillar.AGENT.value == "agent" @@ -19,7 +19,7 @@ class TestEditPillar: assert EditPillar.ENGINE.value == "engine" def test_is_string_enum(self) -> None: - from openjarvis.learning.distillation.models import EditPillar + from openjarvis.learning.spec_search.models import EditPillar assert isinstance(EditPillar.AGENT, str) assert EditPillar("agent") is EditPillar.AGENT @@ -29,7 +29,7 @@ class TestEditRiskTier: """Tests for EditRiskTier enum.""" def test_has_three_tiers(self) -> None: - from openjarvis.learning.distillation.models import EditRiskTier + from openjarvis.learning.spec_search.models import EditRiskTier assert EditRiskTier.AUTO.value == "auto" assert EditRiskTier.REVIEW.value == "review" @@ -40,13 +40,13 @@ class TestEditOp: """Tests for EditOp enum — must contain all v1 ops plus v2 placeholders.""" def test_intelligence_ops(self) -> None: - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp assert EditOp.SET_MODEL_FOR_QUERY_CLASS.value == "set_model_for_query_class" assert EditOp.SET_MODEL_PARAM.value == "set_model_param" def test_agent_ops(self) -> None: - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp assert EditOp.PATCH_SYSTEM_PROMPT.value == "patch_system_prompt" assert EditOp.REPLACE_SYSTEM_PROMPT.value == "replace_system_prompt" @@ -55,14 +55,14 @@ class TestEditOp: assert EditOp.EDIT_FEW_SHOT_EXEMPLARS.value == "edit_few_shot_exemplars" def test_tools_ops(self) -> None: - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp assert EditOp.ADD_TOOL_TO_AGENT.value == "add_tool_to_agent" assert EditOp.REMOVE_TOOL_FROM_AGENT.value == "remove_tool_from_agent" assert EditOp.EDIT_TOOL_DESCRIPTION.value == "edit_tool_description" def test_v2_placeholder_ops(self) -> None: - from openjarvis.learning.distillation.models import EditOp + from openjarvis.learning.spec_search.models import EditOp assert EditOp.LORA_FINETUNE.value == "lora_finetune" @@ -71,7 +71,7 @@ class TestTriggerKind: """Tests for TriggerKind enum.""" def test_four_trigger_kinds(self) -> None: - from openjarvis.learning.distillation.models import TriggerKind + from openjarvis.learning.spec_search.models import TriggerKind assert TriggerKind.SCHEDULED.value == "scheduled" assert TriggerKind.CLUSTER.value == "cluster" @@ -83,7 +83,7 @@ class TestAutonomyMode: """Tests for AutonomyMode enum.""" def test_three_modes(self) -> None: - from openjarvis.learning.distillation.models import AutonomyMode + from openjarvis.learning.spec_search.models import AutonomyMode assert AutonomyMode.AUTO.value == "auto" assert AutonomyMode.TIERED.value == "tiered" @@ -94,7 +94,7 @@ class TestSessionStatus: """Tests for SessionStatus enum.""" def test_all_statuses(self) -> None: - from openjarvis.learning.distillation.models import SessionStatus + from openjarvis.learning.spec_search.models import SessionStatus assert SessionStatus.INITIATED.value == "initiated" assert SessionStatus.DIAGNOSING.value == "diagnosing" @@ -115,7 +115,7 @@ class TestEdit: """Tests for Edit pydantic model.""" def _valid_edit_kwargs(self) -> dict: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( EditOp, EditPillar, EditRiskTier, @@ -134,7 +134,7 @@ class TestEdit: } def test_constructs_with_valid_fields(self) -> None: - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit edit = Edit(**self._valid_edit_kwargs()) @@ -144,7 +144,7 @@ class TestEdit: assert edit.references == ["trace-001", "trace-002"] def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit edit = Edit(**self._valid_edit_kwargs()) as_json = edit.model_dump_json() @@ -156,7 +156,7 @@ class TestEdit: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit kwargs = self._valid_edit_kwargs() kwargs["pillar"] = "not_a_pillar" @@ -168,7 +168,7 @@ class TestEdit: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit kwargs = self._valid_edit_kwargs() kwargs["op"] = "not_an_op" @@ -177,7 +177,7 @@ class TestEdit: Edit(**kwargs) def test_payload_can_be_empty_dict(self) -> None: - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit kwargs = self._valid_edit_kwargs() kwargs["payload"] = {} @@ -186,7 +186,7 @@ class TestEdit: assert edit.payload == {} def test_references_default_empty_list(self) -> None: - from openjarvis.learning.distillation.models import Edit + from openjarvis.learning.spec_search.models import Edit kwargs = self._valid_edit_kwargs() del kwargs["references"] @@ -217,7 +217,7 @@ class TestFailureCluster: } def test_constructs_with_valid_fields(self) -> None: - from openjarvis.learning.distillation.models import FailureCluster + from openjarvis.learning.spec_search.models import FailureCluster cluster = FailureCluster(**self._valid_cluster_kwargs()) @@ -228,7 +228,7 @@ class TestFailureCluster: assert len(cluster.addressed_by_edit_ids) == 2 def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import FailureCluster + from openjarvis.learning.spec_search.models import FailureCluster cluster = FailureCluster(**self._valid_cluster_kwargs()) as_json = cluster.model_dump_json() @@ -237,7 +237,7 @@ class TestFailureCluster: assert restored == cluster def test_addressed_by_edit_ids_defaults_empty(self) -> None: - from openjarvis.learning.distillation.models import FailureCluster + from openjarvis.learning.spec_search.models import FailureCluster kwargs = self._valid_cluster_kwargs() del kwargs["addressed_by_edit_ids"] @@ -249,7 +249,7 @@ class TestFailureCluster: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import FailureCluster + from openjarvis.learning.spec_search.models import FailureCluster kwargs = self._valid_cluster_kwargs() kwargs["student_failure_rate"] = 1.5 @@ -261,7 +261,7 @@ class TestFailureCluster: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import FailureCluster + from openjarvis.learning.spec_search.models import FailureCluster kwargs = self._valid_cluster_kwargs() kwargs["teacher_success_rate"] = -0.1 @@ -281,7 +281,7 @@ class TestLearningPlan: def _valid_plan_kwargs(self) -> dict: from datetime import datetime, timezone - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -320,7 +320,7 @@ class TestLearningPlan: } def test_constructs_with_valid_fields(self) -> None: - from openjarvis.learning.distillation.models import LearningPlan + from openjarvis.learning.spec_search.models import LearningPlan plan = LearningPlan(**self._valid_plan_kwargs()) @@ -331,7 +331,7 @@ class TestLearningPlan: assert plan.estimated_cost_usd == 1.42 def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import LearningPlan + from openjarvis.learning.spec_search.models import LearningPlan plan = LearningPlan(**self._valid_plan_kwargs()) as_json = plan.model_dump_json() @@ -341,7 +341,7 @@ class TestLearningPlan: def test_empty_clusters_and_edits_allowed(self) -> None: # An aborted session may produce a plan with no clusters and no edits. - from openjarvis.learning.distillation.models import LearningPlan + from openjarvis.learning.spec_search.models import LearningPlan kwargs = self._valid_plan_kwargs() kwargs["failure_clusters"] = [] @@ -355,7 +355,7 @@ class TestLearningPlan: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import LearningPlan + from openjarvis.learning.spec_search.models import LearningPlan kwargs = self._valid_plan_kwargs() kwargs["estimated_cost_usd"] = -1.0 @@ -382,7 +382,7 @@ class TestBenchmarkSnapshot: } def test_constructs_with_valid_fields(self) -> None: - from openjarvis.learning.distillation.models import BenchmarkSnapshot + from openjarvis.learning.spec_search.models import BenchmarkSnapshot snap = BenchmarkSnapshot(**self._valid_snapshot_kwargs()) @@ -393,7 +393,7 @@ class TestBenchmarkSnapshot: assert snap.elapsed_seconds == 184.3 def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import BenchmarkSnapshot + from openjarvis.learning.spec_search.models import BenchmarkSnapshot snap = BenchmarkSnapshot(**self._valid_snapshot_kwargs()) restored = BenchmarkSnapshot.model_validate_json(snap.model_dump_json()) @@ -404,7 +404,7 @@ class TestBenchmarkSnapshot: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import BenchmarkSnapshot + from openjarvis.learning.spec_search.models import BenchmarkSnapshot kwargs = self._valid_snapshot_kwargs() kwargs["overall_score"] = 1.5 @@ -416,7 +416,7 @@ class TestBenchmarkSnapshot: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import BenchmarkSnapshot + from openjarvis.learning.spec_search.models import BenchmarkSnapshot kwargs = self._valid_snapshot_kwargs() kwargs["task_count"] = -1 @@ -446,14 +446,14 @@ class TestEditOutcome: } def test_applied_outcome(self) -> None: - from openjarvis.learning.distillation.models import EditOutcome + from openjarvis.learning.spec_search.models import EditOutcome outcome = EditOutcome(**self._valid_outcome_kwargs()) assert outcome.status == "applied" assert outcome.benchmark_delta == 0.04 def test_rejected_outcome_has_no_applied_at(self) -> None: - from openjarvis.learning.distillation.models import EditOutcome + from openjarvis.learning.spec_search.models import EditOutcome outcome = EditOutcome( edit_id="edit-002", @@ -471,7 +471,7 @@ class TestEditOutcome: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import EditOutcome + from openjarvis.learning.spec_search.models import EditOutcome kwargs = self._valid_outcome_kwargs() kwargs["status"] = "totally_made_up" @@ -480,7 +480,7 @@ class TestEditOutcome: EditOutcome(**kwargs) def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import EditOutcome + from openjarvis.learning.spec_search.models import EditOutcome outcome = EditOutcome(**self._valid_outcome_kwargs()) restored = EditOutcome.model_validate_json(outcome.model_dump_json()) @@ -500,7 +500,7 @@ class TestLearningSession: from datetime import datetime, timezone from pathlib import Path - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( AutonomyMode, BenchmarkSnapshot, SessionStatus, @@ -535,7 +535,7 @@ class TestLearningSession: } def test_constructs_with_valid_fields(self) -> None: - from openjarvis.learning.distillation.models import LearningSession + from openjarvis.learning.spec_search.models import LearningSession session = LearningSession(**self._valid_session_kwargs()) assert session.id == "session-001" @@ -544,7 +544,7 @@ class TestLearningSession: assert session.benchmark_after is None def test_round_trip_via_json(self) -> None: - from openjarvis.learning.distillation.models import LearningSession + from openjarvis.learning.spec_search.models import LearningSession session = LearningSession(**self._valid_session_kwargs()) as_json = session.model_dump_json() @@ -553,7 +553,7 @@ class TestLearningSession: assert restored == session def test_supports_parent_session_chain(self) -> None: - from openjarvis.learning.distillation.models import LearningSession + from openjarvis.learning.spec_search.models import LearningSession kwargs = self._valid_session_kwargs() kwargs["parent_session_id"] = "session-000" @@ -565,7 +565,7 @@ class TestLearningSession: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import LearningSession + from openjarvis.learning.spec_search.models import LearningSession kwargs = self._valid_session_kwargs() kwargs["status"] = "not_a_status" @@ -577,7 +577,7 @@ class TestLearningSession: import pytest from pydantic import ValidationError - from openjarvis.learning.distillation.models import LearningSession + from openjarvis.learning.spec_search.models import LearningSession kwargs = self._valid_session_kwargs() kwargs["teacher_cost_usd"] = -0.01 diff --git a/tests/learning/spec_search/test_multi_session.py b/tests/learning/spec_search/test_multi_session.py new file mode 100644 index 00000000..71cafc6a --- /dev/null +++ b/tests/learning/spec_search/test_multi_session.py @@ -0,0 +1,146 @@ +"""Tests for SpecSearchLoop (paper Algorithm 1).""" + +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest + +from openjarvis.learning.spec_search.models import ( + AutonomyMode, + BenchmarkSnapshot, + LearningSession, + SessionStatus, + TriggerKind, +) +from openjarvis.learning.spec_search.multi_session import ( + SpecSearchLoop, +) + + +def _session( + score: float, + cost: float = 0.1, + status: SessionStatus = SessionStatus.COMPLETED, + error: str | None = None, +) -> LearningSession: + return LearningSession( + id=f"s-{score}", + trigger=TriggerKind.ON_DEMAND, + trigger_metadata={}, + status=status, + autonomy_mode=AutonomyMode.AUTO, + started_at=datetime.now(timezone.utc), + diagnosis_path="/tmp/diag.md", + plan_path="/tmp/plan.json", + benchmark_before=BenchmarkSnapshot( + benchmark_version="v1", + overall_score=0.0, + cluster_scores={}, + task_count=10, + elapsed_seconds=1.0, + ), + benchmark_after=BenchmarkSnapshot( + benchmark_version="v1", + overall_score=score, + cluster_scores={}, + task_count=10, + elapsed_seconds=1.0, + ), + git_checkpoint_pre="sha-pre", + teacher_cost_usd=cost, + error=error, + ) + + +def _orch_yielding(sessions: list[LearningSession]) -> MagicMock: + """Mock orchestrator whose .run(trigger) returns the next session.""" + orch = MagicMock() + orch.run.side_effect = sessions + return orch + + +class TestSpecSearchLoop: + def test_validates_stagnation_k(self) -> None: + with pytest.raises(ValueError): + SpecSearchLoop(MagicMock(), stagnation_k=0) + + def test_validates_budget(self) -> None: + with pytest.raises(ValueError): + SpecSearchLoop(MagicMock(), max_total_cost_usd=0) + + def test_stops_after_k_sessions_without_improvement(self) -> None: + # First session improves to 0.6; next 3 do not improve. With k=3 + # the loop should stop after the 4th session. + orch = _orch_yielding( + [ + _session(0.6), + _session(0.6), + _session(0.6), + _session(0.6), + ] + ) + loop = SpecSearchLoop(orch, stagnation_k=3, max_total_cost_usd=10.0) + result = loop.run() + assert result.stop_reason == "stagnation" + assert len(result.sessions) == 4 + assert result.best_overall_score == pytest.approx(0.6) + + def test_keeps_improving_resets_streak(self) -> None: + # Sessions improve each time; loop only stops when budget hits. + orch = _orch_yielding( + [ + _session(0.5, cost=2.0), + _session(0.6, cost=2.0), + _session(0.7, cost=2.0), + _session(0.8, cost=2.0), # cumulative = 8.0 + _session(0.9, cost=3.0), # cumulative = 11.0 -> over budget + ] + ) + loop = SpecSearchLoop(orch, stagnation_k=10, max_total_cost_usd=10.0) + result = loop.run() + assert result.stop_reason == "budget" + assert len(result.sessions) == 5 + assert result.best_overall_score == pytest.approx(0.9) + + def test_failed_session_terminates_loop(self) -> None: + orch = _orch_yielding( + [ + _session(0.5), + _session(0.0, status=SessionStatus.FAILED, error="crash"), + ] + ) + loop = SpecSearchLoop(orch, stagnation_k=5, max_total_cost_usd=10.0) + result = loop.run() + assert result.stop_reason == "failed" + assert len(result.sessions) == 2 + + def test_total_cost_accumulates(self) -> None: + orch = _orch_yielding( + [ + _session(0.5, cost=1.0), + _session(0.5, cost=2.0), + _session(0.5, cost=3.0), + ] + ) + loop = SpecSearchLoop(orch, stagnation_k=2, max_total_cost_usd=100.0) + result = loop.run() + assert result.total_cost_usd == pytest.approx(6.0) + + def test_eps_threshold_filters_noise(self) -> None: + # Improvement smaller than stagnation_eps does not reset the streak. + orch = _orch_yielding( + [ + _session(0.500), + _session(0.5005), # below default eps=0.001 + _session(0.5009), # below eps from current best + ] + ) + loop = SpecSearchLoop( + orch, stagnation_k=2, stagnation_eps=0.001, max_total_cost_usd=10.0 + ) + result = loop.run() + assert result.stop_reason == "stagnation" + assert len(result.sessions) == 3 + assert result.best_overall_score == pytest.approx(0.500) diff --git a/tests/learning/distillation/test_orchestrator.py b/tests/learning/spec_search/test_orchestrator.py similarity index 71% rename from tests/learning/distillation/test_orchestrator.py rename to tests/learning/spec_search/test_orchestrator.py index f489d03d..dd252e18 100644 --- a/tests/learning/distillation/test_orchestrator.py +++ b/tests/learning/spec_search/test_orchestrator.py @@ -1,4 +1,4 @@ -"""Tests for DistillationOrchestrator — full session with mocks.""" +"""Tests for SpecSearchOrchestrator — full session with mocks.""" from __future__ import annotations @@ -6,13 +6,13 @@ import json from pathlib import Path from unittest.mock import MagicMock, patch -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( AutonomyMode, BenchmarkSnapshot, FailureCluster, SessionStatus, ) -from openjarvis.learning.distillation.triggers import OnDemandTrigger +from openjarvis.learning.spec_search.triggers import OnDemandTrigger def _make_snapshot(overall: float = 0.6) -> BenchmarkSnapshot: @@ -26,7 +26,7 @@ def _make_snapshot(overall: float = 0.6) -> BenchmarkSnapshot: def _make_diagnosis_result(): - from openjarvis.learning.distillation.diagnose.runner import DiagnosisResult + from openjarvis.learning.spec_search.diagnose.runner import DiagnosisResult return DiagnosisResult( diagnosis_md="## Diagnosis\nMath routing is broken.", @@ -75,13 +75,13 @@ def _make_mock_engine(): return engine -class TestDistillationOrchestrator: +class TestSpecSearchOrchestrator: def test_full_session_completes(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.orchestrator import ( - DistillationOrchestrator, + from openjarvis.learning.spec_search.orchestrator import ( + SpecSearchOrchestrator, ) - orch = DistillationOrchestrator( + orch = SpecSearchOrchestrator( teacher_engine=_make_mock_engine(), teacher_model="claude-opus-4-6", trace_store=MagicMock(count=MagicMock(return_value=30)), @@ -100,7 +100,7 @@ class TestDistillationOrchestrator: ) with patch( - "openjarvis.learning.distillation.orchestrator.DiagnosisRunner" + "openjarvis.learning.spec_search.orchestrator.DiagnosisRunner" ) as MockDiag: MockDiag.return_value.run.return_value = _make_diagnosis_result() session = orch.run(OnDemandTrigger()) @@ -112,14 +112,14 @@ class TestDistillationOrchestrator: assert session.teacher_cost_usd >= 0 def test_cold_start_returns_failed(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.orchestrator import ( - DistillationOrchestrator, + from openjarvis.learning.spec_search.orchestrator import ( + SpecSearchOrchestrator, ) trace_store = MagicMock() trace_store.count.return_value = 5 # Not enough - orch = DistillationOrchestrator( + orch = SpecSearchOrchestrator( teacher_engine=MagicMock(), teacher_model="claude-opus-4-6", trace_store=trace_store, @@ -141,13 +141,13 @@ class TestDistillationOrchestrator: assert "not enough traces" in (session.error or "").lower() def test_session_persisted_to_store(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.orchestrator import ( - DistillationOrchestrator, + from openjarvis.learning.spec_search.orchestrator import ( + SpecSearchOrchestrator, ) session_store = MagicMock() - orch = DistillationOrchestrator( + orch = SpecSearchOrchestrator( teacher_engine=_make_mock_engine(), teacher_model="claude-opus-4-6", trace_store=MagicMock(count=MagicMock(return_value=30)), @@ -166,9 +166,43 @@ class TestDistillationOrchestrator: ) with patch( - "openjarvis.learning.distillation.orchestrator.DiagnosisRunner" + "openjarvis.learning.spec_search.orchestrator.DiagnosisRunner" ) as MockDiag: MockDiag.return_value.run.return_value = _make_diagnosis_result() orch.run(OnDemandTrigger()) assert session_store.save_session.called + + +class TestFromConfig: + def test_from_config_round_trips_paper_defaults(self, tmp_path: Path) -> None: + from openjarvis.core.config import SpecSearchLearningConfig + from openjarvis.learning.spec_search.orchestrator import ( + SpecSearchOrchestrator, + ) + + cfg = SpecSearchLearningConfig( + enabled=True, + teacher_model="claude-opus-4-6", + teacher_engine="cloud", + autonomy_mode="auto", + max_regression=0.01, # paper default + stagnation_k=5, + ) + + orch = SpecSearchOrchestrator.from_config( + cfg, + teacher_engine=MagicMock(), + trace_store=MagicMock(), + benchmark_samples=[], + student_runner=MagicMock(), + judge=MagicMock(), + session_store=MagicMock(), + checkpoint_store=MagicMock(), + openjarvis_home=tmp_path, + ) + + # Round-trip the knobs: paper defaults must reach the orchestrator. + assert orch._model == "claude-opus-4-6" + assert orch._max_regression == 0.01 + assert orch._autonomy.value == "auto" diff --git a/tests/learning/distillation/test_paths.py b/tests/learning/spec_search/test_paths.py similarity index 69% rename from tests/learning/distillation/test_paths.py rename to tests/learning/spec_search/test_paths.py index b123a3b0..b40b23f4 100644 --- a/tests/learning/distillation/test_paths.py +++ b/tests/learning/spec_search/test_paths.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.storage.paths module.""" +"""Tests for openjarvis.learning.spec_search.storage.paths module.""" from __future__ import annotations @@ -7,18 +7,18 @@ from pathlib import Path import pytest # --------------------------------------------------------------------------- -# resolve_distillation_root +# resolve_spec_search_root # --------------------------------------------------------------------------- -class TestResolveDistillationRoot: - """Tests for resolve_distillation_root().""" +class TestResolveSpecSearchRoot: + """Tests for resolve_spec_search_root().""" def test_default_is_under_home(self, monkeypatch: pytest.MonkeyPatch) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths monkeypatch.delenv("OPENJARVIS_HOME", raising=False) - result = paths.resolve_distillation_root() + result = paths.resolve_spec_search_root() assert result == Path.home() / ".openjarvis" / "learning" def test_respects_openjarvis_home_env_var( @@ -26,11 +26,11 @@ class TestResolveDistillationRoot: tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths custom = tmp_path / "custom_oj" monkeypatch.setenv("OPENJARVIS_HOME", str(custom)) - result = paths.resolve_distillation_root() + result = paths.resolve_spec_search_root() assert result == custom / "learning" def test_returns_absolute_path( @@ -38,17 +38,17 @@ class TestResolveDistillationRoot: tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "rel")) - result = paths.resolve_distillation_root() + result = paths.resolve_spec_search_root() assert result.is_absolute() def test_rejects_path_inside_source_tree( self, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths # Find the OpenJarvis source root by walking up from the paths module. source_root = paths._find_source_root() @@ -58,10 +58,10 @@ class TestResolveDistillationRoot: monkeypatch.setenv("OPENJARVIS_HOME", str(source_root / "junk_dir")) with pytest.raises(paths.ConfigurationError, match="inside the source tree"): - paths.resolve_distillation_root() + paths.resolve_spec_search_root() def test_find_source_root_returns_repo_root(self) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths result = paths._find_source_root() assert result is not None @@ -69,22 +69,22 @@ class TestResolveDistillationRoot: # --------------------------------------------------------------------------- -# ensure_distillation_dirs +# ensure_spec_search_dirs # --------------------------------------------------------------------------- -class TestEnsureDistillationDirs: - """Tests for ensure_distillation_dirs().""" +class TestEnsureSpecSearchDirs: + """Tests for ensure_spec_search_dirs().""" def test_creates_subdirs( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "oj")) - root = paths.ensure_distillation_dirs() + root = paths.ensure_spec_search_dirs() assert root.exists() assert (root / "sessions").exists() @@ -97,11 +97,11 @@ class TestEnsureDistillationDirs: tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - from openjarvis.learning.distillation.storage import paths + from openjarvis.learning.spec_search.storage import paths monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "oj")) - first = paths.ensure_distillation_dirs() - second = paths.ensure_distillation_dirs() + first = paths.ensure_spec_search_dirs() + second = paths.ensure_spec_search_dirs() assert first == second assert first.exists() diff --git a/tests/learning/distillation/test_pending_queue.py b/tests/learning/spec_search/test_pending_queue.py similarity index 80% rename from tests/learning/distillation/test_pending_queue.py rename to tests/learning/spec_search/test_pending_queue.py index ba67d171..91915097 100644 --- a/tests/learning/distillation/test_pending_queue.py +++ b/tests/learning/spec_search/test_pending_queue.py @@ -1,10 +1,10 @@ -"""Tests for openjarvis.learning.distillation.pending_queue module.""" +"""Tests for openjarvis.learning.spec_search.pending_queue module.""" from __future__ import annotations from pathlib import Path -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, @@ -27,7 +27,7 @@ def _make_edit(edit_id: str = "edit-001") -> Edit: class TestPendingQueue: def test_enqueue_creates_file(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.pending_queue import PendingQueue + from openjarvis.learning.spec_search.pending_queue import PendingQueue queue = PendingQueue(tmp_path / "pending_review") queue.enqueue("session-001", _make_edit()) @@ -37,7 +37,7 @@ class TestPendingQueue: assert "edit-001" in files[0].name def test_list_pending(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.pending_queue import PendingQueue + from openjarvis.learning.spec_search.pending_queue import PendingQueue queue = PendingQueue(tmp_path / "pending_review") queue.enqueue("session-001", _make_edit("e1")) @@ -48,7 +48,7 @@ class TestPendingQueue: assert ids == {"e1", "e2"} def test_resolve_removes_file(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.pending_queue import PendingQueue + from openjarvis.learning.spec_search.pending_queue import PendingQueue queue = PendingQueue(tmp_path / "pending_review") queue.enqueue("session-001", _make_edit()) @@ -57,13 +57,13 @@ class TestPendingQueue: assert len(queue.list_pending()) == 0 def test_list_empty_queue(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.pending_queue import PendingQueue + from openjarvis.learning.spec_search.pending_queue import PendingQueue queue = PendingQueue(tmp_path / "pending_review") assert queue.list_pending() == [] def test_get_pending_edit(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.pending_queue import PendingQueue + from openjarvis.learning.spec_search.pending_queue import PendingQueue queue = PendingQueue(tmp_path / "pending_review") queue.enqueue("session-001", _make_edit()) diff --git a/tests/learning/distillation/test_planner.py b/tests/learning/spec_search/test_planner.py similarity index 92% rename from tests/learning/distillation/test_planner.py rename to tests/learning/spec_search/test_planner.py index be7602fb..402a1444 100644 --- a/tests/learning/distillation/test_planner.py +++ b/tests/learning/spec_search/test_planner.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.plan.planner module. +"""Tests for openjarvis.learning.spec_search.plan.planner module. All tests use mocked CloudEngine — no live API calls. """ @@ -9,7 +9,7 @@ import json from pathlib import Path from unittest.mock import MagicMock -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( EditOp, EditRiskTier, FailureCluster, @@ -72,7 +72,7 @@ class TestLearningPlanner: """Tests for LearningPlanner.""" def test_produces_learning_plan(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = _make_teacher_response([_make_edit_dict()]) @@ -95,7 +95,7 @@ class TestLearningPlanner: assert plan.teacher_model == "claude-opus-4-6" def test_assigns_risk_tiers(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() # Teacher incorrectly sets MANUAL for an auto-tier op @@ -119,7 +119,7 @@ class TestLearningPlanner: assert plan.edits[0].risk_tier == EditRiskTier.AUTO def test_persists_plan_json(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = _make_teacher_response([_make_edit_dict()]) @@ -143,7 +143,7 @@ class TestLearningPlanner: assert data["session_id"] == "session-001" def test_drops_cluster_with_zero_rates(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = _make_teacher_response([_make_edit_dict()]) @@ -188,7 +188,7 @@ class TestLearningPlanner: assert "dropped" not in good.skill_gap.lower() def test_all_clusters_dropped_returns_empty_edits(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = _make_teacher_response([]) @@ -220,7 +220,7 @@ class TestLearningPlanner: assert plan.failure_clusters[0].addressed_by_edit_ids == [] def test_persists_teacher_trace_jsonl(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = _make_teacher_response([_make_edit_dict()]) @@ -246,7 +246,7 @@ class TestLearningPlanner: assert "cost_usd" in record def test_handles_malformed_teacher_output(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.plan.planner import LearningPlanner + from openjarvis.learning.spec_search.plan.planner import LearningPlanner engine = MagicMock() engine.generate.return_value = { diff --git a/tests/learning/distillation/test_prompt_diff.py b/tests/learning/spec_search/test_prompt_diff.py similarity index 83% rename from tests/learning/distillation/test_prompt_diff.py rename to tests/learning/spec_search/test_prompt_diff.py index 9ba38a0b..447db201 100644 --- a/tests/learning/distillation/test_prompt_diff.py +++ b/tests/learning/spec_search/test_prompt_diff.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.plan.prompt_diff module.""" +"""Tests for openjarvis.learning.spec_search.plan.prompt_diff module.""" from __future__ import annotations @@ -9,14 +9,14 @@ class TestChangedLineRatio: """Tests for changed_line_ratio().""" def test_identical_strings(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( changed_line_ratio, ) assert changed_line_ratio("hello\nworld\n", "hello\nworld\n") == 0.0 def test_completely_different(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( changed_line_ratio, ) @@ -24,7 +24,7 @@ class TestChangedLineRatio: assert ratio == 1.0 def test_partial_change(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( changed_line_ratio, ) @@ -35,7 +35,7 @@ class TestChangedLineRatio: assert 0.2 <= ratio <= 0.35 def test_empty_original(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( changed_line_ratio, ) @@ -44,7 +44,7 @@ class TestChangedLineRatio: assert ratio == 1.0 def test_empty_both(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( changed_line_ratio, ) @@ -55,7 +55,7 @@ class TestApplyUnifiedDiff: """Tests for apply_unified_diff().""" def test_applies_simple_patch(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( apply_unified_diff, ) @@ -73,7 +73,7 @@ class TestApplyUnifiedDiff: assert result == "line1\nchanged_line2\nline3\n" def test_returns_none_on_bad_diff(self) -> None: - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( apply_unified_diff, ) @@ -85,13 +85,13 @@ class TestMaybeDowngradeToReplace: """Tests for maybe_downgrade_to_replace().""" def test_non_patch_op_passes_through(self) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( maybe_downgrade_to_replace, ) @@ -109,13 +109,13 @@ class TestMaybeDowngradeToReplace: assert result.op == EditOp.SET_MODEL_FOR_QUERY_CLASS def test_small_diff_stays_patch(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( maybe_downgrade_to_replace, ) @@ -145,13 +145,13 @@ class TestMaybeDowngradeToReplace: assert result.op == EditOp.PATCH_SYSTEM_PROMPT def test_large_diff_downgrades_to_replace(self) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( maybe_downgrade_to_replace, ) @@ -185,13 +185,13 @@ class TestMaybeDowngradeToReplace: assert "new1" in result.payload["new_content"] def test_bad_diff_downgrades_to_replace_with_raw(self) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.prompt_diff import ( + from openjarvis.learning.spec_search.plan.prompt_diff import ( maybe_downgrade_to_replace, ) diff --git a/tests/learning/distillation/test_regression.py b/tests/learning/spec_search/test_regression.py similarity index 83% rename from tests/learning/distillation/test_regression.py rename to tests/learning/spec_search/test_regression.py index 7f1c1231..610262f5 100644 --- a/tests/learning/distillation/test_regression.py +++ b/tests/learning/spec_search/test_regression.py @@ -1,8 +1,8 @@ -"""Tests for openjarvis.learning.distillation.gate.regression module.""" +"""Tests for openjarvis.learning.spec_search.gate.regression module.""" from __future__ import annotations -from openjarvis.learning.distillation.models import BenchmarkSnapshot +from openjarvis.learning.spec_search.models import BenchmarkSnapshot def _make_snapshot( @@ -22,7 +22,7 @@ class TestRegressionCheck: """Tests for regression_check().""" def test_no_regression_when_all_improve(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) @@ -32,7 +32,7 @@ class TestRegressionCheck: assert not result.has_regression def test_detects_cluster_regression(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) @@ -43,7 +43,7 @@ class TestRegressionCheck: assert "c2" in result.regressed_clusters def test_small_drop_within_threshold(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) @@ -53,7 +53,7 @@ class TestRegressionCheck: assert not result.has_regression def test_new_cluster_in_after_not_flagged(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) @@ -63,7 +63,7 @@ class TestRegressionCheck: assert not result.has_regression def test_missing_cluster_in_after_flagged(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) @@ -75,7 +75,7 @@ class TestRegressionCheck: assert "c2" in result.regressed_clusters def test_result_has_details(self) -> None: - from openjarvis.learning.distillation.gate.regression import ( + from openjarvis.learning.spec_search.gate.regression import ( regression_check, ) diff --git a/tests/learning/distillation/test_risk_tier.py b/tests/learning/spec_search/test_risk_tier.py similarity index 63% rename from tests/learning/distillation/test_risk_tier.py rename to tests/learning/spec_search/test_risk_tier.py index d7510e6d..e058fb5a 100644 --- a/tests/learning/distillation/test_risk_tier.py +++ b/tests/learning/spec_search/test_risk_tier.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.plan.risk_tier module.""" +"""Tests for openjarvis.learning.spec_search.plan.risk_tier module.""" from __future__ import annotations @@ -7,15 +7,15 @@ class TestTierTable: """Tests for TIER_TABLE completeness.""" def test_every_edit_op_has_a_tier(self) -> None: - from openjarvis.learning.distillation.models import EditOp - from openjarvis.learning.distillation.plan.risk_tier import TIER_TABLE + from openjarvis.learning.spec_search.models import EditOp + from openjarvis.learning.spec_search.plan.risk_tier import TIER_TABLE for op in EditOp: assert op in TIER_TABLE, f"Missing tier for {op}" def test_no_extra_keys(self) -> None: - from openjarvis.learning.distillation.models import EditOp - from openjarvis.learning.distillation.plan.risk_tier import TIER_TABLE + from openjarvis.learning.spec_search.models import EditOp + from openjarvis.learning.spec_search.plan.risk_tier import TIER_TABLE for key in TIER_TABLE: assert key in EditOp, f"Extra key in TIER_TABLE: {key}" @@ -25,48 +25,48 @@ class TestAssignTier: """Tests for assign_tier() function.""" def test_intelligence_ops_are_auto(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.SET_MODEL_FOR_QUERY_CLASS) == EditRiskTier.AUTO assert assign_tier(EditOp.SET_MODEL_PARAM) == EditRiskTier.AUTO def test_tool_ops_are_auto(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.ADD_TOOL_TO_AGENT) == EditRiskTier.AUTO assert assign_tier(EditOp.REMOVE_TOOL_FROM_AGENT) == EditRiskTier.AUTO assert assign_tier(EditOp.EDIT_TOOL_DESCRIPTION) == EditRiskTier.AUTO def test_agent_param_is_auto(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.SET_AGENT_PARAM) == EditRiskTier.AUTO def test_prompt_ops_are_review(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.PATCH_SYSTEM_PROMPT) == EditRiskTier.REVIEW assert assign_tier(EditOp.REPLACE_SYSTEM_PROMPT) == EditRiskTier.REVIEW def test_agent_class_is_review(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.SET_AGENT_CLASS) == EditRiskTier.REVIEW def test_few_shot_is_review(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.EDIT_FEW_SHOT_EXEMPLARS) == EditRiskTier.REVIEW def test_lora_is_manual(self) -> None: - from openjarvis.learning.distillation.models import EditOp, EditRiskTier - from openjarvis.learning.distillation.plan.risk_tier import assign_tier + from openjarvis.learning.spec_search.models import EditOp, EditRiskTier + from openjarvis.learning.spec_search.plan.risk_tier import assign_tier assert assign_tier(EditOp.LORA_FINETUNE) == EditRiskTier.MANUAL @@ -75,13 +75,13 @@ class TestAssignTiers: """Tests for assign_tiers() batch function.""" def test_overwrites_teacher_tier(self) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.risk_tier import assign_tiers + from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers # Teacher incorrectly sets AUTO for a prompt edit edit = Edit( @@ -98,13 +98,13 @@ class TestAssignTiers: assert result[0].risk_tier == EditRiskTier.REVIEW def test_preserves_correct_tier(self) -> None: - from openjarvis.learning.distillation.models import ( + from openjarvis.learning.spec_search.models import ( Edit, EditOp, EditPillar, EditRiskTier, ) - from openjarvis.learning.distillation.plan.risk_tier import assign_tiers + from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers edit = Edit( id="edit-002", diff --git a/tests/learning/distillation/test_rollback.py b/tests/learning/spec_search/test_rollback.py similarity index 94% rename from tests/learning/distillation/test_rollback.py rename to tests/learning/spec_search/test_rollback.py index 4219159a..f3ffb841 100644 --- a/tests/learning/distillation/test_rollback.py +++ b/tests/learning/spec_search/test_rollback.py @@ -25,7 +25,7 @@ def _setup_config_tree(root: Path) -> None: class TestRollbackIntegration: def test_rollback_restores_files(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) @@ -73,7 +73,7 @@ class TestRollbackIntegration: assert log_count >= 5 # baseline + 2 edits + 2 reverts def test_rollback_nonexistent_session(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.checkpoint.store import ( + from openjarvis.learning.spec_search.checkpoint.store import ( CheckpointStore, ) diff --git a/tests/learning/distillation/test_session_store.py b/tests/learning/spec_search/test_session_store.py similarity index 88% rename from tests/learning/distillation/test_session_store.py rename to tests/learning/spec_search/test_session_store.py index 9243a97b..8b7f497f 100644 --- a/tests/learning/distillation/test_session_store.py +++ b/tests/learning/spec_search/test_session_store.py @@ -1,11 +1,11 @@ -"""Tests for openjarvis.learning.distillation.storage.session_store module.""" +"""Tests for openjarvis.learning.spec_search.storage.session_store module.""" from __future__ import annotations from datetime import datetime, timezone from pathlib import Path -from openjarvis.learning.distillation.models import ( +from openjarvis.learning.spec_search.models import ( AutonomyMode, BenchmarkSnapshot, EditOutcome, @@ -87,7 +87,7 @@ class TestSessionStoreInit: """Tests for SessionStore initialization.""" def test_creates_tables(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -100,7 +100,7 @@ class TestSessionStoreInit: store.close() def test_idempotent_init(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -115,7 +115,7 @@ class TestSessionStoreSaveAndGet: """Tests for save_session/get_session round-trip.""" def test_round_trip(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -129,7 +129,7 @@ class TestSessionStoreSaveAndGet: store.close() def test_update_existing_session(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -147,7 +147,7 @@ class TestSessionStoreSaveAndGet: store.close() def test_get_returns_none_for_unknown(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -160,7 +160,7 @@ class TestSessionStoreList: """Tests for list_sessions ordering and filters.""" def test_lists_in_started_at_desc_order(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -179,7 +179,7 @@ class TestSessionStoreList: store.close() def test_filter_by_status(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -196,7 +196,7 @@ class TestSessionStoreList: store.close() def test_limit(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -219,7 +219,7 @@ class TestEditOutcomes: """Tests for save_outcome / list_outcomes.""" def test_round_trip(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -238,7 +238,7 @@ class TestEditOutcomes: store.close() def test_multiple_outcomes_per_session(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -256,7 +256,7 @@ class TestEditOutcomes: store.close() def test_outcomes_for_unknown_session_returns_empty(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) @@ -269,7 +269,7 @@ class TestParentSessionChain: """Tests for parent_session_id foreign key.""" def test_parent_id_round_trips(self, tmp_path: Path) -> None: - from openjarvis.learning.distillation.storage.session_store import ( + from openjarvis.learning.spec_search.storage.session_store import ( SessionStore, ) diff --git a/tests/learning/distillation/test_teacher_agent.py b/tests/learning/spec_search/test_teacher_agent.py similarity index 90% rename from tests/learning/distillation/test_teacher_agent.py rename to tests/learning/spec_search/test_teacher_agent.py index 0d595797..cb804b10 100644 --- a/tests/learning/distillation/test_teacher_agent.py +++ b/tests/learning/spec_search/test_teacher_agent.py @@ -1,4 +1,4 @@ -"""Tests for openjarvis.learning.distillation.diagnose.teacher_agent module. +"""Tests for openjarvis.learning.spec_search.diagnose.teacher_agent module. All tests use a mocked CloudEngine — no live API calls. """ @@ -8,7 +8,7 @@ from __future__ import annotations import json from unittest.mock import MagicMock -from openjarvis.learning.distillation.diagnose.types import DiagnosticTool +from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool def _make_tool( @@ -44,7 +44,7 @@ class TestTeacherAgentNoTools: """Teacher responds without using any tools.""" def test_returns_content_from_single_turn(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -67,7 +67,7 @@ class TestTeacherAgentNoTools: assert result.total_cost_usd > 0 def test_tracks_cost(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -91,7 +91,7 @@ class TestTeacherAgentWithTools: """Teacher uses tools in a multi-turn loop.""" def test_executes_tool_call_and_continues(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -129,7 +129,7 @@ class TestTeacherAgentWithTools: assert result.tool_call_records[0].tool == "test_tool" def test_stops_at_max_turns(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -159,7 +159,7 @@ class TestTeacherAgentWithTools: assert result.turns == 3 def test_stops_at_max_cost(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -192,7 +192,7 @@ class TestTeacherAgentWithTools: assert result.turns < 100 def test_multiple_tool_calls_in_one_turn(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) @@ -227,7 +227,7 @@ class TestTeacherAgentResult: """Tests for TeacherAgentResult structure.""" def test_result_has_all_fields(self) -> None: - from openjarvis.learning.distillation.diagnose.teacher_agent import ( + from openjarvis.learning.spec_search.diagnose.teacher_agent import ( TeacherAgent, ) diff --git a/tests/learning/distillation/test_triggers.py b/tests/learning/spec_search/test_triggers.py similarity index 72% rename from tests/learning/distillation/test_triggers.py rename to tests/learning/spec_search/test_triggers.py index e51c4697..f3beec27 100644 --- a/tests/learning/distillation/test_triggers.py +++ b/tests/learning/spec_search/test_triggers.py @@ -1,18 +1,18 @@ -"""Tests for openjarvis.learning.distillation.triggers module.""" +"""Tests for openjarvis.learning.spec_search.triggers module.""" from __future__ import annotations class TestOnDemandTrigger: def test_constructs(self) -> None: - from openjarvis.learning.distillation.triggers import OnDemandTrigger + from openjarvis.learning.spec_search.triggers import OnDemandTrigger t = OnDemandTrigger() assert t.kind.value == "on_demand" assert t.metadata == {} def test_metadata(self) -> None: - from openjarvis.learning.distillation.triggers import OnDemandTrigger + from openjarvis.learning.spec_search.triggers import OnDemandTrigger t = OnDemandTrigger(metadata={"source": "cli"}) assert t.metadata["source"] == "cli" @@ -20,7 +20,7 @@ class TestOnDemandTrigger: class TestUserFlagTrigger: def test_constructs_with_trace_id(self) -> None: - from openjarvis.learning.distillation.triggers import UserFlagTrigger + from openjarvis.learning.spec_search.triggers import UserFlagTrigger t = UserFlagTrigger(trace_id="trace-001") assert t.kind.value == "user_flag" @@ -30,7 +30,7 @@ class TestUserFlagTrigger: class TestScheduledTrigger: def test_constructs(self) -> None: - from openjarvis.learning.distillation.triggers import ScheduledTrigger + from openjarvis.learning.spec_search.triggers import ScheduledTrigger t = ScheduledTrigger(cron="0 3 * * *", new_trace_count=25) assert t.kind.value == "scheduled" @@ -40,7 +40,7 @@ class TestScheduledTrigger: class TestClusterTrigger: def test_constructs(self) -> None: - from openjarvis.learning.distillation.triggers import ClusterTrigger + from openjarvis.learning.spec_search.triggers import ClusterTrigger t = ClusterTrigger( cluster_description="math failures",