mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-08-14 00:47:52 +00:00
spec_search: drop CLI stubs, rename, and ship paper-aligned tutorial + configs (#346)
This commit is contained in:
Jon Saad-Falcon
@@ -0,0 +1,76 @@
|
||||
# LLM-Guided Spec Search — quickstart configuration
|
||||
# Copy to ~/.openjarvis/config.toml and run:
|
||||
# python -m openjarvis_examples.spec_search_quickstart
|
||||
#
|
||||
# This config has two parts:
|
||||
# 1. The agent system being optimized (intelligence / engine / agent / tools).
|
||||
# Same schema as the other examples in this directory; parsed by
|
||||
# ``openjarvis.core.config.load_config``.
|
||||
# 2. ``[learning.spec_search]`` and its sub-tables — the search hyperparameters
|
||||
# consumed by ``SpecSearchOrchestrator.from_config`` and ``SpecSearchLoop``.
|
||||
#
|
||||
# Defaults below match the paper (Saad-Falcon et al., 2026):
|
||||
# - max_regression = 0.01 (epsilon in GateOK)
|
||||
# - stagnation_k = 5 (Algorithm 1 stopping rule)
|
||||
# - composite_reward weights (alpha, beta, gamma, delta) = (0.5, 0.1, 0.1, 0.3)
|
||||
#
|
||||
# Teacher API keys come from your environment / credentials store, not this file.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Agent system being optimized
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[engine]
|
||||
default = "ollama" # swap to "vllm" on H100/RTX 6000 / DGX Spark
|
||||
|
||||
[intelligence]
|
||||
default_model = "qwen3.5:9b" # the local student
|
||||
# default_model = "qwen3.5:27b-fp8" # workstation tier
|
||||
|
||||
[agent]
|
||||
default_agent = "orchestrator" # multi-turn, tool-using
|
||||
max_turns = 10
|
||||
|
||||
[tools]
|
||||
enabled = [
|
||||
"code_interpreter",
|
||||
"file_read",
|
||||
"web_search",
|
||||
"think",
|
||||
"calculator",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-guided spec search hyperparameters (paper §3.3, Algorithm 1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[learning.spec_search]
|
||||
enabled = true
|
||||
teacher_model = "claude-opus-4-6" # frontier proposer
|
||||
teacher_engine = "cloud" # CloudEngine registry key (uses LiteLLM)
|
||||
autonomy_mode = "tiered" # auto | tiered | manual
|
||||
|
||||
# Per-session bounds (one diagnose / plan / execute / record pass)
|
||||
min_traces = 20
|
||||
max_cost_per_session_usd = 5.0
|
||||
max_tool_calls_per_diagnosis = 30
|
||||
|
||||
# Multi-session loop (paper Algorithm 1 stopping)
|
||||
stagnation_k = 5 # stop after this many sessions with no gate-score gain
|
||||
stagnation_eps = 0.001 # delta below this counts as no improvement
|
||||
max_total_cost_usd = 50.0 # cumulative teacher-cost budget across all sessions
|
||||
|
||||
# GateOK predicate — accept iff target cluster improves AND every other cluster
|
||||
# regresses by at most max_regression (epsilon in the paper).
|
||||
max_regression = 0.01 # paper default: 1%
|
||||
min_improvement = 0.0
|
||||
benchmark_subsample_size = 50
|
||||
benchmark_version = "personal_v1"
|
||||
|
||||
# Composite reward (paper Eq. 1) — used only when an Intelligence edit triggers
|
||||
# LoRA / GRPO training inside an accepted edit. The held-out gate is unaffected.
|
||||
[learning.spec_search.composite_reward]
|
||||
alpha = 0.5 # accuracy weight
|
||||
beta = 0.1 # energy penalty
|
||||
gamma = 0.1 # latency penalty
|
||||
delta = 0.3 # cost penalty
|
||||
@@ -556,13 +556,13 @@ to Python.
|
||||
|
||||
---
|
||||
|
||||
## Distillation (Frontier-Driven Harness Learning)
|
||||
## LLM-Guided Spec Search (Frontier-Driven Harness Learning)
|
||||
|
||||
The distillation subsystem uses a frontier closed-source model (the "teacher") as a meta-engineer for the local student's full harness — not just its weights. Instead of pushing knowledge into a small model's weights, we push a frontier model's engineering judgement into the surrounding configuration: prompts, routing, agent class, tool availability, and tool descriptions.
|
||||
LLM-guided spec search uses a frontier closed-source model (the "teacher") as a meta-engineer for the local student's full harness — not just its weights. Instead of pushing knowledge into a small model's weights, we push a frontier model's engineering judgement into the surrounding configuration: prompts, routing, agent class, tool availability, and tool descriptions.
|
||||
|
||||
### Where it lives
|
||||
|
||||
`learning/distillation/` is the fifth subsystem within the Learning pillar, alongside `learning/routing/`, `learning/optimize/`, `learning/training/`, and `learning/intelligence/`.
|
||||
`learning/spec_search/` is the fifth subsystem within the Learning pillar, alongside `learning/routing/`, `learning/optimize/`, `learning/training/`, and `learning/intelligence/`.
|
||||
|
||||
### Four-phase loop
|
||||
|
||||
@@ -579,7 +579,7 @@ Trigger → Diagnose → Plan → Execute → Record
|
||||
|
||||
| Component | Module | Purpose |
|
||||
|-----------|--------|---------|
|
||||
| `DistillationOrchestrator` | `orchestrator.py` | Top-level session driver |
|
||||
| `SpecSearchOrchestrator` | `orchestrator.py` | Top-level session driver |
|
||||
| `TeacherAgent` | `diagnose/teacher_agent.py` | Frontier model tool-calling loop |
|
||||
| `DiagnosisRunner` | `diagnose/runner.py` | Phase 1 orchestration |
|
||||
| `LearningPlanner` | `plan/planner.py` | Diagnosis → typed LearningPlan |
|
||||
@@ -609,4 +609,4 @@ Every edit is assigned a tier from a deterministic lookup table:
|
||||
| `review` | System prompt edits, agent class, few-shot exemplars | Queue for user approval |
|
||||
| `manual` | LoRA fine-tuning (v2) | Never auto-apply |
|
||||
|
||||
See [Distillation user guide](../user-guide/learning-distillation.md) for CLI usage and configuration.
|
||||
See [LLM-guided spec search guide](../user-guide/llm-guided-spec-search.md) for the architecture and the building blocks.
|
||||
|
||||
@@ -45,7 +45,7 @@ The memory pipeline includes document ingestion, chunking, embedding generation,
|
||||
|
||||
The Learning system is the fifth primitive, connecting the other four through **trace-driven feedback**. Every agent interaction can produce a `Trace` capturing the full sequence of steps — routing decisions, memory retrieval, inference calls, tool invocations, and final responses. The `TraceAnalyzer` computes statistics from accumulated traces, and the `TraceDrivenPolicy` uses these statistics to learn which model/agent/tool combinations produce the best outcomes for different query types.
|
||||
|
||||
The learning system is configured through nested sub-sections in `config.toml`: `[learning.routing]` controls the router policy (heuristic, learned, sft, grpo), `[learning.intelligence]` controls the model-level learning policy, `[learning.agent]` controls agent advisor and ICL updater policies, and `[learning.metrics]` sets the composite reward function weights. The pillar also includes the distillation subsystem, a frontier-driven loop that improves the local harness — see [Learning architecture: Distillation](learning.md#distillation-frontier-driven-harness-learning).
|
||||
The learning system is configured through nested sub-sections in `config.toml`: `[learning.routing]` controls the router policy (heuristic, learned, sft, grpo), `[learning.intelligence]` controls the model-level learning policy, `[learning.agent]` controls agent advisor and ICL updater policies, and `[learning.metrics]` sets the composite reward function weights. The pillar also includes LLM-guided spec search, a frontier-driven loop that improves the local harness — see [Learning architecture: LLM-guided spec search](learning.md#llm-guided-spec-search-frontier-driven-harness-learning).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ These are the areas where active development is happening and contributions are
|
||||
- **Energy-aware routing** — using power consumption data from telemetry to optimize for energy efficiency alongside latency and quality
|
||||
- **Plugin ecosystem** — community-contributed engines, tools, and agents distributed as Python packages
|
||||
- **Federated memory** — memory backends that synchronize across devices
|
||||
- **Distillation:** Frontier-driven harness learning — a frontier model analyzes your traces and proposes config improvements. See [user guide](../user-guide/learning-distillation.md) and [architecture](../architecture/learning.md#distillation-frontier-driven-harness-learning).
|
||||
- **LLM-guided spec search:** Frontier-driven harness learning — a frontier model analyzes your traces and proposes config improvements. See [user guide](../user-guide/llm-guided-spec-search.md) and [architecture](../architecture/learning.md#llm-guided-spec-search-frontier-driven-harness-learning).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1060,21 +1060,21 @@ OpenJarvis respects the following environment variables:
|
||||
|
||||
---
|
||||
|
||||
## Learning & Distillation
|
||||
## Learning & spec search
|
||||
|
||||
The distillation subsystem uses a frontier model to automatically improve your local agent configuration. See the [user guide](../user-guide/learning-distillation.md) for a full walkthrough.
|
||||
LLM-guided spec search uses a frontier model to automatically improve your local agent configuration. See the [user guide](../user-guide/llm-guided-spec-search.md) for a full walkthrough.
|
||||
|
||||
### `[learning.distillation]`
|
||||
### `[learning.spec_search]`
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| `enabled` | bool | `true` | Gate the entire distillation subsystem |
|
||||
| `enabled` | bool | `true` | Gate the entire spec-search subsystem |
|
||||
| `autonomy_mode` | string | `"tiered"` | `auto`, `tiered`, or `manual` |
|
||||
| `teacher_model` | string | `"claude-opus-4-6"` | Frontier model for diagnosis and planning |
|
||||
| `max_cost_per_session_usd` | float | `5.0` | Per-session teacher API budget |
|
||||
| `max_tool_calls_per_diagnosis` | int | `30` | Max teacher tool calls in diagnosis phase |
|
||||
|
||||
### `[learning.distillation.triggers]`
|
||||
### `[learning.spec_search.triggers]`
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
@@ -1086,7 +1086,7 @@ The distillation subsystem uses a frontier model to automatically improve your l
|
||||
| `cluster_min_size` | int | `5` | Minimum traces in a cluster |
|
||||
| `cluster_failure_threshold` | float | `0.3` | Feedback <= this counts as failure |
|
||||
|
||||
### `[learning.distillation.gate]`
|
||||
### `[learning.spec_search.gate]`
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
@@ -1095,7 +1095,7 @@ The distillation subsystem uses a frontier model to automatically improve your l
|
||||
| `benchmark_subsample_size` | int | `50` | Tasks per gate run |
|
||||
| `full_benchmark` | bool | `false` | Disable subsampling (slower, more accurate) |
|
||||
|
||||
### `[learning.distillation.benchmark]`
|
||||
### `[learning.spec_search.benchmark]`
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
@@ -1104,12 +1104,12 @@ The distillation subsystem uses a frontier model to automatically improve your l
|
||||
| `auto_refresh` | bool | `true` | Auto-mine new high-feedback traces |
|
||||
| `max_synthesis_cost_usd_per_refresh` | float | `2.0` | Cost cap per benchmark refresh |
|
||||
|
||||
### `[learning.distillation.tier_overrides]`
|
||||
### `[learning.spec_search.tier_overrides]`
|
||||
|
||||
Override the default risk tier for any operation. Keys are operation names, values are tier strings (`auto`, `review`, `manual`).
|
||||
|
||||
```toml
|
||||
[learning.distillation.tier_overrides]
|
||||
[learning.spec_search.tier_overrides]
|
||||
# patch_system_prompt = "auto" # promote to auto after trust
|
||||
# replace_system_prompt = "auto"
|
||||
```
|
||||
|
||||
+10
-94
@@ -463,98 +463,14 @@ When an agent is configured (e.g., `--agent orchestrator`), non-streaming reques
|
||||
|
||||
---
|
||||
|
||||
## `jarvis learning`
|
||||
## LLM-guided spec search (no CLI yet)
|
||||
|
||||
Frontier-driven harness learning (distillation). Manages learning sessions, reviews pending edits, and controls the benchmark gate.
|
||||
|
||||
### `jarvis learning init`
|
||||
|
||||
Initialize the distillation checkpoint repo and directory layout.
|
||||
|
||||
```bash
|
||||
jarvis learning init
|
||||
```
|
||||
|
||||
### `jarvis learning run`
|
||||
|
||||
Run an on-demand learning session.
|
||||
|
||||
```bash
|
||||
jarvis learning run
|
||||
jarvis learning run --autonomy auto # auto-apply all edits
|
||||
jarvis learning run --autonomy manual # dry-run, everything goes to review
|
||||
```
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--autonomy` | `tiered` | `auto`, `tiered`, or `manual` |
|
||||
|
||||
### `jarvis learning history`
|
||||
|
||||
List past learning sessions.
|
||||
|
||||
```bash
|
||||
jarvis learning history
|
||||
jarvis learning history --limit 5
|
||||
```
|
||||
|
||||
### `jarvis learning show`
|
||||
|
||||
Show details of a learning session (diagnosis, plan, outcomes, cost).
|
||||
|
||||
```bash
|
||||
jarvis learning show <session-id>
|
||||
```
|
||||
|
||||
### `jarvis learning review`
|
||||
|
||||
List all pending edits awaiting approval.
|
||||
|
||||
```bash
|
||||
jarvis learning review
|
||||
```
|
||||
|
||||
### `jarvis learning approve`
|
||||
|
||||
Approve a pending edit (still goes through the benchmark gate).
|
||||
|
||||
```bash
|
||||
jarvis learning approve <edit-id>
|
||||
```
|
||||
|
||||
### `jarvis learning reject`
|
||||
|
||||
Reject a pending edit.
|
||||
|
||||
```bash
|
||||
jarvis learning reject <edit-id>
|
||||
jarvis learning reject <edit-id> --reason "too aggressive"
|
||||
```
|
||||
|
||||
### `jarvis learning rollback`
|
||||
|
||||
Rollback a session's committed edits (creates revert commits).
|
||||
|
||||
```bash
|
||||
jarvis learning rollback <session-id>
|
||||
jarvis learning rollback --last
|
||||
```
|
||||
|
||||
### `jarvis learning benchmark`
|
||||
|
||||
Personal benchmark management.
|
||||
|
||||
```bash
|
||||
jarvis learning benchmark show # current stats
|
||||
jarvis learning benchmark refresh # manual refresh
|
||||
```
|
||||
|
||||
### `jarvis learning daemon`
|
||||
|
||||
Background learning daemon.
|
||||
|
||||
```bash
|
||||
jarvis learning daemon start
|
||||
jarvis learning daemon stop
|
||||
jarvis learning daemon status
|
||||
```
|
||||
LLM-guided spec search (the frontier-driven harness-learning subsystem)
|
||||
is exposed as a Python library only — there is currently no top-level
|
||||
`jarvis` subcommand for it. Construct a `SpecSearchOrchestrator`
|
||||
directly from `openjarvis.learning.spec_search.orchestrator` and call
|
||||
`.run(trigger)` with a trigger from
|
||||
`openjarvis.learning.spec_search.triggers`. See
|
||||
[`docs/user-guide/llm-guided-spec-search.md`](llm-guided-spec-search.md)
|
||||
for the architecture and the building blocks
|
||||
(`splits.py`, external corpora, `external_adapter`).
|
||||
|
||||
@@ -14,7 +14,7 @@ The OpenJarvis evaluation framework (`openjarvis-evals`) measures model **correc
|
||||
|
||||
---
|
||||
|
||||
> **Tip:** The distillation system uses this same eval infrastructure to gate edits against your personal benchmark. See [Learning & Distillation](learning-distillation.md).
|
||||
> **Tip:** LLM-guided spec search uses this same eval infrastructure to gate edits against your personal benchmark. See [LLM-guided spec search](llm-guided-spec-search.md).
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
@@ -1,251 +0,0 @@
|
||||
# Learning & Distillation
|
||||
|
||||
Use a frontier model as a meta-engineer to automatically improve your local agent's prompts, routing, and tools — reversibly, with benchmark-gated quality control.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Initialize
|
||||
|
||||
```bash
|
||||
jarvis learning init
|
||||
```
|
||||
|
||||
This creates the distillation directory layout under `~/.openjarvis/learning/` and initializes a git checkpoint repo at `~/.openjarvis/.git` for tracking config changes.
|
||||
|
||||
### 2. Run your first session
|
||||
|
||||
Once you have at least 20 traces from regular use:
|
||||
|
||||
```bash
|
||||
jarvis learning run
|
||||
```
|
||||
|
||||
The system will:
|
||||
1. **Diagnose** — analyze your traces using a frontier model
|
||||
2. **Plan** — propose typed edits to your config
|
||||
3. **Execute** — apply edits that pass the benchmark gate
|
||||
4. **Record** — persist the session for history and rollback
|
||||
|
||||
### 3. Check results
|
||||
|
||||
```bash
|
||||
jarvis learning history
|
||||
jarvis learning show <session-id>
|
||||
```
|
||||
|
||||
## How a Learning Session Works
|
||||
|
||||
A learning session has four phases:
|
||||
|
||||
### Phase 1: Diagnose
|
||||
|
||||
A frontier model (the "teacher", default `claude-opus-4-6`) analyzes your recent traces using read-only diagnostic tools. It identifies **failure clusters** — groups of related failures with shared root causes. The teacher must actually re-run your student on sample tasks and compare outputs to populate failure rates. This forces evidence-based diagnosis.
|
||||
|
||||
**Output:** `diagnosis.md` with narrative analysis + structured failure clusters.
|
||||
|
||||
### Phase 2: Plan
|
||||
|
||||
A second teacher call converts the diagnosis into a typed `LearningPlan` — a list of `Edit` objects, each targeting a specific part of your configuration (model routing, system prompts, tool availability, etc.). The teacher cannot pick risk tiers — those are assigned deterministically from a lookup table.
|
||||
|
||||
**Output:** `plan.json` frozen and immutable.
|
||||
|
||||
### Phase 3: Execute
|
||||
|
||||
Each edit is applied through its registered `EditApplier`, then scored against your personal benchmark. Edits that improve the benchmark are committed; edits that cause regressions are rolled back. Edits in the `review` tier are queued for your approval instead of being auto-applied.
|
||||
|
||||
**Output:** Git commits in the checkpoint repo + `EditOutcome` records.
|
||||
|
||||
### Phase 4: Record
|
||||
|
||||
The session is persisted to `learning.db` (SQLite index) and `session.json` (authoritative artifact). You can query history, show details, and rollback any session.
|
||||
|
||||
## Configuration
|
||||
|
||||
Add to `~/.openjarvis/config.toml`:
|
||||
|
||||
```toml
|
||||
[learning.distillation]
|
||||
enabled = true # gate the entire subsystem
|
||||
autonomy_mode = "tiered" # auto | tiered | manual
|
||||
teacher_model = "claude-opus-4-6" # any CloudEngine-supported model
|
||||
max_cost_per_session_usd = 5.0 # per-session teacher API budget
|
||||
max_tool_calls_per_diagnosis = 30 # max teacher tool calls in diagnosis
|
||||
```
|
||||
|
||||
### Trigger configuration
|
||||
|
||||
```toml
|
||||
[learning.distillation.triggers]
|
||||
scheduled_enabled = true
|
||||
scheduled_cron = "0 3 * * *" # daily at 03:00 local
|
||||
scheduled_min_new_traces = 20 # minimum new traces to trigger
|
||||
|
||||
cluster_enabled = true
|
||||
cluster_check_interval_minutes = 60
|
||||
cluster_min_size = 5
|
||||
cluster_failure_threshold = 0.3 # feedback <= this counts as failure
|
||||
```
|
||||
|
||||
### Gate configuration
|
||||
|
||||
```toml
|
||||
[learning.distillation.gate]
|
||||
min_improvement = 0.0 # any improvement accepted (raise for margin)
|
||||
max_regression = 0.05 # max per-cluster score drop
|
||||
benchmark_subsample_size = 50 # tasks per gate run
|
||||
full_benchmark = false # set true to disable subsampling
|
||||
```
|
||||
|
||||
### Benchmark configuration
|
||||
|
||||
```toml
|
||||
[learning.distillation.benchmark]
|
||||
synthesis_feedback_threshold = 0.7 # min feedback for benchmark traces
|
||||
max_benchmark_size = 200 # max tasks in the benchmark
|
||||
auto_refresh = true # auto-mine new high-feedback traces
|
||||
max_synthesis_cost_usd_per_refresh = 2.0 # separate from session budget
|
||||
```
|
||||
|
||||
### Risk tier overrides
|
||||
|
||||
Power users can override the default tier for any operation:
|
||||
|
||||
```toml
|
||||
[learning.distillation.tier_overrides]
|
||||
# Promote prompt edits to auto-apply after trust is established:
|
||||
# patch_system_prompt = "auto"
|
||||
# replace_system_prompt = "auto"
|
||||
```
|
||||
|
||||
## Risk Tiers
|
||||
|
||||
Every edit is assigned a risk tier that controls how it's applied:
|
||||
|
||||
| Tier | Behavior | Default ops |
|
||||
|------|----------|-------------|
|
||||
| **auto** | Applied automatically if benchmark gate passes | Model routing, model params, tool add/remove/description, agent params |
|
||||
| **review** | Queued for user approval in `jarvis learning review` | System prompt edits, agent class changes, few-shot exemplars |
|
||||
| **manual** | Never auto-applied; requires explicit approval | LoRA fine-tuning (v2) |
|
||||
|
||||
The tier is assigned deterministically from the edit operation — the teacher cannot override it.
|
||||
|
||||
## Reviewing Edits
|
||||
|
||||
When edits land in the review queue:
|
||||
|
||||
```bash
|
||||
# List all pending edits
|
||||
jarvis learning review
|
||||
|
||||
# Approve an edit (still goes through the benchmark gate)
|
||||
jarvis learning approve <edit-id>
|
||||
|
||||
# Reject an edit with a reason
|
||||
jarvis learning reject <edit-id> --reason "prompt change too aggressive"
|
||||
```
|
||||
|
||||
Even approved edits are gated by the benchmark — approval means "try it", not "force it".
|
||||
|
||||
## Rollback and History
|
||||
|
||||
Every edit creates a git commit in the checkpoint repo at `~/.openjarvis/.git`. This is separate from your OpenJarvis source repo.
|
||||
|
||||
```bash
|
||||
# List past sessions
|
||||
jarvis learning history --limit 20
|
||||
|
||||
# Show session details (diagnosis, plan, outcomes, cost)
|
||||
jarvis learning show <session-id>
|
||||
|
||||
# Rollback a session (creates new revert commits, preserves history)
|
||||
jarvis learning rollback <session-id>
|
||||
jarvis learning rollback --last
|
||||
```
|
||||
|
||||
Rollback never rewrites git history — it creates new revert commits so the audit trail stays intact.
|
||||
|
||||
## Cost Controls
|
||||
|
||||
Three cost boundaries prevent runaway spending:
|
||||
|
||||
1. **`max_cost_per_session_usd`** (default $5.00) — caps the total teacher API cost per session (diagnosis + planning).
|
||||
2. **`max_synthesis_cost_usd_per_refresh`** (default $2.00) — caps the cost of generating gold answers for new benchmark tasks. Separate from the session budget.
|
||||
3. **`teacher_model`** — choose a cheaper model (e.g., `claude-sonnet-4-6`) to reduce per-token costs at the expense of diagnosis quality.
|
||||
|
||||
Cost is tracked on every `LearningSession` as `teacher_cost_usd` and surfaced in `jarvis learning show`.
|
||||
|
||||
## The Personal Benchmark
|
||||
|
||||
The benchmark is your acceptance gate's source of truth — a set of tasks distilled from your high-quality traces, scored by an LLM-as-judge against frontier gold answers.
|
||||
|
||||
**How it's built:**
|
||||
1. Traces with feedback >= 0.7 are candidates
|
||||
2. Tasks are grouped by query class and deduplicated
|
||||
3. For each task, the teacher generates a gold reference answer
|
||||
4. The benchmark is versioned (`personal_v1.json`, `personal_v2.json`, ...)
|
||||
|
||||
**Auto-refresh:** The benchmark grows over time as you accumulate more traces. New tasks are added automatically during background refresh cycles.
|
||||
|
||||
```bash
|
||||
# Manual refresh
|
||||
jarvis learning benchmark refresh
|
||||
|
||||
# Show stats
|
||||
jarvis learning benchmark show
|
||||
```
|
||||
|
||||
## Cold Start: What to Expect on Day One
|
||||
|
||||
The system needs real usage data before it can learn:
|
||||
|
||||
- **< 20 traces:** `jarvis learning run` returns "Not enough traces yet." Triggers are no-ops.
|
||||
- **20+ traces, < 10 high-feedback:** Enough for diagnosis, but no benchmark yet. Sessions will run diagnosis but can't gate edits.
|
||||
- **10+ high-feedback traces:** Bootstrap benchmark is created automatically (`personal_v1.json`). Full learning loop is available.
|
||||
|
||||
**Getting there faster:** Use OpenJarvis normally and provide feedback on results (thumbs up/down in the UI, or `jarvis feedback` in the CLI).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Error | Cause | Fix |
|
||||
|-------|-------|-----|
|
||||
| "Not enough traces yet" | Fewer than 20 traces in the store | Use OpenJarvis more, provide feedback |
|
||||
| "Working tree dirty, cannot stage" | Manual edits to `~/.openjarvis/config.toml` during a session | Commit or revert manual changes first |
|
||||
| "All clusters dropped: insufficient evidence" | Teacher diagnosed clusters but couldn't reproduce failures | Check that the student is actually failing on the flagged tasks |
|
||||
| "ConfigurationError: distillation root inside source tree" | `OPENJARVIS_HOME` points inside the repo | Set `OPENJARVIS_HOME` to `~/.openjarvis` (default) or another external dir |
|
||||
| "Personal benchmark is empty" | Not enough high-feedback traces yet | Provide feedback on 10+ traces with score >= 0.7 |
|
||||
|
||||
## Where Artifacts Live
|
||||
|
||||
All distillation artifacts live under `~/.openjarvis/` (never inside the source repo):
|
||||
|
||||
```
|
||||
~/.openjarvis/
|
||||
├── config.toml # Your configuration (git-tracked by checkpoint)
|
||||
├── agents/ # Agent prompts (git-tracked)
|
||||
├── tools/ # Tool descriptions (git-tracked)
|
||||
├── .git/ # Checkpoint repo for rollback
|
||||
└── learning/
|
||||
├── learning.db # SQLite session index
|
||||
├── benchmarks/ # Personal benchmark versions + gold answers
|
||||
├── sessions/ # Per-session artifacts (diagnosis, plan, traces)
|
||||
└── pending_review/ # Edits awaiting user approval
|
||||
```
|
||||
|
||||
## Background Daemon
|
||||
|
||||
For continuous learning:
|
||||
|
||||
```bash
|
||||
jarvis learning daemon start # Start background watcher
|
||||
jarvis learning daemon status # Check if running
|
||||
jarvis learning daemon stop # Stop the daemon
|
||||
```
|
||||
|
||||
The daemon runs the scheduled trigger (default: daily at 03:00) and the cluster trigger (watches for failure patterns in real-time).
|
||||
|
||||
## See Also
|
||||
|
||||
- [Architecture: Learning](../architecture/learning.md#distillation-frontier-driven-harness-learning) — internal architecture of the distillation subsystem
|
||||
- [User Guide: Evaluations](evaluations.md) — the eval infrastructure that powers the benchmark gate
|
||||
- [User Guide: CLI](cli.md#jarvis-learning) — full CLI reference
|
||||
- [Getting Started: Configuration](../getting-started/configuration.md) — all config knobs
|
||||
@@ -1,121 +1,188 @@
|
||||
# LLM-Guided Spec Search
|
||||
|
||||
LLM-guided spec search is a local–cloud collaboration that uses each side
|
||||
for what it does best: a frontier cloud model (the *teacher*) reads traces
|
||||
from a deployed local agent and proposes typed edits across the agent's
|
||||
LLM-guided spec search (Saad-Falcon et al., 2026) is a local–cloud
|
||||
collaboration: a frontier cloud *teacher* reads traces from a deployed
|
||||
local agent and proposes typed edits across the agent's full
|
||||
configuration; the local hardware runs the resulting configuration with
|
||||
zero marginal API cost at inference time. A held-out gate accepts only
|
||||
edits that improve a target failure mode without unacceptable regression
|
||||
on others.
|
||||
zero marginal API cost at inference time. A held-out *gate* accepts only
|
||||
edits that improve a target failure cluster without unacceptable
|
||||
regression elsewhere.
|
||||
|
||||
The configuration the teacher edits is called the **spec**: a typed object
|
||||
with five primitives (Intelligence, Engine, Agents, Tools & Memory,
|
||||
Learning). The Learning primitive is where the search lives — it specifies
|
||||
which edits are considered, how candidate specs are evaluated, and when
|
||||
optimization stops.
|
||||
This page is a copy-paste tutorial. By the end you will have:
|
||||
|
||||
- a real `SpecSearchOrchestrator` running on your machine,
|
||||
- a multi-session loop with the paper's stagnation rule (Algorithm 1),
|
||||
- an understanding of which knobs to turn for production deployment.
|
||||
|
||||
## TL;DR — run it
|
||||
|
||||
```bash
|
||||
python examples/openjarvis/spec_search_quickstart.py
|
||||
```
|
||||
|
||||
The script is self-contained (no API key, no Ollama) — it wires up real
|
||||
orchestrator + multi-session loop + composite-reward modules with stub
|
||||
teacher/student/judge so you can see one full session and a stagnation
|
||||
loop terminate. Production swap-points are commented inline; see
|
||||
[Going to production](#going-to-production) below.
|
||||
|
||||
## How it works
|
||||
|
||||
A search session repeats four phases:
|
||||
A search session repeats four phases (paper §3.3):
|
||||
|
||||
| Phase | What happens |
|
||||
|---|---|
|
||||
| **Diagnose** | The teacher reads eligible traces and groups failures into clusters, each annotated with a natural-language characterization of the skill gap. |
|
||||
| **Plan** | The teacher proposes typed edits across the editable primitives (Intelligence, Engine, Agents, Tools & Memory). A single proposal can edit multiple slots at once. |
|
||||
| **Execute** | The candidate spec is evaluated on a held-out gate: the targeted cluster must improve, and every other cluster must regress by no more than a per-cluster tolerance. |
|
||||
| **Record** | Accepted edits are committed; rejected edits are rolled back. |
|
||||
| **Diagnose** | Teacher reads eligible traces and groups failures into clusters, each annotated with `(student_failure_rate, teacher_success_rate, skill_gap)`. |
|
||||
| **Plan** | Teacher proposes typed edits across the four editable primitives (Intelligence, Engine, Agents, Tools & Memory). One proposal can edit multiple slots at once. |
|
||||
| **Execute** | Each candidate edit is applied; the gate scores the resulting spec on a held-out subsample. Accepted iff `GateOK` holds (see below). |
|
||||
| **Record** | Accepted edits commit to the checkpoint store; rejected edits roll back. The session is persisted to `SessionStore`. |
|
||||
|
||||
The four-phase loop is implemented by the existing `jarvis learning`
|
||||
command — see [Learning & Distillation](learning-distillation.md) for the
|
||||
end-to-end runnable workflow, edit applier registry, and configuration
|
||||
schema. This document covers the new building blocks added on top.
|
||||
`SpecSearchOrchestrator.run(trigger)` runs **one** session end-to-end.
|
||||
`SpecSearchLoop` (paper Algorithm 1) wraps the orchestrator and repeats
|
||||
sessions until either gate-score stagnation (default *k* = 5 sessions)
|
||||
or budget exhaustion.
|
||||
|
||||
### Alignment with the paper
|
||||
### `GateOK` — the acceptance predicate
|
||||
|
||||
The paper's Algorithm 1 specifies a **multi-session loop** that runs
|
||||
diagnose → plan → execute → record repeatedly until gate-score stagnation
|
||||
(default *k* = 5 sessions) or budget exhaustion, with default per-cluster
|
||||
tolerance ε = 1 %. The current `DistillationOrchestrator` runs
|
||||
**one session per `jarvis learning run` invocation** (one diagnose, one
|
||||
plan, one round of edits + gate decisions, one record); the per-cluster
|
||||
regression check uses a default `max_regression = 0.05` (5 %).
|
||||
Let `G_c(S)` be the held-out gate score of spec `S` restricted to
|
||||
failure cluster `c`. For an edit `e` targeting cluster `c`, with
|
||||
`S' = apply(S, e)`:
|
||||
|
||||
To match the paper's defaults today:
|
||||
|
||||
- pass `max_regression=0.01` when constructing the orchestrator or the
|
||||
`BenchmarkGate` directly, and
|
||||
- run `jarvis learning run` from a wrapper that re-invokes it until
|
||||
per-session gate-score deltas stagnate for *k* sessions.
|
||||
|
||||
Wiring the multi-session stagnation loop into the orchestrator natively,
|
||||
plus changing the default `max_regression` to 1 %, are tracked as
|
||||
follow-ups — they're not part of this PR.
|
||||
|
||||
## What this PR adds
|
||||
|
||||
### `splits.py` — deterministic train / test splits
|
||||
|
||||
`src/openjarvis/evals/core/splits.py` adds a small helper that takes a
|
||||
list of records and a benchmark name and returns a deterministic
|
||||
train / test partition. The split is keyed off a stable hash of each
|
||||
record's id, so the same `(records, train_frac, seed)` always yields the
|
||||
same partition. This is the substrate that makes "evaluate on
|
||||
`split=test`" and "search over `split=train`" reproducible.
|
||||
|
||||
```python
|
||||
from openjarvis.evals.core.splits import apply_split
|
||||
|
||||
train = apply_split(records, split="train", train_frac=0.2, seed=42)
|
||||
test = apply_split(records, split="test", train_frac=0.2, seed=42)
|
||||
all_ = apply_split(records, split="all", train_frac=0.2, seed=42) # passthrough
|
||||
```
|
||||
GateOK(S', S, c, eps) ⟺
|
||||
G_c(S') > G_c(S) # target cluster improves, AND
|
||||
G_c'(S') >= G_c'(S) − eps # every other cluster regresses by ≤ eps
|
||||
```
|
||||
|
||||
The `split` kwarg is now wired through every dataset provider in
|
||||
`openjarvis.evals.datasets` (gaia, livecodebench, liveresearch,
|
||||
liveresearchbench, pinchbench, taubench, toolcall15) so any caller that
|
||||
constructs a `BenchmarkConfig` can request a particular split.
|
||||
Default `eps = 0.01` (1 %) per the paper. The `BenchmarkGate` class
|
||||
implements this; the `max_regression` knob is `eps`.
|
||||
|
||||
### External-corpus dataset providers
|
||||
### Composite reward (Intelligence-edit training only)
|
||||
|
||||
When the diagnose phase wants the teacher to reason over a broader
|
||||
agent-trace corpus instead of just the local student's own traces, it
|
||||
can ingest records from a HuggingFace-backed external corpus. Three
|
||||
providers are included:
|
||||
When an Intelligence edit triggers LoRA / GRPO training inside the
|
||||
execute phase, candidate responses `y` to query `q` are scored by
|
||||
(paper Eq. 1):
|
||||
|
||||
| Corpus | HuggingFace dataset | What it surfaces |
|
||||
```
|
||||
R(q, y) = α · R_acc(q, y)
|
||||
− β · Ê(q, y) # energy
|
||||
− γ · L̂(q, y) # latency
|
||||
− δ · Ĉ(q, y) # cost
|
||||
```
|
||||
|
||||
Defaults `(α, β, γ, δ) = (0.5, 0.1, 0.1, 0.3)`. The efficiency
|
||||
quantities (E, L, C) are z-scored *within batch* before weighting, so
|
||||
the reward trades dimensionless deviations rather than raw joules /
|
||||
seconds / dollars (paper Appendix C.6). Implementation:
|
||||
`openjarvis.learning.spec_search.composite_reward.score_batch`.
|
||||
|
||||
The held-out gate evaluates the resulting spec end-to-end; it is
|
||||
unaffected by these weights.
|
||||
|
||||
## Configuration
|
||||
|
||||
The prebuilt config lives at
|
||||
`configs/openjarvis/examples/spec-search-quickstart.toml`. Copy it to
|
||||
`~/.openjarvis/config.toml` (or set `OPENJARVIS_CONFIG` to it) and the
|
||||
regular loader picks it up:
|
||||
|
||||
```python
|
||||
from openjarvis.core.config import load_config
|
||||
cfg = load_config().learning.spec_search # SpecSearchLearningConfig
|
||||
```
|
||||
|
||||
The `[learning.spec_search]` table maps 1:1 onto the
|
||||
`SpecSearchLearningConfig` dataclass and is read by both
|
||||
`SpecSearchOrchestrator.from_config` and `SpecSearchLoop`:
|
||||
|
||||
```toml
|
||||
[learning.spec_search]
|
||||
enabled = true
|
||||
teacher_model = "claude-opus-4-6"
|
||||
teacher_engine = "cloud"
|
||||
autonomy_mode = "tiered" # auto | tiered | manual
|
||||
|
||||
# Per-session bounds
|
||||
min_traces = 20
|
||||
max_cost_per_session_usd = 5.0
|
||||
max_tool_calls_per_diagnosis = 30
|
||||
|
||||
# Multi-session loop (paper Algorithm 1)
|
||||
stagnation_k = 5 # paper default
|
||||
stagnation_eps = 0.001
|
||||
max_total_cost_usd = 50.0
|
||||
|
||||
# Gate (GateOK)
|
||||
max_regression = 0.01 # paper default: epsilon = 1%
|
||||
min_improvement = 0.0
|
||||
benchmark_subsample_size = 50
|
||||
benchmark_version = "personal_v1"
|
||||
|
||||
[learning.spec_search.composite_reward]
|
||||
alpha = 0.5 # accuracy
|
||||
beta = 0.1 # energy
|
||||
gamma = 0.1 # latency
|
||||
delta = 0.3 # cost
|
||||
```
|
||||
|
||||
## Going to production
|
||||
|
||||
The quickstart uses fakes for the teacher engine, student runner, and
|
||||
judge so it runs without external services. To run a real session,
|
||||
swap each fake for the corresponding production component:
|
||||
|
||||
| Slot | Quickstart | Production |
|
||||
|---|---|---|
|
||||
| `adp` | `neulab/agent-data-collection` | Multi-turn agent trajectories from AgentTuning, CodeAct, OpenHands, and others |
|
||||
| `toolorchestra` | `nvidia/ToolScale` | Tool-use trajectories (the dataset underlying the ToolOrchestra paper) |
|
||||
| `generalthoughts` | `natolambert/GeneralThought-430K-filtered` | A filtered reasoning-trace pool |
|
||||
| `teacher_engine` | `FakeTeacherEngine` | `EngineRegistry.get(cfg.teacher_engine)(model=cfg.teacher_model)` — set `ANTHROPIC_API_KEY` etc. |
|
||||
| `trace_store` | `MagicMock` | `openjarvis.traces.store.TraceStore(home / "traces.db")` |
|
||||
| `student_runner` | `MagicMock` | `openjarvis.learning.spec_search.student_runner.VLLMStudentRunner(host=..., model=...)` |
|
||||
| `judge` | `MagicMock` | `openjarvis.evals.core.scorer.LLMJudgeScorer(...)` (or a deterministic scorer if your benchmark provides one) |
|
||||
| `session_store` | `MagicMock` | `openjarvis.learning.spec_search.storage.session_store.SessionStore(home / "learning" / "sessions.db")` |
|
||||
| `checkpoint_store` | `MagicMock` | `openjarvis.learning.spec_search.checkpoint.store.CheckpointStore(home / "learning" / "checkpoints")` |
|
||||
| `scorer` | climbing-plateau fake | a real `Scorer` callable (typically a `BenchmarkGate.score` adapter) |
|
||||
|
||||
Each provider implements the standard `DatasetProvider.load(...)`
|
||||
interface so corpus records can be loaded the same way benchmark records
|
||||
are. They're labeled "NOT USED FOR EVALUATION" in their docstrings —
|
||||
they exist purely to feed the proposer's diagnose phase.
|
||||
The orchestrator only depends on the *interface* of each slot, not the
|
||||
concrete class — anything implementing the corresponding protocol works.
|
||||
|
||||
### `external_adapter.py` — corpus records as synthetic traces
|
||||
## Adding a new external corpus
|
||||
|
||||
`src/openjarvis/learning/distillation/external_adapter.py` adapts records
|
||||
from any external corpus into rows the proposer's existing trace tools
|
||||
understand. The proposer reads from the SQLite TraceStore via search /
|
||||
get tools; this adapter writes each `EvalRecord` as a synthetic `Trace`
|
||||
with `feedback=0.5` (no ground truth) and a `source_name` tag in
|
||||
metadata so multi-source diagnose runs can filter downstream.
|
||||
Diagnose phase can ingest records from a HuggingFace-backed external
|
||||
corpus. Three providers ship in-tree (`adp`, `toolorchestra`,
|
||||
`generalthoughts`); to add a new one:
|
||||
|
||||
1. Create `src/openjarvis/evals/datasets/<corpus>.py` implementing
|
||||
`DatasetProvider` (`adp.py` is a small reference). The provider's
|
||||
`load(max_samples, seed, split)` must respect `split` via
|
||||
`apply_split` from `openjarvis.evals.core.splits`.
|
||||
2. Register: `@DatasetRegistry.register("<corpus>")`.
|
||||
3. Feed it to the proposer via the trace store:
|
||||
|
||||
```python
|
||||
from openjarvis.evals.datasets.adp import ADPDataset
|
||||
from openjarvis.learning.distillation.external_adapter import (
|
||||
from openjarvis.learning.spec_search.external_adapter import (
|
||||
write_external_records_as_traces,
|
||||
)
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
records = ADPDataset().load(max_samples=200, seed=42, split="all")
|
||||
records = list(ADPDataset().load(max_samples=200, seed=42, split="all"))
|
||||
store = TraceStore("~/.openjarvis/traces.db")
|
||||
n = write_external_records_as_traces(store, records, source_name="adp")
|
||||
# proposer's diagnose phase can now search/filter these traces by source="adp"
|
||||
# proposer can now filter on metadata["source"] == "adp"
|
||||
```
|
||||
|
||||
### Bug fix: agent-backend trace toggle
|
||||
## What runs where
|
||||
|
||||
At inference time, the resulting spec runs entirely on-device — model
|
||||
inference, agent execution, tool invocation. Teacher API calls happen
|
||||
only at search time (diagnose + plan), and only **eligible scrubbed
|
||||
traces** are transmitted (per the trace-eligibility rules in your
|
||||
config).
|
||||
|
||||
Users requiring strict local-only operation can swap a larger local
|
||||
model in as the teacher; this trades search quality for zero cloud
|
||||
exposure.
|
||||
|
||||
## Bug fix bundled with this release
|
||||
|
||||
`src/openjarvis/evals/backends/jarvis_agent.py` previously hardcoded
|
||||
`builder.telemetry(telemetry).traces(True).build()`, ignoring the
|
||||
@@ -125,7 +192,7 @@ intent. A corrupt traces.db then turned every agent eval into "database
|
||||
disk image is malformed" errors that the eval scorer dropped, producing
|
||||
fake high accuracies from a handful of successful samples.
|
||||
|
||||
The fix is one line — match the `JarvisDirectBackend` pattern:
|
||||
The one-line fix:
|
||||
|
||||
```python
|
||||
self._system = builder.telemetry(telemetry).traces(telemetry).build()
|
||||
@@ -134,75 +201,14 @@ self._system = builder.telemetry(telemetry).traces(telemetry).build()
|
||||
Callers that previously expected traces to always be written should pass
|
||||
`telemetry=True` explicitly.
|
||||
|
||||
## Configuration
|
||||
## See also
|
||||
|
||||
`jarvis learning` reads its configuration from `~/.openjarvis/config.toml`:
|
||||
|
||||
```toml
|
||||
[learning.distillation]
|
||||
enabled = true # gate the entire subsystem
|
||||
autonomy_mode = "tiered" # auto | tiered | manual
|
||||
teacher_model = "claude-opus-4-6" # any CloudEngine-supported model
|
||||
max_cost_per_session_usd = 5.0 # per-session teacher API budget
|
||||
max_tool_calls_per_diagnosis = 30 # max teacher tool calls in diagnosis
|
||||
```
|
||||
|
||||
Gate / acceptance knobs (constructor arguments to `BenchmarkGate` and
|
||||
`DistillationOrchestrator`):
|
||||
|
||||
| Argument | Meaning | Current default | Paper default |
|
||||
|---|---|---|---|
|
||||
| `max_regression` | Maximum per-cluster score drop tolerated before rejecting an edit (the ε in the paper's `GateOK`). | `0.05` (5 %) | `0.01` (1 %) |
|
||||
| `min_improvement` | Minimum overall score improvement required to accept an edit. | `0.0` | (paper does not specify; uses `> 0` per `Gc(S') > Gc(S)`) |
|
||||
| `n_tasks` | Number of tasks scored per gate run. | `50` | n/a (gate is per-cluster) |
|
||||
|
||||
Pass `max_regression=0.01` explicitly to match the paper's default. See
|
||||
the *Alignment with the paper* note above for the gap between the
|
||||
single-session `DistillationOrchestrator` and the paper's multi-session
|
||||
loop with stagnation criterion.
|
||||
|
||||
Each `EditApplier` registers an autonomy tier (`auto`, `review`,
|
||||
`manual`); see the
|
||||
[Learning & Distillation](learning-distillation.md) doc for the per-edit
|
||||
configuration schema and the registered appliers.
|
||||
|
||||
## What runs where
|
||||
|
||||
At inference time, the resulting spec runs entirely on-device — model
|
||||
inference, agent execution, and tool invocation. Teacher API calls are
|
||||
made only at search time, for diagnosis and edit proposal. The local
|
||||
spec makes zero teacher calls at inference time.
|
||||
|
||||
When a frontier teacher is used, only **eligible** scrubbed traces are
|
||||
transmitted (per the trace-eligibility rules in the configuration).
|
||||
Users requiring strict local-only operation can swap a larger local
|
||||
model in as the teacher; this trades some search quality for zero cloud
|
||||
exposure.
|
||||
|
||||
## Why decouple primitives at all
|
||||
|
||||
Existing personal AI frameworks bundle agent prompts, tool descriptions,
|
||||
memory configuration, and runtime settings around a specific cloud
|
||||
model. Naive substitution of a local model for the cloud model collapses
|
||||
accuracy because none of the surrounding configuration was tuned for the
|
||||
new model. Optimizing across primitives jointly — instead of
|
||||
single-primitive optimization (LoRA-only, prompt-only) — is what allows
|
||||
LLM-guided spec search to recover that lost accuracy. The decomposition
|
||||
into Intelligence / Engine / Agents / Tools & Memory / Learning, with
|
||||
the spec as the typed configuration object, is what makes the search
|
||||
space well-defined.
|
||||
|
||||
## Adding a new external corpus
|
||||
|
||||
1. Create `src/openjarvis/evals/datasets/<corpus>.py` implementing
|
||||
`DatasetProvider` (look at `adp.py` for a small reference). The
|
||||
provider's `load(max_samples, seed, split)` should respect the
|
||||
`split` kwarg via `apply_split` from `openjarvis.evals.core.splits`.
|
||||
2. Register the new dataset in your local `DatasetRegistry` (typically
|
||||
via `@DatasetRegistry.register("<corpus>")` decorator on the class).
|
||||
3. Use it the same way as the bundled corpora: load records, then call
|
||||
`write_external_records_as_traces(store, records, source_name="<corpus>")`
|
||||
to make them visible to the proposer's diagnose phase.
|
||||
|
||||
The proposer can then filter on `metadata["source"] == "<corpus>"` if
|
||||
you're feeding multiple corpora into the same search session.
|
||||
- `examples/openjarvis/spec_search_quickstart.py` — runnable end-to-end demo.
|
||||
- `configs/openjarvis/examples/spec-search-quickstart.toml` — prebuilt config.
|
||||
- `src/openjarvis/learning/spec_search/orchestrator.py` — `SpecSearchOrchestrator` (single session).
|
||||
- `src/openjarvis/learning/spec_search/multi_session.py` — `SpecSearchLoop` (Algorithm 1).
|
||||
- `src/openjarvis/learning/spec_search/composite_reward.py` — paper Eq. 1.
|
||||
- `src/openjarvis/learning/spec_search/gate/benchmark_gate.py` — `GateOK` predicate.
|
||||
- `src/openjarvis/learning/spec_search/external_adapter.py` — corpus → trace adapter.
|
||||
- `tests/learning/spec_search/test_multi_session.py`, `test_composite_reward.py` — unit tests.
|
||||
- `tests/learning/spec_search/test_orchestrator.py` — full-session test with mocks.
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
"""LLM-Guided Spec Search — runnable quickstart.
|
||||
|
||||
This script wires up every primitive ``SpecSearchOrchestrator`` needs and runs
|
||||
one full session, then a multi-session loop with the paper's stagnation rule
|
||||
(Algorithm 1, k=5, epsilon=1%).
|
||||
|
||||
It is *self-contained*: by default the teacher engine and the student runner
|
||||
are local fakes, so you can ``python examples/openjarvis/spec_search_quickstart.py``
|
||||
with no API keys and no Ollama / vLLM running. Every "swap this for production"
|
||||
hookpoint is called out inline.
|
||||
|
||||
Configuration is read from ``configs/openjarvis/examples/spec-search-quickstart.toml``
|
||||
via the regular ``openjarvis.core.config.load_config`` machinery — you can copy
|
||||
that TOML to ``~/.openjarvis/config.toml`` and tune the gate / stagnation /
|
||||
reward knobs without editing this file.
|
||||
|
||||
Run:
|
||||
|
||||
OPENJARVIS_HOME=/tmp/openjarvis-spec-search-demo \\
|
||||
python examples/openjarvis/spec_search_quickstart.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.core.config import SpecSearchLearningConfig
|
||||
from openjarvis.learning.spec_search.composite_reward import (
|
||||
RewardWeights,
|
||||
TrainingSample,
|
||||
score_batch,
|
||||
)
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
BenchmarkSnapshot,
|
||||
FailureCluster,
|
||||
)
|
||||
from openjarvis.learning.spec_search.multi_session import SpecSearchLoop
|
||||
from openjarvis.learning.spec_search.orchestrator import SpecSearchOrchestrator
|
||||
from openjarvis.learning.spec_search.triggers import OnDemandTrigger
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fakes — replace these with real production components.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeTeacherEngine:
|
||||
"""Stand-in for ``CloudEngine``.
|
||||
|
||||
For production, use the registry::
|
||||
|
||||
from openjarvis.core.registry import EngineRegistry
|
||||
engine_cls = EngineRegistry.get(cfg.teacher_engine) # "cloud"
|
||||
engine = engine_cls(model=cfg.teacher_model) # set ANTHROPIC_API_KEY
|
||||
|
||||
The orchestrator only calls ``engine.generate(...)`` so any object with
|
||||
that method will work.
|
||||
"""
|
||||
|
||||
call_count: int = 0
|
||||
|
||||
def generate(self, **_: Any) -> dict[str, Any]:
|
||||
self.call_count += 1
|
||||
# Propose one auto-tier Tools edit so the gate has something to score.
|
||||
return {
|
||||
"content": json.dumps(
|
||||
{
|
||||
"edits": [
|
||||
{
|
||||
"id": f"edit-{self.call_count:03d}",
|
||||
"pillar": "tools",
|
||||
"op": "edit_tool_description",
|
||||
"target": "tools.web_search",
|
||||
"payload": {
|
||||
"tool_name": "web_search",
|
||||
"new_description": (
|
||||
"Search the web for recent information. "
|
||||
"Prefer this for time-sensitive queries."
|
||||
),
|
||||
},
|
||||
"rationale": (
|
||||
"Student under-invokes web_search on multi-hop queries."
|
||||
),
|
||||
"expected_improvement": "c1",
|
||||
"risk_tier": "auto",
|
||||
"references": ["t-001"],
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
"usage": {"total_tokens": 1200},
|
||||
"cost_usd": 0.04,
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
|
||||
|
||||
def _fake_diagnosis() -> Any:
|
||||
"""A canned DiagnosisResult so the demo doesn't need a real teacher loop."""
|
||||
from openjarvis.learning.spec_search.diagnose.runner import DiagnosisResult
|
||||
|
||||
return DiagnosisResult(
|
||||
diagnosis_md=(
|
||||
"## Diagnosis\n\n"
|
||||
"Cluster `c1` (multi-hop research): student fails to invoke "
|
||||
"`web_search` on time-sensitive queries; teacher invokes it "
|
||||
"consistently. Likely cause: tool description does not signal "
|
||||
"freshness."
|
||||
),
|
||||
clusters=[
|
||||
FailureCluster(
|
||||
id="c1",
|
||||
description="Multi-hop research; web_search under-invocation",
|
||||
sample_trace_ids=["t-001", "t-002", "t-003"],
|
||||
student_failure_rate=0.7,
|
||||
teacher_success_rate=0.95,
|
||||
skill_gap="Student does not invoke web_search on multi-hop research.",
|
||||
)
|
||||
],
|
||||
cost_usd=0.05,
|
||||
tool_call_records=[],
|
||||
)
|
||||
|
||||
|
||||
def _fake_scorer_factory(start_score: float = 0.60, step: float = 0.06):
|
||||
"""Return a scorer whose overall score climbs by ``step`` on each call.
|
||||
|
||||
Mimics the loop the paper describes: each accepted edit lifts the gate
|
||||
score, and the multi-session loop stops once gains plateau.
|
||||
"""
|
||||
state = {"score": start_score - step, "calls": 0}
|
||||
|
||||
def scorer(**_: Any) -> BenchmarkSnapshot:
|
||||
state["calls"] += 1
|
||||
# Every other call (the "after" snapshot) bumps the score; gain
|
||||
# tapers after 3 sessions so the stagnation rule fires.
|
||||
if state["calls"] % 2 == 0:
|
||||
bump = step if state["calls"] // 2 <= 3 else 0.0
|
||||
state["score"] = min(1.0, state["score"] + bump)
|
||||
return BenchmarkSnapshot(
|
||||
benchmark_version="personal_v1",
|
||||
overall_score=state["score"],
|
||||
cluster_scores={"c1": state["score"]},
|
||||
task_count=10,
|
||||
elapsed_seconds=5.0,
|
||||
)
|
||||
|
||||
return scorer
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wire-up
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_orchestrator(
|
||||
config: SpecSearchLearningConfig,
|
||||
home: Path,
|
||||
) -> SpecSearchOrchestrator:
|
||||
"""Construct a SpecSearchOrchestrator from a SpecSearchLearningConfig.
|
||||
|
||||
All five injected primitives below are demo fakes; the comments next to
|
||||
each one show the production replacement.
|
||||
"""
|
||||
return SpecSearchOrchestrator.from_config(
|
||||
config,
|
||||
# Teacher: replace with EngineRegistry.get("cloud")(model=cfg.teacher_model)
|
||||
teacher_engine=FakeTeacherEngine(),
|
||||
# TraceStore: production = TraceStore(home / "traces.db")
|
||||
trace_store=MagicMock(count=MagicMock(return_value=config.min_traces + 10)),
|
||||
benchmark_samples=[],
|
||||
# StudentRunner: production = VLLMStudentRunner(host=..., model=...)
|
||||
student_runner=MagicMock(),
|
||||
# Judge: production = openjarvis.evals.core.scorer.LLMJudgeScorer(...)
|
||||
judge=MagicMock(),
|
||||
# SessionStore + CheckpointStore: production = real on-disk stores
|
||||
session_store=MagicMock(),
|
||||
checkpoint_store=MagicMock(
|
||||
current_sha=MagicMock(return_value="demo-sha"),
|
||||
begin_stage=MagicMock(
|
||||
return_value=MagicMock(pre_stage_sha="demo-sha"),
|
||||
),
|
||||
),
|
||||
openjarvis_home=home,
|
||||
scorer=_fake_scorer_factory(),
|
||||
)
|
||||
|
||||
|
||||
def demo_composite_reward(weights: RewardWeights) -> None:
|
||||
"""Show how the paper Eq. 1 reward ranks Intelligence-edit candidates."""
|
||||
candidates = [
|
||||
TrainingSample(
|
||||
accuracy=1.0, energy_joules=200, latency_seconds=5.0, cost_usd=0.0
|
||||
),
|
||||
TrainingSample(
|
||||
accuracy=1.0, energy_joules=400, latency_seconds=8.0, cost_usd=0.0
|
||||
),
|
||||
TrainingSample(
|
||||
accuracy=0.0, energy_joules=100, latency_seconds=2.0, cost_usd=0.0
|
||||
),
|
||||
]
|
||||
rewards = score_batch(candidates, weights=weights)
|
||||
print("\nComposite reward (paper Eq. 1) — ranking 3 candidates:")
|
||||
print(
|
||||
f" weights = (alpha={weights.alpha}, beta={weights.beta}, "
|
||||
f"gamma={weights.gamma}, delta={weights.delta})"
|
||||
)
|
||||
for i, (c, r) in enumerate(zip(candidates, rewards)):
|
||||
print(
|
||||
f" candidate {i}: acc={c.accuracy} energy={c.energy_joules}J "
|
||||
f"latency={c.latency_seconds}s -> reward={r:+.3f}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# In a real deployment, ``load_config()`` reads from ``~/.openjarvis/config.toml``.
|
||||
# For a self-contained demo we synthesize the spec-search config inline so the
|
||||
# script is runnable without copying any files. To use the prebuilt TOML:
|
||||
#
|
||||
# from openjarvis.core.config import load_config
|
||||
# cfg = load_config().learning.spec_search
|
||||
#
|
||||
# (after copying ``configs/openjarvis/examples/spec-search-quickstart.toml``
|
||||
# to ``~/.openjarvis/config.toml``)
|
||||
|
||||
cfg = SpecSearchLearningConfig(
|
||||
enabled=True,
|
||||
teacher_model="claude-opus-4-6",
|
||||
teacher_engine="cloud",
|
||||
autonomy_mode="auto",
|
||||
min_traces=20,
|
||||
max_cost_per_session_usd=5.0,
|
||||
max_tool_calls_per_diagnosis=30,
|
||||
stagnation_k=5, # paper default
|
||||
stagnation_eps=0.001,
|
||||
max_total_cost_usd=50.0,
|
||||
max_regression=0.01, # paper default: epsilon = 1%
|
||||
min_improvement=0.0,
|
||||
benchmark_subsample_size=10,
|
||||
)
|
||||
|
||||
home = Path(
|
||||
os.environ.get("OPENJARVIS_HOME")
|
||||
or tempfile.mkdtemp(prefix="openjarvis-spec-search-")
|
||||
)
|
||||
print(f"OPENJARVIS_HOME = {home}")
|
||||
|
||||
# ----- Single session ---------------------------------------------------
|
||||
orch = build_orchestrator(cfg, home)
|
||||
|
||||
# The orchestrator's diagnose phase calls a real DiagnosisRunner that
|
||||
# invokes the teacher. We swap it out for the canned diagnosis above so
|
||||
# the demo does not need an API key.
|
||||
from unittest.mock import patch
|
||||
|
||||
print("\n=== Single session (one diagnose / plan / execute / record) ===")
|
||||
with patch(
|
||||
"openjarvis.learning.spec_search.orchestrator.DiagnosisRunner"
|
||||
) as MockDiag:
|
||||
MockDiag.return_value.run.return_value = _fake_diagnosis()
|
||||
session = orch.run(OnDemandTrigger())
|
||||
|
||||
print(f" status = {session.status.value}")
|
||||
print(f" teacher_cost_usd = ${session.teacher_cost_usd:.4f}")
|
||||
print(f" edit_outcomes = {[(o.edit_id, o.status) for o in session.edit_outcomes]}")
|
||||
if session.benchmark_after is not None:
|
||||
print(
|
||||
f" before -> after = "
|
||||
f"{session.benchmark_before.overall_score:.3f} -> "
|
||||
f"{session.benchmark_after.overall_score:.3f}"
|
||||
)
|
||||
|
||||
# ----- Multi-session loop (paper Algorithm 1) ---------------------------
|
||||
orch = build_orchestrator(cfg, home) # fresh fakes for clean state
|
||||
loop = SpecSearchLoop(
|
||||
orch,
|
||||
stagnation_k=cfg.stagnation_k,
|
||||
stagnation_eps=cfg.stagnation_eps,
|
||||
max_total_cost_usd=cfg.max_total_cost_usd,
|
||||
)
|
||||
|
||||
print(
|
||||
"\n=== Multi-session loop (stagnation_k = "
|
||||
f"{cfg.stagnation_k}, max_total_cost = ${cfg.max_total_cost_usd}) ==="
|
||||
)
|
||||
with patch(
|
||||
"openjarvis.learning.spec_search.orchestrator.DiagnosisRunner"
|
||||
) as MockDiag:
|
||||
MockDiag.return_value.run.return_value = _fake_diagnosis()
|
||||
result = loop.run()
|
||||
|
||||
print(f" sessions = {len(result.sessions)}")
|
||||
print(f" stop_reason = {result.stop_reason}")
|
||||
print(f" total cost = ${result.total_cost_usd:.4f}")
|
||||
print(f" best score = {result.best_overall_score:.3f}")
|
||||
|
||||
demo_composite_reward(RewardWeights())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -183,7 +183,7 @@ nav:
|
||||
- Scheduler: user-guide/scheduler.md
|
||||
- Telemetry: user-guide/telemetry.md
|
||||
- Security: user-guide/security.md
|
||||
- Learning & Distillation: user-guide/learning-distillation.md
|
||||
- LLM-guided spec search: user-guide/llm-guided-spec-search.md
|
||||
- Leaderboard: leaderboard.md
|
||||
- Roadmap: development/roadmap.md
|
||||
- Development:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Load system prompt and few-shot overrides from $OPENJARVIS_HOME.
|
||||
|
||||
Distillation (M1) proposes edits that get written to disk by appliers.
|
||||
LLM-guided spec search (M1) proposes edits that get written to disk by appliers.
|
||||
This module lets agents pick those overrides up at runtime:
|
||||
|
||||
- System prompts: ``$OPENJARVIS_HOME/agents/{name}/system_prompt.md``
|
||||
|
||||
@@ -41,7 +41,6 @@ from openjarvis.cli.telemetry_cmd import telemetry
|
||||
from openjarvis.cli.tool_cmd import tool
|
||||
from openjarvis.cli.vault_cmd import vault
|
||||
from openjarvis.cli.workflow_cmd import workflow
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
|
||||
@click.group(
|
||||
@@ -113,7 +112,6 @@ cli.add_command(connect, "connect")
|
||||
cli.add_command(digest, "digest")
|
||||
cli.add_command(deep_research_setup, "deep-research-setup")
|
||||
cli.add_command(deep_research_setup, "research")
|
||||
cli.add_command(learning_group, "learning")
|
||||
cli.add_command(bootstrap_cmd, "_bootstrap")
|
||||
|
||||
# Gateway CLI commands (lazy import to avoid pulling starlette)
|
||||
|
||||
@@ -684,6 +684,54 @@ class MetricsConfig:
|
||||
efficiency_weight: float = 0.1
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SpecSearchCompositeRewardConfig:
|
||||
"""Composite reward weights for Intelligence-edit training (paper Eq. 1).
|
||||
|
||||
R(q, y) = alpha * R_acc - beta * E_hat - gamma * L_hat - delta * C_hat
|
||||
"""
|
||||
|
||||
alpha: float = 0.5
|
||||
beta: float = 0.1
|
||||
gamma: float = 0.1
|
||||
delta: float = 0.3
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SpecSearchLearningConfig:
|
||||
"""LLM-guided spec search config (paper §3.3, Algorithm 1).
|
||||
|
||||
Maps to ``[learning.spec_search]`` and is consumed by
|
||||
``SpecSearchOrchestrator.from_config`` and ``SpecSearchLoop``.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
teacher_model: str = "claude-opus-4-6"
|
||||
teacher_engine: str = "cloud" # registry key for the cloud engine
|
||||
autonomy_mode: str = "tiered" # auto | tiered | manual
|
||||
|
||||
# Per-session bounds (one diagnose/plan/execute pass)
|
||||
min_traces: int = 20
|
||||
max_cost_per_session_usd: float = 5.0
|
||||
max_tool_calls_per_diagnosis: int = 30
|
||||
|
||||
# Multi-session loop (paper Algorithm 1)
|
||||
stagnation_k: int = 5
|
||||
max_total_cost_usd: float = 50.0
|
||||
stagnation_eps: float = 0.001 # gate-score delta below this counts as no progress
|
||||
|
||||
# Gate (GateOK predicate)
|
||||
max_regression: float = 0.01 # paper default: epsilon = 1%
|
||||
min_improvement: float = 0.0
|
||||
benchmark_subsample_size: int = 50
|
||||
benchmark_version: str = "personal_v1"
|
||||
|
||||
# Composite reward (only used when an Intelligence edit triggers training)
|
||||
composite_reward: SpecSearchCompositeRewardConfig = field(
|
||||
default_factory=SpecSearchCompositeRewardConfig,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LearningConfig:
|
||||
"""Learning system settings with per-primitive sub-policies."""
|
||||
@@ -697,6 +745,9 @@ class LearningConfig:
|
||||
)
|
||||
agent: AgentLearningConfig = field(default_factory=AgentLearningConfig)
|
||||
skills: SkillsLearningConfig = field(default_factory=SkillsLearningConfig)
|
||||
spec_search: SpecSearchLearningConfig = field(
|
||||
default_factory=SpecSearchLearningConfig,
|
||||
)
|
||||
metrics: MetricsConfig = field(default_factory=MetricsConfig)
|
||||
|
||||
# Training pipeline
|
||||
@@ -1578,7 +1629,7 @@ def _parse_mining_section(data: dict) -> Optional["MiningConfig"]:
|
||||
pearld_rpc_url=extra.get("pearld_rpc_url", "http://localhost:44107")
|
||||
)
|
||||
elif isinstance(target_str, str) and target_str.startswith("pool:"):
|
||||
submit_target = PoolTarget(url=target_str[len("pool:"):])
|
||||
submit_target = PoolTarget(url=target_str[len("pool:") :])
|
||||
else:
|
||||
raise ValueError(
|
||||
f"[mining].submit_target must be 'solo' or 'pool:<url>', got {target_str!r}"
|
||||
|
||||
@@ -339,9 +339,9 @@ def _build_t6(frame: ResultsFrame) -> Tuple[str, str]:
|
||||
def _build_t7(frame: ResultsFrame) -> Tuple[str, str]:
|
||||
"""T7: Edit category x framework — preliminary (raw deltas).
|
||||
|
||||
Spec §10 says richer edit-attribution data needs the spec-distillation
|
||||
pipeline to tag accepted edits. For now, raw per-benchmark accuracy
|
||||
deltas across frameworks.
|
||||
Spec §10 says richer edit-attribution data needs the LLM-guided
|
||||
spec-search pipeline to tag accepted edits. For now, raw per-benchmark
|
||||
accuracy deltas across frameworks.
|
||||
"""
|
||||
df = frame.df.filter(pl.col("metric_name") == "accuracy")
|
||||
pivot = (
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
NOT USED FOR EVALUATION. Surfaces multi-turn agent trajectories to the
|
||||
LLM-guided spec search proposer via
|
||||
``openjarvis.learning.distillation.external_adapter`` so the diagnose
|
||||
``openjarvis.learning.spec_search.external_adapter`` so the diagnose
|
||||
phase can reason over a broad pool of agent traces without depending on
|
||||
the per-cell student's own trace history.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
NOT USED FOR EVALUATION. Provides the LLM-guided spec search proposer
|
||||
with a large pool of reasoning trajectories to reason over (via
|
||||
`openjarvis.learning.distillation.external_adapter`) when the diagnose
|
||||
`openjarvis.learning.spec_search.external_adapter`) when the diagnose
|
||||
phase wants signal from a broad reasoning corpus rather than the
|
||||
per-cell student's own trace history.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
NOT USED FOR EVALUATION. Surfaces tool-use trajectories from
|
||||
``nvidia/ToolScale`` (the dataset underlying the ToolOrchestra paper) to
|
||||
the LLM-guided spec search proposer via
|
||||
``openjarvis.learning.distillation.external_adapter`` so the diagnose
|
||||
``openjarvis.learning.spec_search.external_adapter`` so the diagnose
|
||||
phase can reason over a broad pool of tool-use traces.
|
||||
|
||||
The ``dataset_id`` is kept as ``"toolorchestra"`` (matching the published
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""Harness distillation: frontier-driven learning subsystem."""
|
||||
@@ -1,132 +0,0 @@
|
||||
"""``jarvis learning`` — distillation learning CLI subcommands.
|
||||
|
||||
See spec §12.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group("learning")
|
||||
def learning_group() -> None:
|
||||
"""Frontier-driven harness learning (distillation)."""
|
||||
|
||||
|
||||
@learning_group.command("init")
|
||||
def learning_init() -> None:
|
||||
"""Initialize the distillation checkpoint repo and directory layout."""
|
||||
from openjarvis.learning.distillation.checkpoint.store import CheckpointStore
|
||||
from openjarvis.learning.distillation.storage.paths import (
|
||||
ensure_distillation_dirs,
|
||||
resolve_distillation_root,
|
||||
)
|
||||
|
||||
root = resolve_distillation_root()
|
||||
ensure_distillation_dirs()
|
||||
home = root.parent # ~/.openjarvis
|
||||
store = CheckpointStore(home)
|
||||
store.init()
|
||||
console.print(f"[green]Initialized distillation at {root}[/green]")
|
||||
|
||||
|
||||
@learning_group.command("run")
|
||||
@click.option(
|
||||
"--autonomy",
|
||||
type=click.Choice(["auto", "tiered", "manual"]),
|
||||
default="tiered",
|
||||
)
|
||||
def learning_run(autonomy: str) -> None:
|
||||
"""Run an on-demand learning session."""
|
||||
console.print("[yellow]On-demand session started.[/yellow]")
|
||||
console.print("[dim]Use 'jarvis learning history' to check results.[/dim]")
|
||||
console.print(f"[dim]Autonomy mode: {autonomy}[/dim]")
|
||||
# Full wiring deferred to integration — this registers the CLI surface.
|
||||
console.print("[dim]Full orchestration requires configured teacher engine.[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("history")
|
||||
@click.option("--limit", type=int, default=10, help="Max sessions to show.")
|
||||
def learning_history(limit: int) -> None:
|
||||
"""List past learning sessions."""
|
||||
console.print(f"[dim]Showing last {limit} sessions (requires learning.db).[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("show")
|
||||
@click.argument("session_id")
|
||||
def learning_show(session_id: str) -> None:
|
||||
"""Show details of a learning session."""
|
||||
console.print(f"[dim]Session: {session_id}[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("review")
|
||||
def learning_review() -> None:
|
||||
"""Review pending edits awaiting approval."""
|
||||
console.print("[dim]Pending review queue.[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("approve")
|
||||
@click.argument("edit_id")
|
||||
def learning_approve(edit_id: str) -> None:
|
||||
"""Approve a pending edit."""
|
||||
console.print(f"[dim]Approving edit: {edit_id}[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("reject")
|
||||
@click.argument("edit_id")
|
||||
@click.option("--reason", type=str, default="", help="Rejection reason.")
|
||||
def learning_reject(edit_id: str, reason: str) -> None:
|
||||
"""Reject a pending edit."""
|
||||
console.print(f"[dim]Rejecting edit: {edit_id}[/dim]")
|
||||
|
||||
|
||||
@learning_group.command("rollback")
|
||||
@click.argument("session_id", required=False)
|
||||
@click.option("--last", is_flag=True, help="Rollback the most recent session.")
|
||||
def learning_rollback(session_id: str | None, last: bool) -> None:
|
||||
"""Rollback a learning session's commits."""
|
||||
target = session_id or ("last session" if last else "none")
|
||||
console.print(f"[dim]Rolling back: {target}[/dim]")
|
||||
|
||||
|
||||
@learning_group.group("benchmark")
|
||||
def benchmark_group() -> None:
|
||||
"""Personal benchmark management."""
|
||||
|
||||
|
||||
@benchmark_group.command("refresh")
|
||||
def benchmark_refresh() -> None:
|
||||
"""Manually refresh the personal benchmark."""
|
||||
console.print("[dim]Refreshing personal benchmark.[/dim]")
|
||||
|
||||
|
||||
@benchmark_group.command("show")
|
||||
def benchmark_show() -> None:
|
||||
"""Show current benchmark statistics."""
|
||||
console.print("[dim]Benchmark stats.[/dim]")
|
||||
|
||||
|
||||
@learning_group.group("daemon")
|
||||
def daemon_group() -> None:
|
||||
"""Background learning daemon."""
|
||||
|
||||
|
||||
@daemon_group.command("start")
|
||||
def daemon_start() -> None:
|
||||
"""Start the learning daemon."""
|
||||
console.print("[dim]Starting daemon.[/dim]")
|
||||
|
||||
|
||||
@daemon_group.command("stop")
|
||||
def daemon_stop() -> None:
|
||||
"""Stop the learning daemon."""
|
||||
console.print("[dim]Stopping daemon.[/dim]")
|
||||
|
||||
|
||||
@daemon_group.command("status")
|
||||
def daemon_status() -> None:
|
||||
"""Check daemon status."""
|
||||
console.print("[dim]Daemon status.[/dim]")
|
||||
@@ -0,0 +1 @@
|
||||
"""LLM-guided spec search: frontier-driven learning subsystem."""
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
"""Git-backed checkpoint store for distillation config rollback.
|
||||
"""Git-backed checkpoint store for spec-search config rollback.
|
||||
|
||||
A thin wrapper over a local git repository at ``<openjarvis_home>/.git``.
|
||||
The repo tracks ``config.toml``, ``agents/``, and ``tools/`` so that the
|
||||
@@ -19,7 +19,7 @@ import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.storage.paths import (
|
||||
from openjarvis.learning.spec_search.storage.paths import (
|
||||
ConfigurationError,
|
||||
_find_source_root,
|
||||
)
|
||||
@@ -47,7 +47,7 @@ class StageHandle:
|
||||
|
||||
|
||||
class CheckpointStore:
|
||||
"""Thin git wrapper for the distillation checkpoint repo.
|
||||
"""Thin git wrapper for the spec-search checkpoint repo.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -74,7 +74,7 @@ class CheckpointStore:
|
||||
|
||||
Refuses to initialize if ``self.root`` is inside the OpenJarvis source
|
||||
tree — this is the same defense-in-depth check as
|
||||
``resolve_distillation_root``: we never want a stray git repo writing
|
||||
``resolve_spec_search_root``: we never want a stray git repo writing
|
||||
config snapshots into the working copy.
|
||||
|
||||
Idempotent: if ``.git`` already exists and contains a baseline commit,
|
||||
@@ -100,8 +100,8 @@ class CheckpointStore:
|
||||
|
||||
if not (self._root / ".git").exists():
|
||||
self._git("init", "-q")
|
||||
self._git("config", "user.email", "distillation@openjarvis.local")
|
||||
self._git("config", "user.name", "OpenJarvis Distillation")
|
||||
self._git("config", "user.email", "spec-search@openjarvis.local")
|
||||
self._git("config", "user.name", "OpenJarvis Spec Search")
|
||||
|
||||
# Stage whatever tracked paths currently exist (it's OK if some
|
||||
# don't yet — the user may not have agents or tools dirs at first
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Composite reward for Intelligence-edit training (paper §3.3, Eq. 1).
|
||||
|
||||
R(q, y) = alpha * R_acc(q, y) - beta * E_hat(q, y)
|
||||
- gamma * L_hat(q, y)
|
||||
- delta * C_hat(q, y)
|
||||
|
||||
The efficiency quantities (E, L, C) are normalised within the evaluated
|
||||
benchmark before weighting (z-score), so the reward trades dimensionless
|
||||
deviations rather than raw joules / seconds / dollars (paper Appendix C.6).
|
||||
|
||||
Default weights (alpha, beta, gamma, delta) = (0.5, 0.1, 0.1, 0.3).
|
||||
|
||||
This reward is consumed only inside an Intelligence edit that triggers
|
||||
training (LoRA/GRPO). The held-out gate (BenchmarkGate) evaluates the
|
||||
resulting spec end-to-end and is not affected by these weights.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics
|
||||
from dataclasses import dataclass
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RewardWeights:
|
||||
"""Composite-reward weights from paper Eq. 1."""
|
||||
|
||||
alpha: float = 0.5
|
||||
beta: float = 0.1
|
||||
gamma: float = 0.1
|
||||
delta: float = 0.3
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class TrainingSample:
|
||||
"""One (query, response) candidate scored during Intelligence training.
|
||||
|
||||
All efficiency quantities are raw (un-normalised); normalisation is
|
||||
applied across the batch by ``score_batch``.
|
||||
"""
|
||||
|
||||
accuracy: float # 0..1; for binary tasks, 0 or 1
|
||||
energy_joules: float
|
||||
latency_seconds: float
|
||||
cost_usd: float
|
||||
|
||||
|
||||
def _zscore(values: Sequence[float]) -> list[float]:
|
||||
"""Per-batch z-score; returns zeros if the batch has zero variance.
|
||||
|
||||
Within-batch normalisation is the paper's choice (Appendix C.6) so
|
||||
the composite reward trades dimensionless deviations.
|
||||
"""
|
||||
if not values:
|
||||
return []
|
||||
mean = statistics.fmean(values)
|
||||
if len(values) < 2:
|
||||
return [0.0] * len(values)
|
||||
stdev = statistics.pstdev(values)
|
||||
if stdev == 0.0:
|
||||
return [0.0] * len(values)
|
||||
return [(v - mean) / stdev for v in values]
|
||||
|
||||
|
||||
def score_batch(
|
||||
samples: Sequence[TrainingSample],
|
||||
weights: RewardWeights | None = None,
|
||||
) -> list[float]:
|
||||
"""Score a batch of candidates with the paper's composite reward.
|
||||
|
||||
Energy / latency / cost are z-scored within the batch before being
|
||||
weighted, so the reward magnitudes are comparable across benchmarks
|
||||
and hardware platforms.
|
||||
|
||||
Args:
|
||||
samples: candidate (query, response) pairs to score.
|
||||
weights: composite-reward weights; uses paper defaults if None.
|
||||
|
||||
Returns:
|
||||
One scalar reward per sample, in the same order.
|
||||
"""
|
||||
w = weights or RewardWeights()
|
||||
if not samples:
|
||||
return []
|
||||
|
||||
e_norm = _zscore([s.energy_joules for s in samples])
|
||||
l_norm = _zscore([s.latency_seconds for s in samples])
|
||||
c_norm = _zscore([s.cost_usd for s in samples])
|
||||
|
||||
return [
|
||||
w.alpha * s.accuracy - w.beta * e - w.gamma * lat - w.delta * cost
|
||||
for s, e, lat, cost in zip(samples, e_norm, l_norm, c_norm)
|
||||
]
|
||||
|
||||
|
||||
__all__ = ["RewardWeights", "TrainingSample", "score_batch"]
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
"""DiagnosisRunner: orchestrates phase 1 of the distillation loop.
|
||||
"""DiagnosisRunner: orchestrates phase 1 of the spec-search loop.
|
||||
|
||||
Builds diagnostic tools, runs the TeacherAgent, parses failure clusters
|
||||
from the teacher's output, and persists artifacts.
|
||||
@@ -15,14 +15,14 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
from openjarvis.learning.distillation.diagnose.types import ToolCallRecord
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -85,7 +85,7 @@ class DiagnosisResult:
|
||||
|
||||
|
||||
class DiagnosisRunner:
|
||||
"""Orchestrates phase 1 of the distillation loop.
|
||||
"""Orchestrates phase 1 of the spec-search loop.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
+1
-1
@@ -24,7 +24,7 @@ from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.core.types import Message, Role, ToolCall
|
||||
from openjarvis.learning.distillation.diagnose.types import (
|
||||
from openjarvis.learning.spec_search.diagnose.types import (
|
||||
DiagnosticTool,
|
||||
ToolCallRecord,
|
||||
)
|
||||
+2
-2
@@ -5,7 +5,7 @@ All tools are **read-only** relative to the user's config. They do not mutate
|
||||
|
||||
Tools that execute code (``run_student_on_task``, ``run_self_on_task``) append
|
||||
new traces to ``TraceStore`` as a side effect. These traces are tagged with
|
||||
``source=distillation_session:<id>`` so they can be excluded from future
|
||||
``source=spec_search_session:<id>`` so they can be excluded from future
|
||||
learning input.
|
||||
|
||||
See spec §5.2.
|
||||
@@ -16,7 +16,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.learning.distillation.diagnose.types import DiagnosticTool
|
||||
from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool
|
||||
|
||||
|
||||
def build_diagnostic_tools(
|
||||
+3
-3
@@ -9,14 +9,14 @@ import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
ApplyResult,
|
||||
EditApplier,
|
||||
ValidationResult,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import apply_unified_diff
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import apply_unified_diff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+2
-2
@@ -5,13 +5,13 @@ See spec §4.1 op semantics for SET_MODEL_FOR_QUERY_CLASS and SET_MODEL_PARAM.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
ApplyResult,
|
||||
EditApplier,
|
||||
ValidationResult,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
|
||||
class SetModelForQueryClassApplier(EditApplier):
|
||||
+2
-2
@@ -9,13 +9,13 @@ See spec §4.1.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
ApplyResult,
|
||||
EditApplier,
|
||||
ValidationResult,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
|
||||
class LoraStubApplier(EditApplier):
|
||||
+2
-2
@@ -7,13 +7,13 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
ApplyResult,
|
||||
EditApplier,
|
||||
ValidationResult,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
|
||||
class AddToolToAgentApplier(EditApplier):
|
||||
+1
-1
@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
|
||||
@dataclass
|
||||
+7
-7
@@ -1,4 +1,4 @@
|
||||
"""Per-edit execution loop for the distillation execute phase.
|
||||
"""Per-edit execution loop for the spec-search execute phase.
|
||||
|
||||
Iterates over a plan's edits, handles tier routing, validates, and applies.
|
||||
Does NOT include the benchmark gate — that's wired in M5.
|
||||
@@ -11,30 +11,30 @@ from __future__ import annotations
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
EditFewShotExemplarsApplier,
|
||||
PatchSystemPromptApplier,
|
||||
ReplaceSystemPromptApplier,
|
||||
SetAgentClassApplier,
|
||||
SetAgentParamApplier,
|
||||
)
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelForQueryClassApplier,
|
||||
SetModelParamApplier,
|
||||
)
|
||||
from openjarvis.learning.distillation.execute.appliers.lora_stub import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.lora_stub import (
|
||||
LoraStubApplier,
|
||||
)
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
AddToolToAgentApplier,
|
||||
EditToolDescriptionApplier,
|
||||
RemoveToolFromAgentApplier,
|
||||
)
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
EditApplierRegistry,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
Edit,
|
||||
EditOutcome,
|
||||
+2
-2
@@ -12,8 +12,8 @@ import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from openjarvis.learning.distillation.gate.regression import regression_check
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.gate.regression import regression_check
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Cold start detection and bootstrap for the distillation subsystem.
|
||||
"""Cold start detection and bootstrap for the spec-search subsystem.
|
||||
|
||||
Day one: no traces, no benchmark. The system must not crash and must
|
||||
give the user a clear message about what's needed. This module provides
|
||||
+1
-1
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
|
||||
@dataclass
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
"""Pydantic models and enums for the distillation subsystem.
|
||||
"""Pydantic models and enums for the spec-search subsystem.
|
||||
|
||||
This module defines the typed vocabulary used by the diagnose, plan, execute,
|
||||
and record phases. Three model families:
|
||||
@@ -328,7 +328,7 @@ class EditOutcome(BaseModel):
|
||||
|
||||
|
||||
class LearningSession(BaseModel):
|
||||
"""The durable record of one distillation session.
|
||||
"""The durable record of one spec-search session.
|
||||
|
||||
Persisted in two places: `<session_dir>/session.json` (authoritative) and
|
||||
the SQLite SessionStore (queryable index). When in doubt, prefer the JSON
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Multi-session loop for LLM-guided spec search (paper Algorithm 1).
|
||||
|
||||
The single-session ``SpecSearchOrchestrator.run(trigger)`` does one
|
||||
diagnose / plan / execute / record pass. Algorithm 1 in the paper
|
||||
specifies a *multi-session* loop that repeats this pass until either
|
||||
gate-score stagnation (default *k* = 5 sessions with no improvement)
|
||||
or budget exhaustion.
|
||||
|
||||
``SpecSearchLoop`` wraps an existing ``SpecSearchOrchestrator`` and
|
||||
implements that stopping logic without modifying the orchestrator
|
||||
itself, so existing single-session callers and tests are unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
LearningSession,
|
||||
SessionStatus,
|
||||
)
|
||||
from openjarvis.learning.spec_search.triggers import OnDemandTrigger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LoopResult:
|
||||
"""Final state of a multi-session run.
|
||||
|
||||
``stop_reason`` is one of:
|
||||
|
||||
- ``"stagnation"`` — gate score did not improve for ``stagnation_k``
|
||||
consecutive sessions.
|
||||
- ``"budget"`` — cumulative teacher cost reached ``max_total_cost_usd``.
|
||||
- ``"failed"`` — a session returned ``SessionStatus.FAILED``; the loop
|
||||
exits rather than burning more budget on a broken state.
|
||||
"""
|
||||
|
||||
sessions: list[LearningSession] = field(default_factory=list)
|
||||
stop_reason: str = "stagnation"
|
||||
total_cost_usd: float = 0.0
|
||||
best_overall_score: float = 0.0
|
||||
|
||||
|
||||
class SpecSearchLoop:
|
||||
"""Paper Algorithm 1: greedy gated edits across primitives, multi-session.
|
||||
|
||||
Args:
|
||||
orchestrator: a constructed ``SpecSearchOrchestrator``. Each tick
|
||||
of the loop calls ``orchestrator.run(trigger)``.
|
||||
stagnation_k: stop after this many consecutive sessions with no
|
||||
gate-score improvement (paper default: 5).
|
||||
stagnation_eps: gate-score delta below this counts as "no
|
||||
improvement" — guards against floating-point noise.
|
||||
max_total_cost_usd: cumulative teacher-cost budget across all
|
||||
sessions. The loop stops as soon as this is exceeded.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
orchestrator: Any,
|
||||
*,
|
||||
stagnation_k: int = 5,
|
||||
stagnation_eps: float = 0.001,
|
||||
max_total_cost_usd: float = 50.0,
|
||||
) -> None:
|
||||
if stagnation_k < 1:
|
||||
raise ValueError("stagnation_k must be >= 1")
|
||||
if max_total_cost_usd <= 0:
|
||||
raise ValueError("max_total_cost_usd must be > 0")
|
||||
self._orch = orchestrator
|
||||
self._k = stagnation_k
|
||||
self._eps = stagnation_eps
|
||||
self._budget = max_total_cost_usd
|
||||
|
||||
def run(self, trigger: Any | None = None) -> LoopResult:
|
||||
"""Run sessions until stagnation, budget, or failure."""
|
||||
result = LoopResult()
|
||||
no_improve_streak = 0
|
||||
|
||||
while True:
|
||||
session_trigger = trigger if trigger is not None else OnDemandTrigger()
|
||||
session = self._orch.run(session_trigger)
|
||||
result.sessions.append(session)
|
||||
result.total_cost_usd += session.teacher_cost_usd or 0.0
|
||||
|
||||
if session.status == SessionStatus.FAILED:
|
||||
result.stop_reason = "failed"
|
||||
logger.info(
|
||||
"spec-search loop stopping: session failed (%s)",
|
||||
session.error,
|
||||
)
|
||||
break
|
||||
|
||||
after_score = (
|
||||
session.benchmark_after.overall_score
|
||||
if session.benchmark_after is not None
|
||||
else 0.0
|
||||
)
|
||||
if after_score > result.best_overall_score + self._eps:
|
||||
result.best_overall_score = after_score
|
||||
no_improve_streak = 0
|
||||
else:
|
||||
no_improve_streak += 1
|
||||
|
||||
if no_improve_streak >= self._k:
|
||||
result.stop_reason = "stagnation"
|
||||
logger.info(
|
||||
"spec-search loop stopping: %d sessions without improvement",
|
||||
no_improve_streak,
|
||||
)
|
||||
break
|
||||
|
||||
if result.total_cost_usd >= self._budget:
|
||||
result.stop_reason = "budget"
|
||||
logger.info(
|
||||
"spec-search loop stopping: cumulative cost $%.2f >= budget $%.2f",
|
||||
result.total_cost_usd,
|
||||
self._budget,
|
||||
)
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
__all__ = ["LoopResult", "SpecSearchLoop"]
|
||||
+60
-15
@@ -1,4 +1,4 @@
|
||||
"""DistillationOrchestrator: top-level driver for a learning session.
|
||||
"""SpecSearchOrchestrator: top-level driver for a learning session.
|
||||
|
||||
Wires diagnose (M2) → plan (M3) → execute (M4) → gate (M5) into a
|
||||
single ``run(trigger)`` method. All dependencies are injected.
|
||||
@@ -14,12 +14,12 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from openjarvis.learning.distillation.diagnose.runner import DiagnosisRunner
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.execute.loop import _build_registry
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import BenchmarkGate
|
||||
from openjarvis.learning.distillation.gate.cold_start import check_readiness
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import DiagnosisRunner
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.execute.loop import _build_registry
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import BenchmarkGate
|
||||
from openjarvis.learning.spec_search.gate.cold_start import check_readiness
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
EditOutcome,
|
||||
@@ -27,18 +27,65 @@ from openjarvis.learning.distillation.models import (
|
||||
LearningSession,
|
||||
SessionStatus,
|
||||
)
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DistillationOrchestrator:
|
||||
"""Top-level driver for a distillation learning session.
|
||||
class SpecSearchOrchestrator:
|
||||
"""Top-level driver for a spec-search learning session.
|
||||
|
||||
All dependencies are injected so tests can mock everything.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls,
|
||||
config: Any, # SpecSearchLearningConfig
|
||||
*,
|
||||
teacher_engine: Any,
|
||||
trace_store: Any,
|
||||
benchmark_samples: list,
|
||||
student_runner: Any,
|
||||
judge: Any,
|
||||
session_store: Any,
|
||||
checkpoint_store: Any,
|
||||
openjarvis_home: Path,
|
||||
scorer: Callable[..., BenchmarkSnapshot] | None = None,
|
||||
) -> "SpecSearchOrchestrator":
|
||||
"""Build a single-session orchestrator from a SpecSearchLearningConfig.
|
||||
|
||||
Hyperparameters (teacher model, gate tolerance, autonomy mode, etc.)
|
||||
come from ``config``; runtime primitives that cannot be expressed in
|
||||
TOML (engine instances, trace store, judge, etc.) must be injected.
|
||||
|
||||
See ``configs/openjarvis/examples/spec-search-quickstart.toml`` for
|
||||
the TOML schema and ``examples/openjarvis/spec_search_quickstart.py``
|
||||
for an end-to-end wiring example.
|
||||
"""
|
||||
autonomy = AutonomyMode(config.autonomy_mode)
|
||||
return cls(
|
||||
teacher_engine=teacher_engine,
|
||||
teacher_model=config.teacher_model,
|
||||
trace_store=trace_store,
|
||||
benchmark_samples=benchmark_samples,
|
||||
student_runner=student_runner,
|
||||
judge=judge,
|
||||
session_store=session_store,
|
||||
checkpoint_store=checkpoint_store,
|
||||
openjarvis_home=openjarvis_home,
|
||||
autonomy_mode=autonomy,
|
||||
scorer=scorer,
|
||||
benchmark_version=config.benchmark_version,
|
||||
min_traces=config.min_traces,
|
||||
max_cost_usd=config.max_cost_per_session_usd,
|
||||
max_tool_calls=config.max_tool_calls_per_diagnosis,
|
||||
min_improvement=config.min_improvement,
|
||||
max_regression=config.max_regression,
|
||||
subsample_size=config.benchmark_subsample_size,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -81,7 +128,7 @@ class DistillationOrchestrator:
|
||||
self._subsample_size = subsample_size
|
||||
|
||||
def run(self, trigger: Any) -> LearningSession:
|
||||
"""Execute a full distillation session.
|
||||
"""Execute a full spec-search session.
|
||||
|
||||
Returns the completed LearningSession.
|
||||
"""
|
||||
@@ -340,9 +387,7 @@ class DistillationOrchestrator:
|
||||
try:
|
||||
applier.rollback(edit, ctx)
|
||||
except Exception as rb_exc:
|
||||
logger.warning(
|
||||
"Edit %s rollback failed: %s", edit.id, rb_exc
|
||||
)
|
||||
logger.warning("Edit %s rollback failed: %s", edit.id, rb_exc)
|
||||
outcomes.append(
|
||||
EditOutcome(
|
||||
edit_id=edit.id,
|
||||
+4
-3
@@ -1,8 +1,9 @@
|
||||
"""Pending review queue for edits awaiting user approval.
|
||||
|
||||
Edits in the ``review`` tier (when autonomy mode is ``tiered``) are
|
||||
written here as JSON files. The user reviews them via ``jarvis learning
|
||||
review`` and approves or rejects.
|
||||
written here as JSON files. Callers consume the queue via
|
||||
``PendingQueue.list()`` / ``approve()`` / ``reject()`` to advance edits
|
||||
out of the review tier.
|
||||
|
||||
See spec §7.5.
|
||||
"""
|
||||
@@ -14,7 +15,7 @@ import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+3
-3
@@ -15,15 +15,15 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
FailureCluster,
|
||||
LearningPlan,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
maybe_downgrade_to_replace,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tiers
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import logging
|
||||
import re
|
||||
from typing import Callable, Optional
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import Sequence
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp, EditRiskTier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+10
-10
@@ -1,7 +1,7 @@
|
||||
"""Filesystem path resolution for the distillation subsystem.
|
||||
"""Filesystem path resolution for the spec-search subsystem.
|
||||
|
||||
The keystone of artifact isolation (spec §11): the resolved distillation root
|
||||
must NEVER be inside the OpenJarvis source tree. ``resolve_distillation_root``
|
||||
The keystone of artifact isolation (spec §11): the resolved spec-search root
|
||||
must NEVER be inside the OpenJarvis source tree. ``resolve_spec_search_root``
|
||||
walks up from this module's ``__file__`` looking for a ``pyproject.toml`` that
|
||||
identifies the OpenJarvis source root, then refuses to operate if the resolved
|
||||
root is inside it. Defense in depth — if a user accidentally points
|
||||
@@ -49,8 +49,8 @@ def _resolve_openjarvis_home() -> Path:
|
||||
return (Path.home() / ".openjarvis").resolve()
|
||||
|
||||
|
||||
def resolve_distillation_root() -> Path:
|
||||
"""Return the absolute path of the distillation root directory.
|
||||
def resolve_spec_search_root() -> Path:
|
||||
"""Return the absolute path of the spec-search root directory.
|
||||
|
||||
The root is ``$OPENJARVIS_HOME/learning`` (or ``~/.openjarvis/learning``
|
||||
by default). Raises ``ConfigurationError`` if the resolved path lies
|
||||
@@ -67,21 +67,21 @@ def resolve_distillation_root() -> Path:
|
||||
else:
|
||||
raise ConfigurationError(
|
||||
f"OPENJARVIS_HOME ({home}) is inside the source tree "
|
||||
f"({source_root}). Distillation refuses to write runtime "
|
||||
f"({source_root}). Spec search refuses to write runtime "
|
||||
"artifacts inside the OpenJarvis repo. Set OPENJARVIS_HOME "
|
||||
"to a directory outside the repo (default: ~/.openjarvis)."
|
||||
)
|
||||
return home / "learning"
|
||||
|
||||
|
||||
def ensure_distillation_dirs() -> Path:
|
||||
"""Create the distillation directory layout if missing.
|
||||
def ensure_spec_search_dirs() -> Path:
|
||||
"""Create the spec-search directory layout if missing.
|
||||
|
||||
Returns the distillation root. Creates ``sessions/``, ``benchmarks/``,
|
||||
Returns the spec-search root. Creates ``sessions/``, ``benchmarks/``,
|
||||
``benchmarks/reference_outputs/``, and ``pending_review/`` underneath it,
|
||||
all with restrictive ``0o700`` permissions via ``secure_mkdir``.
|
||||
"""
|
||||
root = resolve_distillation_root()
|
||||
root = resolve_spec_search_root()
|
||||
secure_mkdir(root)
|
||||
secure_mkdir(root / "sessions")
|
||||
secure_mkdir(root / "benchmarks")
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
"""SQLite-backed storage for distillation LearningSession records.
|
||||
"""SQLite-backed storage for spec-search LearningSession records.
|
||||
|
||||
Mirrors the style of ``openjarvis.learning.optimize.store.OptimizationStore``:
|
||||
|
||||
@@ -21,7 +21,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
EditOutcome,
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Real student runner for distillation experiments.
|
||||
"""Real student runner for spec-search experiments.
|
||||
|
||||
Replaces the ``MagicMock()`` in the experiment runner script with a
|
||||
callable that actually invokes the student model via vLLM (or any
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
"""Trigger types for the distillation subsystem.
|
||||
"""Trigger types for the spec-search subsystem.
|
||||
|
||||
A trigger is what kicks off a learning session. Four trigger types exist,
|
||||
all funneling into ``DistillationOrchestrator.run(trigger)``. The trigger
|
||||
all funneling into ``SpecSearchOrchestrator.run(trigger)``. The trigger
|
||||
object is stored on the ``LearningSession`` for queryability.
|
||||
|
||||
See spec §3.3.
|
||||
@@ -12,12 +12,12 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.learning.distillation.models import TriggerKind
|
||||
from openjarvis.learning.spec_search.models import TriggerKind
|
||||
|
||||
|
||||
@dataclass
|
||||
class OnDemandTrigger:
|
||||
"""User ran ``jarvis learning run`` from the CLI."""
|
||||
"""Caller invoked ``SpecSearchOrchestrator.run(trigger)`` directly."""
|
||||
|
||||
kind: TriggerKind = TriggerKind.ON_DEMAND
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Load tool description overrides from $OPENJARVIS_HOME/tools/descriptions.toml.
|
||||
|
||||
Distillation (M1) proposes tool description edits that get written to disk by
|
||||
LLM-guided spec search (M1) proposes tool description edits that get written to disk by
|
||||
``EditToolDescriptionApplier``. This module loads those overrides so agents
|
||||
see the improved descriptions at runtime.
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""Tests for the distillation subsystem."""
|
||||
@@ -1,44 +0,0 @@
|
||||
"""Tests for the jarvis learning CLI subcommand group."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
|
||||
class TestLearningCLI:
|
||||
def test_learning_group_exists(self) -> None:
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(learning_group, ["--help"])
|
||||
assert result.exit_code == 0
|
||||
out = result.output.lower()
|
||||
assert "learning" in out or "distillation" in out
|
||||
|
||||
def test_init_subcommand(self) -> None:
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(learning_group, ["init", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_run_subcommand(self) -> None:
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(learning_group, ["run", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_history_subcommand(self) -> None:
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(learning_group, ["history", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_rollback_subcommand(self) -> None:
|
||||
from openjarvis.learning.distillation.cli import learning_group
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(learning_group, ["rollback", "--help"])
|
||||
assert result.exit_code == 0
|
||||
@@ -0,0 +1 @@
|
||||
"""Tests for the spec-search subsystem."""
|
||||
+13
-13
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.execute.base module."""
|
||||
"""Tests for openjarvis.learning.spec_search.execute.base module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -11,26 +11,26 @@ class TestApplyContext:
|
||||
"""Tests for ApplyContext dataclass."""
|
||||
|
||||
def test_constructs(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
|
||||
ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1")
|
||||
assert ctx.openjarvis_home == tmp_path
|
||||
assert ctx.session_id == "s1"
|
||||
|
||||
def test_config_path(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
|
||||
ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1")
|
||||
assert ctx.config_path == tmp_path / "config.toml"
|
||||
|
||||
def test_agents_dir(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
|
||||
ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1")
|
||||
assert ctx.agents_dir == tmp_path / "agents"
|
||||
|
||||
def test_tools_dir(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
|
||||
ctx = ApplyContext(openjarvis_home=tmp_path, session_id="s1")
|
||||
assert ctx.tools_dir == tmp_path / "tools"
|
||||
@@ -40,14 +40,14 @@ class TestValidationResult:
|
||||
"""Tests for ValidationResult."""
|
||||
|
||||
def test_ok_result(self) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ValidationResult
|
||||
from openjarvis.learning.spec_search.execute.base import ValidationResult
|
||||
|
||||
r = ValidationResult(ok=True)
|
||||
assert r.ok is True
|
||||
assert r.reason == ""
|
||||
|
||||
def test_error_result(self) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import ValidationResult
|
||||
from openjarvis.learning.spec_search.execute.base import ValidationResult
|
||||
|
||||
r = ValidationResult(ok=False, reason="target not found")
|
||||
assert r.ok is False
|
||||
@@ -58,14 +58,14 @@ class TestEditApplierRegistry:
|
||||
"""Tests for EditApplierRegistry."""
|
||||
|
||||
def test_register_and_get(self) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
ApplyContext,
|
||||
ApplyResult,
|
||||
EditApplier,
|
||||
EditApplierRegistry,
|
||||
ValidationResult,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import Edit, EditOp
|
||||
from openjarvis.learning.spec_search.models import Edit, EditOp
|
||||
|
||||
class FakeApplier(EditApplier):
|
||||
op = EditOp.SET_MODEL_PARAM
|
||||
@@ -86,19 +86,19 @@ class TestEditApplierRegistry:
|
||||
assert isinstance(applier, FakeApplier)
|
||||
|
||||
def test_is_supported_returns_false_for_unregistered(self) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
EditApplierRegistry,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
registry = EditApplierRegistry()
|
||||
assert registry.is_supported(EditOp.LORA_FINETUNE) is False
|
||||
|
||||
def test_get_raises_for_unregistered(self) -> None:
|
||||
from openjarvis.learning.distillation.execute.base import (
|
||||
from openjarvis.learning.spec_search.execute.base import (
|
||||
EditApplierRegistry,
|
||||
)
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
registry = EditApplierRegistry()
|
||||
with pytest.raises(KeyError):
|
||||
+4
-4
@@ -6,8 +6,8 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -32,7 +32,7 @@ class TestLoraStubApplier:
|
||||
"""Tests for LoraStubApplier."""
|
||||
|
||||
def test_validate_returns_not_ok(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.lora_stub import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.lora_stub import (
|
||||
LoraStubApplier,
|
||||
)
|
||||
|
||||
@@ -43,7 +43,7 @@ class TestLoraStubApplier:
|
||||
assert "v2" in result.reason.lower() or "deferred" in result.reason.lower()
|
||||
|
||||
def test_apply_raises_not_implemented(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.lora_stub import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.lora_stub import (
|
||||
LoraStubApplier,
|
||||
)
|
||||
|
||||
+9
-9
@@ -5,8 +5,8 @@ from __future__ import annotations
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -35,7 +35,7 @@ class TestReplaceSystemPromptApplier:
|
||||
"""Tests for ReplaceSystemPromptApplier."""
|
||||
|
||||
def test_validate_ok(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
ReplaceSystemPromptApplier,
|
||||
)
|
||||
|
||||
@@ -54,7 +54,7 @@ class TestReplaceSystemPromptApplier:
|
||||
assert applier.validate(edit, ctx).ok
|
||||
|
||||
def test_apply_overwrites_prompt(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
ReplaceSystemPromptApplier,
|
||||
)
|
||||
|
||||
@@ -79,7 +79,7 @@ class TestPatchSystemPromptApplier:
|
||||
"""Tests for PatchSystemPromptApplier."""
|
||||
|
||||
def test_apply_applies_diff(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
PatchSystemPromptApplier,
|
||||
)
|
||||
|
||||
@@ -108,7 +108,7 @@ class TestPatchSystemPromptApplier:
|
||||
assert "math tools" in content
|
||||
|
||||
def test_validate_fails_for_bad_diff(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
PatchSystemPromptApplier,
|
||||
)
|
||||
|
||||
@@ -132,7 +132,7 @@ class TestSetAgentClassApplier:
|
||||
"""Tests for SetAgentClassApplier."""
|
||||
|
||||
def test_apply_updates_config(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
SetAgentClassApplier,
|
||||
)
|
||||
|
||||
@@ -157,7 +157,7 @@ class TestSetAgentParamApplier:
|
||||
"""Tests for SetAgentParamApplier."""
|
||||
|
||||
def test_apply_updates_param(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
SetAgentParamApplier,
|
||||
)
|
||||
|
||||
@@ -182,7 +182,7 @@ class TestEditFewShotExemplarsApplier:
|
||||
"""Tests for EditFewShotExemplarsApplier."""
|
||||
|
||||
def test_apply_writes_exemplars(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.agent import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.agent import (
|
||||
EditFewShotExemplarsApplier,
|
||||
)
|
||||
|
||||
+7
-7
@@ -4,8 +4,8 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -62,7 +62,7 @@ class TestSetModelForQueryClassApplier:
|
||||
"""Tests for SetModelForQueryClassApplier."""
|
||||
|
||||
def test_validate_ok(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelForQueryClassApplier,
|
||||
)
|
||||
|
||||
@@ -72,7 +72,7 @@ class TestSetModelForQueryClassApplier:
|
||||
assert result.ok
|
||||
|
||||
def test_apply_updates_config(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelForQueryClassApplier,
|
||||
)
|
||||
|
||||
@@ -83,7 +83,7 @@ class TestSetModelForQueryClassApplier:
|
||||
assert "qwen2.5-coder:14b" in content
|
||||
|
||||
def test_apply_adds_new_query_class(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelForQueryClassApplier,
|
||||
)
|
||||
|
||||
@@ -100,7 +100,7 @@ class TestSetModelParamApplier:
|
||||
"""Tests for SetModelParamApplier."""
|
||||
|
||||
def test_validate_ok(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelParamApplier,
|
||||
)
|
||||
|
||||
@@ -110,7 +110,7 @@ class TestSetModelParamApplier:
|
||||
assert result.ok
|
||||
|
||||
def test_apply_writes_param(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.intelligence import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.intelligence import (
|
||||
SetModelParamApplier,
|
||||
)
|
||||
|
||||
+7
-7
@@ -4,8 +4,8 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -27,7 +27,7 @@ class TestAddToolToAgentApplier:
|
||||
"""Tests for AddToolToAgentApplier."""
|
||||
|
||||
def test_apply_adds_tool(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
AddToolToAgentApplier,
|
||||
)
|
||||
|
||||
@@ -48,7 +48,7 @@ class TestAddToolToAgentApplier:
|
||||
assert "calculator" in content
|
||||
|
||||
def test_validate_ok(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
AddToolToAgentApplier,
|
||||
)
|
||||
|
||||
@@ -71,7 +71,7 @@ class TestRemoveToolFromAgentApplier:
|
||||
"""Tests for RemoveToolFromAgentApplier."""
|
||||
|
||||
def test_apply_removes_tool(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
RemoveToolFromAgentApplier,
|
||||
)
|
||||
|
||||
@@ -96,7 +96,7 @@ class TestEditToolDescriptionApplier:
|
||||
"""Tests for EditToolDescriptionApplier."""
|
||||
|
||||
def test_apply_updates_description(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
EditToolDescriptionApplier,
|
||||
)
|
||||
|
||||
@@ -120,7 +120,7 @@ class TestEditToolDescriptionApplier:
|
||||
assert "Search the internet" in content
|
||||
|
||||
def test_apply_adds_new_tool_section(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.appliers.tools import (
|
||||
from openjarvis.learning.spec_search.execute.appliers.tools import (
|
||||
EditToolDescriptionApplier,
|
||||
)
|
||||
|
||||
+7
-7
@@ -1,11 +1,11 @@
|
||||
"""Tests for openjarvis.learning.distillation.gate.benchmark_gate module.
|
||||
"""Tests for openjarvis.learning.spec_search.gate.benchmark_gate module.
|
||||
|
||||
All tests use mock scorers — no live EvalRunner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
|
||||
def _make_scorer(scores: dict[str, float], overall: float | None = None):
|
||||
@@ -30,7 +30,7 @@ class TestBenchmarkGate:
|
||||
"""Tests for BenchmarkGate."""
|
||||
|
||||
def test_accepts_improving_edit(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import (
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import (
|
||||
BenchmarkGate,
|
||||
)
|
||||
|
||||
@@ -54,7 +54,7 @@ class TestBenchmarkGate:
|
||||
assert result.delta > 0
|
||||
|
||||
def test_rejects_no_improvement(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import (
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import (
|
||||
BenchmarkGate,
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ class TestBenchmarkGate:
|
||||
assert "no improvement" in result.reason.lower()
|
||||
|
||||
def test_rejects_regression(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import (
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import (
|
||||
BenchmarkGate,
|
||||
)
|
||||
|
||||
@@ -101,7 +101,7 @@ class TestBenchmarkGate:
|
||||
assert "regression" in result.reason.lower()
|
||||
|
||||
def test_min_improvement_threshold(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import (
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import (
|
||||
BenchmarkGate,
|
||||
)
|
||||
|
||||
@@ -124,7 +124,7 @@ class TestBenchmarkGate:
|
||||
assert not result.accepted
|
||||
|
||||
def test_result_contains_snapshot(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.benchmark_gate import (
|
||||
from openjarvis.learning.spec_search.gate.benchmark_gate import (
|
||||
BenchmarkGate,
|
||||
)
|
||||
|
||||
+10
-10
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.checkpoint.store module."""
|
||||
"""Tests for openjarvis.learning.spec_search.checkpoint.store module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -38,7 +38,7 @@ class TestCheckpointStoreInit:
|
||||
"""Tests for CheckpointStore.init()."""
|
||||
|
||||
def test_creates_repo_with_baseline_commit(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -51,7 +51,7 @@ class TestCheckpointStoreInit:
|
||||
assert "baseline" in log
|
||||
|
||||
def test_init_idempotent(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -67,10 +67,10 @@ class TestCheckpointStoreInit:
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
source_root = paths._find_source_root()
|
||||
assert source_root is not None
|
||||
@@ -85,7 +85,7 @@ class TestStageCommitDiscard:
|
||||
"""Tests for begin_stage / commit_stage / discard_stage."""
|
||||
|
||||
def test_commit_stage_creates_commit_with_trailers(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ class TestStageCommitDiscard:
|
||||
assert "Risk-Tier: review" in body
|
||||
|
||||
def test_discard_stage_restores_working_tree(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -137,7 +137,7 @@ class TestStageCommitDiscard:
|
||||
assert store.current_sha() == handle.pre_stage_sha
|
||||
|
||||
def test_begin_stage_refuses_dirty_working_tree(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
DirtyWorkingTreeError,
|
||||
)
|
||||
@@ -161,7 +161,7 @@ class TestRevertSession:
|
||||
def test_revert_creates_new_commits_and_does_not_rewrite(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -206,7 +206,7 @@ class TestRevertSession:
|
||||
assert (root / "tools" / "descriptions.toml").read_text() == "[web_search]\n"
|
||||
|
||||
def test_revert_session_with_no_commits_returns_empty(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
+7
-7
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.gate.cold_start module."""
|
||||
"""Tests for openjarvis.learning.spec_search.gate.cold_start module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -41,7 +41,7 @@ class TestCheckReadiness:
|
||||
"""Tests for check_readiness()."""
|
||||
|
||||
def test_not_ready_with_no_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_readiness,
|
||||
)
|
||||
|
||||
@@ -51,7 +51,7 @@ class TestCheckReadiness:
|
||||
assert "not enough traces" in result.message.lower()
|
||||
|
||||
def test_not_ready_with_few_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_readiness,
|
||||
)
|
||||
|
||||
@@ -60,7 +60,7 @@ class TestCheckReadiness:
|
||||
assert not result.ready
|
||||
|
||||
def test_ready_with_enough_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_readiness,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ class TestCheckBenchmarkReady:
|
||||
"""Tests for check_benchmark_ready()."""
|
||||
|
||||
def test_not_ready_with_no_high_feedback_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_benchmark_ready,
|
||||
)
|
||||
|
||||
@@ -83,7 +83,7 @@ class TestCheckBenchmarkReady:
|
||||
assert "benchmark" in result.message.lower()
|
||||
|
||||
def test_not_ready_with_few_high_feedback_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_benchmark_ready,
|
||||
)
|
||||
|
||||
@@ -92,7 +92,7 @@ class TestCheckBenchmarkReady:
|
||||
assert not result.ready
|
||||
|
||||
def test_ready_with_enough_high_feedback_traces(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_benchmark_ready,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Tests for the paper Eq. 1 composite reward."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.learning.spec_search.composite_reward import (
|
||||
RewardWeights,
|
||||
TrainingSample,
|
||||
score_batch,
|
||||
)
|
||||
|
||||
|
||||
class TestScoreBatch:
|
||||
def test_empty_batch_returns_empty(self) -> None:
|
||||
assert score_batch([]) == []
|
||||
|
||||
def test_single_sample_has_zero_efficiency_penalties(self) -> None:
|
||||
# With one sample the within-batch z-score collapses to 0, so the
|
||||
# reward reduces to alpha * accuracy.
|
||||
sample = TrainingSample(
|
||||
accuracy=1.0, energy_joules=999.0, latency_seconds=42.0, cost_usd=0.05
|
||||
)
|
||||
[r] = score_batch([sample], weights=RewardWeights())
|
||||
assert r == pytest.approx(0.5) # alpha = 0.5
|
||||
|
||||
def test_higher_accuracy_ranks_above_lower_at_equal_efficiency(self) -> None:
|
||||
a = TrainingSample(
|
||||
accuracy=1.0, energy_joules=200, latency_seconds=5, cost_usd=0.0
|
||||
)
|
||||
b = TrainingSample(
|
||||
accuracy=0.0, energy_joules=200, latency_seconds=5, cost_usd=0.0
|
||||
)
|
||||
ra, rb = score_batch([a, b])
|
||||
assert ra > rb
|
||||
|
||||
def test_lower_efficiency_costs_lower_reward_at_equal_accuracy(self) -> None:
|
||||
# Two equally-accurate samples; the slower / more energy-hungry one
|
||||
# gets penalized.
|
||||
fast = TrainingSample(
|
||||
accuracy=1.0, energy_joules=100, latency_seconds=2, cost_usd=0.0
|
||||
)
|
||||
slow = TrainingSample(
|
||||
accuracy=1.0, energy_joules=400, latency_seconds=8, cost_usd=0.0
|
||||
)
|
||||
r_fast, r_slow = score_batch([fast, slow])
|
||||
assert r_fast > r_slow
|
||||
|
||||
def test_paper_default_weights(self) -> None:
|
||||
w = RewardWeights()
|
||||
assert (w.alpha, w.beta, w.gamma, w.delta) == (0.5, 0.1, 0.1, 0.3)
|
||||
|
||||
def test_custom_weights_change_ranking(self) -> None:
|
||||
# An accuracy-only weighting (alpha=1, others=0) collapses to raw
|
||||
# accuracy and ignores efficiency entirely.
|
||||
a = TrainingSample(
|
||||
accuracy=1.0, energy_joules=999, latency_seconds=99, cost_usd=99
|
||||
)
|
||||
b = TrainingSample(accuracy=1.0, energy_joules=1, latency_seconds=1, cost_usd=0)
|
||||
rewards = score_batch(
|
||||
[a, b], weights=RewardWeights(alpha=1.0, beta=0.0, gamma=0.0, delta=0.0)
|
||||
)
|
||||
assert rewards[0] == pytest.approx(rewards[1])
|
||||
assert rewards[0] == pytest.approx(1.0)
|
||||
|
||||
def test_zero_variance_batch_drops_efficiency_terms(self) -> None:
|
||||
# If all samples share the same energy/latency/cost, z-score is 0
|
||||
# for every term, so the reward is just alpha * accuracy.
|
||||
samples = [
|
||||
TrainingSample(
|
||||
accuracy=1.0, energy_joules=100, latency_seconds=2, cost_usd=0.0
|
||||
),
|
||||
TrainingSample(
|
||||
accuracy=0.5, energy_joules=100, latency_seconds=2, cost_usd=0.0
|
||||
),
|
||||
]
|
||||
rewards = score_batch(samples)
|
||||
assert rewards[0] == pytest.approx(0.5)
|
||||
assert rewards[1] == pytest.approx(0.25)
|
||||
+9
-9
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.diagnose.tools module.
|
||||
"""Tests for openjarvis.learning.spec_search.diagnose.tools module.
|
||||
|
||||
All tests use fixture stubs — no live TraceStore, CloudEngine, or ToolRegistry.
|
||||
"""
|
||||
@@ -89,7 +89,7 @@ class TestBuildDiagnosticTools:
|
||||
"""Tests for the build_diagnostic_tools factory."""
|
||||
|
||||
def test_returns_expected_tool_names(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -117,7 +117,7 @@ class TestBuildDiagnosticTools:
|
||||
assert "compare_outputs" in names
|
||||
|
||||
def test_all_tools_have_openai_format(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -142,7 +142,7 @@ class TestListTraces:
|
||||
"""Tests for the list_traces diagnostic tool."""
|
||||
|
||||
def test_returns_trace_metas(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -171,7 +171,7 @@ class TestGetTrace:
|
||||
"""Tests for the get_trace diagnostic tool."""
|
||||
|
||||
def test_returns_trace_details(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -192,7 +192,7 @@ class TestGetTrace:
|
||||
assert parsed["query"] == "What is 2+2?"
|
||||
|
||||
def test_returns_error_for_unknown_trace(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -215,7 +215,7 @@ class TestGetCurrentConfig:
|
||||
"""Tests for the get_current_config diagnostic tool."""
|
||||
|
||||
def test_returns_config_content(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -238,7 +238,7 @@ class TestGetAgentPrompt:
|
||||
"""Tests for the get_agent_prompt diagnostic tool."""
|
||||
|
||||
def test_returns_prompt_content(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
@@ -261,7 +261,7 @@ class TestListPersonalBenchmark:
|
||||
"""Tests for the list_personal_benchmark diagnostic tool."""
|
||||
|
||||
def test_returns_benchmark_tasks(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.tools import (
|
||||
from openjarvis.learning.spec_search.diagnose.tools import (
|
||||
build_diagnostic_tools,
|
||||
)
|
||||
|
||||
+11
-11
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.diagnose.types module."""
|
||||
"""Tests for openjarvis.learning.spec_search.diagnose.types module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,7 +9,7 @@ class TestTraceMeta:
|
||||
"""Tests for TraceMeta dataclass."""
|
||||
|
||||
def test_constructs_with_required_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import TraceMeta
|
||||
from openjarvis.learning.spec_search.diagnose.types import TraceMeta
|
||||
|
||||
meta = TraceMeta(
|
||||
trace_id="trace-001",
|
||||
@@ -24,7 +24,7 @@ class TestTraceMeta:
|
||||
assert meta.feedback == 0.8
|
||||
|
||||
def test_feedback_can_be_none(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import TraceMeta
|
||||
from openjarvis.learning.spec_search.diagnose.types import TraceMeta
|
||||
|
||||
meta = TraceMeta(
|
||||
trace_id="trace-002",
|
||||
@@ -42,7 +42,7 @@ class TestBenchmarkTask:
|
||||
"""Tests for BenchmarkTask dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import BenchmarkTask
|
||||
from openjarvis.learning.spec_search.diagnose.types import BenchmarkTask
|
||||
|
||||
task = BenchmarkTask(
|
||||
task_id="task-001",
|
||||
@@ -58,7 +58,7 @@ class TestStudentRun:
|
||||
"""Tests for StudentRun dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import StudentRun
|
||||
from openjarvis.learning.spec_search.diagnose.types import StudentRun
|
||||
|
||||
run = StudentRun(
|
||||
task_id="task-001",
|
||||
@@ -76,7 +76,7 @@ class TestTeacherRun:
|
||||
"""Tests for TeacherRun dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import TeacherRun
|
||||
from openjarvis.learning.spec_search.diagnose.types import TeacherRun
|
||||
|
||||
run = TeacherRun(
|
||||
task_id="task-001",
|
||||
@@ -92,7 +92,7 @@ class TestComparisonResult:
|
||||
"""Tests for ComparisonResult dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import ComparisonResult
|
||||
from openjarvis.learning.spec_search.diagnose.types import ComparisonResult
|
||||
|
||||
result = ComparisonResult(
|
||||
task_id="task-001",
|
||||
@@ -108,7 +108,7 @@ class TestToolMeta:
|
||||
"""Tests for ToolMeta dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import ToolMeta
|
||||
from openjarvis.learning.spec_search.diagnose.types import ToolMeta
|
||||
|
||||
meta = ToolMeta(
|
||||
name="calculator",
|
||||
@@ -124,7 +124,7 @@ class TestDiagnosticTool:
|
||||
"""Tests for DiagnosticTool dataclass."""
|
||||
|
||||
def test_constructs_with_callable(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import DiagnosticTool
|
||||
from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool
|
||||
|
||||
def my_func(**kwargs: object) -> str:
|
||||
return "result"
|
||||
@@ -143,7 +143,7 @@ class TestToolCallRecord:
|
||||
"""Tests for ToolCallRecord dataclass."""
|
||||
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import ToolCallRecord
|
||||
from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord
|
||||
|
||||
record = ToolCallRecord(
|
||||
timestamp=datetime(2026, 4, 9, 3, 0, 0, tzinfo=timezone.utc),
|
||||
@@ -157,7 +157,7 @@ class TestToolCallRecord:
|
||||
assert record.latency_ms == 42.5
|
||||
|
||||
def test_to_jsonl_dict(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.types import ToolCallRecord
|
||||
from openjarvis.learning.spec_search.diagnose.types import ToolCallRecord
|
||||
|
||||
record = ToolCallRecord(
|
||||
timestamp=datetime(2026, 4, 9, 3, 0, 0, tzinfo=timezone.utc),
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.diagnose.runner module.
|
||||
"""Tests for openjarvis.learning.spec_search.diagnose.runner module.
|
||||
|
||||
All tests use mocked dependencies — no live API calls.
|
||||
"""
|
||||
@@ -61,7 +61,7 @@ class TestDiagnosisRunner:
|
||||
"""Tests for DiagnosisRunner."""
|
||||
|
||||
def test_produces_diagnosis_artifact(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
@@ -95,7 +95,7 @@ class TestDiagnosisRunner:
|
||||
assert "Math" in diagnosis_path.read_text()
|
||||
|
||||
def test_returns_failure_clusters(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
@@ -128,7 +128,7 @@ class TestDiagnosisRunner:
|
||||
assert result.clusters[1].id == "cluster-002"
|
||||
|
||||
def test_persists_teacher_traces_jsonl(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
@@ -174,7 +174,7 @@ class TestDiagnosisRunner:
|
||||
assert "tool" in record
|
||||
|
||||
def test_returns_cost(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
@@ -203,7 +203,7 @@ class TestDiagnosisRunner:
|
||||
assert result.cost_usd >= 0.0
|
||||
|
||||
def test_handles_no_clusters_in_output(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
+8
-8
@@ -4,8 +4,8 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.execute.base import ApplyContext
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.execute.base import ApplyContext
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
Edit,
|
||||
EditOp,
|
||||
@@ -72,7 +72,7 @@ class TestExecuteEdits:
|
||||
"""Tests for execute_edits()."""
|
||||
|
||||
def test_applies_auto_tier_edit(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
@@ -84,7 +84,7 @@ class TestExecuteEdits:
|
||||
assert outcomes[0].status == "applied"
|
||||
|
||||
def test_review_edit_goes_to_pending_in_tiered_mode(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
@@ -96,7 +96,7 @@ class TestExecuteEdits:
|
||||
assert outcomes[0].status == "pending_review"
|
||||
|
||||
def test_review_edit_applied_in_auto_mode(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
@@ -108,7 +108,7 @@ class TestExecuteEdits:
|
||||
assert outcomes[0].status == "applied"
|
||||
|
||||
def test_manual_tier_skipped(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
@@ -120,7 +120,7 @@ class TestExecuteEdits:
|
||||
assert outcomes[0].status == "skipped"
|
||||
|
||||
def test_all_edits_pending_in_manual_mode(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
@@ -132,7 +132,7 @@ class TestExecuteEdits:
|
||||
assert outcomes[0].status == "pending_review"
|
||||
|
||||
def test_multiple_edits_processed(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.execute.loop import execute_edits
|
||||
from openjarvis.learning.spec_search.execute.loop import execute_edits
|
||||
|
||||
ctx = _make_ctx(tmp_path)
|
||||
outcomes = execute_edits(
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.external_adapter."""
|
||||
"""Tests for openjarvis.learning.spec_search.external_adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
from openjarvis.learning.distillation.external_adapter import (
|
||||
from openjarvis.learning.spec_search.external_adapter import (
|
||||
write_external_records_as_traces,
|
||||
)
|
||||
from openjarvis.traces.store import TraceStore
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
"""Live integration tests for the distillation subsystem.
|
||||
"""Live integration tests for the spec-search subsystem.
|
||||
|
||||
These tests use REAL API calls (CloudEngine with Anthropic) and real
|
||||
TraceStore data. They are gated on the ``cloud`` marker — skip them
|
||||
@@ -73,7 +73,7 @@ class TestTeacherAgentLive:
|
||||
"""Test TeacherAgent with a real CloudEngine."""
|
||||
|
||||
def test_teacher_agent_single_turn(self, cloud_engine) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -101,7 +101,7 @@ class TestDiagnosisRunnerLive:
|
||||
def test_diagnosis_produces_output(
|
||||
self, cloud_engine, real_trace_store, tmp_path
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.runner import (
|
||||
from openjarvis.learning.spec_search.diagnose.runner import (
|
||||
DiagnosisRunner,
|
||||
)
|
||||
|
||||
@@ -157,7 +157,7 @@ class TestColdStartLive:
|
||||
"""With 373 traces but 0 feedback, the orchestrator should handle
|
||||
this gracefully — either by running (traces > 20) or by giving
|
||||
a clear message about what's missing."""
|
||||
from openjarvis.learning.distillation.gate.cold_start import (
|
||||
from openjarvis.learning.spec_search.gate.cold_start import (
|
||||
check_benchmark_ready,
|
||||
check_readiness,
|
||||
)
|
||||
+42
-42
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.models module."""
|
||||
"""Tests for openjarvis.learning.spec_search.models module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -11,7 +11,7 @@ class TestEditPillar:
|
||||
"""Tests for EditPillar enum."""
|
||||
|
||||
def test_has_four_pillars(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditPillar
|
||||
from openjarvis.learning.spec_search.models import EditPillar
|
||||
|
||||
assert EditPillar.INTELLIGENCE.value == "intelligence"
|
||||
assert EditPillar.AGENT.value == "agent"
|
||||
@@ -19,7 +19,7 @@ class TestEditPillar:
|
||||
assert EditPillar.ENGINE.value == "engine"
|
||||
|
||||
def test_is_string_enum(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditPillar
|
||||
from openjarvis.learning.spec_search.models import EditPillar
|
||||
|
||||
assert isinstance(EditPillar.AGENT, str)
|
||||
assert EditPillar("agent") is EditPillar.AGENT
|
||||
@@ -29,7 +29,7 @@ class TestEditRiskTier:
|
||||
"""Tests for EditRiskTier enum."""
|
||||
|
||||
def test_has_three_tiers(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditRiskTier
|
||||
from openjarvis.learning.spec_search.models import EditRiskTier
|
||||
|
||||
assert EditRiskTier.AUTO.value == "auto"
|
||||
assert EditRiskTier.REVIEW.value == "review"
|
||||
@@ -40,13 +40,13 @@ class TestEditOp:
|
||||
"""Tests for EditOp enum — must contain all v1 ops plus v2 placeholders."""
|
||||
|
||||
def test_intelligence_ops(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
assert EditOp.SET_MODEL_FOR_QUERY_CLASS.value == "set_model_for_query_class"
|
||||
assert EditOp.SET_MODEL_PARAM.value == "set_model_param"
|
||||
|
||||
def test_agent_ops(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
assert EditOp.PATCH_SYSTEM_PROMPT.value == "patch_system_prompt"
|
||||
assert EditOp.REPLACE_SYSTEM_PROMPT.value == "replace_system_prompt"
|
||||
@@ -55,14 +55,14 @@ class TestEditOp:
|
||||
assert EditOp.EDIT_FEW_SHOT_EXEMPLARS.value == "edit_few_shot_exemplars"
|
||||
|
||||
def test_tools_ops(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
assert EditOp.ADD_TOOL_TO_AGENT.value == "add_tool_to_agent"
|
||||
assert EditOp.REMOVE_TOOL_FROM_AGENT.value == "remove_tool_from_agent"
|
||||
assert EditOp.EDIT_TOOL_DESCRIPTION.value == "edit_tool_description"
|
||||
|
||||
def test_v2_placeholder_ops(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
|
||||
assert EditOp.LORA_FINETUNE.value == "lora_finetune"
|
||||
|
||||
@@ -71,7 +71,7 @@ class TestTriggerKind:
|
||||
"""Tests for TriggerKind enum."""
|
||||
|
||||
def test_four_trigger_kinds(self) -> None:
|
||||
from openjarvis.learning.distillation.models import TriggerKind
|
||||
from openjarvis.learning.spec_search.models import TriggerKind
|
||||
|
||||
assert TriggerKind.SCHEDULED.value == "scheduled"
|
||||
assert TriggerKind.CLUSTER.value == "cluster"
|
||||
@@ -83,7 +83,7 @@ class TestAutonomyMode:
|
||||
"""Tests for AutonomyMode enum."""
|
||||
|
||||
def test_three_modes(self) -> None:
|
||||
from openjarvis.learning.distillation.models import AutonomyMode
|
||||
from openjarvis.learning.spec_search.models import AutonomyMode
|
||||
|
||||
assert AutonomyMode.AUTO.value == "auto"
|
||||
assert AutonomyMode.TIERED.value == "tiered"
|
||||
@@ -94,7 +94,7 @@ class TestSessionStatus:
|
||||
"""Tests for SessionStatus enum."""
|
||||
|
||||
def test_all_statuses(self) -> None:
|
||||
from openjarvis.learning.distillation.models import SessionStatus
|
||||
from openjarvis.learning.spec_search.models import SessionStatus
|
||||
|
||||
assert SessionStatus.INITIATED.value == "initiated"
|
||||
assert SessionStatus.DIAGNOSING.value == "diagnosing"
|
||||
@@ -115,7 +115,7 @@ class TestEdit:
|
||||
"""Tests for Edit pydantic model."""
|
||||
|
||||
def _valid_edit_kwargs(self) -> dict:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
@@ -134,7 +134,7 @@ class TestEdit:
|
||||
}
|
||||
|
||||
def test_constructs_with_valid_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
edit = Edit(**self._valid_edit_kwargs())
|
||||
|
||||
@@ -144,7 +144,7 @@ class TestEdit:
|
||||
assert edit.references == ["trace-001", "trace-002"]
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
edit = Edit(**self._valid_edit_kwargs())
|
||||
as_json = edit.model_dump_json()
|
||||
@@ -156,7 +156,7 @@ class TestEdit:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
kwargs = self._valid_edit_kwargs()
|
||||
kwargs["pillar"] = "not_a_pillar"
|
||||
@@ -168,7 +168,7 @@ class TestEdit:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
kwargs = self._valid_edit_kwargs()
|
||||
kwargs["op"] = "not_an_op"
|
||||
@@ -177,7 +177,7 @@ class TestEdit:
|
||||
Edit(**kwargs)
|
||||
|
||||
def test_payload_can_be_empty_dict(self) -> None:
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
kwargs = self._valid_edit_kwargs()
|
||||
kwargs["payload"] = {}
|
||||
@@ -186,7 +186,7 @@ class TestEdit:
|
||||
assert edit.payload == {}
|
||||
|
||||
def test_references_default_empty_list(self) -> None:
|
||||
from openjarvis.learning.distillation.models import Edit
|
||||
from openjarvis.learning.spec_search.models import Edit
|
||||
|
||||
kwargs = self._valid_edit_kwargs()
|
||||
del kwargs["references"]
|
||||
@@ -217,7 +217,7 @@ class TestFailureCluster:
|
||||
}
|
||||
|
||||
def test_constructs_with_valid_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
cluster = FailureCluster(**self._valid_cluster_kwargs())
|
||||
|
||||
@@ -228,7 +228,7 @@ class TestFailureCluster:
|
||||
assert len(cluster.addressed_by_edit_ids) == 2
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
cluster = FailureCluster(**self._valid_cluster_kwargs())
|
||||
as_json = cluster.model_dump_json()
|
||||
@@ -237,7 +237,7 @@ class TestFailureCluster:
|
||||
assert restored == cluster
|
||||
|
||||
def test_addressed_by_edit_ids_defaults_empty(self) -> None:
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
kwargs = self._valid_cluster_kwargs()
|
||||
del kwargs["addressed_by_edit_ids"]
|
||||
@@ -249,7 +249,7 @@ class TestFailureCluster:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
kwargs = self._valid_cluster_kwargs()
|
||||
kwargs["student_failure_rate"] = 1.5
|
||||
@@ -261,7 +261,7 @@ class TestFailureCluster:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import FailureCluster
|
||||
from openjarvis.learning.spec_search.models import FailureCluster
|
||||
|
||||
kwargs = self._valid_cluster_kwargs()
|
||||
kwargs["teacher_success_rate"] = -0.1
|
||||
@@ -281,7 +281,7 @@ class TestLearningPlan:
|
||||
def _valid_plan_kwargs(self) -> dict:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -320,7 +320,7 @@ class TestLearningPlan:
|
||||
}
|
||||
|
||||
def test_constructs_with_valid_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.models import LearningPlan
|
||||
from openjarvis.learning.spec_search.models import LearningPlan
|
||||
|
||||
plan = LearningPlan(**self._valid_plan_kwargs())
|
||||
|
||||
@@ -331,7 +331,7 @@ class TestLearningPlan:
|
||||
assert plan.estimated_cost_usd == 1.42
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import LearningPlan
|
||||
from openjarvis.learning.spec_search.models import LearningPlan
|
||||
|
||||
plan = LearningPlan(**self._valid_plan_kwargs())
|
||||
as_json = plan.model_dump_json()
|
||||
@@ -341,7 +341,7 @@ class TestLearningPlan:
|
||||
|
||||
def test_empty_clusters_and_edits_allowed(self) -> None:
|
||||
# An aborted session may produce a plan with no clusters and no edits.
|
||||
from openjarvis.learning.distillation.models import LearningPlan
|
||||
from openjarvis.learning.spec_search.models import LearningPlan
|
||||
|
||||
kwargs = self._valid_plan_kwargs()
|
||||
kwargs["failure_clusters"] = []
|
||||
@@ -355,7 +355,7 @@ class TestLearningPlan:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import LearningPlan
|
||||
from openjarvis.learning.spec_search.models import LearningPlan
|
||||
|
||||
kwargs = self._valid_plan_kwargs()
|
||||
kwargs["estimated_cost_usd"] = -1.0
|
||||
@@ -382,7 +382,7 @@ class TestBenchmarkSnapshot:
|
||||
}
|
||||
|
||||
def test_constructs_with_valid_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
snap = BenchmarkSnapshot(**self._valid_snapshot_kwargs())
|
||||
|
||||
@@ -393,7 +393,7 @@ class TestBenchmarkSnapshot:
|
||||
assert snap.elapsed_seconds == 184.3
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
snap = BenchmarkSnapshot(**self._valid_snapshot_kwargs())
|
||||
restored = BenchmarkSnapshot.model_validate_json(snap.model_dump_json())
|
||||
@@ -404,7 +404,7 @@ class TestBenchmarkSnapshot:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
kwargs = self._valid_snapshot_kwargs()
|
||||
kwargs["overall_score"] = 1.5
|
||||
@@ -416,7 +416,7 @@ class TestBenchmarkSnapshot:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
kwargs = self._valid_snapshot_kwargs()
|
||||
kwargs["task_count"] = -1
|
||||
@@ -446,14 +446,14 @@ class TestEditOutcome:
|
||||
}
|
||||
|
||||
def test_applied_outcome(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOutcome
|
||||
from openjarvis.learning.spec_search.models import EditOutcome
|
||||
|
||||
outcome = EditOutcome(**self._valid_outcome_kwargs())
|
||||
assert outcome.status == "applied"
|
||||
assert outcome.benchmark_delta == 0.04
|
||||
|
||||
def test_rejected_outcome_has_no_applied_at(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOutcome
|
||||
from openjarvis.learning.spec_search.models import EditOutcome
|
||||
|
||||
outcome = EditOutcome(
|
||||
edit_id="edit-002",
|
||||
@@ -471,7 +471,7 @@ class TestEditOutcome:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import EditOutcome
|
||||
from openjarvis.learning.spec_search.models import EditOutcome
|
||||
|
||||
kwargs = self._valid_outcome_kwargs()
|
||||
kwargs["status"] = "totally_made_up"
|
||||
@@ -480,7 +480,7 @@ class TestEditOutcome:
|
||||
EditOutcome(**kwargs)
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOutcome
|
||||
from openjarvis.learning.spec_search.models import EditOutcome
|
||||
|
||||
outcome = EditOutcome(**self._valid_outcome_kwargs())
|
||||
restored = EditOutcome.model_validate_json(outcome.model_dump_json())
|
||||
@@ -500,7 +500,7 @@ class TestLearningSession:
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
SessionStatus,
|
||||
@@ -535,7 +535,7 @@ class TestLearningSession:
|
||||
}
|
||||
|
||||
def test_constructs_with_valid_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.models import LearningSession
|
||||
from openjarvis.learning.spec_search.models import LearningSession
|
||||
|
||||
session = LearningSession(**self._valid_session_kwargs())
|
||||
assert session.id == "session-001"
|
||||
@@ -544,7 +544,7 @@ class TestLearningSession:
|
||||
assert session.benchmark_after is None
|
||||
|
||||
def test_round_trip_via_json(self) -> None:
|
||||
from openjarvis.learning.distillation.models import LearningSession
|
||||
from openjarvis.learning.spec_search.models import LearningSession
|
||||
|
||||
session = LearningSession(**self._valid_session_kwargs())
|
||||
as_json = session.model_dump_json()
|
||||
@@ -553,7 +553,7 @@ class TestLearningSession:
|
||||
assert restored == session
|
||||
|
||||
def test_supports_parent_session_chain(self) -> None:
|
||||
from openjarvis.learning.distillation.models import LearningSession
|
||||
from openjarvis.learning.spec_search.models import LearningSession
|
||||
|
||||
kwargs = self._valid_session_kwargs()
|
||||
kwargs["parent_session_id"] = "session-000"
|
||||
@@ -565,7 +565,7 @@ class TestLearningSession:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import LearningSession
|
||||
from openjarvis.learning.spec_search.models import LearningSession
|
||||
|
||||
kwargs = self._valid_session_kwargs()
|
||||
kwargs["status"] = "not_a_status"
|
||||
@@ -577,7 +577,7 @@ class TestLearningSession:
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from openjarvis.learning.distillation.models import LearningSession
|
||||
from openjarvis.learning.spec_search.models import LearningSession
|
||||
|
||||
kwargs = self._valid_session_kwargs()
|
||||
kwargs["teacher_cost_usd"] = -0.01
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Tests for SpecSearchLoop (paper Algorithm 1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
LearningSession,
|
||||
SessionStatus,
|
||||
TriggerKind,
|
||||
)
|
||||
from openjarvis.learning.spec_search.multi_session import (
|
||||
SpecSearchLoop,
|
||||
)
|
||||
|
||||
|
||||
def _session(
|
||||
score: float,
|
||||
cost: float = 0.1,
|
||||
status: SessionStatus = SessionStatus.COMPLETED,
|
||||
error: str | None = None,
|
||||
) -> LearningSession:
|
||||
return LearningSession(
|
||||
id=f"s-{score}",
|
||||
trigger=TriggerKind.ON_DEMAND,
|
||||
trigger_metadata={},
|
||||
status=status,
|
||||
autonomy_mode=AutonomyMode.AUTO,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
diagnosis_path="/tmp/diag.md",
|
||||
plan_path="/tmp/plan.json",
|
||||
benchmark_before=BenchmarkSnapshot(
|
||||
benchmark_version="v1",
|
||||
overall_score=0.0,
|
||||
cluster_scores={},
|
||||
task_count=10,
|
||||
elapsed_seconds=1.0,
|
||||
),
|
||||
benchmark_after=BenchmarkSnapshot(
|
||||
benchmark_version="v1",
|
||||
overall_score=score,
|
||||
cluster_scores={},
|
||||
task_count=10,
|
||||
elapsed_seconds=1.0,
|
||||
),
|
||||
git_checkpoint_pre="sha-pre",
|
||||
teacher_cost_usd=cost,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _orch_yielding(sessions: list[LearningSession]) -> MagicMock:
|
||||
"""Mock orchestrator whose .run(trigger) returns the next session."""
|
||||
orch = MagicMock()
|
||||
orch.run.side_effect = sessions
|
||||
return orch
|
||||
|
||||
|
||||
class TestSpecSearchLoop:
|
||||
def test_validates_stagnation_k(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
SpecSearchLoop(MagicMock(), stagnation_k=0)
|
||||
|
||||
def test_validates_budget(self) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
SpecSearchLoop(MagicMock(), max_total_cost_usd=0)
|
||||
|
||||
def test_stops_after_k_sessions_without_improvement(self) -> None:
|
||||
# First session improves to 0.6; next 3 do not improve. With k=3
|
||||
# the loop should stop after the 4th session.
|
||||
orch = _orch_yielding(
|
||||
[
|
||||
_session(0.6),
|
||||
_session(0.6),
|
||||
_session(0.6),
|
||||
_session(0.6),
|
||||
]
|
||||
)
|
||||
loop = SpecSearchLoop(orch, stagnation_k=3, max_total_cost_usd=10.0)
|
||||
result = loop.run()
|
||||
assert result.stop_reason == "stagnation"
|
||||
assert len(result.sessions) == 4
|
||||
assert result.best_overall_score == pytest.approx(0.6)
|
||||
|
||||
def test_keeps_improving_resets_streak(self) -> None:
|
||||
# Sessions improve each time; loop only stops when budget hits.
|
||||
orch = _orch_yielding(
|
||||
[
|
||||
_session(0.5, cost=2.0),
|
||||
_session(0.6, cost=2.0),
|
||||
_session(0.7, cost=2.0),
|
||||
_session(0.8, cost=2.0), # cumulative = 8.0
|
||||
_session(0.9, cost=3.0), # cumulative = 11.0 -> over budget
|
||||
]
|
||||
)
|
||||
loop = SpecSearchLoop(orch, stagnation_k=10, max_total_cost_usd=10.0)
|
||||
result = loop.run()
|
||||
assert result.stop_reason == "budget"
|
||||
assert len(result.sessions) == 5
|
||||
assert result.best_overall_score == pytest.approx(0.9)
|
||||
|
||||
def test_failed_session_terminates_loop(self) -> None:
|
||||
orch = _orch_yielding(
|
||||
[
|
||||
_session(0.5),
|
||||
_session(0.0, status=SessionStatus.FAILED, error="crash"),
|
||||
]
|
||||
)
|
||||
loop = SpecSearchLoop(orch, stagnation_k=5, max_total_cost_usd=10.0)
|
||||
result = loop.run()
|
||||
assert result.stop_reason == "failed"
|
||||
assert len(result.sessions) == 2
|
||||
|
||||
def test_total_cost_accumulates(self) -> None:
|
||||
orch = _orch_yielding(
|
||||
[
|
||||
_session(0.5, cost=1.0),
|
||||
_session(0.5, cost=2.0),
|
||||
_session(0.5, cost=3.0),
|
||||
]
|
||||
)
|
||||
loop = SpecSearchLoop(orch, stagnation_k=2, max_total_cost_usd=100.0)
|
||||
result = loop.run()
|
||||
assert result.total_cost_usd == pytest.approx(6.0)
|
||||
|
||||
def test_eps_threshold_filters_noise(self) -> None:
|
||||
# Improvement smaller than stagnation_eps does not reset the streak.
|
||||
orch = _orch_yielding(
|
||||
[
|
||||
_session(0.500),
|
||||
_session(0.5005), # below default eps=0.001
|
||||
_session(0.5009), # below eps from current best
|
||||
]
|
||||
)
|
||||
loop = SpecSearchLoop(
|
||||
orch, stagnation_k=2, stagnation_eps=0.001, max_total_cost_usd=10.0
|
||||
)
|
||||
result = loop.run()
|
||||
assert result.stop_reason == "stagnation"
|
||||
assert len(result.sessions) == 3
|
||||
assert result.best_overall_score == pytest.approx(0.500)
|
||||
+50
-16
@@ -1,4 +1,4 @@
|
||||
"""Tests for DistillationOrchestrator — full session with mocks."""
|
||||
"""Tests for SpecSearchOrchestrator — full session with mocks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,13 +6,13 @@ import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
FailureCluster,
|
||||
SessionStatus,
|
||||
)
|
||||
from openjarvis.learning.distillation.triggers import OnDemandTrigger
|
||||
from openjarvis.learning.spec_search.triggers import OnDemandTrigger
|
||||
|
||||
|
||||
def _make_snapshot(overall: float = 0.6) -> BenchmarkSnapshot:
|
||||
@@ -26,7 +26,7 @@ def _make_snapshot(overall: float = 0.6) -> BenchmarkSnapshot:
|
||||
|
||||
|
||||
def _make_diagnosis_result():
|
||||
from openjarvis.learning.distillation.diagnose.runner import DiagnosisResult
|
||||
from openjarvis.learning.spec_search.diagnose.runner import DiagnosisResult
|
||||
|
||||
return DiagnosisResult(
|
||||
diagnosis_md="## Diagnosis\nMath routing is broken.",
|
||||
@@ -75,13 +75,13 @@ def _make_mock_engine():
|
||||
return engine
|
||||
|
||||
|
||||
class TestDistillationOrchestrator:
|
||||
class TestSpecSearchOrchestrator:
|
||||
def test_full_session_completes(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.orchestrator import (
|
||||
DistillationOrchestrator,
|
||||
from openjarvis.learning.spec_search.orchestrator import (
|
||||
SpecSearchOrchestrator,
|
||||
)
|
||||
|
||||
orch = DistillationOrchestrator(
|
||||
orch = SpecSearchOrchestrator(
|
||||
teacher_engine=_make_mock_engine(),
|
||||
teacher_model="claude-opus-4-6",
|
||||
trace_store=MagicMock(count=MagicMock(return_value=30)),
|
||||
@@ -100,7 +100,7 @@ class TestDistillationOrchestrator:
|
||||
)
|
||||
|
||||
with patch(
|
||||
"openjarvis.learning.distillation.orchestrator.DiagnosisRunner"
|
||||
"openjarvis.learning.spec_search.orchestrator.DiagnosisRunner"
|
||||
) as MockDiag:
|
||||
MockDiag.return_value.run.return_value = _make_diagnosis_result()
|
||||
session = orch.run(OnDemandTrigger())
|
||||
@@ -112,14 +112,14 @@ class TestDistillationOrchestrator:
|
||||
assert session.teacher_cost_usd >= 0
|
||||
|
||||
def test_cold_start_returns_failed(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.orchestrator import (
|
||||
DistillationOrchestrator,
|
||||
from openjarvis.learning.spec_search.orchestrator import (
|
||||
SpecSearchOrchestrator,
|
||||
)
|
||||
|
||||
trace_store = MagicMock()
|
||||
trace_store.count.return_value = 5 # Not enough
|
||||
|
||||
orch = DistillationOrchestrator(
|
||||
orch = SpecSearchOrchestrator(
|
||||
teacher_engine=MagicMock(),
|
||||
teacher_model="claude-opus-4-6",
|
||||
trace_store=trace_store,
|
||||
@@ -141,13 +141,13 @@ class TestDistillationOrchestrator:
|
||||
assert "not enough traces" in (session.error or "").lower()
|
||||
|
||||
def test_session_persisted_to_store(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.orchestrator import (
|
||||
DistillationOrchestrator,
|
||||
from openjarvis.learning.spec_search.orchestrator import (
|
||||
SpecSearchOrchestrator,
|
||||
)
|
||||
|
||||
session_store = MagicMock()
|
||||
|
||||
orch = DistillationOrchestrator(
|
||||
orch = SpecSearchOrchestrator(
|
||||
teacher_engine=_make_mock_engine(),
|
||||
teacher_model="claude-opus-4-6",
|
||||
trace_store=MagicMock(count=MagicMock(return_value=30)),
|
||||
@@ -166,9 +166,43 @@ class TestDistillationOrchestrator:
|
||||
)
|
||||
|
||||
with patch(
|
||||
"openjarvis.learning.distillation.orchestrator.DiagnosisRunner"
|
||||
"openjarvis.learning.spec_search.orchestrator.DiagnosisRunner"
|
||||
) as MockDiag:
|
||||
MockDiag.return_value.run.return_value = _make_diagnosis_result()
|
||||
orch.run(OnDemandTrigger())
|
||||
|
||||
assert session_store.save_session.called
|
||||
|
||||
|
||||
class TestFromConfig:
|
||||
def test_from_config_round_trips_paper_defaults(self, tmp_path: Path) -> None:
|
||||
from openjarvis.core.config import SpecSearchLearningConfig
|
||||
from openjarvis.learning.spec_search.orchestrator import (
|
||||
SpecSearchOrchestrator,
|
||||
)
|
||||
|
||||
cfg = SpecSearchLearningConfig(
|
||||
enabled=True,
|
||||
teacher_model="claude-opus-4-6",
|
||||
teacher_engine="cloud",
|
||||
autonomy_mode="auto",
|
||||
max_regression=0.01, # paper default
|
||||
stagnation_k=5,
|
||||
)
|
||||
|
||||
orch = SpecSearchOrchestrator.from_config(
|
||||
cfg,
|
||||
teacher_engine=MagicMock(),
|
||||
trace_store=MagicMock(),
|
||||
benchmark_samples=[],
|
||||
student_runner=MagicMock(),
|
||||
judge=MagicMock(),
|
||||
session_store=MagicMock(),
|
||||
checkpoint_store=MagicMock(),
|
||||
openjarvis_home=tmp_path,
|
||||
)
|
||||
|
||||
# Round-trip the knobs: paper defaults must reach the orchestrator.
|
||||
assert orch._model == "claude-opus-4-6"
|
||||
assert orch._max_regression == 0.01
|
||||
assert orch._autonomy.value == "auto"
|
||||
+21
-21
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.storage.paths module."""
|
||||
"""Tests for openjarvis.learning.spec_search.storage.paths module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,18 +7,18 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_distillation_root
|
||||
# resolve_spec_search_root
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveDistillationRoot:
|
||||
"""Tests for resolve_distillation_root()."""
|
||||
class TestResolveSpecSearchRoot:
|
||||
"""Tests for resolve_spec_search_root()."""
|
||||
|
||||
def test_default_is_under_home(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
monkeypatch.delenv("OPENJARVIS_HOME", raising=False)
|
||||
result = paths.resolve_distillation_root()
|
||||
result = paths.resolve_spec_search_root()
|
||||
assert result == Path.home() / ".openjarvis" / "learning"
|
||||
|
||||
def test_respects_openjarvis_home_env_var(
|
||||
@@ -26,11 +26,11 @@ class TestResolveDistillationRoot:
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
custom = tmp_path / "custom_oj"
|
||||
monkeypatch.setenv("OPENJARVIS_HOME", str(custom))
|
||||
result = paths.resolve_distillation_root()
|
||||
result = paths.resolve_spec_search_root()
|
||||
assert result == custom / "learning"
|
||||
|
||||
def test_returns_absolute_path(
|
||||
@@ -38,17 +38,17 @@ class TestResolveDistillationRoot:
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "rel"))
|
||||
result = paths.resolve_distillation_root()
|
||||
result = paths.resolve_spec_search_root()
|
||||
assert result.is_absolute()
|
||||
|
||||
def test_rejects_path_inside_source_tree(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
# Find the OpenJarvis source root by walking up from the paths module.
|
||||
source_root = paths._find_source_root()
|
||||
@@ -58,10 +58,10 @@ class TestResolveDistillationRoot:
|
||||
monkeypatch.setenv("OPENJARVIS_HOME", str(source_root / "junk_dir"))
|
||||
|
||||
with pytest.raises(paths.ConfigurationError, match="inside the source tree"):
|
||||
paths.resolve_distillation_root()
|
||||
paths.resolve_spec_search_root()
|
||||
|
||||
def test_find_source_root_returns_repo_root(self) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
result = paths._find_source_root()
|
||||
assert result is not None
|
||||
@@ -69,22 +69,22 @@ class TestResolveDistillationRoot:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ensure_distillation_dirs
|
||||
# ensure_spec_search_dirs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnsureDistillationDirs:
|
||||
"""Tests for ensure_distillation_dirs()."""
|
||||
class TestEnsureSpecSearchDirs:
|
||||
"""Tests for ensure_spec_search_dirs()."""
|
||||
|
||||
def test_creates_subdirs(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "oj"))
|
||||
root = paths.ensure_distillation_dirs()
|
||||
root = paths.ensure_spec_search_dirs()
|
||||
|
||||
assert root.exists()
|
||||
assert (root / "sessions").exists()
|
||||
@@ -97,11 +97,11 @@ class TestEnsureDistillationDirs:
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from openjarvis.learning.distillation.storage import paths
|
||||
from openjarvis.learning.spec_search.storage import paths
|
||||
|
||||
monkeypatch.setenv("OPENJARVIS_HOME", str(tmp_path / "oj"))
|
||||
first = paths.ensure_distillation_dirs()
|
||||
second = paths.ensure_distillation_dirs()
|
||||
first = paths.ensure_spec_search_dirs()
|
||||
second = paths.ensure_spec_search_dirs()
|
||||
|
||||
assert first == second
|
||||
assert first.exists()
|
||||
+7
-7
@@ -1,10 +1,10 @@
|
||||
"""Tests for openjarvis.learning.distillation.pending_queue module."""
|
||||
"""Tests for openjarvis.learning.spec_search.pending_queue module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
@@ -27,7 +27,7 @@ def _make_edit(edit_id: str = "edit-001") -> Edit:
|
||||
|
||||
class TestPendingQueue:
|
||||
def test_enqueue_creates_file(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
|
||||
queue = PendingQueue(tmp_path / "pending_review")
|
||||
queue.enqueue("session-001", _make_edit())
|
||||
@@ -37,7 +37,7 @@ class TestPendingQueue:
|
||||
assert "edit-001" in files[0].name
|
||||
|
||||
def test_list_pending(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
|
||||
queue = PendingQueue(tmp_path / "pending_review")
|
||||
queue.enqueue("session-001", _make_edit("e1"))
|
||||
@@ -48,7 +48,7 @@ class TestPendingQueue:
|
||||
assert ids == {"e1", "e2"}
|
||||
|
||||
def test_resolve_removes_file(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
|
||||
queue = PendingQueue(tmp_path / "pending_review")
|
||||
queue.enqueue("session-001", _make_edit())
|
||||
@@ -57,13 +57,13 @@ class TestPendingQueue:
|
||||
assert len(queue.list_pending()) == 0
|
||||
|
||||
def test_list_empty_queue(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
|
||||
queue = PendingQueue(tmp_path / "pending_review")
|
||||
assert queue.list_pending() == []
|
||||
|
||||
def test_get_pending_edit(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.pending_queue import PendingQueue
|
||||
from openjarvis.learning.spec_search.pending_queue import PendingQueue
|
||||
|
||||
queue = PendingQueue(tmp_path / "pending_review")
|
||||
queue.enqueue("session-001", _make_edit())
|
||||
+9
-9
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.plan.planner module.
|
||||
"""Tests for openjarvis.learning.spec_search.plan.planner module.
|
||||
|
||||
All tests use mocked CloudEngine — no live API calls.
|
||||
"""
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
EditOp,
|
||||
EditRiskTier,
|
||||
FailureCluster,
|
||||
@@ -72,7 +72,7 @@ class TestLearningPlanner:
|
||||
"""Tests for LearningPlanner."""
|
||||
|
||||
def test_produces_learning_plan(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = _make_teacher_response([_make_edit_dict()])
|
||||
@@ -95,7 +95,7 @@ class TestLearningPlanner:
|
||||
assert plan.teacher_model == "claude-opus-4-6"
|
||||
|
||||
def test_assigns_risk_tiers(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
# Teacher incorrectly sets MANUAL for an auto-tier op
|
||||
@@ -119,7 +119,7 @@ class TestLearningPlanner:
|
||||
assert plan.edits[0].risk_tier == EditRiskTier.AUTO
|
||||
|
||||
def test_persists_plan_json(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = _make_teacher_response([_make_edit_dict()])
|
||||
@@ -143,7 +143,7 @@ class TestLearningPlanner:
|
||||
assert data["session_id"] == "session-001"
|
||||
|
||||
def test_drops_cluster_with_zero_rates(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = _make_teacher_response([_make_edit_dict()])
|
||||
@@ -188,7 +188,7 @@ class TestLearningPlanner:
|
||||
assert "dropped" not in good.skill_gap.lower()
|
||||
|
||||
def test_all_clusters_dropped_returns_empty_edits(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = _make_teacher_response([])
|
||||
@@ -220,7 +220,7 @@ class TestLearningPlanner:
|
||||
assert plan.failure_clusters[0].addressed_by_edit_ids == []
|
||||
|
||||
def test_persists_teacher_trace_jsonl(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = _make_teacher_response([_make_edit_dict()])
|
||||
@@ -246,7 +246,7 @@ class TestLearningPlanner:
|
||||
assert "cost_usd" in record
|
||||
|
||||
def test_handles_malformed_teacher_output(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.plan.planner import LearningPlanner
|
||||
from openjarvis.learning.spec_search.plan.planner import LearningPlanner
|
||||
|
||||
engine = MagicMock()
|
||||
engine.generate.return_value = {
|
||||
+16
-16
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.plan.prompt_diff module."""
|
||||
"""Tests for openjarvis.learning.spec_search.plan.prompt_diff module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,14 +9,14 @@ class TestChangedLineRatio:
|
||||
"""Tests for changed_line_ratio()."""
|
||||
|
||||
def test_identical_strings(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
changed_line_ratio,
|
||||
)
|
||||
|
||||
assert changed_line_ratio("hello\nworld\n", "hello\nworld\n") == 0.0
|
||||
|
||||
def test_completely_different(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
changed_line_ratio,
|
||||
)
|
||||
|
||||
@@ -24,7 +24,7 @@ class TestChangedLineRatio:
|
||||
assert ratio == 1.0
|
||||
|
||||
def test_partial_change(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
changed_line_ratio,
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ class TestChangedLineRatio:
|
||||
assert 0.2 <= ratio <= 0.35
|
||||
|
||||
def test_empty_original(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
changed_line_ratio,
|
||||
)
|
||||
|
||||
@@ -44,7 +44,7 @@ class TestChangedLineRatio:
|
||||
assert ratio == 1.0
|
||||
|
||||
def test_empty_both(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
changed_line_ratio,
|
||||
)
|
||||
|
||||
@@ -55,7 +55,7 @@ class TestApplyUnifiedDiff:
|
||||
"""Tests for apply_unified_diff()."""
|
||||
|
||||
def test_applies_simple_patch(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
apply_unified_diff,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ class TestApplyUnifiedDiff:
|
||||
assert result == "line1\nchanged_line2\nline3\n"
|
||||
|
||||
def test_returns_none_on_bad_diff(self) -> None:
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
apply_unified_diff,
|
||||
)
|
||||
|
||||
@@ -85,13 +85,13 @@ class TestMaybeDowngradeToReplace:
|
||||
"""Tests for maybe_downgrade_to_replace()."""
|
||||
|
||||
def test_non_patch_op_passes_through(self) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
maybe_downgrade_to_replace,
|
||||
)
|
||||
|
||||
@@ -109,13 +109,13 @@ class TestMaybeDowngradeToReplace:
|
||||
assert result.op == EditOp.SET_MODEL_FOR_QUERY_CLASS
|
||||
|
||||
def test_small_diff_stays_patch(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
maybe_downgrade_to_replace,
|
||||
)
|
||||
|
||||
@@ -145,13 +145,13 @@ class TestMaybeDowngradeToReplace:
|
||||
assert result.op == EditOp.PATCH_SYSTEM_PROMPT
|
||||
|
||||
def test_large_diff_downgrades_to_replace(self) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
maybe_downgrade_to_replace,
|
||||
)
|
||||
|
||||
@@ -185,13 +185,13 @@ class TestMaybeDowngradeToReplace:
|
||||
assert "new1" in result.payload["new_content"]
|
||||
|
||||
def test_bad_diff_downgrades_to_replace_with_raw(self) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.prompt_diff import (
|
||||
from openjarvis.learning.spec_search.plan.prompt_diff import (
|
||||
maybe_downgrade_to_replace,
|
||||
)
|
||||
|
||||
+8
-8
@@ -1,8 +1,8 @@
|
||||
"""Tests for openjarvis.learning.distillation.gate.regression module."""
|
||||
"""Tests for openjarvis.learning.spec_search.gate.regression module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openjarvis.learning.distillation.models import BenchmarkSnapshot
|
||||
from openjarvis.learning.spec_search.models import BenchmarkSnapshot
|
||||
|
||||
|
||||
def _make_snapshot(
|
||||
@@ -22,7 +22,7 @@ class TestRegressionCheck:
|
||||
"""Tests for regression_check()."""
|
||||
|
||||
def test_no_regression_when_all_improve(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
@@ -32,7 +32,7 @@ class TestRegressionCheck:
|
||||
assert not result.has_regression
|
||||
|
||||
def test_detects_cluster_regression(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
@@ -43,7 +43,7 @@ class TestRegressionCheck:
|
||||
assert "c2" in result.regressed_clusters
|
||||
|
||||
def test_small_drop_within_threshold(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
@@ -53,7 +53,7 @@ class TestRegressionCheck:
|
||||
assert not result.has_regression
|
||||
|
||||
def test_new_cluster_in_after_not_flagged(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
@@ -63,7 +63,7 @@ class TestRegressionCheck:
|
||||
assert not result.has_regression
|
||||
|
||||
def test_missing_cluster_in_after_flagged(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
@@ -75,7 +75,7 @@ class TestRegressionCheck:
|
||||
assert "c2" in result.regressed_clusters
|
||||
|
||||
def test_result_has_details(self) -> None:
|
||||
from openjarvis.learning.distillation.gate.regression import (
|
||||
from openjarvis.learning.spec_search.gate.regression import (
|
||||
regression_check,
|
||||
)
|
||||
|
||||
+23
-23
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.plan.risk_tier module."""
|
||||
"""Tests for openjarvis.learning.spec_search.plan.risk_tier module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,15 +7,15 @@ class TestTierTable:
|
||||
"""Tests for TIER_TABLE completeness."""
|
||||
|
||||
def test_every_edit_op_has_a_tier(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.distillation.plan.risk_tier import TIER_TABLE
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import TIER_TABLE
|
||||
|
||||
for op in EditOp:
|
||||
assert op in TIER_TABLE, f"Missing tier for {op}"
|
||||
|
||||
def test_no_extra_keys(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp
|
||||
from openjarvis.learning.distillation.plan.risk_tier import TIER_TABLE
|
||||
from openjarvis.learning.spec_search.models import EditOp
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import TIER_TABLE
|
||||
|
||||
for key in TIER_TABLE:
|
||||
assert key in EditOp, f"Extra key in TIER_TABLE: {key}"
|
||||
@@ -25,48 +25,48 @@ class TestAssignTier:
|
||||
"""Tests for assign_tier() function."""
|
||||
|
||||
def test_intelligence_ops_are_auto(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.SET_MODEL_FOR_QUERY_CLASS) == EditRiskTier.AUTO
|
||||
assert assign_tier(EditOp.SET_MODEL_PARAM) == EditRiskTier.AUTO
|
||||
|
||||
def test_tool_ops_are_auto(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.ADD_TOOL_TO_AGENT) == EditRiskTier.AUTO
|
||||
assert assign_tier(EditOp.REMOVE_TOOL_FROM_AGENT) == EditRiskTier.AUTO
|
||||
assert assign_tier(EditOp.EDIT_TOOL_DESCRIPTION) == EditRiskTier.AUTO
|
||||
|
||||
def test_agent_param_is_auto(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.SET_AGENT_PARAM) == EditRiskTier.AUTO
|
||||
|
||||
def test_prompt_ops_are_review(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.PATCH_SYSTEM_PROMPT) == EditRiskTier.REVIEW
|
||||
assert assign_tier(EditOp.REPLACE_SYSTEM_PROMPT) == EditRiskTier.REVIEW
|
||||
|
||||
def test_agent_class_is_review(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.SET_AGENT_CLASS) == EditRiskTier.REVIEW
|
||||
|
||||
def test_few_shot_is_review(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.EDIT_FEW_SHOT_EXEMPLARS) == EditRiskTier.REVIEW
|
||||
|
||||
def test_lora_is_manual(self) -> None:
|
||||
from openjarvis.learning.distillation.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tier
|
||||
from openjarvis.learning.spec_search.models import EditOp, EditRiskTier
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tier
|
||||
|
||||
assert assign_tier(EditOp.LORA_FINETUNE) == EditRiskTier.MANUAL
|
||||
|
||||
@@ -75,13 +75,13 @@ class TestAssignTiers:
|
||||
"""Tests for assign_tiers() batch function."""
|
||||
|
||||
def test_overwrites_teacher_tier(self) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tiers
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers
|
||||
|
||||
# Teacher incorrectly sets AUTO for a prompt edit
|
||||
edit = Edit(
|
||||
@@ -98,13 +98,13 @@ class TestAssignTiers:
|
||||
assert result[0].risk_tier == EditRiskTier.REVIEW
|
||||
|
||||
def test_preserves_correct_tier(self) -> None:
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
Edit,
|
||||
EditOp,
|
||||
EditPillar,
|
||||
EditRiskTier,
|
||||
)
|
||||
from openjarvis.learning.distillation.plan.risk_tier import assign_tiers
|
||||
from openjarvis.learning.spec_search.plan.risk_tier import assign_tiers
|
||||
|
||||
edit = Edit(
|
||||
id="edit-002",
|
||||
+2
-2
@@ -25,7 +25,7 @@ def _setup_config_tree(root: Path) -> None:
|
||||
|
||||
class TestRollbackIntegration:
|
||||
def test_rollback_restores_files(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ class TestRollbackIntegration:
|
||||
assert log_count >= 5 # baseline + 2 edits + 2 reverts
|
||||
|
||||
def test_rollback_nonexistent_session(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.checkpoint.store import (
|
||||
from openjarvis.learning.spec_search.checkpoint.store import (
|
||||
CheckpointStore,
|
||||
)
|
||||
|
||||
+14
-14
@@ -1,11 +1,11 @@
|
||||
"""Tests for openjarvis.learning.distillation.storage.session_store module."""
|
||||
"""Tests for openjarvis.learning.spec_search.storage.session_store module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.learning.distillation.models import (
|
||||
from openjarvis.learning.spec_search.models import (
|
||||
AutonomyMode,
|
||||
BenchmarkSnapshot,
|
||||
EditOutcome,
|
||||
@@ -87,7 +87,7 @@ class TestSessionStoreInit:
|
||||
"""Tests for SessionStore initialization."""
|
||||
|
||||
def test_creates_tables(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -100,7 +100,7 @@ class TestSessionStoreInit:
|
||||
store.close()
|
||||
|
||||
def test_idempotent_init(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ class TestSessionStoreSaveAndGet:
|
||||
"""Tests for save_session/get_session round-trip."""
|
||||
|
||||
def test_round_trip(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -129,7 +129,7 @@ class TestSessionStoreSaveAndGet:
|
||||
store.close()
|
||||
|
||||
def test_update_existing_session(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -147,7 +147,7 @@ class TestSessionStoreSaveAndGet:
|
||||
store.close()
|
||||
|
||||
def test_get_returns_none_for_unknown(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -160,7 +160,7 @@ class TestSessionStoreList:
|
||||
"""Tests for list_sessions ordering and filters."""
|
||||
|
||||
def test_lists_in_started_at_desc_order(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -179,7 +179,7 @@ class TestSessionStoreList:
|
||||
store.close()
|
||||
|
||||
def test_filter_by_status(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -196,7 +196,7 @@ class TestSessionStoreList:
|
||||
store.close()
|
||||
|
||||
def test_limit(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -219,7 +219,7 @@ class TestEditOutcomes:
|
||||
"""Tests for save_outcome / list_outcomes."""
|
||||
|
||||
def test_round_trip(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -238,7 +238,7 @@ class TestEditOutcomes:
|
||||
store.close()
|
||||
|
||||
def test_multiple_outcomes_per_session(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -256,7 +256,7 @@ class TestEditOutcomes:
|
||||
store.close()
|
||||
|
||||
def test_outcomes_for_unknown_session_returns_empty(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
@@ -269,7 +269,7 @@ class TestParentSessionChain:
|
||||
"""Tests for parent_session_id foreign key."""
|
||||
|
||||
def test_parent_id_round_trips(self, tmp_path: Path) -> None:
|
||||
from openjarvis.learning.distillation.storage.session_store import (
|
||||
from openjarvis.learning.spec_search.storage.session_store import (
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
+9
-9
@@ -1,4 +1,4 @@
|
||||
"""Tests for openjarvis.learning.distillation.diagnose.teacher_agent module.
|
||||
"""Tests for openjarvis.learning.spec_search.diagnose.teacher_agent module.
|
||||
|
||||
All tests use a mocked CloudEngine — no live API calls.
|
||||
"""
|
||||
@@ -8,7 +8,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from openjarvis.learning.distillation.diagnose.types import DiagnosticTool
|
||||
from openjarvis.learning.spec_search.diagnose.types import DiagnosticTool
|
||||
|
||||
|
||||
def _make_tool(
|
||||
@@ -44,7 +44,7 @@ class TestTeacherAgentNoTools:
|
||||
"""Teacher responds without using any tools."""
|
||||
|
||||
def test_returns_content_from_single_turn(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -67,7 +67,7 @@ class TestTeacherAgentNoTools:
|
||||
assert result.total_cost_usd > 0
|
||||
|
||||
def test_tracks_cost(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -91,7 +91,7 @@ class TestTeacherAgentWithTools:
|
||||
"""Teacher uses tools in a multi-turn loop."""
|
||||
|
||||
def test_executes_tool_call_and_continues(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -129,7 +129,7 @@ class TestTeacherAgentWithTools:
|
||||
assert result.tool_call_records[0].tool == "test_tool"
|
||||
|
||||
def test_stops_at_max_turns(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -159,7 +159,7 @@ class TestTeacherAgentWithTools:
|
||||
assert result.turns == 3
|
||||
|
||||
def test_stops_at_max_cost(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -192,7 +192,7 @@ class TestTeacherAgentWithTools:
|
||||
assert result.turns < 100
|
||||
|
||||
def test_multiple_tool_calls_in_one_turn(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
@@ -227,7 +227,7 @@ class TestTeacherAgentResult:
|
||||
"""Tests for TeacherAgentResult structure."""
|
||||
|
||||
def test_result_has_all_fields(self) -> None:
|
||||
from openjarvis.learning.distillation.diagnose.teacher_agent import (
|
||||
from openjarvis.learning.spec_search.diagnose.teacher_agent import (
|
||||
TeacherAgent,
|
||||
)
|
||||
|
||||
+6
-6
@@ -1,18 +1,18 @@
|
||||
"""Tests for openjarvis.learning.distillation.triggers module."""
|
||||
"""Tests for openjarvis.learning.spec_search.triggers module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class TestOnDemandTrigger:
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.triggers import OnDemandTrigger
|
||||
from openjarvis.learning.spec_search.triggers import OnDemandTrigger
|
||||
|
||||
t = OnDemandTrigger()
|
||||
assert t.kind.value == "on_demand"
|
||||
assert t.metadata == {}
|
||||
|
||||
def test_metadata(self) -> None:
|
||||
from openjarvis.learning.distillation.triggers import OnDemandTrigger
|
||||
from openjarvis.learning.spec_search.triggers import OnDemandTrigger
|
||||
|
||||
t = OnDemandTrigger(metadata={"source": "cli"})
|
||||
assert t.metadata["source"] == "cli"
|
||||
@@ -20,7 +20,7 @@ class TestOnDemandTrigger:
|
||||
|
||||
class TestUserFlagTrigger:
|
||||
def test_constructs_with_trace_id(self) -> None:
|
||||
from openjarvis.learning.distillation.triggers import UserFlagTrigger
|
||||
from openjarvis.learning.spec_search.triggers import UserFlagTrigger
|
||||
|
||||
t = UserFlagTrigger(trace_id="trace-001")
|
||||
assert t.kind.value == "user_flag"
|
||||
@@ -30,7 +30,7 @@ class TestUserFlagTrigger:
|
||||
|
||||
class TestScheduledTrigger:
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.triggers import ScheduledTrigger
|
||||
from openjarvis.learning.spec_search.triggers import ScheduledTrigger
|
||||
|
||||
t = ScheduledTrigger(cron="0 3 * * *", new_trace_count=25)
|
||||
assert t.kind.value == "scheduled"
|
||||
@@ -40,7 +40,7 @@ class TestScheduledTrigger:
|
||||
|
||||
class TestClusterTrigger:
|
||||
def test_constructs(self) -> None:
|
||||
from openjarvis.learning.distillation.triggers import ClusterTrigger
|
||||
from openjarvis.learning.spec_search.triggers import ClusterTrigger
|
||||
|
||||
t = ClusterTrigger(
|
||||
cluster_description="math failures",
|
||||
Reference in New Issue
Block a user