feat(evals): evaluation pipeline improvements and DeepResearch benchmark

- Add resume functionality to skip completed samples in eval runs
- Store problem/reference fields in all EvalResult records for traceability
- Add DeepResearch benchmark dataset and LLM-as-judge scorer
- Improve TerminalBench native backend with better error handling
- Add eval_cmd.py for CLI evaluation commands
- Enhance cloud engine and backend implementations
- Always include trace_data field for consistent output format
- Copy config file to results directory for reproducibility

src/openjarvis/agents/_stubs.py

@@ -72,6 +72,9 @@ class BaseAgent(ABC):
         self._model = model
         self._bus = bus
         self._prompt_builder = prompt_builder
+        self._total_prompt_tokens: int = 0
+        self._total_completion_tokens: int = 0
+        self._total_cost_usd: float = 0.0
 
         # Three-tier resolution: explicit arg > config > class default > hardcoded
         if temperature is not None and max_tokens is not None:
@@ -183,8 +186,12 @@ class BaseAgent(ABC):
             **extra_kwargs,
         )
 
+        usage = result.get("usage", {})
+        self._total_prompt_tokens += usage.get("prompt_tokens", 0)
+        self._total_completion_tokens += usage.get("completion_tokens", 0)
+        self._total_cost_usd += result.get("cost_usd", 0.0)
+
         if self._bus and not getattr(self._engine, "_publishes_events", False):
-            usage = result.get("usage", {})
             self._bus.publish(
                 EventType.INFERENCE_END,
                 {

src/openjarvis/cli/eval_cmd.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import os
 import sys
 from pathlib import Path
 from typing import Optional
@@ -11,6 +12,128 @@ import click
 from rich.console import Console
 from rich.table import Table
 
+# ---------------------------------------------------------------------------
+# API key validation helpers
+# ---------------------------------------------------------------------------
+
+_ANTHROPIC_PREFIXES = ("claude-",)
+_OPENAI_PREFIXES = ("gpt-", "o1-", "o3-", "o4-")
+_GOOGLE_PREFIXES = ("gemini-",)
+_MINIMAX_PREFIXES = ("minimax",)
+
+_PROVIDER_ENV = {
+    "anthropic": "ANTHROPIC_API_KEY",
+    "openai": "OPENAI_API_KEY",
+    "google": "GEMINI_API_KEY",
+    "minimax": "MINIMAX_API_KEY",
+}
+
+# Cheapest model per provider used for the live ping
+_PROBE_MODELS = {
+    "anthropic": "claude-haiku-4-5-20251001",
+    "openai": "gpt-4o-mini",
+    "google": "gemini-3.1-flash-lite-preview",
+}
+
+
+def _provider_for_model(model_name: str) -> Optional[str]:
+    m = model_name.lower()
+    if any(m.startswith(p) for p in _ANTHROPIC_PREFIXES):
+        return "anthropic"
+    if any(m.startswith(p) for p in _OPENAI_PREFIXES):
+        return "openai"
+    if any(m.startswith(p) for p in _GOOGLE_PREFIXES):
+        return "google"
+    if any(m.startswith(p) for p in _MINIMAX_PREFIXES):
+        return "minimax"
+    return None
+
+
+def _live_ping(provider: str, key: str) -> tuple[bool, str]:
+    """Make the smallest possible call to verify a key is accepted."""
+    try:
+        if provider == "anthropic":
+            import anthropic  # type: ignore[import]
+            anthropic.Anthropic(api_key=key).messages.create(
+                model=_PROBE_MODELS["anthropic"],
+                max_tokens=1,
+                messages=[{"role": "user", "content": "hi"}],
+            )
+        elif provider == "openai":
+            import openai  # type: ignore[import]
+            openai.OpenAI(api_key=key).chat.completions.create(
+                model=_PROBE_MODELS["openai"],
+                max_tokens=1,
+                messages=[{"role": "user", "content": "hi"}],
+            )
+        elif provider == "google":
+            from google import genai  # type: ignore[import]
+            with genai.Client(api_key=key) as _client:
+                _client.models.generate_content(
+                    model=_PROBE_MODELS["google"],
+                    contents="hi",
+                )
+        else:
+            # No live check available for this provider — trust the key is set
+            return True, "key set (no live probe)"
+        return True, "ok"
+    except Exception as exc:  # noqa: BLE001
+        return False, str(exc)[:100]
+
+
+def _resolve_key(provider: str) -> Optional[str]:
+    key = os.environ.get(_PROVIDER_ENV.get(provider, ""))
+    if not key and provider == "google":
+        key = os.environ.get("GOOGLE_API_KEY")
+    return key or None
+
+
+def check_api_keys(providers_needed: set[str], console: Console) -> None:
+    """Verify API keys for every required provider.
+
+    Prints a status table, then aborts if any key is missing or rejected.
+    """
+    table = Table(
+        title="[bold]API Key Check[/bold]",
+        border_style="bright_blue",
+        title_style="bold cyan",
+    )
+    table.add_column("Provider", style="cyan", no_wrap=True)
+    table.add_column("Env Var", style="dim")
+    table.add_column("Set", justify="center")
+    table.add_column("Live Test", justify="center")
+    table.add_column("Details", style="dim")
+
+    all_ok = True
+    for provider in sorted(providers_needed):
+        env_var = _PROVIDER_ENV.get(provider, "?")
+        key = _resolve_key(provider)
+
+        if not key:
+            table.add_row(
+                provider, env_var, "[red]✗[/red]", "[red]–[/red]", "not set"
+            )
+            all_ok = False
+        else:
+            ok, detail = _live_ping(provider, key)
+            set_cell = "[green]✓[/green]"
+            live_cell = "[green]✓[/green]" if ok else "[red]✗[/red]"
+            if not ok:
+                all_ok = False
+            table.add_row(provider, env_var, set_cell, live_cell, detail)
+
+    console.print(table)
+
+    if not all_ok:
+        console.print(
+            "[red bold]One or more required API keys are missing or invalid. "
+            "Set the env vars above and retry.[/red bold]"
+        )
+        sys.exit(1)
+
+    console.print("[green]All required API keys verified.[/green]")
+    # click.confirm("\nProceed with evaluation?", default=True, abort=True)  # Disabled for non-interactive runs
+
 # Known benchmarks and backends — mirrored from the evals framework so the
 # CLI can display them even when the (optional) evals package is not installed.
 KNOWN_BENCHMARKS = {
@@ -311,6 +434,23 @@ def eval_run(
             f"{len(suite.benchmarks)} benchmark(s) = {len(run_configs)} run(s)"
         )
 
+        # Collect all cloud providers needed (models + judge)
+        cloud_providers: set[str] = set()
+        for rc in run_configs:
+            if rc.engine_key in (None, "cloud"):
+                p = _provider_for_model(rc.model)
+                if p:
+                    cloud_providers.add(p)
+            # Judge always runs in the cloud
+            p = _provider_for_model(rc.judge_model)
+            if p:
+                cloud_providers.add(p)
+
+        if cloud_providers:
+            console.print()
+            check_api_keys(cloud_providers, console)
+            console.print()
+
         try:
             from openjarvis.evals.cli import _run_single
         except ImportError:
@@ -377,6 +517,19 @@ def eval_run(
         sheets_credentials_path=sheets_credentials_path,
     )
 
+    # Determine which providers are needed for this single run
+    single_providers: set[str] = set()
+    if engine_key in (None, "cloud"):
+        p = _provider_for_model(model)
+        if p:
+            single_providers.add(p)
+    # Judge defaults to gpt-5-mini (OpenAI)
+    single_providers.add("openai")
+
+    console.print()
+    check_api_keys(single_providers, console)
+    console.print()
+
     try:
         from openjarvis.evals.cli import _run_single
 

src/openjarvis/engine/_openai_compat.py

@@ -88,8 +88,10 @@ class _OpenAICompatibleEngine(InferenceEngine):
         estimated_prompt = estimate_prompt_tokens(messages)
         prompt_tokens = max(reported_prompt, estimated_prompt)
         completion_tokens = usage.get("completion_tokens", 0)
+        # Handle reasoning models (DeepSeek R1, etc.) that put output in "reasoning" field
+        message_content = choice["message"].get("content") or choice["message"].get("reasoning") or ""
         result: Dict[str, Any] = {
-            "content": choice["message"].get("content") or "",
+            "content": message_content,
             "usage": {
                 "prompt_tokens": prompt_tokens,
                 "prompt_tokens_evaluated": reported_prompt or prompt_tokens,

src/openjarvis/engine/cloud.py

@@ -627,36 +627,37 @@ class CloudEngine(InferenceEngine):
                 "GEMINI_API_KEY or GOOGLE_API_KEY and install "
                 "openjarvis[inference-google]"
             )
-        # Build contents from messages, converting tool roles for Gemini
+        # Build contents from messages using native genai types for SDK v1+
+        from google.genai import types as _gt
+
         system_text = ""
-        contents: List[Dict[str, Any]] = []
+        contents: List[Any] = []
         for m in messages:
             if m.role.value == "system":
                 system_text = m.content
             elif m.role.value == "tool":
-                # Gemini expects function responses as role="user" with
-                # function_response parts
-                fn_resp_part = {
-                    "function_response": {
-                        "name": m.name or "unknown",
-                        "response": {"result": m.content},
-                    }
-                }
-                # Merge consecutive tool results into a single user message
+                # Gemini expects function responses as role="user"
+                fn_part = _gt.Part(
+                    function_response=_gt.FunctionResponse(
+                        name=m.name or "unknown",
+                        response={"result": m.content},
+                    )
+                )
+                # Merge consecutive tool results into a single user Content
                 if (
                     contents
-                    and contents[-1]["role"] == "user"
-                    and contents[-1]["parts"]
-                    and "function_response" in contents[-1]["parts"][-1]
+                    and contents[-1].role == "user"
+                    and contents[-1].parts
+                    and contents[-1].parts[-1].function_response is not None
                 ):
-                    contents[-1]["parts"].append(fn_resp_part)
+                    contents[-1].parts.append(fn_part)
                 else:
-                    contents.append({"role": "user", "parts": [fn_resp_part]})
+                    contents.append(_gt.Content(role="user", parts=[fn_part]))
             elif m.role.value == "assistant" and m.tool_calls:
                 # Convert assistant tool_calls to function_call parts
-                parts: List[Dict[str, Any]] = []
+                parts: List[Any] = []
                 if m.content:
-                    parts.append({"text": m.content})
+                    parts.append(_gt.Part(text=m.content))
                 for tc in m.tool_calls:
                     args = tc.arguments
                     if isinstance(args, str):
@@ -664,22 +665,25 @@ class CloudEngine(InferenceEngine):
                             args = json.loads(args)
                         except (json.JSONDecodeError, TypeError):
                             args = {"input": args}
-                    fc_part: Dict[str, Any] = {
-                        "function_call": {
-                            "name": tc.name,
-                            "args": args if isinstance(args, dict) else {},
-                        }
+                    fc_part_kwargs: Dict[str, Any] = {
+                        "function_call": _gt.FunctionCall(
+                            name=tc.name,
+                            args=args if isinstance(args, dict) else {},
+                        )
                     }
-                    # Replay thought_signature for Gemini reasoning models
                     sig = self._thought_sigs.get(tc.id)
                     if sig is not None:
-                        fc_part["thought_signature"] = sig
-                    parts.append(fc_part)
-                contents.append({"role": "model", "parts": parts})
+                        fc_part_kwargs["thought_signature"] = sig
+                    parts.append(_gt.Part(**fc_part_kwargs))
+                contents.append(_gt.Content(role="model", parts=parts))
             elif m.role.value == "assistant":
-                contents.append({"role": "model", "parts": [{"text": m.content}]})
+                contents.append(
+                    _gt.Content(role="model", parts=[_gt.Part(text=m.content or "")])
+                )
             else:
-                contents.append({"role": "user", "parts": [{"text": m.content}]})
+                contents.append(
+                    _gt.Content(role="user", parts=[_gt.Part(text=m.content or "")])
+                )
 
         from google.genai import types as genai_types
 
@@ -727,13 +731,16 @@ class CloudEngine(InferenceEngine):
                     tc_dict: Dict[str, Any] = {
                         "id": f"google_{fc.name}",
                         "name": fc.name,
-                        "arguments": json.dumps(fc_args),
+                        "arguments": json.dumps(fc_args, default=str),
                     }
                     # Preserve thought_signature for Gemini reasoning models
                     sig = getattr(part, "thought_signature", None)
                     if sig is not None:
-                        tc_dict["thought_signature"] = sig
+                        # Keep raw bytes in _thought_sigs for replay; store
+                        # base64 string in tc_dict for JSON serialization
                         self._thought_sigs[tc_dict["id"]] = sig
+                        import base64
+                        tc_dict["thought_signature"] = base64.b64encode(sig).decode() if isinstance(sig, bytes) else str(sig)
                     tool_calls.append(tc_dict)
                 elif hasattr(part, "text") and part.text:
                     text_parts.append(part.text)

src/openjarvis/evals/backends/jarvis_agent.py

@@ -21,6 +21,7 @@ class JarvisAgentBackend(InferenceBackend):
     def __init__(
         self,
         engine_key: Optional[str] = None,
+        engine_config: Optional[Dict[str, Any]] = None,
         agent_name: str = "orchestrator",
         tools: Optional[List[str]] = None,
         telemetry: bool = False,
@@ -40,6 +41,16 @@ class JarvisAgentBackend(InferenceBackend):
         builder = SystemBuilder()
         if engine_key:
             builder.engine(engine_key)
+
+        # Apply engine-specific config overrides from eval TOML
+        if engine_config and engine_key:
+            # Apply engine config to the builder's config
+            # For vllm: config.engine.vllm_host
+            # For ollama: config.engine.ollama_host, etc.
+            host_attr = f"{engine_key}_host"
+            if "host" in engine_config:
+                setattr(builder._config.engine, host_attr, engine_config["host"])
+
         if model:
             builder.model(model)
         builder.agent(agent_name)

src/openjarvis/evals/backends/jarvis_direct.py

@@ -20,6 +20,7 @@ class JarvisDirectBackend(InferenceBackend):
     def __init__(
         self,
         engine_key: Optional[str] = None,
+        engine_config: Optional[Dict[str, Any]] = None,
         telemetry: bool = False,
         gpu_metrics: bool = False,
     ) -> None:
@@ -31,11 +32,22 @@ class JarvisDirectBackend(InferenceBackend):
         builder = SystemBuilder()
         if engine_key:
             builder.engine(engine_key)
+
+        # Apply engine-specific config overrides from eval TOML
+        if engine_config and engine_key:
+            # Apply engine config to the builder's config
+            # For vllm: config.engine.vllm_host
+            # For ollama: config.engine.ollama_host, etc.
+            host_attr = f"{engine_key}_host"
+            if "host" in engine_config:
+                setattr(builder._config.engine, host_attr, engine_config["host"])
+
         # Propagate gpu_metrics to the runtime config so SystemBuilder
         # creates an EnergyMonitor / GpuMonitor for the InstrumentedEngine.
         if gpu_metrics:
             builder._config.telemetry.gpu_metrics = True
-        self._system = builder.telemetry(telemetry).traces(telemetry).build()
+        # Always enable traces to collect trace_data for eval analysis
+        self._system = builder.telemetry(telemetry).traces(True).build()
 
     def generate(
         self,
@@ -80,6 +92,34 @@ class JarvisDirectBackend(InferenceBackend):
         )
         elapsed = time.monotonic() - t0
 
+        # Extract trace data from the TraceCollector if available
+        trace_data = None
+        collector = getattr(self._system, "trace_collector", None)
+        if collector is not None:
+            trace = getattr(collector, "last_trace", None)
+            if trace is not None:
+                trace_data = {
+                    "trace_id": trace.trace_id,
+                    "steps": [
+                        {
+                            "step_type": (
+                                step.step_type.value
+                                if hasattr(step.step_type, "value")
+                                else step.step_type
+                            ),
+                            "timestamp": step.timestamp,
+                            "duration_seconds": step.duration_seconds,
+                            "input": step.input,
+                            "output": step.output,
+                            "metadata": step.metadata,
+                        }
+                        for step in trace.steps
+                    ],
+                    "messages": trace.messages,
+                    "total_tokens": trace.total_tokens,
+                    "total_latency_seconds": trace.total_latency_seconds,
+                }
+
         usage = result.get("usage", {})
         telemetry_data = result.get("_telemetry", {})
         return {
@@ -93,6 +133,7 @@ class JarvisDirectBackend(InferenceBackend):
             "power_watts": telemetry_data.get("power_watts", 0.0),
             "gpu_utilization_pct": telemetry_data.get("gpu_utilization_pct", 0.0),
             "throughput_tok_per_sec": telemetry_data.get("throughput_tok_per_sec", 0.0),
+            "trace_data": trace_data,
         }
 
     def close(self) -> None:

src/openjarvis/evals/backends/terminalbench_native.py

@@ -1,11 +1,15 @@
-"""Native terminal-bench v2 backend.
+"""Native terminal-bench backend.
 
-Uses Harness for Docker-based execution and scoring.
+Each call to generate_full() runs a single terminal-bench task through the
+official Harness (Docker + terminus-2 agent + test scripts), then stores
+is_resolved in the record metadata for the scorer to read.
 """
 
 from __future__ import annotations
 
 import logging
+import re
+import time
 from pathlib import Path
 from typing import Any, Dict, Optional
 
@@ -15,107 +19,182 @@ LOGGER = logging.getLogger(__name__)
 
 try:
     from terminal_bench import BenchmarkResults, Harness
+    from terminal_bench.agents.agent_name import AgentName
 
     _HAS_TB = True
 except ImportError:
     _HAS_TB = False
 
+# LiteLLM model prefix and api_base by engine type.
+_ENGINE_DEFAULTS: Dict[str, Dict[str, Any]] = {
+    "vllm": {"prefix": "openai/", "api_base": "http://localhost:8000/v1"},
+    "ollama": {"prefix": "openai/", "api_base": "http://localhost:11434/v1"},
+    "cloud": {"prefix": "", "api_base": None},
+}
+
+
+def _litellm_model(model: str, engine_key: str) -> tuple[str, Optional[str]]:
+    """Return (litellm_model_string, api_base) for the given engine."""
+    cfg = _ENGINE_DEFAULTS.get(engine_key, _ENGINE_DEFAULTS["cloud"])
+    return cfg["prefix"] + model, cfg["api_base"]
+
+
+def _run_id(task_id: str) -> str:
+    """Docker-safe run_id from task_id (lowercase alphanumeric + hyphens)."""
+    return re.sub(r"[^a-z0-9-]", "-", f"oj-{task_id}".lower())[:60]
+
 
 class TerminalBenchNativeBackend(InferenceBackend):
     """Runs terminal-bench tasks natively via Harness with Docker execution.
 
-    Uses terminal-bench's own agent + LiteLLM to call the model,
-    Docker containers for task execution, and built-in test scripts
-    for scoring. This gives real agentic evaluation, not text-only.
+    generate_full() runs one task per call through the terminal-bench Harness
+    (terminus-2 agent + Docker containers + test scripts), writing is_resolved
+    into the record metadata so TerminalBenchNativeScorer can read it.
+
+    EvalRunner must call set_current_record(record) immediately before each
+    generate_full() call so the backend knows which task directory to target.
     """
 
     backend_id = "terminalbench-native"
 
     def __init__(
         self,
-        model: str = "openai/default",
-        api_base: str = "http://localhost:8000/v1",
-        temperature: float = 0.2,
-        agent_name: str = "naive",
-        output_dir: str = "results/terminalbench/",
-        max_samples: Optional[int] = None,
+        model: str = "claude-3-5-haiku-20241022",
+        engine_key: str = "cloud",
+        output_dir: str = "results/terminalbench-native/",
+        temperature: float = 0.6,
         dataset_name: str = "terminal-bench-core",
         dataset_version: str = "0.1.1",
-        system_prompt: str = "",
-        max_tokens: int = 16384,
+        max_samples: Optional[int] = None,
         n_concurrent: int = 4,
     ) -> None:
         if not _HAS_TB:
             raise ImportError("terminal-bench is required: pip install terminal-bench")
 
-        self._model = model
-        self._api_base = api_base
+        self._litellm_model, self._api_base = _litellm_model(model, engine_key)
         self._temperature = temperature
-        self._agent_name = agent_name
         self._output_dir = Path(output_dir)
-        self._max_samples = max_samples
         self._dataset_name = dataset_name
         self._dataset_version = dataset_version
-        self._system_prompt = system_prompt
-        self._max_tokens = max_tokens
+        self._max_samples = max_samples
         self._n_concurrent = n_concurrent
-        self._results: Optional[BenchmarkResults] = None
+        self._current_record: Any = None
 
-    def run_harness(self, run_id: str) -> BenchmarkResults:
-        """Run the full terminal-bench harness and return results."""
-        output_path = self._output_dir / run_id
-        output_path.mkdir(parents=True, exist_ok=True)
+    # ------------------------------------------------------------------
+    # Pipeline integration
+    # ------------------------------------------------------------------
 
-        harness_kwargs: Dict[str, Any] = {
-            "output_path": output_path,
-            "run_id": run_id,
-            "dataset_name": self._dataset_name,
-            "dataset_version": self._dataset_version,
-            "model_name": self._model,
-            "n_concurrent_trials": self._n_concurrent,
-            "cleanup": True,
-        }
-
-        # Use terminus-2 agent which accepts model_name + api_base as
-        # serializable strings (avoids Pydantic serialization issues with
-        # LLM objects in the harness lock file).
-        from terminal_bench.agents.agent_name import AgentName
-
-        harness_kwargs["agent_name"] = AgentName("terminus-2")
-        harness_kwargs["agent_kwargs"] = {
-            "model_name": self._model,
-            "api_base": self._api_base,
-            "temperature": self._temperature,
-        }
-
-        if self._max_samples is not None:
-            harness_kwargs["n_tasks"] = self._max_samples
-
-        harness = Harness(**harness_kwargs)
-        self._results = harness.run()
-        return self._results
-
-    def generate(
-        self,
-        prompt: str,
-        *,
-        model: str,
-        system: str = "",
-        temperature: float = 0.0,
-        max_tokens: int = 2048,
-    ) -> str:
-        return ""
+    def set_current_record(self, record: Any) -> None:
+        """Called by EvalRunner before each generate_full() to identify the task."""
+        self._current_record = record
 
     def generate_full(
         self,
         prompt: str,
         *,
-        model: str,
+        model: str = "",
         system: str = "",
         temperature: float = 0.0,
         max_tokens: int = 2048,
+        **kwargs: Any,
     ) -> Dict[str, Any]:
-        return {"content": "", "usage": {}, "model": model, "latency_seconds": 0.0}
+        """Run a single terminal-bench task through the official Harness.
+
+        Requires set_current_record() to have been called first so we know
+        the task directory. Stores is_resolved + test_results in the record
+        metadata for the scorer.
+        """
+        if self._current_record is None:
+            LOGGER.warning("set_current_record() not called; returning empty result.")
+            return {"content": "", "usage": {}, "latency_seconds": 0.0}
+
+        task_dir = Path(self._current_record.metadata["task_dir"])
+        task_id = task_dir.name
+
+        # Use caller-supplied temperature if non-zero, else fall back to default.
+        effective_temp = temperature if temperature else self._temperature
+        # Use caller-supplied model if provided and non-empty.
+        effective_model = model or self._litellm_model
+
+        output_path = self._output_dir / "trials"
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        harness = Harness(
+            output_path=output_path,
+            run_id=_run_id(task_id),
+            dataset_path=task_dir.parent,
+            task_ids=[task_id],
+            model_name=effective_model,
+            agent_name=AgentName("terminus-2"),
+            agent_kwargs={
+                "model_name": effective_model,
+                "api_base": self._api_base,
+                "temperature": effective_temp,
+            },
+            n_concurrent_trials=1,
+            n_attempts=1,
+            cleanup=True,
+        )
+
+        t0 = time.monotonic()
+        results = harness.run()
+        latency = time.monotonic() - t0
+
+        trial = results.results[0] if results.results else None
+        is_resolved = bool(trial.is_resolved) if trial else False
+
+        # Write into record metadata so TerminalBenchNativeScorer can read it.
+        self._current_record.metadata["is_resolved"] = is_resolved
+        self._current_record.metadata["test_results"] = {
+            "parser_results": (
+                {k: v.value for k, v in (trial.parser_results or {}).items()}
+                if trial else {}
+            ),
+            "failure_mode": (
+                trial.failure_mode.value
+                if trial and trial.failure_mode else None
+            ),
+        }
+
+        return {
+            "content": "",
+            "usage": {
+                "prompt_tokens": (trial.total_input_tokens or 0) if trial else 0,
+                "completion_tokens": (trial.total_output_tokens or 0) if trial else 0,
+            },
+            "latency_seconds": latency,
+            "cost_usd": 0.0,
+        }
+
+    def generate(self, prompt: str, **kwargs: Any) -> str:
+        return self.generate_full(prompt, **kwargs).get("content", "")
+
+    # ------------------------------------------------------------------
+    # Bulk / legacy path
+    # ------------------------------------------------------------------
+
+    def run_harness(self, run_id: str) -> "BenchmarkResults":
+        """Run the full dataset through the Harness in one shot (legacy/bulk)."""
+        output_path = self._output_dir / run_id
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        harness = Harness(
+            output_path=output_path,
+            run_id=run_id,
+            dataset_name=self._dataset_name,
+            dataset_version=self._dataset_version,
+            model_name=self._litellm_model,
+            agent_name=AgentName("terminus-2"),
+            agent_kwargs={
+                "model_name": self._litellm_model,
+                "api_base": self._api_base,
+                "temperature": self._temperature,
+            },
+            n_concurrent_trials=self._n_concurrent,
+            n_tasks=self._max_samples,
+            cleanup=True,
+        )
+        return harness.run()
 
     def close(self) -> None:
         pass

src/openjarvis/evals/cli.py

@@ -132,10 +132,6 @@ BENCHMARKS = {
         "category": "coding",
         "description": "LiveCodeBench competitive programming",
     },
-    "liveresearch": {
-        "category": "agentic",
-        "description": "DeepResearchBench report generation (alias: deepresearch)",
-    },
     "deepresearch": {
         "category": "agentic",
         "description": "DeepResearchBench deep research report generation",
@@ -174,6 +170,7 @@ def _build_backend(
     gpu_metrics: bool = False,
     model: Optional[str] = None,
     max_turns: Optional[int] = None,
+    engine_config: Optional[dict] = None,
 ):
     """Construct the appropriate backend."""
     if backend_name == "jarvis-agent":
@@ -181,6 +178,7 @@ def _build_backend(
 
         return JarvisAgentBackend(
             engine_key=engine_key,
+            engine_config=engine_config,
             agent_name=agent_name,
             tools=tools,
             telemetry=telemetry,
@@ -193,6 +191,7 @@ def _build_backend(
 
         return JarvisDirectBackend(
             engine_key=engine_key,
+            engine_config=engine_config,
             telemetry=telemetry,
             gpu_metrics=gpu_metrics,
         )
@@ -343,10 +342,10 @@ def _build_dataset(benchmark: str, subset: str | None = None):
         from openjarvis.evals.datasets.livecodebench import LiveCodeBenchDataset
 
         return LiveCodeBenchDataset()
-    elif benchmark in ("liveresearch", "deepresearch"):
-        from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset
+    elif benchmark == "deepresearch":
+        from openjarvis.evals.datasets.deepresearch import DeepResearchBenchDataset
 
-        return LiveResearchBenchDataset(path=subset)
+        return DeepResearchBenchDataset(path=subset)
     elif benchmark == "liveresearchbench":
         from openjarvis.evals.datasets.liveresearchbench import (
             LiveResearchBenchDataset as LRBDataset,
@@ -501,10 +500,10 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
         from openjarvis.evals.scorers.livecodebench import LiveCodeBenchScorer
 
         return LiveCodeBenchScorer(judge_backend, judge_model)
-    elif benchmark in ("liveresearch", "deepresearch"):
-        from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorer
+    elif benchmark == "deepresearch":
+        from openjarvis.evals.scorers.deepresearch import DeepResearchBenchScorer
 
-        return LiveResearchBenchScorer(judge_backend, judge_model)
+        return DeepResearchBenchScorer(judge_backend, judge_model)
     elif benchmark == "liveresearchbench":
         from openjarvis.evals.scorers.liveresearchbench import (
             LiveResearchBenchScorer as LRBScorer,
@@ -674,6 +673,7 @@ def _run_single(config, console: Optional[Console] = None) -> object:
         gpu_metrics=getattr(config, "gpu_metrics", False),
         model=config.model,
         max_turns=getattr(config, "max_turns", None),
+        engine_config=getattr(config, "engine_config", {}),
     )
     dataset = _build_dataset(config.benchmark)
     # Inject engine config for benchmarks that run their own simulation
@@ -965,7 +965,7 @@ def _run_from_config(
     console = Console()
 
     suite = load_eval_config(config_path)
-    run_configs = expand_suite(suite)
+    run_configs = expand_suite(suite, config_path=config_path)
 
     # Filter by model name substring if requested
     if model_filter:

src/openjarvis/evals/core/config.py

@@ -199,9 +199,16 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig:
                 temperature=float(b["temperature"]) if "temperature" in b else None,
                 max_tokens=int(b["max_tokens"]) if "max_tokens" in b else None,
                 subset=b.get("subset"),
+                max_turns=int(b["max_turns"]) if "max_turns" in b else None,
             )
         )
 
+    # Parse [engine.*] sections for engine-specific configurations
+    # In TOML, [engine.vllm] creates {"engine": {"vllm": {...}}}
+    engine_configs = {}
+    if "engine" in raw and isinstance(raw["engine"], dict):
+        engine_configs = dict(raw["engine"])
+
     return EvalSuiteConfig(
         meta=meta,
         defaults=defaults,
@@ -209,10 +216,11 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig:
         run=execution,
         models=models,
         benchmarks=benchmarks,
+        engine_configs=engine_configs,
     )
 
 
-def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
+def expand_suite(suite: EvalSuiteConfig, config_path: Optional[str] = None) -> List[RunConfig]:
     """Expand an EvalSuiteConfig into a list of RunConfigs (models x benchmarks).
 
     Merge precedence (highest wins):
@@ -220,6 +228,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
 
     Args:
         suite: The parsed eval suite config.
+        config_path: Optional path to the original config file.
 
     Returns:
         List of RunConfig, one per model-benchmark pair.
@@ -268,6 +277,11 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
             # Judge engine: suite.judge.engine > "cloud"
             judge_engine = suite.judge.engine or "cloud"
 
+            # Get engine-specific config for this model's engine
+            engine_config = {}
+            if model.engine and model.engine in suite.engine_configs:
+                engine_config = suite.engine_configs[model.engine]
+
             configs.append(
                 RunConfig(
                     benchmark=bench.name,
@@ -280,8 +294,10 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
                     judge_model=judge_model,
                     judge_engine=judge_engine,
                     engine_key=model.engine,
+                    engine_config=engine_config,
                     agent_name=bench.agent,
                     tools=list(bench.tools),
+                    max_turns=bench.max_turns,
                     output_path=output_path,
                     seed=suite.run.seed,
                     dataset_split=bench.split,
@@ -297,7 +313,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
                     sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id,
                     sheets_worksheet=suite.run.sheets_worksheet,
                     sheets_credentials_path=suite.run.sheets_credentials_path,
-                    max_turns=suite.run.max_turns,
+                    config_path=config_path,
                 )
             )
 

src/openjarvis/evals/core/runner.py

@@ -18,6 +18,7 @@ import json
 import logging
 import math
 import re
+import shutil
 import statistics
 import time
 from collections import defaultdict
@@ -149,29 +150,107 @@ class EvalRunner:
                 LOGGER.debug("Task env thread-safety probe failed: %s", exc)
 
         records = list(self._dataset.iter_records())
+        all_records = records  # Keep reference to all records for summary computation
+
+        # --- Resume logic: check for existing results and skip completed samples ---
+        output_path = self._resolve_output_path()
+        resuming = False
+        original_record_count = len(records)
+
+        if output_path and output_path.exists():
+            completed_ids = set()
+            try:
+                with open(output_path) as f:
+                    for line in f:
+                        line = line.strip()
+                        if line:
+                            try:
+                                record_dict = json.loads(line)
+                                if record_dict.get("record_id"):
+                                    completed_ids.add(record_dict["record_id"])
+                            except json.JSONDecodeError:
+                                continue
+
+                if completed_ids:
+                    # Load existing results into memory for summary computation
+                    with open(output_path) as f:
+                        for line in f:
+                            line = line.strip()
+                            if line:
+                                try:
+                                    record_dict = json.loads(line)
+                                    # Convert dict back to EvalResult
+                                    result = EvalResult(
+                                        record_id=record_dict.get("record_id", ""),
+                                        model_answer=record_dict.get("model_answer", ""),
+                                        problem=record_dict.get("problem", ""),
+                                        reference=record_dict.get("reference", ""),
+                                        is_correct=record_dict.get("is_correct"),
+                                        score=record_dict.get("score"),
+                                        latency_seconds=record_dict.get("latency_seconds", 0.0),
+                                        prompt_tokens=record_dict.get("prompt_tokens", 0),
+                                        completion_tokens=record_dict.get("completion_tokens", 0),
+                                        cost_usd=record_dict.get("cost_usd", 0.0),
+                                        error=record_dict.get("error"),
+                                        scoring_metadata=record_dict.get("scoring_metadata"),
+                                        ttft=record_dict.get("ttft", 0.0),
+                                        energy_joules=record_dict.get("energy_joules", 0.0),
+                                        power_watts=record_dict.get("power_watts", 0.0),
+                                        gpu_utilization_pct=record_dict.get("gpu_utilization_pct", 0.0),
+                                        throughput_tok_per_sec=record_dict.get("throughput_tok_per_sec", 0.0),
+                                        mfu_pct=record_dict.get("mfu_pct", 0.0),
+                                        mbu_pct=record_dict.get("mbu_pct", 0.0),
+                                        ipw=record_dict.get("ipw", 0.0),
+                                        ipj=record_dict.get("ipj", 0.0),
+                                        energy_per_output_token_joules=record_dict.get("energy_per_output_token_joules", 0.0),
+                                        throughput_per_watt=record_dict.get("throughput_per_watt", 0.0),
+                                        mean_itl_ms=record_dict.get("mean_itl_ms", 0.0),
+                                        estimated_flops=record_dict.get("estimated_flops", 0.0),
+                                        trace_data=record_dict.get("trace_data"),
+                                    )
+                                    self._results.append(result)
+                                except (json.JSONDecodeError, KeyError, TypeError):
+                                    continue
+
+                    # Filter records to exclude already-completed samples (but keep all_records intact)
+                    records = [r for r in records if r.record_id not in completed_ids]
+                    skipped = original_record_count - len(records)
+                    LOGGER.info(
+                        "Resume mode: found %d completed sample(s) in existing results. "
+                        "Skipping already-processed samples. %d remaining to process.",
+                        skipped,
+                        len(records),
+                    )
+                    resuming = True
+            except Exception as exc:
+                LOGGER.warning("Failed to read existing results for resume: %s", exc)
+                resuming = False
+
         LOGGER.info(
-            "Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s",
+            "Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s%s",
             cfg.benchmark,
             len(records),
             cfg.backend,
             cfg.model,
             cfg.max_workers,
             cfg.episode_mode,
+            " (resume mode)" if resuming else "",
         )
 
-        # --- Warmup phase (discard results) ---
-        warmup_count = cfg.warmup_samples
-        if warmup_count > 0 and records:
-            warmup_records = records[:warmup_count]
-            for rec in warmup_records:
-                self._process_one(rec)
-            LOGGER.info("Warmup complete: %d samples discarded", len(warmup_records))
+        # --- Warmup phase (discard results) - skip if resuming ---
+        if not resuming:
+            warmup_count = cfg.warmup_samples
+            if warmup_count > 0 and records:
+                warmup_records = records[:warmup_count]
+                for rec in warmup_records:
+                    self._process_one(rec)
+                LOGGER.info("Warmup complete: %d samples discarded", len(warmup_records))
 
         # Open output file for incremental JSONL writing
-        output_path = self._resolve_output_path()
         if output_path:
             output_path.parent.mkdir(parents=True, exist_ok=True)
-            self._output_file = open(output_path, "w")
+            # Use append mode to preserve existing results when resuming
+            self._output_file = open(output_path, "a")
 
         # Notify trackers of run start
         for tracker in self._trackers:
@@ -184,7 +263,8 @@ class EvalRunner:
                     exc,
                 )
 
-        total = len(records)
+        # Use original record count for progress tracking (includes completed + remaining)
+        total = original_record_count
         try:
             if cfg.episode_mode:
                 self._run_episode_mode(records, progress_callback, total)
@@ -214,7 +294,7 @@ class EvalRunner:
                 self._output_file = None
 
         ended_at = time.time()
-        summary = self._compute_summary(records, started_at, ended_at)
+        summary = self._compute_summary(all_records, started_at, ended_at)
 
         # Notify trackers of summary and run end
         for tracker in self._trackers:
@@ -244,6 +324,13 @@ class EvalRunner:
             LOGGER.info("Results written to %s", output_path)
             LOGGER.info("Summary written to %s", summary_path)
 
+            # Copy config file if available
+            if self._config.config_path:
+                config_source = Path(self._config.config_path)
+                config_dest = output_path.parent / config_source.name
+                shutil.copy2(self._config.config_path, config_dest)
+                LOGGER.info("Config copied to %s", config_dest)
+
             # Write per-trace data
             traces_dir = self._write_traces(output_path)
 
@@ -285,6 +372,8 @@ class EvalRunner:
                 task_env = self._dataset.create_task_env(record)
                 ctx = task_env if task_env is not None else nullcontext()
                 with ctx:
+                    if hasattr(self._backend, "set_current_record"):
+                        self._backend.set_current_record(record)
                     full = self._backend.generate_full(
                         record.problem,
                         **gen_kwargs,
@@ -321,6 +410,8 @@ class EvalRunner:
                         content,
                     )
             else:
+                if hasattr(self._backend, "set_current_record"):
+                    self._backend.set_current_record(record)
                 full = self._backend.generate_full(
                     record.problem,
                     **gen_kwargs,
@@ -392,6 +483,8 @@ class EvalRunner:
             return EvalResult(
                 record_id=record.record_id,
                 model_answer=content,
+                problem=record.problem,
+                reference=record.reference,
                 is_correct=is_correct,
                 score=1.0 if is_correct else (0.0 if is_correct is not None else None),
                 latency_seconds=latency,
@@ -419,6 +512,8 @@ class EvalRunner:
             return EvalResult(
                 record_id=record.record_id,
                 model_answer="",
+                problem=record.problem,
+                reference=record.reference,
                 error=str(exc),
             )
 
@@ -687,6 +782,8 @@ class EvalRunner:
             return EvalResult(
                 record_id=record.record_id,
                 model_answer="\n---\n".join(all_responses),
+                problem=record.problem,
+                reference=record.reference,
                 is_correct=is_correct,
                 score=1.0 if is_correct else (0.0 if is_correct is not None else None),
                 latency_seconds=total_latency,
@@ -705,6 +802,8 @@ class EvalRunner:
             return EvalResult(
                 record_id=record.record_id,
                 model_answer="",
+                problem=record.problem,
+                reference=record.reference,
                 error=str(exc),
                 scoring_metadata={"interactive": True, "error": str(exc)},
             )
@@ -745,6 +844,8 @@ class EvalRunner:
             "benchmark": self._config.benchmark,
             "model": self._config.model,
             "backend": self._config.backend,
+            "problem": result.problem,
+            "reference": result.reference,
             "model_answer": result.model_answer,
             "is_correct": result.is_correct,
             "score": result.score,
@@ -767,6 +868,7 @@ class EvalRunner:
             "throughput_per_watt": result.throughput_per_watt,
             "mean_itl_ms": result.mean_itl_ms,
             "estimated_flops": result.estimated_flops,
+            "trace_data": result.trace_data,
         }
         try:
             line = json.dumps(record_dict, default=str)
@@ -1147,9 +1249,9 @@ def _result_to_trace_dict(result: EvalResult) -> Dict[str, Any]:
         "throughput_per_watt": result.throughput_per_watt,
         "mean_itl_ms": result.mean_itl_ms,
         "estimated_flops": result.estimated_flops,
+        # Always include trace_data for consistent output format, even if None/empty
+        "trace_data": result.trace_data,
     }
-    if result.trace_data is not None:
-        d["trace_data"] = result.trace_data
     return d
 
 

src/openjarvis/evals/core/types.py

@@ -25,6 +25,8 @@ class EvalResult:
 
     record_id: str
     model_answer: str
+    problem: str = ""
+    reference: str = ""
     is_correct: Optional[bool] = None
     score: Optional[float] = None
     latency_seconds: float = 0.0
@@ -65,6 +67,7 @@ class RunConfig:
     judge_model: str = "gpt-5-mini-2025-08-07"
     judge_engine: str = "cloud"
     engine_key: Optional[str] = None
+    engine_config: Dict[str, Any] = field(default_factory=dict)
     agent_name: Optional[str] = None
     tools: List[str] = field(default_factory=list)
     output_path: Optional[str] = None
@@ -73,6 +76,7 @@ class RunConfig:
     telemetry: bool = False
     gpu_metrics: bool = False
     metadata: Dict[str, Any] = field(default_factory=dict)
+    max_turns: Optional[int] = None
     warmup_samples: int = 0
     wandb_project: str = ""
     wandb_entity: str = ""
@@ -84,6 +88,8 @@ class RunConfig:
     system_prompt: str = ""
     episode_mode: bool = False
     dataset_subset: Optional[str] = None
+    # Path to the original config file used for this run
+    config_path: Optional[str] = None
     # Override the agent harness's max_turns budget. Default None means use
     # the JarvisConfig.agent.max_turns value (typically 10). Set higher when
     # running thinking/reasoning models that consume turns on intermediate
@@ -243,6 +249,7 @@ class BenchmarkConfig:
     temperature: Optional[float] = None
     max_tokens: Optional[int] = None
     subset: Optional[str] = None
+    max_turns: Optional[int] = None
 
 
 @dataclass(slots=True)
@@ -255,6 +262,7 @@ class EvalSuiteConfig:
     run: ExecutionConfig = field(default_factory=ExecutionConfig)
     models: List[ModelConfig] = field(default_factory=list)
     benchmarks: List[BenchmarkConfig] = field(default_factory=list)
+    engine_configs: Dict[str, Dict[str, Any]] = field(default_factory=dict)
 
 
 __all__ = [

src/openjarvis/evals/datasets/deepresearch.py

@@ -0,0 +1,191 @@
+"""DeepResearchBench dataset provider — deep research benchmark.
+
+Clones the deep_research_bench repo at runtime and parses query + criteria
+JSONL files into EvalRecords for use with AgenticRunner.
+
+Reference: https://github.com/Ayanami0730/deep_research_bench
+Paper: https://arxiv.org/abs/2510.14240
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from openjarvis.evals.core.dataset import DatasetProvider
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+DEEPRESEARCH_REPO = "https://github.com/Ayanami0730/deep_research_bench.git"
+CACHE_DIR = Path.home() / ".cache" / "deepresearch_bench"
+
+
+def _load_jsonl(path: Path) -> List[Dict[str, Any]]:
+    """Load a JSONL file into a list of dicts."""
+    records: List[Dict[str, Any]] = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    return records
+
+
+def _build_criteria_index(
+    criteria_records: List[Dict[str, Any]],
+) -> Dict[int, Dict[str, Any]]:
+    """Index criteria records by their integer id."""
+    index: Dict[int, Dict[str, Any]] = {}
+    for rec in criteria_records:
+        rec_id = rec.get("id")
+        if rec_id is not None:
+            index[int(rec_id)] = rec
+    return index
+
+
+class DeepResearchBenchDataset(DatasetProvider):
+    """DeepResearchBench — deep research with 100 expert-curated tasks.
+
+    Clones Ayanami0730/deep_research_bench from GitHub (or uses a local
+    path) and parses query + criteria JSONL files into EvalRecords.
+    """
+
+    dataset_id = "deepresearch"
+    dataset_name = "DeepResearchBench"
+
+    def __init__(self, path: Optional[str] = None) -> None:
+        self._local_path = Path(path) if path else None
+        self._repo_dir: Path = self._local_path or CACHE_DIR
+        self._records: List[EvalRecord] = []
+
+    def verify_requirements(self) -> List[str]:
+        issues: List[str] = []
+        if self._local_path is None and shutil.which("git") is None:
+            issues.append(
+                "git binary not found. Install git to clone DeepResearchBench."
+            )
+        return issues
+
+    def _ensure_repo(self) -> Path:
+        """Clone the repo if not already cached. Returns repo dir."""
+        if self._local_path is not None:
+            if not self._local_path.exists():
+                raise FileNotFoundError(
+                    f"DeepResearchBench path not found: {self._local_path}"
+                )
+            return self._local_path
+
+        if not self._repo_dir.exists():
+            LOGGER.info("Cloning DeepResearchBench from %s ...", DEEPRESEARCH_REPO)
+            self._repo_dir.parent.mkdir(parents=True, exist_ok=True)
+            subprocess.run(
+                [
+                    "git",
+                    "clone",
+                    "--depth",
+                    "1",
+                    DEEPRESEARCH_REPO,
+                    str(self._repo_dir),
+                ],
+                check=True,
+                capture_output=True,
+            )
+            LOGGER.info("DeepResearchBench cloned to %s", self._repo_dir)
+
+        return self._repo_dir
+
+    def load(
+        self,
+        *,
+        max_samples: Optional[int] = None,
+        split: Optional[str] = None,
+        seed: Optional[int] = None,
+    ) -> None:
+        repo_dir = self._ensure_repo()
+
+        # Load queries
+        query_path = repo_dir / "data" / "prompt_data" / "query.jsonl"
+        if not query_path.exists():
+            raise FileNotFoundError(f"Query file not found: {query_path}")
+
+        queries = _load_jsonl(query_path)
+        if not queries:
+            raise FileNotFoundError(f"No queries found in {query_path}")
+
+        # Load criteria (optional — used for rubric-based scoring)
+        criteria_path = repo_dir / "data" / "criteria_data" / "criteria.jsonl"
+        criteria_index: Dict[int, Dict[str, Any]] = {}
+        if criteria_path.exists():
+            criteria_records = _load_jsonl(criteria_path)
+            criteria_index = _build_criteria_index(criteria_records)
+            LOGGER.info(
+                "Loaded %d criteria records for rubric scoring", len(criteria_index)
+            )
+
+        # Optionally filter by language via split (e.g. split="en" or split="zh")
+        if split and split in ("en", "zh"):
+            queries = [q for q in queries if q.get("language") == split]
+
+        if seed is not None:
+            random.Random(seed).shuffle(queries)
+        if max_samples is not None:
+            queries = queries[:max_samples]
+
+        self._records = []
+        for query in queries:
+            q_id = query.get("id")
+            topic = query.get("topic", "")
+            language = query.get("language", "en")
+            prompt = query.get("prompt", "")
+
+            if not prompt:
+                LOGGER.warning("Skipping query %s: empty prompt", q_id)
+                continue
+
+            # Build the research task prompt
+            research_prompt = (
+                "You are a deep research assistant. Conduct thorough research "
+                "on the following topic and produce a comprehensive, well-structured "
+                "research report with citations and analysis.\n\n"
+                f"## Research Task\n\n{prompt}"
+            )
+
+            # Attach criteria metadata if available
+            criteria = criteria_index.get(int(q_id)) if q_id is not None else None
+            metadata: Dict[str, Any] = {
+                "topic": topic,
+                "language": language,
+                "original_id": q_id,
+            }
+
+            if criteria:
+                metadata["dimension_weight"] = criteria.get("dimension_weight", {})
+                metadata["criterions"] = criteria.get("criterions", {})
+
+            self._records.append(
+                EvalRecord(
+                    record_id=f"deepresearch-{q_id or len(self._records)}",
+                    problem=research_prompt,
+                    reference="",  # No single reference answer; rubric-based
+                    category="agentic",
+                    subject=topic,
+                    metadata=metadata,
+                )
+            )
+
+        LOGGER.info("DeepResearchBench: loaded %d tasks", len(self._records))
+
+    def iter_records(self) -> Iterable[EvalRecord]:
+        return iter(self._records)
+
+    def size(self) -> int:
+        return len(self._records)
+
+
+__all__ = ["DeepResearchBenchDataset"]

src/openjarvis/evals/datasets/terminalbench_native.py

@@ -145,7 +145,7 @@ class TerminalBenchNativeDataset(DatasetProvider):
 
         metadata: Dict[str, Any] = {
             "task_id": task_id,
-            "task_dir": str(task_dir),
+            "task_dir": task_dir,  # Path, not string — needed by env and backend
             "category": category_val,
             "difficulty": task_data.get("difficulty"),
             "tags": task_data.get("tags"),
@@ -162,15 +162,10 @@ class TerminalBenchNativeDataset(DatasetProvider):
         )
 
     def create_task_env(self, record):
-        """Return a TerminalBenchTaskEnv for the given record."""
-        try:
-            from openjarvis.evals.execution.terminalbench_env import (
-                TerminalBenchTaskEnv,
-            )
-
-            return TerminalBenchTaskEnv(record.metadata)
-        except ImportError:
-            return None
+        # The TerminalBenchNativeBackend runs the full trial (Docker + agent +
+        # tests) via the terminal-bench Harness inside generate_full().
+        # No separate task environment is needed.
+        return None
 
     def verify_requirements(self):
         """Check that terminal-bench and docker are available."""

src/openjarvis/evals/execution/terminalbench_env.py

@@ -29,17 +29,24 @@ class TerminalBenchTaskEnv:
     # ------------------------------------------------------------------
 
     def __enter__(self) -> TerminalBenchTaskEnv:
+        from terminal_bench.handlers.trial_handler import Task, TaskPaths
         from terminal_bench.terminal.terminal import spin_up_terminal
 
         task = self._metadata.get("task")
         task_paths = self._metadata.get("task_paths")
         task_id = self._metadata.get("task_id", "unknown")
 
+        # Lazily construct Task and TaskPaths from task_dir when not pre-populated.
         if task is None or task_paths is None:
-            raise ValueError(
-                "Task metadata missing 'task' or 'task_paths'. "
-                "Use the 'terminalbench-native' dataset."
-            )
+            task_dir = self._metadata.get("task_dir")
+            if task_dir is None:
+                raise ValueError(
+                    "TerminalBenchTaskEnv requires 'task_dir' in record metadata."
+                )
+            task_paths = TaskPaths(Path(task_dir))
+            task = Task.from_yaml(task_paths.task_config_path)
+            self._metadata["task"] = task
+            self._metadata["task_paths"] = task_paths
 
         docker_image_prefix = f"tb__{task_id}".replace(".", "-")
         client_image_name = f"{docker_image_prefix}__client"

src/openjarvis/evals/scorers/deepresearch.py

@@ -0,0 +1,316 @@
+"""DeepResearchBench scorer — LLM-as-judge for deep research quality.
+
+Evaluates research output quality across four dimensions from the
+DeepResearchBench rubric: comprehensiveness, insight, instruction_following,
+and readability. Uses LLM-as-judge with per-task criteria when available,
+falling back to a generic research quality rubric.
+
+Reference: https://github.com/Ayanami0730/deep_research_bench
+Paper: https://arxiv.org/abs/2510.14240
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from openjarvis.evals.core.scorer import LLMJudgeScorer
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+# The four scoring dimensions from DeepResearchBench
+DIMENSIONS = ["comprehensiveness", "insight", "instruction_following", "readability"]
+
+# Default dimension weights when task-specific weights are unavailable
+DEFAULT_WEIGHTS = {
+    "comprehensiveness": 0.25,
+    "insight": 0.30,
+    "instruction_following": 0.25,
+    "readability": 0.20,
+}
+
+_GENERIC_RUBRIC = """Evaluate the research report across these four dimensions on a 0-10 scale:
+
+1. **Comprehensiveness** (information coverage, depth, data support, balanced perspectives):
+   - 0-3: Minimal coverage, misses major aspects of the topic
+   - 4-6: Covers main points but lacks depth or misses important subtopics
+   - 7-8: Thorough coverage with good depth and supporting evidence
+   - 9-10: Exceptional coverage, addresses all facets with rich detail
+
+2. **Insight** (analysis depth, critical thinking, original perspectives, forward-thinking):
+   - 0-3: Surface-level description only, no analysis
+   - 4-6: Some analysis but mostly descriptive, limited original thinking
+   - 7-8: Strong analytical depth with meaningful insights and reasoning
+   - 9-10: Exceptional analysis with novel perspectives and deep understanding
+
+3. **Instruction Following** (task adherence, scope compliance, requirement completeness):
+   - 0-3: Fails to address the core research question
+   - 4-6: Partially addresses the question but misses key requirements
+   - 7-8: Addresses all major requirements with minor omissions
+   - 9-10: Fully addresses every aspect of the research task
+
+4. **Readability** (structure, language fluency, technical terminology, presentation):
+   - 0-3: Poorly organized, difficult to follow
+   - 4-6: Reasonable structure but could be clearer
+   - 7-8: Well-structured with clear writing and good flow
+   - 9-10: Exceptionally clear, professional structure and presentation"""
+
+
+def _format_criteria_rubric(criterions: Dict[str, List[Dict[str, Any]]]) -> str:
+    """Format task-specific criteria into a rubric string for the judge prompt."""
+    parts: List[str] = []
+    for dimension in DIMENSIONS:
+        criteria_list = criterions.get(dimension, [])
+        if not criteria_list:
+            continue
+        parts.append(f"\n### {dimension.replace('_', ' ').title()}")
+        for i, crit in enumerate(criteria_list, 1):
+            criterion = crit.get("criterion", "")
+            explanation = crit.get("explanation", "")
+            weight = crit.get("weight", 0.0)
+            parts.append(
+                f"  {i}. [{weight:.0%}] {criterion}"
+                + (f"\n     {explanation}" if explanation else "")
+            )
+    return "\n".join(parts)
+
+
+def _build_judge_prompt(
+    *,
+    task_prompt: str,
+    article: str,
+    rubric: str,
+) -> str:
+    """Build the LLM judge prompt for evaluating a research report."""
+    return f"""You are an expert evaluator assessing the quality of an AI-generated research report.
+
+## Original Research Task
+{task_prompt}
+
+## Research Report to Evaluate
+{article}
+
+## Evaluation Rubric
+{rubric}
+
+## Instructions
+Evaluate the research report against the rubric criteria across four dimensions:
+comprehensiveness, insight, instruction_following, and readability.
+
+Score each dimension on a 0-10 scale.
+
+Return your evaluation as JSON with this exact structure:
+```json
+{{
+  "scores": {{
+    "comprehensiveness": <score 0-10>,
+    "insight": <score 0-10>,
+    "instruction_following": <score 0-10>,
+    "readability": <score 0-10>
+  }},
+  "weighted_total": <weighted score 0-10>,
+  "notes": "brief justification for each dimension score"
+}}
+```
+
+Be a rigorous evaluator. Reserve scores of 9-10 for genuinely excellent work.
+A score of 5 represents adequate but unremarkable quality."""
+
+
+def _parse_judge_response(raw: str) -> Dict[str, Any]:
+    """Parse LLM judge response, extracting dimension scores.
+
+    Tries: JSON code block -> balanced braces -> regex fallback.
+    """
+    if not raw or not raw.strip():
+        return {
+            "scores": {},
+            "weighted_total": 0.0,
+            "notes": "Empty judge response",
+        }
+
+    # Try JSON code block
+    code_block = re.search(r"```json\s*(.*?)\s*```", raw, re.DOTALL)
+    if code_block:
+        try:
+            parsed = json.loads(code_block.group(1))
+            if isinstance(parsed, dict):
+                return _normalize_response(parsed)
+        except json.JSONDecodeError:
+            pass
+
+    # Try balanced braces extraction
+    candidates: List[str] = []
+    depth = 0
+    current: List[str] = []
+    for char in raw:
+        if char == "{":
+            if depth == 0:
+                current = []
+            depth += 1
+        if depth > 0:
+            current.append(char)
+        if char == "}":
+            depth -= 1
+            if depth == 0 and current:
+                candidates.append("".join(current))
+
+    for candidate in reversed(candidates):
+        try:
+            parsed = json.loads(candidate)
+            if isinstance(parsed, dict) and "scores" in parsed:
+                return _normalize_response(parsed)
+        except json.JSONDecodeError:
+            continue
+
+    for candidate in reversed(candidates):
+        try:
+            parsed = json.loads(candidate)
+            if isinstance(parsed, dict):
+                return _normalize_response(parsed)
+        except json.JSONDecodeError:
+            continue
+
+    # Regex fallback: look for individual dimension scores
+    scores: Dict[str, float] = {}
+    for dim in DIMENSIONS:
+        match = re.search(
+            rf"{dim}[:\s]*([\d.]+)",
+            raw,
+            re.IGNORECASE,
+        )
+        if match:
+            try:
+                val = float(match.group(1))
+                if 0.0 <= val <= 10.0:
+                    scores[dim] = val
+            except ValueError:
+                pass
+
+    if scores:
+        LOGGER.warning("Fell back to regex score extraction")
+        mean_score = sum(scores.values()) / len(scores) if scores else 0.0
+        return {
+            "scores": scores,
+            "weighted_total": mean_score,
+            "notes": "Scores extracted from prose via regex",
+        }
+
+    LOGGER.warning("Failed to parse judge response")
+    return {
+        "scores": {},
+        "weighted_total": 0.0,
+        "notes": "Failed to parse judge response",
+    }
+
+
+def _normalize_response(parsed: Dict[str, Any]) -> Dict[str, Any]:
+    """Normalize judge response to standard structure."""
+    result: Dict[str, Any] = {"scores": {}, "weighted_total": 0.0, "notes": ""}
+
+    # Extract scores
+    scores_data = parsed.get("scores", {})
+    if isinstance(scores_data, dict):
+        for dim in DIMENSIONS:
+            val = scores_data.get(dim)
+            if isinstance(val, (int, float)):
+                result["scores"][dim] = float(val)
+
+    # Extract weighted_total
+    for key in ("weighted_total", "total", "overall_score", "score"):
+        if key in parsed and isinstance(parsed[key], (int, float)):
+            result["weighted_total"] = float(parsed[key])
+            break
+    else:
+        # Compute mean if no explicit total
+        if result["scores"]:
+            values = list(result["scores"].values())
+            result["weighted_total"] = sum(values) / len(values)
+
+    # Extract notes
+    for key in ("notes", "justification", "reasoning"):
+        if key in parsed:
+            result["notes"] = str(parsed[key])
+            break
+
+    return result
+
+
+class DeepResearchBenchScorer(LLMJudgeScorer):
+    """LLM-as-judge scorer for DeepResearchBench deep research tasks.
+
+    Evaluates research reports across four dimensions:
+    comprehensiveness, insight, instruction_following, readability.
+    Uses task-specific criteria when available from the benchmark data.
+    """
+
+    scorer_id = "deepresearch"
+
+    def score(
+        self,
+        record: EvalRecord,
+        model_answer: str,
+    ) -> Tuple[Optional[bool], Dict[str, Any]]:
+        if not model_answer or not model_answer.strip():
+            return False, {"reason": "empty_response", "score": 0.0}
+
+        # Build rubric — use task-specific criteria if available
+        criterions = record.metadata.get("criterions")
+        if criterions and isinstance(criterions, dict):
+            rubric = _format_criteria_rubric(criterions)
+        else:
+            rubric = _GENERIC_RUBRIC
+
+        # Get the original research task (strip our wrapper prompt)
+        task_prompt = record.problem
+
+        # Build judge prompt
+        prompt = _build_judge_prompt(
+            task_prompt=task_prompt,
+            article=model_answer,
+            rubric=rubric,
+        )
+
+        try:
+            raw = self._ask_judge(prompt, temperature=0.0, max_tokens=4096)
+        except Exception as exc:
+            LOGGER.error("LLM judge call failed for %s: %s", record.record_id, exc)
+            return None, {"error": str(exc), "score": 0.0}
+
+        parsed = _parse_judge_response(raw)
+        scores = parsed.get("scores", {})
+
+        # Compute weighted total using task-specific or default weights
+        dimension_weights = record.metadata.get("dimension_weight", DEFAULT_WEIGHTS)
+        weighted_total = 0.0
+        total_weight = 0.0
+        for dim in DIMENSIONS:
+            dim_score = scores.get(dim, 0.0)
+            dim_weight = dimension_weights.get(dim, DEFAULT_WEIGHTS.get(dim, 0.25))
+            weighted_total += dim_score * dim_weight
+            total_weight += dim_weight
+
+        if total_weight > 0:
+            weighted_total /= total_weight
+        # Normalize to 0-1 range (scores are 0-10)
+        normalized_score = weighted_total / 10.0
+
+        # Threshold: score >= 0.5 (i.e., 5/10 weighted average) is considered passing
+        is_correct = normalized_score >= 0.5
+
+        metadata: Dict[str, Any] = {
+            "score": normalized_score,
+            "dimension_scores": scores,
+            "dimension_weights": dimension_weights,
+            "weighted_total_0_10": weighted_total,
+            "notes": parsed.get("notes", ""),
+            "raw_judge_output": raw,
+        }
+
+        return is_correct, metadata
+
+
+__all__ = ["DeepResearchBenchScorer"]

src/openjarvis/sdk.py

@@ -549,7 +549,11 @@ class Jarvis:
         result = agent_obj.run(query, context=ctx)
         return {
             "content": result.content,
-            "usage": {},
+            "usage": {
+                "prompt_tokens": agent_obj._total_prompt_tokens,
+                "completion_tokens": agent_obj._total_completion_tokens,
+            },
+            "cost_usd": agent_obj._total_cost_usd,
             "tool_results": [
                 {
                     "tool_name": tr.tool_name,

src/openjarvis/system/builder.py

@@ -70,6 +70,10 @@ class SystemBuilder:
         self._traces = enabled
         return self
 
+    def max_turns(self, n: int) -> SystemBuilder:
+        self._config.agent.max_turns = n
+        return self
+
     def sandbox(self, enabled: bool) -> SystemBuilder:
         self._sandbox = enabled
         return self

src/openjarvis/system/orchestrator.py

@@ -208,16 +208,17 @@ class QueryOrchestrator:
 
         s.bus.subscribe(EventType.INFERENCE_END, _on_inference_end)
 
-        # Check trace_store (set at build time) instead of config.traces.enabled
-        # because the shared config singleton can be mutated by other SystemBuilder
-        # instances (e.g. the judge backend).
+        # Check if traces are enabled. Even if trace_store is None (e.g. due to
+        # initialization failure), we still create a TraceCollector so that
+        # trace data can be extracted programmatically (e.g. by eval backends).
+        # The collector will simply skip persisting if store is None.
         try:
-            if s.trace_store is not None:
+            if s.config.traces.enabled:
                 from openjarvis.traces.collector import TraceCollector
 
                 collector = TraceCollector(
                     ag,
-                    store=s.trace_store,
+                    store=s.trace_store,  # Can be None - collector handles it
                     bus=s.bus,
                 )
                 result = collector.run(query, context=ctx)
@@ -274,7 +275,11 @@ class QueryOrchestrator:
 
         return {
             "content": result.content,
-            "usage": getattr(result, "usage", {}),
+            "usage": {
+                "prompt_tokens": getattr(ag, "_total_prompt_tokens", 0),
+                "completion_tokens": getattr(ag, "_total_completion_tokens", 0),
+            },
+            "cost_usd": getattr(ag, "_total_cost_usd", 0.0),
             "tool_results": [
                 {
                     "tool_name": tr.tool_name,