feat(evals): add TerminalBench V2.1 benchmark (#345)

2026-08-14 08:52:06 +00:00 · 2026-05-14 20:23:14 -07:00
parent b8c3b417ec
commit 44815e8544
11 changed files with 844 additions and 8 deletions
@@ -78,6 +78,7 @@ These benchmarks test multi-step agent capabilities including tool use, code gen
 | **SWEfficiency** | `swefficiency` | agentic | Software optimization tasks |
 | **TerminalBench** | `terminalbench` | agentic | Terminal-based task completion |
 | **TerminalBench Native** | `terminalbench-native` | agentic | TerminalBench with native Docker execution |
+| **TerminalBench V2.1** | `terminalbench-v2.1` | agentic | TB v2.1 Harbor-style Docker tasks |
 | **LifelongAgent** | `lifelong-agent` | agentic | Sequential task learning across sessions |
 | **PaperArena** | `paperarena` | agentic | Scientific paper analysis |
 | **DeepPlanning** | `deepplanning` | agentic | Shopping constraint planning |
@@ -1,4 +1,4 @@
-"""Native terminal-bench v2 backend.
+"""Native TerminalBench V2.1 backend.

 Uses Harness for Docker-based execution and scoring.
 """
@@ -53,6 +53,10 @@ BENCHMARKS = {
        "category": "agentic",
        "description": "TerminalBench Native (Docker)",
    },
+    "terminalbench-v2.1": {
+        "category": "agentic",
+        "description": "TerminalBench V2.1 (Harbor-style Docker tasks)",
+    },
    "email_triage": {
        "category": "use-case",
        "description": "Email triage classification + draft",
@@ -303,6 +307,12 @@ def _build_dataset(benchmark: str, subset: str | None = None):
        )

        return TerminalBenchNativeDataset()
+    elif benchmark == "terminalbench-v2.1":
+        from openjarvis.evals.datasets.terminalbench_v2_1 import (
+            TerminalBenchV21Dataset,
+        )
+
+        return TerminalBenchV21Dataset()
    elif benchmark == "email_triage":
        from openjarvis.evals.datasets.email_triage import EmailTriageDataset

@@ -462,6 +472,12 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
        )

        return TerminalBenchNativeScorer(judge_backend, judge_model)
+    elif benchmark == "terminalbench-v2.1":
+        from openjarvis.evals.scorers.terminalbench_v2_1 import (
+            TerminalBenchV21Scorer,
+        )
+
+        return TerminalBenchV21Scorer(judge_backend, judge_model)
    elif benchmark == "email_triage":
        from openjarvis.evals.scorers.email_triage import EmailTriageScorer

@@ -641,7 +657,7 @@ def _build_trackers(config) -> list:


 def _run_terminalbench_native(config, console: Console) -> object:
-    """Run TerminalBench V2 natively via terminal-bench Harness."""
+    """Run TerminalBench V2.1 natively via terminal-bench Harness."""
    from openjarvis.evals.backends.terminalbench_native import (
        TerminalBenchNativeBackend,
    )
@@ -665,8 +681,8 @@ def _run_terminalbench_native(config, console: Console) -> object:

    # Docker compose project names must be lowercase alphanumeric + hyphens/underscores
    model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-"))
-    run_id = f"tb2-{model_slug}"
-    console.print(f"  Running TerminalBench V2 natively: {model}")
+    run_id = f"tb21-{model_slug}"
+    console.print(f"  Running TerminalBench V2.1 natively: {model}")
    console.print(f"  Harness run_id: {run_id}")

    results = backend.run_harness(run_id)
@@ -702,7 +718,7 @@ def _run_single(config, console: Optional[Console] = None) -> object:
    if console is None:
        console = Console()

-    # TerminalBench V2 native: use terminal-bench Harness directly
+    # TerminalBench V2.1 native: use terminal-bench Harness directly
    if config.benchmark == "terminalbench-native":
        return _run_terminalbench_native(config, console)

@@ -1,7 +1,7 @@
-# TerminalBench V2 (native Docker) on Qwen/Qwen3.5-122B-A10B-FP8
+# TerminalBench V2.1 (native Docker) on Qwen/Qwen3.5-122B-A10B-FP8
 [meta]
 name = "terminalbench-native-qwen122b"
-description = "TerminalBench V2 native on Qwen/Qwen3.5-122B-A10B-FP8"
+description = "TerminalBench V2.1 native on Qwen/Qwen3.5-122B-A10B-FP8"

 [defaults]
 temperature = 0.6
@@ -57,6 +57,7 @@ KNOWN_BENCHMARKS = {
    "swefficiency",
    "terminalbench",
    "terminalbench-native",
+    "terminalbench-v2.1",
    "email_triage",
    "morning_brief",
    "research_mining",
@@ -0,0 +1,239 @@
+"""TerminalBench V2.1 dataset provider.
+
+Loads tasks from the terminal-bench-2.1 repo layout (ekellbuch/terminal-bench-2,
+branch terminal-bench-2.1). Each task lives in a top-level directory containing:
+
+    <task_name>/
+      task.toml          # metadata + docker image + timeouts
+      instruction.md     # the agent prompt
+      environment/       # Dockerfile + supporting files (pre-built into task.toml's docker_image)
+      solution/          # oracle solve.sh (not used by eval)
+      tests/             # test.sh + test_outputs.py (pytest) used by the verifier
+
+Reference: https://github.com/ekellbuch/terminal-bench-2/tree/terminal-bench-2.1
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from openjarvis.evals.core.dataset import DatasetProvider
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_DEFAULT_REPO = "https://github.com/ekellbuch/terminal-bench-2.git"
+_DEFAULT_BRANCH = "terminal-bench-2.1"
+# Shared cache across isolated HOMEs; falls back to $HOME/.cache if env override set
+import os as _os  # noqa: E402
+
+_DEFAULT_CACHE = Path(
+    _os.environ.get("TBV21_REPO_DIR") or "/home/ubuntu/.cache/terminalbench-v2.1/repo"
+)
+
+
+def _load_task_toml(task_dir: Path) -> Dict[str, Any]:
+    """Parse task.toml using tomllib (3.11+) or tomli fallback."""
+    task_file = task_dir / "task.toml"
+    if not task_file.exists():
+        return {}
+    try:
+        import tomllib  # Python 3.11+
+    except ImportError:
+        try:
+            import tomli as tomllib  # type: ignore
+        except ImportError:
+            LOGGER.warning("tomllib/tomli not available; skipping %s", task_dir)
+            return {}
+    try:
+        return tomllib.loads(task_file.read_text()) or {}
+    except Exception as exc:  # noqa: BLE001 - defensive
+        LOGGER.warning("Failed to parse %s: %s", task_file, exc)
+        return {}
+
+
+class TerminalBenchV21Dataset(DatasetProvider):
+    """TerminalBench V2.1 dataset (89 Docker-based terminal tasks)."""
+
+    dataset_id = "terminalbench-v2.1"
+    dataset_name = "TerminalBench V2.1"
+
+    def __init__(
+        self,
+        repo_url: str = _DEFAULT_REPO,
+        branch: str = _DEFAULT_BRANCH,
+        path: Optional[str] = None,
+        task_ids: Optional[List[str]] = None,
+    ) -> None:
+        self._repo_url = repo_url
+        self._branch = branch
+        self._repo_dir: Path = Path(path) if path else _DEFAULT_CACHE
+        self._task_ids = task_ids
+        self._records: List[EvalRecord] = []
+
+    def _ensure_repo(self) -> Path:
+        """Clone the TB v2.1 repo once and cache it locally."""
+        if self._repo_dir.exists() and (self._repo_dir / ".git").is_dir():
+            return self._repo_dir
+        if shutil.which("git") is None:
+            raise RuntimeError(
+                "git binary not found. Install git to clone TerminalBench V2.1 tasks."
+            )
+        self._repo_dir.parent.mkdir(parents=True, exist_ok=True)
+        LOGGER.info("Cloning %s (branch %s) into %s", self._repo_url, self._branch, self._repo_dir)
+        subprocess.run(
+            [
+                "git",
+                "clone",
+                "--branch",
+                self._branch,
+                "--depth",
+                "1",
+                self._repo_url,
+                str(self._repo_dir),
+            ],
+            check=True,
+        )
+        return self._repo_dir
+
+    def load(
+        self,
+        *,
+        max_samples: Optional[int] = None,
+        split: Optional[str] = None,
+        seed: Optional[int] = None,
+    ) -> None:
+        repo = self._ensure_repo()
+        task_dirs = sorted(
+            d
+            for d in repo.iterdir()
+            if d.is_dir() and (d / "task.toml").exists()
+        )
+
+        if self._task_ids:
+            wanted = set(self._task_ids)
+            task_dirs = [d for d in task_dirs if d.name in wanted]
+
+        if seed is not None:
+            rng = random.Random(seed)
+            task_dirs = list(task_dirs)
+            rng.shuffle(task_dirs)
+
+        if max_samples is not None:
+            task_dirs = task_dirs[:max_samples]
+
+        self._records = []
+        for idx, task_dir in enumerate(task_dirs):
+            record = self._convert_task(task_dir, idx)
+            if record is not None:
+                self._records.append(record)
+
+    def iter_records(self) -> Iterable[EvalRecord]:
+        return iter(self._records)
+
+    def size(self) -> int:
+        return len(self._records)
+
+    def _convert_task(self, task_dir: Path, idx: int) -> Optional[EvalRecord]:
+        task_data = _load_task_toml(task_dir)
+        instruction_file = task_dir / "instruction.md"
+        if not instruction_file.exists():
+            return None
+
+        raw_instruction = instruction_file.read_text().strip()
+        if not raw_instruction:
+            return None
+
+        # Agentic framing — works for both the one-shot direct backend
+        # (model emits a single bash script) and the multi-turn agent
+        # backend (model calls `docker_shell_exec` repeatedly). When the
+        # agentic backend is in use, the container is live, and every
+        # `docker_shell_exec` call lands there. The test verifier runs
+        # after the agent finishes.
+        instruction = (
+            "You are solving a TerminalBench V2.1 task inside a Linux "
+            "container (working dir /app, root access, internet available, "
+            "common tools: bash, python3, pip, curl, apt, git). Use the "
+            "`docker_shell_exec` tool to inspect the environment, install "
+            "packages, run commands, and create files. Iterate until the "
+            "task is complete and the required output files are in place. "
+            "Do not claim you are done until you have verified the outputs "
+            "exist. If you are limited to a single response (no tool use), "
+            "output a full bash script inside a ```bash ... ``` fence that "
+            "solves the task end-to-end.\n\n"
+            "--- TASK ---\n" + raw_instruction
+        )
+
+        meta = task_data.get("metadata", {}) or {}
+        env = task_data.get("environment", {}) or {}
+        verifier = task_data.get("verifier", {}) or {}
+        agent = task_data.get("agent", {}) or {}
+
+        task_id = task_dir.name or f"tbv21_{idx}"
+        category = meta.get("category", "terminal")
+
+        metadata: Dict[str, Any] = {
+            "task_id": task_id,
+            "task_dir": str(task_dir),
+            "category": category,
+            "difficulty": meta.get("difficulty"),
+            "tags": meta.get("tags", []),
+            "docker_image": env.get("docker_image"),
+            "cpus": env.get("cpus"),
+            "memory": env.get("memory"),
+            "storage": env.get("storage"),
+            "build_timeout_sec": env.get("build_timeout_sec"),
+            "verifier_timeout_sec": verifier.get("timeout_sec"),
+            "agent_timeout_sec": agent.get("timeout_sec"),
+            "author_name": meta.get("author_name"),
+            "expert_time_estimate_min": meta.get("expert_time_estimate_min"),
+        }
+
+        return EvalRecord(
+            record_id=f"terminalbench-v2.1-{task_id}",
+            problem=instruction,
+            reference="",
+            category="agentic",
+            subject=category,
+            metadata=metadata,
+        )
+
+    def create_task_env(self, record):
+        """Return a per-task Docker environment (context manager).
+
+        The runner enters this around the agent call so that tools like
+        ``docker_shell_exec`` can target the running container.
+        """
+        try:
+            from openjarvis.evals.execution.terminalbench_v2_1_env import (
+                TerminalBenchV21TaskEnv,
+            )
+        except ImportError:
+            return None
+        return TerminalBenchV21TaskEnv(record.metadata)
+
+    def verify_requirements(self) -> List[str]:
+        """Check runtime prerequisites (docker, git, tomllib)."""
+        issues: List[str] = []
+        if shutil.which("docker") is None:
+            issues.append("docker not found in PATH (required to run TB v2.1 tasks)")
+        if shutil.which("git") is None:
+            issues.append("git not found in PATH (required to clone TB v2.1 repo)")
+        try:
+            import tomllib  # noqa: F401
+        except ImportError:
+            try:
+                import tomli  # noqa: F401
+            except ImportError:
+                issues.append(
+                    "tomllib (3.11+) or tomli not available — cannot parse task.toml"
+                )
+        return issues
+
+
+__all__ = ["TerminalBenchV21Dataset"]
@@ -0,0 +1,126 @@
+"""TerminalBench V2.1 task environment.
+
+Per-task Docker container + scoring lifecycle. Intended to be used as a
+context manager by the eval runner so that the agent has a live
+container to interact with through :mod:`openjarvis.tools.docker_shell_exec`.
+
+On ``__enter__``:
+    * Pulls / runs the task's docker image with ``sleep infinity``.
+    * Mounts the task's ``tests/`` directory read-only at ``/tests``.
+    * Creates ``/logs/verifier/`` for reward output.
+    * Binds the container name into :mod:`docker_shell_exec`'s thread-local
+      state so the agent's ``docker_shell_exec`` tool targets this container.
+
+On ``__exit__``:
+    * Runs ``/tests/test.sh`` to produce ``/logs/verifier/reward.txt``.
+    * Reads the reward, stashes it on ``record.metadata``.
+    * Clears the ``docker_shell_exec`` thread-local.
+    * Tears down the container.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+import uuid
+from pathlib import Path
+from types import TracebackType
+from typing import Any, MutableMapping, Optional, Type
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TerminalBenchV21TaskEnv:
+    """Per-task Docker + scoring lifecycle for TerminalBench V2.1."""
+
+    def __init__(self, metadata: MutableMapping[str, Any]) -> None:
+        self._metadata = metadata
+        self._container: Optional[str] = None
+        self._started = False
+
+    # ------------------------------------------------------------------
+    # Context manager
+    # ------------------------------------------------------------------
+    def __enter__(self) -> "TerminalBenchV21TaskEnv":
+        from openjarvis.tools.docker_shell_exec import set_active_container
+
+        docker_image = self._metadata.get("docker_image")
+        task_dir = self._metadata.get("task_dir")
+        if not docker_image or not task_dir:
+            raise ValueError(
+                "TerminalBenchV21TaskEnv missing 'docker_image' or 'task_dir' "
+                "metadata"
+            )
+
+        tests_dir = Path(task_dir) / "tests"
+        if not tests_dir.is_dir():
+            raise ValueError(f"Missing tests/ dir in {task_dir}")
+
+        task_id = str(self._metadata.get("task_id") or "task")
+        name = f"tbv21-{task_id}-{uuid.uuid4().hex[:8]}"
+        name = re.sub(r"[^a-zA-Z0-9_-]", "-", name)[:63]
+        self._container = name
+
+        cpus = str(self._metadata.get("cpus") or 2)
+        memory = str(self._metadata.get("memory") or "4G")
+
+        start = subprocess.run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                name,
+                "--cpus",
+                cpus,
+                "--memory",
+                memory,
+                "-v",
+                f"{tests_dir}:/tests:ro",
+                "--entrypoint",
+                "/bin/bash",
+                docker_image,
+                "-c",
+                "mkdir -p /logs/verifier && sleep infinity",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=600,
+        )
+        if start.returncode != 0:
+            self._metadata["tbv21_env_error"] = start.stderr[:500]
+            raise RuntimeError(
+                f"docker run failed for {task_id}: {start.stderr[:300]}"
+            )
+
+        self._started = True
+        self._metadata["tbv21_container"] = name
+        set_active_container(name)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        """Clean up the container + thread-local binding.
+
+        Test execution + reward reading lives on the scorer, which runs
+        *inside* this context manager (while the container is still up).
+        """
+        from openjarvis.tools.docker_shell_exec import set_active_container
+
+        set_active_container(None)
+        if self._container:
+            subprocess.run(
+                ["docker", "rm", "-f", self._container],
+                capture_output=True,
+                text=True,
+                timeout=60,
+            )
+            self._container = None
+
+
+__all__ = ["TerminalBenchV21TaskEnv"]
@@ -0,0 +1,258 @@
+"""TerminalBench V2.1 scorer.
+
+Two modes:
+
+1. **Agentic mode (preferred)** — the dataset's ``create_task_env`` spun up a
+   container, the agent interacted with it through ``docker_shell_exec``, and
+   on context exit the env ran ``tests/test.sh`` and wrote ``tbv21_reward``
+   into ``record.metadata``. The scorer just reads that value.
+
+2. **One-shot mode (fallback)** — no env was attached. The model answer is
+   treated as a bash script (extracted from a ```bash ... ``` fence if
+   present). The scorer runs the script in the task container and then the
+   tests, same as before.
+
+This means the same scorer supports both ``backend = "jarvis-direct"`` and
+``backend = "jarvis-agent"`` TB v2.1 configs.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+import uuid
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.evals.core.scorer import Scorer
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+_DEFAULT_AGENT_TIMEOUT = 900.0
+_DEFAULT_VERIFIER_TIMEOUT = 900.0
+
+
+def _extract_bash(model_answer: str) -> str:
+    for pat in (
+        r"```(?:bash|sh|shell)\s*\n(.*?)```",
+        r"```\s*\n(.*?)```",
+    ):
+        m = re.search(pat, model_answer, re.DOTALL)
+        if m:
+            return m.group(1).strip()
+    stripped = model_answer.strip()
+    if stripped.startswith("#!") or stripped.startswith("set -"):
+        return stripped
+    return stripped
+
+
+class TerminalBenchV21Scorer(Scorer):
+    """Reward = 1 if the task's tests pass, 0 otherwise."""
+
+    scorer_id = "terminalbench-v2.1"
+
+    def __init__(
+        self,
+        judge_backend=None,
+        judge_model: str = "",
+    ) -> None:
+        self._judge_backend = judge_backend
+        self._judge_model = judge_model
+
+    def score(
+        self,
+        record: EvalRecord,
+        model_answer: str,
+    ) -> Tuple[Optional[bool], Dict[str, Any]]:
+        md = record.metadata or {}
+
+        # ---- Agentic mode: container is live (from task env ctx) ----
+        agentic_container = md.get("tbv21_container")
+        if agentic_container:
+            verifier_timeout = float(
+                md.get("verifier_timeout_sec") or _DEFAULT_VERIFIER_TIMEOUT
+            )
+            meta_a: Dict[str, Any] = {"mode": "agentic", "container": agentic_container}
+            try:
+                tests = subprocess.run(
+                    [
+                        "docker",
+                        "exec",
+                        agentic_container,
+                        "bash",
+                        "/tests/test.sh",
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=verifier_timeout + 60,
+                )
+                meta_a["tests_exit_code"] = tests.returncode
+                meta_a["tests_stdout_tail"] = tests.stdout[-2000:]
+                meta_a["tests_stderr_tail"] = tests.stderr[-1500:]
+            except subprocess.TimeoutExpired:
+                meta_a["tests_exit_code"] = -1
+                meta_a["tests_stdout_tail"] = "timeout"
+
+            try:
+                reward_out = subprocess.run(
+                    [
+                        "docker",
+                        "exec",
+                        agentic_container,
+                        "bash",
+                        "-c",
+                        "cat /logs/verifier/reward.txt 2>/dev/null || echo 0",
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+                raw = (reward_out.stdout or "").strip()
+                meta_a["reward_raw"] = raw
+                try:
+                    reward = float(raw)
+                except ValueError:
+                    reward = 0.0
+                meta_a["reward"] = reward
+                meta_a["score"] = reward
+                return reward >= 0.5, meta_a
+            except Exception as exc:  # noqa: BLE001
+                meta_a["reason"] = f"reward_read_failed: {exc}"
+                meta_a["score"] = 0.0
+                return False, meta_a
+
+        # ---- One-shot fallback ----
+        if not model_answer or not model_answer.strip():
+            return False, {"reason": "empty_response", "score": 0.0}
+
+        docker_image: Optional[str] = md.get("docker_image")
+        task_dir_str: Optional[str] = md.get("task_dir")
+        if not docker_image or not task_dir_str:
+            return None, {"reason": "missing_metadata", "mode": "oneshot"}
+        task_dir = Path(task_dir_str)
+        tests_dir = task_dir / "tests"
+        if not tests_dir.is_dir():
+            return None, {"reason": "no_tests_dir", "mode": "oneshot"}
+
+        agent_timeout = float(md.get("agent_timeout_sec") or _DEFAULT_AGENT_TIMEOUT)
+        verifier_timeout = float(
+            md.get("verifier_timeout_sec") or _DEFAULT_VERIFIER_TIMEOUT
+        )
+        cpus = str(md.get("cpus") or 2)
+        memory = str(md.get("memory") or "4G")
+
+        solve_script = _extract_bash(model_answer)
+        container = f"tbv21-{md.get('task_id', 'task')}-{uuid.uuid4().hex[:8]}"
+        container = re.sub(r"[^a-zA-Z0-9_-]", "-", container)[:63]
+        meta = {"mode": "oneshot"}
+
+        try:
+            start = subprocess.run(
+                [
+                    "docker",
+                    "run",
+                    "-d",
+                    "--name",
+                    container,
+                    "--cpus",
+                    cpus,
+                    "--memory",
+                    memory,
+                    "-v",
+                    f"{tests_dir}:/tests:ro",
+                    "--entrypoint",
+                    "/bin/bash",
+                    docker_image,
+                    "-c",
+                    "mkdir -p /logs/verifier && sleep infinity",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=600,
+            )
+            if start.returncode != 0:
+                meta["docker_start_error"] = start.stderr[:500]
+                return False, meta
+
+            subprocess.run(
+                [
+                    "docker",
+                    "exec",
+                    "-i",
+                    container,
+                    "bash",
+                    "-c",
+                    "cat > /app/solve.sh && chmod +x /app/solve.sh",
+                ],
+                input=solve_script,
+                text=True,
+                capture_output=True,
+                timeout=60,
+            )
+            agent = subprocess.run(
+                [
+                    "docker",
+                    "exec",
+                    container,
+                    "bash",
+                    "-c",
+                    "cd /app && bash /app/solve.sh",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=agent_timeout + 30,
+            )
+            meta["agent_exit_code"] = agent.returncode
+            meta["agent_stdout_tail"] = agent.stdout[-1500:]
+            meta["agent_stderr_tail"] = agent.stderr[-1500:]
+
+            tests = subprocess.run(
+                ["docker", "exec", container, "bash", "/tests/test.sh"],
+                capture_output=True,
+                text=True,
+                timeout=verifier_timeout + 30,
+            )
+            meta["tests_exit_code"] = tests.returncode
+            meta["tests_stdout_tail"] = tests.stdout[-1500:]
+            meta["tests_stderr_tail"] = tests.stderr[-1500:]
+
+            reward_out = subprocess.run(
+                [
+                    "docker",
+                    "exec",
+                    container,
+                    "bash",
+                    "-c",
+                    "cat /logs/verifier/reward.txt 2>/dev/null || echo 0",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            raw_reward = (reward_out.stdout or "").strip()
+            meta["reward_raw"] = raw_reward
+            try:
+                reward = float(raw_reward)
+            except ValueError:
+                reward = 0.0
+            meta["reward"] = reward
+            meta["score"] = reward
+            return reward >= 0.5, meta
+        except subprocess.TimeoutExpired as exc:
+            meta["reason"] = f"timeout: {exc.cmd[:3]}"
+            return False, meta
+        except Exception as exc:  # noqa: BLE001
+            meta["reason"] = f"{type(exc).__name__}: {exc}"
+            return False, meta
+        finally:
+            subprocess.run(
+                ["docker", "rm", "-f", container],
+                capture_output=True,
+                text=True,
+                timeout=60,
+            )
+
+
+__all__ = ["TerminalBenchV21Scorer"]
@@ -73,6 +73,7 @@ except ImportError:
    pass

 try:
+    import openjarvis.tools.docker_shell_exec  # noqa: F401
    import openjarvis.tools.shell_exec  # noqa: F401
 except ImportError:
    pass
@@ -0,0 +1,176 @@
+"""Container-scoped shell executor.
+
+Runs commands inside the currently-active TB v2.1 task container (set via
+:func:`openjarvis.tools.docker_shell_exec.set_active_container`). When no
+container is active the tool refuses to run — it is explicitly not a
+host-shell alternative.
+
+The expected lifecycle is:
+
+    from openjarvis.tools.docker_shell_exec import set_active_container
+
+    set_active_container(container_name)
+    try:
+        # run agent — every shell command goes through `docker exec`
+    finally:
+        set_active_container(None)
+
+The TB v2.1 task environment sets/clears this context automatically.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import threading
+from typing import Any, Optional
+
+from openjarvis.core.registry import ToolRegistry
+from openjarvis.tools._stubs import BaseTool, ToolResult, ToolSpec
+
+# NOTE: We use a module-level (process-wide) variable rather than a
+# threading.local, because ToolExecutor dispatches each tool call onto a
+# fresh ThreadPoolExecutor worker — thread-locals set on the runner
+# thread are not visible there. The eval runner processes one task at
+# a time per process, so there is no cross-task race. Parallelism
+# across models is achieved through isolated HOMEs (separate processes).
+_active_container: Optional[str] = None
+_lock = threading.Lock()
+_DEFAULT_TIMEOUT = 60
+_MAX_TIMEOUT = 600
+
+
+def set_active_container(name: Optional[str]) -> None:
+    """Bind (or clear) the Docker container for this process."""
+    global _active_container
+    with _lock:
+        _active_container = name
+
+
+def get_active_container() -> Optional[str]:
+    with _lock:
+        return _active_container
+
+
+@ToolRegistry.register("docker_shell_exec")
+class DockerShellExecTool(BaseTool):
+    """Execute a shell command inside the active TB v2.1 task container."""
+
+    tool_id = "docker_shell_exec"
+
+    @property
+    def spec(self) -> ToolSpec:
+        return ToolSpec(
+            name="docker_shell_exec",
+            description=(
+                "Execute a shell command inside the task's Linux container "
+                "(the /app working directory is mounted and writable). Use "
+                "this for ALL filesystem operations, running scripts, and "
+                "installing packages. Returns stdout/stderr/exit-code."
+            ),
+            parameters={
+                "type": "object",
+                "properties": {
+                    "command": {
+                        "type": "string",
+                        "description": (
+                            "Shell command to execute. Runs through "
+                            "`bash -c` inside the task container."
+                        ),
+                    },
+                    "timeout": {
+                        "type": "integer",
+                        "description": (
+                            "Timeout in seconds (default 60, max 600)."
+                        ),
+                    },
+                    "working_dir": {
+                        "type": "string",
+                        "description": (
+                            "Container-side working directory (default /app)."
+                        ),
+                    },
+                },
+                "required": ["command"],
+            },
+            category="system",
+            requires_confirmation=False,
+            timeout_seconds=float(_MAX_TIMEOUT),
+            required_capabilities=["code:execute"],
+        )
+
+    def execute(self, **params: Any) -> ToolResult:
+        container = get_active_container()
+        if not container:
+            return ToolResult(
+                tool_name="docker_shell_exec",
+                content=(
+                    "No active task container. This tool can only run "
+                    "inside a TerminalBench V2.1 task environment."
+                ),
+                success=False,
+            )
+
+        command = params.get("command", "")
+        if not command:
+            return ToolResult(
+                tool_name="docker_shell_exec",
+                content="No command provided.",
+                success=False,
+            )
+
+        try:
+            timeout = int(params.get("timeout", _DEFAULT_TIMEOUT))
+        except (TypeError, ValueError):
+            timeout = _DEFAULT_TIMEOUT
+        timeout = max(1, min(timeout, _MAX_TIMEOUT))
+
+        working_dir = params.get("working_dir") or "/app"
+
+        cmd = [
+            "docker",
+            "exec",
+            "-w",
+            str(working_dir),
+            container,
+            "bash",
+            "-c",
+            command,
+        ]
+        try:
+            r = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+        except subprocess.TimeoutExpired:
+            return ToolResult(
+                tool_name="docker_shell_exec",
+                content=(
+                    f"Command timed out after {timeout}s: {command!r}"
+                ),
+                success=False,
+            )
+
+        body = (
+            f"[exit {r.returncode}]\n"
+            + (f"--- stdout ---\n{r.stdout}\n" if r.stdout else "")
+            + (f"--- stderr ---\n{r.stderr}\n" if r.stderr else "")
+        )
+        return ToolResult(
+            tool_name="docker_shell_exec",
+            content=body.strip() or f"[exit {r.returncode}]",
+            success=(r.returncode == 0),
+            metadata={
+                "exit_code": r.returncode,
+                "stdout": r.stdout,
+                "stderr": r.stderr,
+            },
+        )
+
+
+__all__ = [
+    "DockerShellExecTool",
+    "get_active_container",
+    "set_active_container",
+]
@@ -129,6 +129,15 @@ class TestDatasetInstantiation:
        assert ds.dataset_id == "terminalbench-native"
        assert ds.dataset_name == "TerminalBench Native"

+    def test_terminalbench_v2_1(self) -> None:
+        from openjarvis.evals.datasets.terminalbench_v2_1 import (
+            TerminalBenchV21Dataset,
+        )
+
+        ds = TerminalBenchV21Dataset()
+        assert ds.dataset_id == "terminalbench-v2.1"
+        assert ds.dataset_name == "TerminalBench V2.1"
+
    def test_livecodebench(self) -> None:
        from openjarvis.evals.datasets.livecodebench import LiveCodeBenchDataset

@@ -254,6 +263,14 @@ class TestScorerInstantiation:
        s = TerminalBenchNativeScorer(_mock_backend(), "test-model")
        assert s.scorer_id == "terminalbench-native"

+    def test_terminalbench_v2_1_scorer(self) -> None:
+        from openjarvis.evals.scorers.terminalbench_v2_1 import (
+            TerminalBenchV21Scorer,
+        )
+
+        s = TerminalBenchV21Scorer(_mock_backend(), "test-model")
+        assert s.scorer_id == "terminalbench-v2.1"
+
    def test_livecodebench_scorer(self) -> None:
        from openjarvis.evals.scorers.livecodebench import LiveCodeBenchScorer

@@ -294,6 +311,7 @@ ALL_BENCHMARKS = [
    "swefficiency",
    "terminalbench",
    "terminalbench-native",
+    "terminalbench-v2.1",
    "livecodebench",
    "liveresearch",
    "toolcall15",
@@ -355,7 +373,7 @@ class TestConfigBenchmarks:
    def test_benchmarks_count(self) -> None:
        from openjarvis.evals.core.config import KNOWN_BENCHMARKS

-        assert len(KNOWN_BENCHMARKS) == 31
+        assert len(KNOWN_BENCHMARKS) == 32


 # ---------------------------------------------------------------------------