fix(evals): make TauBench dependency explicit (#739)

* fix(evals): make TauBench dependency explicit

* fix(evals): verify TauBench install provenance

docs/user-guide/evaluations.md

@@ -31,6 +31,16 @@ uv sync --extra dev --extra eval-wandb     # Weights & Biases run tracking
 uv sync --extra dev --extra eval-sheets    # Google Sheets results export
 ```
 
+TauBench additionally requires Python 3.12 or newer and the upstream `tau2`
+package. Install the pinned revision explicitly before running that benchmark:
+
+```bash
+uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@fc0055dc4e0a316c3f83133267fbd6faaa770992"
+```
+
+OpenJarvis does not install third-party packages automatically when an
+evaluation is imported or run.
+
 !!! note "Python version requirement"
     Python 3.10 requires the `tomli` package for TOML config parsing. `openjarvis` declares it as a conditional dependency, so it is installed automatically.
 

src/openjarvis/evals/datasets/taubench.py

@@ -8,13 +8,12 @@ Reference: https://github.com/sierra-research/tau2-bench
 
 from __future__ import annotations
 
+import json
 import logging
 import os
-import subprocess
-import sys
+from importlib import metadata
 from typing import Iterable, List, Optional
 
-from openjarvis.core.paths import get_cache_dir
 from openjarvis.evals.core.dataset import DatasetProvider
 from openjarvis.evals.core.splits import apply_split
 from openjarvis.evals.core.types import EvalRecord
@@ -22,48 +21,50 @@ from openjarvis.evals.core.types import EvalRecord
 LOGGER = logging.getLogger(__name__)
 
 TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"
-CACHE_DIR = get_cache_dir() / "tau2-bench"
+# v1.0.1. Keep the full commit SHA here (rather than a movable tag) so every
+# TauBench setup uses the same third-party code.
+TAU2_REVISION = "fc0055dc4e0a316c3f83133267fbd6faaa770992"
+TAU2_INSTALL_SPEC = f"tau2 @ git+{TAU2_REPO}@{TAU2_REVISION}"
 
 DOMAINS = ("airline", "retail", "telecom")
 
 
 def _ensure_tau2() -> None:
-    """Ensure tau2 package is importable; install from cache if needed."""
+    """Ensure the explicitly installed, pinned tau2 package is importable."""
+    try:
+        distribution = metadata.distribution("tau2")
+    except metadata.PackageNotFoundError as exc:
+        raise ImportError(
+            "TauBench requires tau2, which OpenJarvis does not install at "
+            "runtime. Install the pinned dependency explicitly (Python >=3.12): "
+            f'uv pip install "{TAU2_INSTALL_SPEC}"'
+        ) from exc
+
+    try:
+        direct_url_text = distribution.read_text("direct_url.json")
+        direct_url = json.loads(direct_url_text or "")
+        vcs_info = direct_url.get("vcs_info", {})
+        installed_repo = direct_url.get("url")
+        installed_revision = vcs_info.get("commit_id")
+    except (json.JSONDecodeError, AttributeError):
+        installed_repo = None
+        installed_revision = None
+
+    if installed_repo != TAU2_REPO or installed_revision != TAU2_REVISION:
+        raise ImportError(
+            "The installed tau2 package does not match OpenJarvis's pinned "
+            "source revision. Reinstall it explicitly (Python >=3.12): "
+            f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+        )
+
     try:
         import tau2  # noqa: F401
-    except ImportError:
-        # Clone and install from source
-        if not CACHE_DIR.exists():
-            LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO)
-            CACHE_DIR.parent.mkdir(parents=True, exist_ok=True)
-            subprocess.run(
-                ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)],
-                check=True,
-                capture_output=True,
-            )
-        LOGGER.info("Installing tau2-bench ...")
-        # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs
-        # which don't ship pip by default.
-        try:
-            subprocess.run(
-                [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
-                check=True,
-                capture_output=True,
-            )
-        except (subprocess.CalledProcessError, FileNotFoundError):
-            subprocess.run(
-                [
-                    "uv",
-                    "pip",
-                    "install",
-                    "--python",
-                    sys.executable,
-                    "-e",
-                    str(CACHE_DIR),
-                ],
-                check=True,
-                capture_output=True,
-            )
+    except ImportError as exc:
+        raise ImportError(
+            "The pinned tau2 package is installed but cannot be imported. "
+            "Reinstall it explicitly (Python >=3.12): "
+            f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+        ) from exc
 
 
 class TauBenchDataset(DatasetProvider):

tests/evals/datasets/test_taubench.py

@@ -0,0 +1,98 @@
+"""Tests for the TauBench optional dependency boundary."""
+
+from __future__ import annotations
+
+import builtins
+import sys
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+
+from openjarvis.evals.datasets import taubench
+
+
+def _mock_direct_url(monkeypatch, direct_url):
+    distribution = Mock()
+    distribution.read_text.return_value = direct_url
+    monkeypatch.setattr(
+        taubench.metadata, "distribution", Mock(return_value=distribution)
+    )
+
+
+def test_ensure_tau2_accepts_the_pinned_source_revision(monkeypatch):
+    monkeypatch.setitem(sys.modules, "tau2", ModuleType("tau2"))
+    _mock_direct_url(
+        monkeypatch,
+        (
+            '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+            '"vcs_info": {"vcs": "git", '
+            f'"commit_id": "{taubench.TAU2_REVISION}"}}}}'
+        ),
+    )
+
+    taubench._ensure_tau2()
+
+
+def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch):
+    monkeypatch.setitem(sys.modules, "tau2", None)
+    monkeypatch.setattr(
+        taubench.metadata,
+        "distribution",
+        Mock(side_effect=taubench.metadata.PackageNotFoundError),
+    )
+
+    with pytest.raises(ImportError) as exc_info:
+        taubench._ensure_tau2()
+
+    message = str(exc_info.value)
+    assert "does not install at runtime" in message
+    assert taubench.TAU2_REVISION in message
+    assert "uv pip install" in message
+
+
+@pytest.mark.parametrize(
+    "direct_url",
+    [
+        # Editable install left behind by the previous runtime installer.
+        '{"url": "file:///home/user/.openjarvis/cache/tau2-bench", '
+        '"dir_info": {"editable": true}}',
+        # A git install from an arbitrary upstream revision.
+        '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+        '"vcs_info": {"vcs": "git", "commit_id": "deadbeef"}}',
+        # Registry installs do not carry PEP 610 direct-origin metadata.
+        None,
+    ],
+)
+def test_ensure_tau2_rejects_unpinned_install(monkeypatch, direct_url):
+    _mock_direct_url(monkeypatch, direct_url)
+    original_import = builtins.__import__
+
+    def guarded_import(name, *args, **kwargs):
+        if name == "tau2":
+            raise AssertionError("unverified tau2 package was imported")
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", guarded_import)
+
+    with pytest.raises(ImportError) as exc_info:
+        taubench._ensure_tau2()
+
+    message = str(exc_info.value)
+    assert "does not match" in message
+    assert taubench.TAU2_REVISION in message
+    assert "--force-reinstall" in message
+
+
+def test_verify_requirements_reports_install_instruction(monkeypatch):
+    monkeypatch.setitem(sys.modules, "tau2", None)
+    monkeypatch.setattr(
+        taubench.metadata,
+        "distribution",
+        Mock(side_effect=taubench.metadata.PackageNotFoundError),
+    )
+
+    issues = taubench.TauBenchDataset().verify_requirements()
+
+    assert len(issues) == 1
+    assert taubench.TAU2_REVISION in issues[0]