fix(evals): verify TauBench install provenance

docs/user-guide/evaluations.md

@@ -35,7 +35,7 @@ TauBench additionally requires Python 3.12 or newer and the upstream `tau2`
 package. Install the pinned revision explicitly before running that benchmark:
 
 ```bash
-uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@b711c1ead46f55111bf765cf44d5da8bacc2d28c"
+uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@fc0055dc4e0a316c3f83133267fbd6faaa770992"
 ```
 
 OpenJarvis does not install third-party packages automatically when an

src/openjarvis/evals/datasets/taubench.py

@@ -8,8 +8,10 @@ Reference: https://github.com/sierra-research/tau2-bench
 
 from __future__ import annotations
 
+import json
 import logging
 import os
+from importlib import metadata
 from typing import Iterable, List, Optional
 
 from openjarvis.evals.core.dataset import DatasetProvider
@@ -21,7 +23,7 @@ LOGGER = logging.getLogger(__name__)
 TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"
 # v1.0.1. Keep the full commit SHA here (rather than a movable tag) so every
 # TauBench setup uses the same third-party code.
-TAU2_REVISION = "b711c1ead46f55111bf765cf44d5da8bacc2d28c"
+TAU2_REVISION = "fc0055dc4e0a316c3f83133267fbd6faaa770992"
 TAU2_INSTALL_SPEC = f"tau2 @ git+{TAU2_REPO}@{TAU2_REVISION}"
 
 DOMAINS = ("airline", "retail", "telecom")
@@ -30,14 +32,40 @@ DOMAINS = ("airline", "retail", "telecom")
 def _ensure_tau2() -> None:
     """Ensure the explicitly installed, pinned tau2 package is importable."""
     try:
-        import tau2  # noqa: F401
-    except ImportError as exc:
+        distribution = metadata.distribution("tau2")
+    except metadata.PackageNotFoundError as exc:
         raise ImportError(
             "TauBench requires tau2, which OpenJarvis does not install at "
             "runtime. Install the pinned dependency explicitly (Python >=3.12): "
             f'uv pip install "{TAU2_INSTALL_SPEC}"'
         ) from exc
 
+    try:
+        direct_url_text = distribution.read_text("direct_url.json")
+        direct_url = json.loads(direct_url_text or "")
+        vcs_info = direct_url.get("vcs_info", {})
+        installed_repo = direct_url.get("url")
+        installed_revision = vcs_info.get("commit_id")
+    except (json.JSONDecodeError, AttributeError):
+        installed_repo = None
+        installed_revision = None
+
+    if installed_repo != TAU2_REPO or installed_revision != TAU2_REVISION:
+        raise ImportError(
+            "The installed tau2 package does not match OpenJarvis's pinned "
+            "source revision. Reinstall it explicitly (Python >=3.12): "
+            f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+        )
+
+    try:
+        import tau2  # noqa: F401
+    except ImportError as exc:
+        raise ImportError(
+            "The pinned tau2 package is installed but cannot be imported. "
+            "Reinstall it explicitly (Python >=3.12): "
+            f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+        ) from exc
+
 
 class TauBenchDataset(DatasetProvider):
     """TauBench V2 multi-turn customer service benchmark.

tests/evals/datasets/test_taubench.py

@@ -2,22 +2,45 @@
 
 from __future__ import annotations
 
+import builtins
 import sys
 from types import ModuleType
+from unittest.mock import Mock
 
 import pytest
 
 from openjarvis.evals.datasets import taubench
 
 
-def test_ensure_tau2_accepts_an_installed_package(monkeypatch):
+def _mock_direct_url(monkeypatch, direct_url):
+    distribution = Mock()
+    distribution.read_text.return_value = direct_url
+    monkeypatch.setattr(
+        taubench.metadata, "distribution", Mock(return_value=distribution)
+    )
+
+
+def test_ensure_tau2_accepts_the_pinned_source_revision(monkeypatch):
     monkeypatch.setitem(sys.modules, "tau2", ModuleType("tau2"))
+    _mock_direct_url(
+        monkeypatch,
+        (
+            '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+            '"vcs_info": {"vcs": "git", '
+            f'"commit_id": "{taubench.TAU2_REVISION}"}}}}'
+        ),
+    )
 
     taubench._ensure_tau2()
 
 
 def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch):
     monkeypatch.setitem(sys.modules, "tau2", None)
+    monkeypatch.setattr(
+        taubench.metadata,
+        "distribution",
+        Mock(side_effect=taubench.metadata.PackageNotFoundError),
+    )
 
     with pytest.raises(ImportError) as exc_info:
         taubench._ensure_tau2()
@@ -28,8 +51,46 @@ def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch):
     assert "uv pip install" in message
 
 
+@pytest.mark.parametrize(
+    "direct_url",
+    [
+        # Editable install left behind by the previous runtime installer.
+        '{"url": "file:///home/user/.openjarvis/cache/tau2-bench", '
+        '"dir_info": {"editable": true}}',
+        # A git install from an arbitrary upstream revision.
+        '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+        '"vcs_info": {"vcs": "git", "commit_id": "deadbeef"}}',
+        # Registry installs do not carry PEP 610 direct-origin metadata.
+        None,
+    ],
+)
+def test_ensure_tau2_rejects_unpinned_install(monkeypatch, direct_url):
+    _mock_direct_url(monkeypatch, direct_url)
+    original_import = builtins.__import__
+
+    def guarded_import(name, *args, **kwargs):
+        if name == "tau2":
+            raise AssertionError("unverified tau2 package was imported")
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", guarded_import)
+
+    with pytest.raises(ImportError) as exc_info:
+        taubench._ensure_tau2()
+
+    message = str(exc_info.value)
+    assert "does not match" in message
+    assert taubench.TAU2_REVISION in message
+    assert "--force-reinstall" in message
+
+
 def test_verify_requirements_reports_install_instruction(monkeypatch):
     monkeypatch.setitem(sys.modules, "tau2", None)
+    monkeypatch.setattr(
+        taubench.metadata,
+        "distribution",
+        Mock(side_effect=taubench.metadata.PackageNotFoundError),
+    )
 
     issues = taubench.TauBenchDataset().verify_requirements()