diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.md index c6179a13..7f0587c3 100644 --- a/docs/user-guide/evaluations.md +++ b/docs/user-guide/evaluations.md @@ -35,7 +35,7 @@ TauBench additionally requires Python 3.12 or newer and the upstream `tau2` package. Install the pinned revision explicitly before running that benchmark: ```bash -uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@b711c1ead46f55111bf765cf44d5da8bacc2d28c" +uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@fc0055dc4e0a316c3f83133267fbd6faaa770992" ``` OpenJarvis does not install third-party packages automatically when an diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py index fee4ca59..d56cdfeb 100644 --- a/src/openjarvis/evals/datasets/taubench.py +++ b/src/openjarvis/evals/datasets/taubench.py @@ -8,8 +8,10 @@ Reference: https://github.com/sierra-research/tau2-bench from __future__ import annotations +import json import logging import os +from importlib import metadata from typing import Iterable, List, Optional from openjarvis.evals.core.dataset import DatasetProvider @@ -21,7 +23,7 @@ LOGGER = logging.getLogger(__name__) TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git" # v1.0.1. Keep the full commit SHA here (rather than a movable tag) so every # TauBench setup uses the same third-party code. -TAU2_REVISION = "b711c1ead46f55111bf765cf44d5da8bacc2d28c" +TAU2_REVISION = "fc0055dc4e0a316c3f83133267fbd6faaa770992" TAU2_INSTALL_SPEC = f"tau2 @ git+{TAU2_REPO}@{TAU2_REVISION}" DOMAINS = ("airline", "retail", "telecom") @@ -30,14 +32,40 @@ DOMAINS = ("airline", "retail", "telecom") def _ensure_tau2() -> None: """Ensure the explicitly installed, pinned tau2 package is importable.""" try: - import tau2 # noqa: F401 - except ImportError as exc: + distribution = metadata.distribution("tau2") + except metadata.PackageNotFoundError as exc: raise ImportError( "TauBench requires tau2, which OpenJarvis does not install at " "runtime. Install the pinned dependency explicitly (Python >=3.12): " f'uv pip install "{TAU2_INSTALL_SPEC}"' ) from exc + try: + direct_url_text = distribution.read_text("direct_url.json") + direct_url = json.loads(direct_url_text or "") + vcs_info = direct_url.get("vcs_info", {}) + installed_repo = direct_url.get("url") + installed_revision = vcs_info.get("commit_id") + except (json.JSONDecodeError, AttributeError): + installed_repo = None + installed_revision = None + + if installed_repo != TAU2_REPO or installed_revision != TAU2_REVISION: + raise ImportError( + "The installed tau2 package does not match OpenJarvis's pinned " + "source revision. Reinstall it explicitly (Python >=3.12): " + f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"' + ) + + try: + import tau2 # noqa: F401 + except ImportError as exc: + raise ImportError( + "The pinned tau2 package is installed but cannot be imported. " + "Reinstall it explicitly (Python >=3.12): " + f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"' + ) from exc + class TauBenchDataset(DatasetProvider): """TauBench V2 multi-turn customer service benchmark. diff --git a/tests/evals/datasets/test_taubench.py b/tests/evals/datasets/test_taubench.py index ddf87747..b54e074f 100644 --- a/tests/evals/datasets/test_taubench.py +++ b/tests/evals/datasets/test_taubench.py @@ -2,22 +2,45 @@ from __future__ import annotations +import builtins import sys from types import ModuleType +from unittest.mock import Mock import pytest from openjarvis.evals.datasets import taubench -def test_ensure_tau2_accepts_an_installed_package(monkeypatch): +def _mock_direct_url(monkeypatch, direct_url): + distribution = Mock() + distribution.read_text.return_value = direct_url + monkeypatch.setattr( + taubench.metadata, "distribution", Mock(return_value=distribution) + ) + + +def test_ensure_tau2_accepts_the_pinned_source_revision(monkeypatch): monkeypatch.setitem(sys.modules, "tau2", ModuleType("tau2")) + _mock_direct_url( + monkeypatch, + ( + '{"url": "https://github.com/sierra-research/tau2-bench.git", ' + '"vcs_info": {"vcs": "git", ' + f'"commit_id": "{taubench.TAU2_REVISION}"}}}}' + ), + ) taubench._ensure_tau2() def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch): monkeypatch.setitem(sys.modules, "tau2", None) + monkeypatch.setattr( + taubench.metadata, + "distribution", + Mock(side_effect=taubench.metadata.PackageNotFoundError), + ) with pytest.raises(ImportError) as exc_info: taubench._ensure_tau2() @@ -28,8 +51,46 @@ def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch): assert "uv pip install" in message +@pytest.mark.parametrize( + "direct_url", + [ + # Editable install left behind by the previous runtime installer. + '{"url": "file:///home/user/.openjarvis/cache/tau2-bench", ' + '"dir_info": {"editable": true}}', + # A git install from an arbitrary upstream revision. + '{"url": "https://github.com/sierra-research/tau2-bench.git", ' + '"vcs_info": {"vcs": "git", "commit_id": "deadbeef"}}', + # Registry installs do not carry PEP 610 direct-origin metadata. + None, + ], +) +def test_ensure_tau2_rejects_unpinned_install(monkeypatch, direct_url): + _mock_direct_url(monkeypatch, direct_url) + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "tau2": + raise AssertionError("unverified tau2 package was imported") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + + with pytest.raises(ImportError) as exc_info: + taubench._ensure_tau2() + + message = str(exc_info.value) + assert "does not match" in message + assert taubench.TAU2_REVISION in message + assert "--force-reinstall" in message + + def test_verify_requirements_reports_install_instruction(monkeypatch): monkeypatch.setitem(sys.modules, "tau2", None) + monkeypatch.setattr( + taubench.metadata, + "distribution", + Mock(side_effect=taubench.metadata.PackageNotFoundError), + ) issues = taubench.TauBenchDataset().verify_requirements()