diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdc09d78..de995b2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: - name: Run tests run: | - uv run pytest tests/ -v --tb=short -m "not live and not cloud" \ + uv run pytest tests/ -v --tb=short -m "not live and not cloud and not hub" \ --cov=openjarvis \ --cov-report=term-missing \ --cov-report=xml \ diff --git a/pyproject.toml b/pyproject.toml index 7c718d9b..5b5a9120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,7 @@ markers = [ "live_channel: requires real channel credentials (env vars)", "nvidia: requires NVIDIA GPU", "slow: long-running test", + "hub: downloads real datasets from the HuggingFace Hub at runtime; excluded from the default CI lane (run with -m hub)", "live_external: requires HERMES_AGENT_PATH and OPENCLAW_PATH; spawns real foreign-framework subprocesses", "modal: requires Modal token + network; runs real swebench harness on Modal", ] diff --git a/tests/evals/datasets/test_external_agent_datasets.py b/tests/evals/datasets/test_external_agent_datasets.py index bbfdc2df..c5b8ca1c 100644 --- a/tests/evals/datasets/test_external_agent_datasets.py +++ b/tests/evals/datasets/test_external_agent_datasets.py @@ -10,6 +10,14 @@ These tests download each dataset once to ~/.cache/huggingface and verify the provider loads, iterates, and honours the split kwarg. The ModuleNotFoundError skip branch is defensive — currently unreachable since all three provider modules exist. + +Marked ``hub``: they hit the live HuggingFace Hub, so they are excluded +from the default CI lane (which runs ``-m "not live and not cloud and not +hub"``) — a transient Hub outage or rate-limit must not redden ``main``. +Run them on demand with ``pytest -m hub``. The ADP provider swallows +per-config download errors and returns 0 records on a network failure +(see ``adp.py``), so a Hub outage surfaces here as ``assert 1 <= 0`` +rather than an exception — another reason these can't run unguarded in CI. """ from __future__ import annotations @@ -18,6 +26,8 @@ import importlib import pytest +pytestmark = pytest.mark.hub + PROVIDERS = [ ("openjarvis.evals.datasets.adp", "ADPDataset"), ("openjarvis.evals.datasets.toolorchestra", "ToolOrchestraDataset"), diff --git a/tests/evals/test_dataset_splits_integration.py b/tests/evals/test_dataset_splits_integration.py index e312ec21..b0955f17 100644 --- a/tests/evals/test_dataset_splits_integration.py +++ b/tests/evals/test_dataset_splits_integration.py @@ -1,4 +1,13 @@ -"""Integration test: each provider's split kwarg produces disjoint train/test slices.""" +"""Integration test: each provider's split kwarg produces disjoint train/test slices. + +Marked ``hub``: every provider here downloads a real corpus from the +HuggingFace Hub, so the module is excluded from the default CI lane (which +runs ``-m "not live and not cloud and not hub"``). A transient Hub outage +or rate-limit raises connectivity errors (``LocalEntryNotFoundError``, +``HfHubHTTPError``) that the gated/not-found skip branch below does not +catch — so running these unguarded made ``main`` flaky-red. Run on demand +with ``pytest -m hub``. +""" from __future__ import annotations @@ -6,6 +15,8 @@ import importlib import pytest +pytestmark = pytest.mark.hub + PROVIDERS = [ ("openjarvis.evals.datasets.pinchbench", "PinchBenchDataset"), ("openjarvis.evals.datasets.liveresearch", "LiveResearchBenchDataset"),