diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 41309ef0a..e1ac1fa39 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -187,7 +187,7 @@ jobs:
 
       - name: Install min requirements
         run: |
-          uv pip install -e. --group=test --resolution=lowest-direct --system
+          uv pip install -e. --group=test-pybind11 --resolution=lowest-direct --system
 
       - name: Setup CMake 3.15
         uses: jwlawson/actions-setup-cmake@v2.0
@@ -223,7 +223,7 @@ jobs:
         run: python3.13t -m venv /venv
 
       - name: Install deps
-        run: /venv/bin/pip install -e . --group=test ninja
+        run: /venv/bin/pip install -e . --group=test-pybind11 ninja
 
       - name: Test package
         run: /venv/bin/pytest
@@ -246,7 +246,7 @@ jobs:
             cmake ninja git make gcc-g++ python39 python39-devel python39-pip
 
       - name: Install
-        run: python3.9 -m pip install . --group=test
+        run: python3.9 -m pip install . --group=test-pybind11
 
       - name: Test package
         run:
@@ -279,7 +279,7 @@ jobs:
           persist-credentials: false
 
       - name: Install
-        run: python -m pip install . --group=test
+        run: python -m pip install . --group=test-pybind11
 
       - name: Test package
         run: >-
@@ -312,7 +312,7 @@ jobs:
           persist-credentials: false
 
       - name: Install
-        run: python -m pip install . --group=test
+        run: python -m pip install . --group=test-pybind11
 
       - name: Test package
         run: >-
diff --git a/pyproject.toml b/pyproject.toml
index 1326a1ed5..cdc3cbe4e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling", "hatch-vcs"]
+requires = ["hatchling >=1.24", "hatch-vcs >=0.4"]
 build-backend = "hatchling.build"
 
 [project]
@@ -77,6 +77,9 @@ hatch.scikit-build = "scikit_build_core.hatch.hooks"
 test = [
     "build >=0.8",
     "cattrs >=22.2.0",
+    "filelock >=3.8.0",
+    "hatchling >=1.24",
+    "hatch-vcs >=0.4",
     "pip>=23; python_version<'3.13'",
     "pip>=24.1; python_version>='3.13'",
     "pytest >=7.2",
@@ -89,10 +92,6 @@ test = [
     "virtualenv >=20.20",
     "wheel >=0.40",
 ]
-test-hatchling = [
-    { include-group = "test" },
-    "hatchling >=1.24.0",
-]
 test-meta = [
     { include-group = "test" },
     "hatch-fancy-pypi-readme>=22.3",
@@ -120,7 +119,6 @@ cov = [
 ]
 dev = [
     { include-group = "cov" },
-    { include-group = "test-hatchling" },
     { include-group = "test-meta" },
     { include-group = "test-numpy" },
     { include-group = "test-pybind11" },
diff --git a/tests/conftest.py b/tests/conftest.py
index c197d7864..1d871f485 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -14,6 +14,7 @@
 from typing import Any, Literal, overload
 
 import virtualenv as _virtualenv
+from filelock import FileLock
 
 if sys.version_info < (3, 11):
     import tomli as tomllib
@@ -26,6 +27,7 @@
     from typing import TypeGuard
 
 
+import download_wheels
 import pytest
 from packaging.requirements import Requirement
 from packaging.version import Version
@@ -37,53 +39,32 @@
 
 
 @pytest.fixture(scope="session")
-def pep518_wheelhouse(tmp_path_factory: pytest.TempPathFactory) -> Path:
-    wheelhouse = tmp_path_factory.mktemp("wheelhouse")
-
-    subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "wheel",
-            "--wheel-dir",
-            str(wheelhouse),
-            f"{BASE}",
-        ],
-        check=True,
-    )
-    packages = [
-        "build",
-        "cython",
-        "hatchling",
-        "pip",
-        "setuptools",
-        "virtualenv",
-        "wheel",
-    ]
+def pep518_wheelhouse(pytestconfig: pytest.Config) -> Path:
+    wheelhouse = pytestconfig.cache.mkdir("wheelhouse")
+
+    main_lock = FileLock(wheelhouse / "main.lock")
+    with main_lock:
+        subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "wheel",
+                "--wheel-dir",
+                str(wheelhouse),
+                "--no-build-isolation",
+                f"{BASE}",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+    wheels_lock = FileLock(wheelhouse / "wheels.lock")
+    with wheels_lock:
+        if not all(list(wheelhouse.glob(f"{p}*.whl")) for p in download_wheels.WHEELS):
+            download_wheels.prepare(wheelhouse)
 
-    if importlib.util.find_spec("cmake") is not None:
-        packages.append("cmake")
-
-    if importlib.util.find_spec("ninja") is not None:
-        packages.append("ninja")
-
-    if importlib.util.find_spec("pybind11") is not None:
-        packages.append("pybind11")
-
-    subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "download",
-            "-q",
-            "-d",
-            str(wheelhouse),
-            *packages,
-        ],
-        check=True,
-    )
     return wheelhouse
 
 
diff --git a/tests/plans.fmf b/tests/plans.fmf
index 6a12e7fb6..4189e4d76 100644
--- a/tests/plans.fmf
+++ b/tests/plans.fmf
@@ -27,7 +27,7 @@
         node-date: 2025-02-27T16:18:39-05:00
         describe-name: v0.11.0
         EOF
-        pip install --user . --group=test
+        pip install --user . --group=test-pybind11
   discover:
     how: fmf
     filter: "tag: pytest"
diff --git a/tests/test_hatchling.py b/tests/test_hatchling.py
index 8fc193447..8e832b3a0 100644
--- a/tests/test_hatchling.py
+++ b/tests/test_hatchling.py
@@ -3,6 +3,7 @@
 import zipfile
 from pathlib import Path
 
+import download_wheels
 import pytest
 
 pytest.importorskip("hatchling")
@@ -42,7 +43,9 @@ def test_hatchling_sdist(isolated, tmp_path: Path) -> None:
 )
 def test_hatchling_wheel(isolated, build_args, tmp_path: Path) -> None:
     dist = tmp_path / "dist"
-    isolated.install("build[virtualenv]", "scikit-build-core", "hatchling", "pybind11")
+    isolated.install(
+        "build[virtualenv]", "scikit-build-core", "hatchling", *download_wheels.EXTRA
+    )
     isolated.module("build", "--no-isolation", f"--outdir={dist}", *build_args)
     ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
 
diff --git a/tests/utils/download_wheels.py b/tests/utils/download_wheels.py
new file mode 100644
index 000000000..29c4b44e3
--- /dev/null
+++ b/tests/utils/download_wheels.py
@@ -0,0 +1,59 @@
+# /// script
+# dependencies = ["pip"]
+# ///
+
+"""
+Download wheels into the pytest cache. Must be run from the pytest directory
+(project root, usually). If run in an environment, requires pip. Only downloads
+pybind11, ninja, or cmake if those are in the environment already.
+"""
+
+import importlib.util
+import subprocess
+import sys
+from pathlib import Path
+
+EXTRA = []
+
+if importlib.util.find_spec("cmake") is not None:
+    EXTRA.append("cmake")
+
+if importlib.util.find_spec("ninja") is not None:
+    EXTRA.append("ninja")
+
+if importlib.util.find_spec("pybind11") is not None:
+    EXTRA.append("pybind11")
+
+WHEELS = [
+    "build",
+    "cython",
+    "hatchling",
+    "pip",
+    "setuptools",
+    "virtualenv",
+    "wheel",
+    *EXTRA,
+]
+
+
+def prepare(wheelhouse: Path) -> None:
+    subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "download",
+            "-q",
+            "-d",
+            str(wheelhouse),
+            *WHEELS,
+        ],
+        check=True,
+    )
+    print(f"Downloaded wheels to {wheelhouse}")
+
+
+if __name__ == "__main__":
+    wheelhouse = Path(".pytest_cache/d/wheelhouse")
+    wheelhouse.mkdir(parents=True, exist_ok=True)
+    prepare(wheelhouse)