Testing cleanup

.github/workflows/prepare_test_data.yaml

@@ -4,11 +4,29 @@ on:
   schedule:
     - cron: "0 0 1 * *" # run once a month to prevent artifact expiration
   workflow_dispatch:
-  # Uncomment and adjust the branch name if you need to add new datasets to the artifact.
-  # It needs to be a branch in the spatialdata-io origin repository, not from a fork.
-#  push:
-#    branches:
-#      - main
+    inputs:
+      force_all:
+        description: "Download all registered datasets. Set to false to use dataset_keys."
+        required: true
+        type: boolean
+        default: true
+      dataset_keys:
+        description: "Dataset keys to download when force_all is false. Separate keys with spaces or commas."
+        required: false
+        type: string
+        default: ""
+      force_redownload:
+        description: "Redownload and replace existing selected datasets."
+        required: true
+        type: boolean
+        default: false
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/prepare_test_data.yaml"
+      - "pyproject.toml"
+      - "scripts/test_data_downloader/**"
 
 permissions:
   contents: read
@@ -18,72 +36,44 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           filter: blob:none
           persist-credentials: false
 
-      - name: Download test datasets
-        run: |
-          mkdir -p ./data
-          cd ./data
-
-          # -------
-          # the Xenium datasets are licensed as CC BY 4.0, as shown here
-          # https://www.10xgenomics.com/support/software/xenium-onboard-analysis/latest/resources/xenium-example-data
-
-          # 10x Genomics Xenium 2.0.0
-          curl -O https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_human_Breast_2fov/Xenium_V1_human_Breast_2fov_outs.zip
-          curl -O https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_human_Lung_2fov/Xenium_V1_human_Lung_2fov_outs.zip
-
-          # 10x Genomics Xenium 3.0.0 (5K) Mouse ileum, multimodal cell segmentation
-          # this file seems to be corrupted; skipping it for now
-          # curl -O https://cf.10xgenomics.com/samples/xenium/3.0.0/Xenium_Prime_MultiCellSeg_Mouse_Ileum_tiny/Xenium_Prime_MultiCellSeg_Mouse_Ileum_tiny.zip
-
-          # 10x Genomics Xenium 3.0.0 (5K) Mouse ileum, nuclear expansion
-          curl -O https://cf.10xgenomics.com/samples/xenium/3.0.0/Xenium_Prime_Mouse_Ileum_tiny/Xenium_Prime_Mouse_Ileum_tiny_outs.zip
-
-          # 10x Genomics Xenium 4.0.0 (v1) Human ovary, nuclear expansion
-          curl -O https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_Human_Ovary_tiny/Xenium_V1_Human_Ovary_tiny_outs.zip
-
-          # 10x Genomics Xenium 4.0.0 (v1) Human ovary, multimodal cell segmentation
-          curl -O https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_MultiCellSeg_Human_Ovary_tiny/Xenium_V1_MultiCellSeg_Human_Ovary_tiny_outs.zip
-
-          # 10x Genomics Xenium 4.0.0 (v1+Protein) Human kidney, multimodal cell segmentation
-          curl -O https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_Protein_Human_Kidney_tiny/Xenium_V1_Protein_Human_Kidney_tiny_outs.zip
-
-          # -------
-          # the Visium HD dataset is licensed as CC BY 4.0, as shown here
-          # https://www.10xgenomics.com/support/software/space-ranger/latest/resources/visium-hd-example-data
-
-          # 10x Genomics Visium HD 4.0.1 3' Mouse Brain Chunk
-          curl -O https://cf.10xgenomics.com/samples/spatial-exp/4.0.1/Visium_HD_Tiny_3prime_Dataset/Visium_HD_Tiny_3prime_Dataset_outs.zip
-
-          # -------
-          # we received written permission to make the following dataset public and integrate it in the CI system of spatialdata-io
-          # Spatial Genomics seqFISH v2
-          curl -O https://s3.embl.de/spatialdata/raw_data/seqfish-2-test-dataset.zip
-
-          # -------
-          # MACSima OMAP datasets are licensed as CC BY 4.0
-          # OMAP23 for format v1.x.x
-          curl -o OMAP23_small.zip "https://zenodo.org/api/records/18196452/files-archive"
-
-          # OMAP10 for format v0.x.x
-          curl -o OMAP10_small.zip "https://zenodo.org/api/records/18196366/files-archive"
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+          python-version: "3.13"
 
-      - name: Unzip files
+      - name: Download test datasets
+        env:
+          DATASET_KEYS: ${{ inputs.dataset_keys }}
         run: |
-          cd ./data
-          for file in *.zip; do
-              dir="${file%.zip}"
-              mkdir -p "$dir"
-              unzip "$file" -d "$dir"
-              rm "$file"
-          done
+          args=(--output ./data)
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force_all }}" == "false" ]]; then
+            dataset_keys="${DATASET_KEYS}"
+            if [[ -z "${dataset_keys}" ]]; then
+              echo "::error::dataset_keys must be provided when force_all is false."
+              exit 1
+            fi
+            dataset_keys="${dataset_keys//,/ }"
+            for dataset_key in ${dataset_keys}; do
+              args+=(--dataset "${dataset_key}")
+            done
+          fi
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force_redownload }}" == "true" ]]; then
+            args+=(--force)
+          fi
+          uv run --only-group dev python scripts/test_data_downloader "${args[@]}"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: data
           path: ./data
+          if-no-files-found: error
+          retention-days: 64

docs/contributing.md

@@ -462,6 +462,43 @@ If the `download.py` and `to_zarr.py` scripts require Python imports for package
 
 We encourage testing the reader function and any helper function.
 
+Tests are split by scope:
+
+- Unit tests live in `tests/unit/` and should not require downloaded test data.
+- Integration tests live in `tests/integration/` and cover reader workflows, CLI commands, file I/O, and zarr roundtrips.
+- Integration tests that require external datasets use dataset keys from `scripts/test_data_downloader/datasets.toml`.
+  They resolve data under `SPATIALDATA_IO_TEST_DATA_DIR` when set, otherwise `data/` in the repository root. If the required dataset is unavailable, the test should skip with a clear message.
+- Reader tests are marked by reader name. When modifying one reader, use `pytest -m <reader>` to run the tests
+  specific to that reader, including shared parametrized checks for that reader.
+
+Useful local commands:
+
+```bash
+pytest tests/unit
+pytest tests/integration
+pytest -m "integration and data"
+pytest -m xenium
+pytest -m "xenium and data"
+pytest -m "xenium and not slow"
+pytest -m "xenium and cli"
+python scripts/test_data_downloader --group xenium
+SPATIALDATA_IO_TEST_DATA_DIR=/path/to/data pytest -m data
+```
+
+To download the same optional datasets used by CI, run:
+
+```bash
+python scripts/test_data_downloader
+```
+
+By default, the downloader skips datasets that already exist. Use `--force` to redownload selected datasets, `--dataset` for a single dataset key, and `--list` to show the available keys.
+The downloader verifies every downloaded file with [Pooch](https://www.fatiando.org/pooch/). The dataset manifest lives in
+`scripts/test_data_downloader/datasets.toml`; append new entries there when adding or updating test datasets. This manifest
+stores project-specific metadata such as dataset keys, groups, output directories, and sources; it is not a separate Pooch
+registry. Prefer a stable repository DOI when one is available. For DOI entries, Pooch loads the repository's per-file hashes
+at runtime and verifies every downloaded file. Otherwise, register the archive URL and its SHA-256 hash as
+`known_hash = "sha256:..."`.
+
 #### Testing multiple versions
 
 When multiple versions of the raw data format are present, we encourage testing the reader on all of them to ensure backward compatibility. This task is greatly simplified if small test datasets are used for the CI tests. If this is not available, we suggest running the tests locally on multiple versions of the data before the PR is ready for review.

pyproject.toml

@@ -52,7 +52,9 @@ urls.Source = "https://github.com/scverse/spatialdata-io"
 dev = [
   "mypy",
   "pandas-stubs",
+  "pooch>=1.9",
   "prek",
+  "requests",
   "scipy-stubs",
   "twine>=4.0.2",
   "types-geopandas",
@@ -150,9 +152,11 @@ lint.pydocstyle.convention = "numpy"
 # globally, so that a newly added untyped dependency is still reported.
 [[tool.mypy.overrides]]
 module = [
+  "anndata.*",
   "dask_image.*",
   "h5py.*",
   "multiscale_spatial_image.*",
+  "pooch.*",
   "pyarrow.*",
   "rasterio.*",
   "readfcs.*",
@@ -163,6 +167,15 @@ module = [
 ]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+# PyArrow's generated compute functions are runtime attributes that are omitted
+# from the pyarrow 24 type information.
+module = [
+  "spatialdata_io.readers.cosmx",
+  "spatialdata_io.readers.xenium",
+]
+disable_error_code = [ "attr-defined" ]
+
 [tool.pytest]
 addopts = [
   "--import-mode=importlib", # allow using test files with same name
@@ -171,9 +184,28 @@ addopts = [
 # `spatialdata.testing` pull in, which strict marker checking would otherwise reject
 markers = [
   "array_api",
+  "cli: command-line interface tests",
+  "codex: tests for the codex reader",
+  "cosmx: tests for the cosmx reader",
+  "curio: tests for the curio reader",
+  "data: tests that require optional downloaded test datasets",
+  "dbit: tests for the dbit reader",
+  "generic: tests for the generic reader module",
   "gpu",
+  "integration: multi-component tests, file I/O tests, or reader workflow tests",
+  "iss: tests for the iss reader",
+  "macsima: tests for the macsima reader",
+  "mcmicro: tests for the mcmicro reader",
+  "merscope: tests for the merscope reader",
+  "seqfish: tests for the seqfish reader",
   "skip_with_pyarrow_strings",
-  "slow",
+  "slow: tests with comparatively high runtime",
+  "steinbock: tests for the steinbock reader",
+  "stereoseq: tests for the stereoseq reader",
+  "unit: fast isolated tests that do not require external datasets",
+  "visium: tests for the visium reader",
+  "visium_hd: tests for the visium_hd reader",
+  "xenium: tests for the xenium reader",
 ]
 strict = true
 testpaths = [ "tests" ]

scripts/test_data_downloader/__main__.py

@@ -0,0 +1,8 @@
+"""Command-line entrypoint for the optional test data downloader."""
+
+from __future__ import annotations
+
+from downloader import main
+
+if __name__ == "__main__":
+    main()

scripts/test_data_downloader/datasets.toml

@@ -0,0 +1,105 @@
+# -------
+# the Xenium datasets are licensed as CC BY 4.0, as shown here
+# https://www.10xgenomics.com/support/software/xenium-onboard-analysis/latest/resources/xenium-example-data
+
+# 10x Genomics Xenium 2.0.0
+[[datasets]]
+key = "xenium_breast"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_human_Breast_2fov/Xenium_V1_human_Breast_2fov_outs.zip"
+known_hash = "sha256:cc1e987b06aa748a6b24d3d6f51fc0d6765daa4836c483f14ec4f1bd18b1779b"
+extracted_dir = "Xenium_V1_human_Breast_2fov_outs"
+source = "10x Genomics Xenium 2.0.0, CC BY 4.0"
+
+# 10x Genomics Xenium 2.0.0
+[[datasets]]
+key = "xenium_lung"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/2.0.0/Xenium_V1_human_Lung_2fov/Xenium_V1_human_Lung_2fov_outs.zip"
+known_hash = "sha256:acc353069871eeda9977fc80f0d11e81eef8b7a683212819cd892d983c4e4d91"
+extracted_dir = "Xenium_V1_human_Lung_2fov_outs"
+source = "10x Genomics Xenium 2.0.0, CC BY 4.0"
+
+# 10x Genomics Xenium 3.0.0 (5K) Mouse ileum, multimodal cell segmentation
+# this file seems to be corrupted; skipping it for now
+# https://cf.10xgenomics.com/samples/xenium/3.0.0/Xenium_Prime_MultiCellSeg_Mouse_Ileum_tiny/Xenium_Prime_MultiCellSeg_Mouse_Ileum_tiny.zip
+
+# 10x Genomics Xenium 3.0.0 (5K) Mouse ileum, nuclear expansion
+[[datasets]]
+key = "xenium_prime_mouse_ileum"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/3.0.0/Xenium_Prime_Mouse_Ileum_tiny/Xenium_Prime_Mouse_Ileum_tiny_outs.zip"
+known_hash = "sha256:72dc2353825f6049959fa85e3c0617abd2a7cfb1c19531a2bba2d246fa276cd3"
+extracted_dir = "Xenium_Prime_Mouse_Ileum_tiny_outs"
+source = "10x Genomics Xenium 3.0.0, CC BY 4.0"
+
+# 10x Genomics Xenium 4.0.0 (v1) Human ovary, nuclear expansion
+[[datasets]]
+key = "xenium_ovary"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_Human_Ovary_tiny/Xenium_V1_Human_Ovary_tiny_outs.zip"
+known_hash = "sha256:72b9a6d73ec428dd823e9a0d1e3b70d6f0538ec79f67fa7591f034af0894c764"
+extracted_dir = "Xenium_V1_Human_Ovary_tiny_outs"
+source = "10x Genomics Xenium 4.0.0, CC BY 4.0"
+
+# 10x Genomics Xenium 4.0.0 (v1) Human ovary, multimodal cell segmentation
+[[datasets]]
+key = "xenium_multicell_ovary"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_MultiCellSeg_Human_Ovary_tiny/Xenium_V1_MultiCellSeg_Human_Ovary_tiny_outs.zip"
+known_hash = "sha256:9b154a3c08360fe690c83e0e6a6710d0c5656ec93a6e3ff969b3950d291479b0"
+extracted_dir = "Xenium_V1_MultiCellSeg_Human_Ovary_tiny_outs"
+source = "10x Genomics Xenium 4.0.0, CC BY 4.0"
+
+# 10x Genomics Xenium 4.0.0 (v1+Protein) Human kidney, multimodal cell segmentation
+[[datasets]]
+key = "xenium_protein_kidney"
+group = "xenium"
+url = "https://cf.10xgenomics.com/samples/xenium/4.0.0/Xenium_V1_Protein_Human_Kidney_tiny/Xenium_V1_Protein_Human_Kidney_tiny_outs.zip"
+known_hash = "sha256:abd7e8f7fd047dcc6afdb1e9eece90d4533d3ead053c6f05c482be050bdf79d2"
+extracted_dir = "Xenium_V1_Protein_Human_Kidney_tiny_outs"
+source = "10x Genomics Xenium 4.0.0, CC BY 4.0"
+
+# -------
+# the Visium HD dataset is licensed as CC BY 4.0, as shown here
+# https://www.10xgenomics.com/support/software/space-ranger/latest/resources/visium-hd-example-data
+
+# 10x Genomics Visium HD 4.0.1 3' Mouse Brain Chunk
+[[datasets]]
+key = "visium_hd_tiny"
+group = "visium_hd"
+url = "https://cf.10xgenomics.com/samples/spatial-exp/4.0.1/Visium_HD_Tiny_3prime_Dataset/Visium_HD_Tiny_3prime_Dataset_outs.zip"
+known_hash = "sha256:38be766fc4fa077f083b74d9a6746ab6db985838dbbba3006b5a36c51a3c5a50"
+extracted_dir = "Visium_HD_Tiny_3prime_Dataset_outs"
+source = "10x Genomics Visium HD 4.0.1, CC BY 4.0"
+
+# -------
+# we received written permission to make the following dataset public and integrate it in the CI system of spatialdata-io
+# Spatial Genomics seqFISH v2
+[[datasets]]
+key = "seqfish"
+group = "seqfish"
+url = "https://s3.embl.de/spatialdata/raw_data/seqfish-2-test-dataset.zip"
+known_hash = "sha256:553208b2bab71e45fe0fd3799478aa03894270c8e447cecc6366c0bd412256c0"
+extracted_dir = "seqfish-2-test-dataset"
+source = "Spatial Genomics seqFISH v2, public test data"
+test_path = "instrument 2 official"
+
+# -------
+# MACSima OMAP datasets are licensed as CC BY 4.0
+# Pooch loads and verifies the published per-file MD5 checksums from each stable DOI at runtime.
+# OMAP23 for format v1.x.x
+[[datasets]]
+key = "macsima_omap23"
+group = "macsima"
+doi = "10.5281/zenodo.18196452"
+extracted_dir = "OMAP23_small"
+source = "MACSima OMAP23, CC BY 4.0"
+
+# OMAP10 for format v0.x.x
+[[datasets]]
+key = "macsima_omap10"
+group = "macsima"
+doi = "10.5281/zenodo.18196366"
+extracted_dir = "OMAP10_small"
+source = "MACSima OMAP10, CC BY 4.0"