diff --git a/src/napari_spatialdata/_widgets.py b/src/napari_spatialdata/_widgets.py
index 7cab1d64..153fab4d 100644
--- a/src/napari_spatialdata/_widgets.py
+++ b/src/napari_spatialdata/_widgets.py
@@ -226,7 +226,21 @@ def _(self, vec: pd.Series, **kwargs: Any) -> dict[str, Any]:
                 color_dict.update({np.nan: "#808080ff"})
             else:
                 colors = self.model.adata.uns[vec_color_name]
-                color_dict = dict(zip(vec.cat.categories, colors.tolist(), strict=True))
+                # One color per category: pair them in order.
+                if len(colors) == len(vec.cat.categories):
+                    color_dict = dict(zip(vec.cat.categories, colors.tolist(), strict=True))
+                else:
+                    observed = vec.cat.remove_unused_categories().cat.categories
+                    # The table rows annotating this element do not use every category, so the colors
+                    # of the unused categories were already dropped.
+                    if len(colors) == len(observed):
+                        color_dict = dict(zip(observed, colors.tolist(), strict=True))
+                    else:
+                        raise ValueError(
+                            f"The palette in uns[{vec_color_name!r}] has {len(colors)} colors, but "
+                            f"{vec.name!r} has {len(vec.cat.categories)} categories, so colors "
+                            f"cannot be matched to categories."
+                        )
         else:
             df = layer.metadata["_columns_df"]
             if vec_color_name not in df.columns:
diff --git a/tests/test_widgets.py b/tests/test_widgets.py
index f0db306e..7e2c1123 100644
--- a/tests/test_widgets.py
+++ b/tests/test_widgets.py
@@ -6,6 +6,7 @@
 import pytest
 from anndata import AnnData
 from anndata.tests.helpers import assert_equal
+from matplotlib.colors import to_rgba
 from napari.layers import Image, Labels
 from napari.utils.events import EventedList
 from qtpy import QtWidgets
@@ -15,6 +16,7 @@
 from napari_spatialdata._model import DataModel
 from napari_spatialdata._sdata_widgets import SdataWidget
 from napari_spatialdata._view import QtAdataScatterWidget, QtAdataViewWidget
+from napari_spatialdata._widgets import AListWidget
 
 
 # make_napari_viewer is a pytest fixture that returns a napari viewer object
@@ -322,3 +324,26 @@ def test_old_annotation(adata_labels, annotation_values, mocker):
 
     assert np.array_equal(scatter_widget._model.adata.obs[col_name], annotation_values)
     assert scatter_widget.status_label.text() == "Status: Annotation updated."
+
+
+def test_palette_length_mismatch(
+    make_napari_viewer: Any,
+    adata_labels: AnnData,
+    labels: ArrayLike,
+) -> None:
+    """A palette holding colors only for the used categories is matched to those categories."""
+    color_by_category = {"beta": "#222222", "delta": "#444444"}
+    values = np.resize(list(color_by_category), adata_labels.n_obs)
+    adata_labels.obs["annotation"] = pd.Categorical(values, categories=["alpha", "beta", "gamma", "delta"])
+    adata_labels.uns["annotation_colors"] = np.array(list(color_by_category.values()))
+
+    viewer = make_napari_viewer()
+    layer = viewer.add_labels(labels, metadata={"adata": adata_labels, "indices": np.unique(labels)[1:]})
+
+    model = DataModel()
+    model.adata = adata_labels
+    model.layer = layer
+    AListWidget(viewer, model, attr="obs")._onAction(items=["annotation"])
+
+    for cell_id, category in zip(adata_labels.obs["cell_id"], values, strict=True):
+        assert np.allclose(layer.colormap.color_dict[cell_id], to_rgba(color_by_category[category]))