From 895156b5838ed47bc23e98b461b052ceacb09847 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Fri, 2 Oct 2026 16:48:00 +0200 Subject: [PATCH 1/3] perf: build resource groups in one pass groups filtered the resources once per group label, and each filter checked membership against a list, so mapping a datapackage with many small groups took quadratic time. filter_by_attribute and exclude now share a helper that indexes the lists directly. --- src/bw_processing/datapackage.py | 48 +++++++++++++++----------------- tests/test_datapackage.py | 17 +++++++++++ 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/bw_processing/datapackage.py b/src/bw_processing/datapackage.py index 7d9ae7a..2bc0840 100644 --- a/src/bw_processing/datapackage.py +++ b/src/bw_processing/datapackage.py @@ -78,14 +78,14 @@ def groups(self) -> dict: Ignores resources which don't have group labels. """ - labels = [] - for resource in self.resources: - if not resource.get("group"): - continue - elif resource["group"] not in labels: - labels.append(resource["group"]) + # One pass over the resources: filtering once per label made this + # quadratic in the number of resources. + indices = {} + for i, resource in enumerate(self.resources): + if resource.get("group"): + indices.setdefault(resource["group"], []).append(i) - return {label: self.filter_by_attribute("group", label) for label in labels} + return {label: self._filtered(lst) for label, lst in indices.items()} def _get_index(self, name_or_index: Union[str, int]) -> int: """Get index of a resource by name or index. @@ -188,16 +188,9 @@ def filter_by_attribute(self, key: str, value: Any) -> "FilteredDatapackage": This method was introduced to allow for the efficient construction of matrices; each datapackage can have data for multiple matrices, and we can then create filtered datapackages which exclusively have data for the matrix of interest. As such, they should be considered read-only, though this is not enforced. """ - fdp = FilteredDatapackage() - fdp.fs = self.fs - fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"} - fdp.metadata["resources"] = [] - to_include = [i for i, resource in enumerate(self.resources) if resource.get(key) == value] - fdp.data = [o for i, o in enumerate(self.data) if i in to_include] - fdp.resources = [o for i, o in enumerate(self.resources) if i in to_include] - if hasattr(self, "indexer"): - fdp.indexer = self.indexer - return fdp + return self._filtered( + [i for i, resource in enumerate(self.resources) if resource.get(key) == value] + ) def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage": """Filter a datapackage to exclude resources matching a filter. @@ -213,21 +206,24 @@ def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage": exclude_generic({"group': "some_group", "kind": "some_kind"}) """ + return self._filtered( + [ + i + for i, resource in enumerate(self.resources) + if any(resource.get(key) != value for key, value in filters.items()) + ] + ) + + def _filtered(self, indices: List[int]) -> "FilteredDatapackage": + """Create a ``FilteredDatapackage`` with the resources at ``indices``, sharing their data.""" fdp = FilteredDatapackage() fdp.fs = self.fs fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"} fdp.metadata["resources"] = [] - + fdp.data = [self.data[i] for i in indices] + fdp.resources = [self.resources[i] for i in indices] if hasattr(self, "indexer"): fdp.indexer = self.indexer - - indices_to_include = [ - i - for i, resource in enumerate(self.resources) - if any(resource.get(key) != value for key, value in filters.items()) - ] - fdp.data = [o for i, o in enumerate(self.data) if i in indices_to_include] - fdp.resources = [o for i, o in enumerate(self.resources) if i in indices_to_include] return fdp def _dehydrate_interfaces(self) -> None: diff --git a/tests/test_datapackage.py b/tests/test_datapackage.py index 5dcee83..6f212de 100644 --- a/tests/test_datapackage.py +++ b/tests/test_datapackage.py @@ -90,6 +90,23 @@ def test_group_ordering_consistent(): ] +def test_groups_match_filter_by_attribute_when_interleaved(): + dp = create_datapackage() + add_data(dp) + # interleave the resources of different groups + order = list(range(0, len(dp), 2)) + list(range(1, len(dp), 2)) + dp.resources = [dp.resources[i] for i in order] + dp.data = [dp.data[i] for i in order] + + labels = list(dict.fromkeys(o["group"] for o in dp.resources if o.get("group"))) + assert list(dp.groups) == labels + for label, fdp in dp.groups.items(): + expected = dp.filter_by_attribute("group", label) + assert fdp.resources == expected.resources + assert all(a is b for a, b in zip(fdp.data, expected.data)) + assert len(fdp.data) == len(expected.data) + + def test_get_max_index_value(): dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture.zip")) assert dp.metadata.get("64_bit_indices") From 3edfb10b5340a9aa1452996eaeeecedcb7e3e9e1 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Fri, 2 Oct 2026 16:53:45 +0200 Subject: [PATCH 2/3] test: check interleaved groups against the resource order directly --- tests/test_datapackage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_datapackage.py b/tests/test_datapackage.py index 6f212de..c9ff6ce 100644 --- a/tests/test_datapackage.py +++ b/tests/test_datapackage.py @@ -90,7 +90,7 @@ def test_group_ordering_consistent(): ] -def test_groups_match_filter_by_attribute_when_interleaved(): +def test_groups_keep_resource_order_when_interleaved(): dp = create_datapackage() add_data(dp) # interleave the resources of different groups @@ -101,10 +101,10 @@ def test_groups_match_filter_by_attribute_when_interleaved(): labels = list(dict.fromkeys(o["group"] for o in dp.resources if o.get("group"))) assert list(dp.groups) == labels for label, fdp in dp.groups.items(): - expected = dp.filter_by_attribute("group", label) - assert fdp.resources == expected.resources - assert all(a is b for a, b in zip(fdp.data, expected.data)) - assert len(fdp.data) == len(expected.data) + positions = [i for i, o in enumerate(dp.resources) if o.get("group") == label] + assert [o["name"] for o in fdp.resources] == [dp.resources[i]["name"] for i in positions] + assert len(fdp.data) == len(positions) + assert all(a is dp.data[i] for a, i in zip(fdp.data, positions)) def test_get_max_index_value(): From 1a6f8982adc6ad9197a3b154a82bb747d6c0a374 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Fri, 2 Oct 2026 17:06:36 +0200 Subject: [PATCH 3/3] perf: check group labels and deleted indices without filtering - __contains__ looks up group labels in a set instead of building a filtered package per group - del_resource_group checks indices against a set --- src/bw_processing/datapackage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bw_processing/datapackage.py b/src/bw_processing/datapackage.py index 2bc0840..d6fb49c 100644 --- a/src/bw_processing/datapackage.py +++ b/src/bw_processing/datapackage.py @@ -68,7 +68,8 @@ def __len__(self): return len(self.data) def __contains__(self, key): - return key in {o["name"] for o in self.resources} or key in self.groups + groups = {o.get("group") for o in self.resources if o.get("group")} + return key in {o["name"] for o in self.resources} or key in groups resources = property(__get_resources, __set_resources) @@ -144,7 +145,7 @@ def del_resource_group(self, name: str) -> None: if self._modified: raise PotentialInconsistency("Datapackage is modified; save modifications or reload") - indices = [i for i, resource in enumerate(self.resources) if resource.get("group") == name] + indices = {i for i, resource in enumerate(self.resources) if resource.get("group") == name} for obj in (obj for i, obj in enumerate(self.resources) if i in indices): try: