diff --git a/src/bw_processing/datapackage.py b/src/bw_processing/datapackage.py index 7d9ae7a..d6fb49c 100644 --- a/src/bw_processing/datapackage.py +++ b/src/bw_processing/datapackage.py @@ -68,7 +68,8 @@ def __len__(self): return len(self.data) def __contains__(self, key): - return key in {o["name"] for o in self.resources} or key in self.groups + groups = {o.get("group") for o in self.resources if o.get("group")} + return key in {o["name"] for o in self.resources} or key in groups resources = property(__get_resources, __set_resources) @@ -78,14 +79,14 @@ def groups(self) -> dict: Ignores resources which don't have group labels. """ - labels = [] - for resource in self.resources: - if not resource.get("group"): - continue - elif resource["group"] not in labels: - labels.append(resource["group"]) + # One pass over the resources: filtering once per label made this + # quadratic in the number of resources. + indices = {} + for i, resource in enumerate(self.resources): + if resource.get("group"): + indices.setdefault(resource["group"], []).append(i) - return {label: self.filter_by_attribute("group", label) for label in labels} + return {label: self._filtered(lst) for label, lst in indices.items()} def _get_index(self, name_or_index: Union[str, int]) -> int: """Get index of a resource by name or index. @@ -144,7 +145,7 @@ def del_resource_group(self, name: str) -> None: if self._modified: raise PotentialInconsistency("Datapackage is modified; save modifications or reload") - indices = [i for i, resource in enumerate(self.resources) if resource.get("group") == name] + indices = {i for i, resource in enumerate(self.resources) if resource.get("group") == name} for obj in (obj for i, obj in enumerate(self.resources) if i in indices): try: @@ -188,16 +189,9 @@ def filter_by_attribute(self, key: str, value: Any) -> "FilteredDatapackage": This method was introduced to allow for the efficient construction of matrices; each datapackage can have data for multiple matrices, and we can then create filtered datapackages which exclusively have data for the matrix of interest. As such, they should be considered read-only, though this is not enforced. """ - fdp = FilteredDatapackage() - fdp.fs = self.fs - fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"} - fdp.metadata["resources"] = [] - to_include = [i for i, resource in enumerate(self.resources) if resource.get(key) == value] - fdp.data = [o for i, o in enumerate(self.data) if i in to_include] - fdp.resources = [o for i, o in enumerate(self.resources) if i in to_include] - if hasattr(self, "indexer"): - fdp.indexer = self.indexer - return fdp + return self._filtered( + [i for i, resource in enumerate(self.resources) if resource.get(key) == value] + ) def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage": """Filter a datapackage to exclude resources matching a filter. @@ -213,21 +207,24 @@ def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage": exclude_generic({"group': "some_group", "kind": "some_kind"}) """ + return self._filtered( + [ + i + for i, resource in enumerate(self.resources) + if any(resource.get(key) != value for key, value in filters.items()) + ] + ) + + def _filtered(self, indices: List[int]) -> "FilteredDatapackage": + """Create a ``FilteredDatapackage`` with the resources at ``indices``, sharing their data.""" fdp = FilteredDatapackage() fdp.fs = self.fs fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"} fdp.metadata["resources"] = [] - + fdp.data = [self.data[i] for i in indices] + fdp.resources = [self.resources[i] for i in indices] if hasattr(self, "indexer"): fdp.indexer = self.indexer - - indices_to_include = [ - i - for i, resource in enumerate(self.resources) - if any(resource.get(key) != value for key, value in filters.items()) - ] - fdp.data = [o for i, o in enumerate(self.data) if i in indices_to_include] - fdp.resources = [o for i, o in enumerate(self.resources) if i in indices_to_include] return fdp def _dehydrate_interfaces(self) -> None: diff --git a/tests/test_datapackage.py b/tests/test_datapackage.py index 5dcee83..c9ff6ce 100644 --- a/tests/test_datapackage.py +++ b/tests/test_datapackage.py @@ -90,6 +90,23 @@ def test_group_ordering_consistent(): ] +def test_groups_keep_resource_order_when_interleaved(): + dp = create_datapackage() + add_data(dp) + # interleave the resources of different groups + order = list(range(0, len(dp), 2)) + list(range(1, len(dp), 2)) + dp.resources = [dp.resources[i] for i in order] + dp.data = [dp.data[i] for i in order] + + labels = list(dict.fromkeys(o["group"] for o in dp.resources if o.get("group"))) + assert list(dp.groups) == labels + for label, fdp in dp.groups.items(): + positions = [i for i, o in enumerate(dp.resources) if o.get("group") == label] + assert [o["name"] for o in fdp.resources] == [dp.resources[i]["name"] for i in positions] + assert len(fdp.data) == len(positions) + assert all(a is dp.data[i] for a, i in zip(fdp.data, positions)) + + def test_get_max_index_value(): dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture.zip")) assert dp.metadata.get("64_bit_indices")