Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 25 additions & 28 deletions src/bw_processing/datapackage.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ def __len__(self):
return len(self.data)

def __contains__(self, key):
return key in {o["name"] for o in self.resources} or key in self.groups
groups = {o.get("group") for o in self.resources if o.get("group")}
return key in {o["name"] for o in self.resources} or key in groups

resources = property(__get_resources, __set_resources)

Expand All @@ -78,14 +79,14 @@ def groups(self) -> dict:

Ignores resources which don't have group labels.
"""
labels = []
for resource in self.resources:
if not resource.get("group"):
continue
elif resource["group"] not in labels:
labels.append(resource["group"])
# One pass over the resources: filtering once per label made this
# quadratic in the number of resources.
indices = {}
for i, resource in enumerate(self.resources):
if resource.get("group"):
indices.setdefault(resource["group"], []).append(i)

return {label: self.filter_by_attribute("group", label) for label in labels}
return {label: self._filtered(lst) for label, lst in indices.items()}

def _get_index(self, name_or_index: Union[str, int]) -> int:
"""Get index of a resource by name or index.
Expand Down Expand Up @@ -144,7 +145,7 @@ def del_resource_group(self, name: str) -> None:
if self._modified:
raise PotentialInconsistency("Datapackage is modified; save modifications or reload")

indices = [i for i, resource in enumerate(self.resources) if resource.get("group") == name]
indices = {i for i, resource in enumerate(self.resources) if resource.get("group") == name}

for obj in (obj for i, obj in enumerate(self.resources) if i in indices):
try:
Expand Down Expand Up @@ -188,16 +189,9 @@ def filter_by_attribute(self, key: str, value: Any) -> "FilteredDatapackage":

This method was introduced to allow for the efficient construction of matrices; each datapackage can have data for multiple matrices, and we can then create filtered datapackages which exclusively have data for the matrix of interest. As such, they should be considered read-only, though this is not enforced.
"""
fdp = FilteredDatapackage()
fdp.fs = self.fs
fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"}
fdp.metadata["resources"] = []
to_include = [i for i, resource in enumerate(self.resources) if resource.get(key) == value]
fdp.data = [o for i, o in enumerate(self.data) if i in to_include]
fdp.resources = [o for i, o in enumerate(self.resources) if i in to_include]
if hasattr(self, "indexer"):
fdp.indexer = self.indexer
return fdp
return self._filtered(
[i for i, resource in enumerate(self.resources) if resource.get(key) == value]
)

def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage":
"""Filter a datapackage to exclude resources matching a filter.
Expand All @@ -213,21 +207,24 @@ def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage":
exclude_generic({"group': "some_group", "kind": "some_kind"})

"""
return self._filtered(
[
i
for i, resource in enumerate(self.resources)
if any(resource.get(key) != value for key, value in filters.items())
]
)

def _filtered(self, indices: List[int]) -> "FilteredDatapackage":
"""Create a ``FilteredDatapackage`` with the resources at ``indices``, sharing their data."""
fdp = FilteredDatapackage()
fdp.fs = self.fs
fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"}
fdp.metadata["resources"] = []

fdp.data = [self.data[i] for i in indices]
fdp.resources = [self.resources[i] for i in indices]
if hasattr(self, "indexer"):
fdp.indexer = self.indexer

indices_to_include = [
i
for i, resource in enumerate(self.resources)
if any(resource.get(key) != value for key, value in filters.items())
]
fdp.data = [o for i, o in enumerate(self.data) if i in indices_to_include]
fdp.resources = [o for i, o in enumerate(self.resources) if i in indices_to_include]
return fdp

def _dehydrate_interfaces(self) -> None:
Expand Down
17 changes: 17 additions & 0 deletions tests/test_datapackage.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,23 @@ def test_group_ordering_consistent():
]


def test_groups_keep_resource_order_when_interleaved():
dp = create_datapackage()
add_data(dp)
# interleave the resources of different groups
order = list(range(0, len(dp), 2)) + list(range(1, len(dp), 2))
dp.resources = [dp.resources[i] for i in order]
dp.data = [dp.data[i] for i in order]

labels = list(dict.fromkeys(o["group"] for o in dp.resources if o.get("group")))
assert list(dp.groups) == labels
for label, fdp in dp.groups.items():
positions = [i for i, o in enumerate(dp.resources) if o.get("group") == label]
assert [o["name"] for o in fdp.resources] == [dp.resources[i]["name"] for i in positions]
assert len(fdp.data) == len(positions)
assert all(a is dp.data[i] for a, i in zip(fdp.data, positions))


def test_get_max_index_value():
dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture.zip"))
assert dp.metadata.get("64_bit_indices")
Expand Down