From 419e355d69cc681a1f5ea92ab3f34fdf83d058af Mon Sep 17 00:00:00 2001 From: Michael Szell Date: Mon, 12 Oct 2026 07:30:29 +0200 Subject: [PATCH 1/5] Refactor functions into utils and _procedure --- growbikenet/_procedure.py | 779 +++++++++++++++++++++++++ growbikenet/growbikenet.py | 84 ++- growbikenet/{functions.py => utils.py} | 740 +---------------------- 3 files changed, 849 insertions(+), 754 deletions(-) create mode 100644 growbikenet/_procedure.py rename growbikenet/{functions.py => utils.py} (59%) diff --git a/growbikenet/_procedure.py b/growbikenet/_procedure.py new file mode 100644 index 0000000..0d44e2c --- /dev/null +++ b/growbikenet/_procedure.py @@ -0,0 +1,779 @@ +"""Main calls from the `growbikenet` procedure. These functions are called +once from the `growbikenet` module. +""" + +import os +import re +import warnings +from collections import defaultdict + +import numpy as np +import pandas as pd + +from . import constants, settings, osm + +pd.set_option('display.max_columns', None) # for debugging +import datetime + +import geopandas as gpd +import networkx as nx +import osmnx as ox +from tqdm import tqdm + +from growbikenet.visualization import generate_plots +from growbikenet.utils import ( + add_point_data_to_net, + add_trip_data_to_net, + assign_pbi_to_edges, + bike_infra_mapping_gdf, + create_gdf_with_geoms, + df_from_graph + download_network, + filter_network_by_component_length, + filter_points_distant_from_osm_nodes, + get_principal_bearing, + import_network, + initialize_progress_bar, + nodelist_to_edgetuples, + node_to_edge_attributes, + nx_to_nodes_edges, + orientation_order, + prepare_nodes_edges, + route_nodepairs, + set_path_to_pbi, + slugify, + snap_points_to_osm_nodes, + update_with_existing_bike_network, + weigh_edges, + _create_delaunay_edges, + _get_existing_network_seed_points, + _get_grid_seed_points, + _get_tags_seed_points, + _get_weighted_distances, + _order_df, + _prepare_seed_points, + _remove_edge_overlaps, + _resolve_crs_calculations, + _update_seed_points_with_existing_bike_network, +) + + +def _gbn_validate_settings(): + """ Check if user settings input is valid. If not, raise an exception or + warning. + + Returns + ------- + setting_was_auto : dict + Dictionary remembering which setting or constant was set to auto, so + it can be reset to auto in the end. + """ + + if type(constants._CRS_CALCULATIONS) is not str: + raise TypeError("constants._CRS_CALCULATIONS must be a string") + if settings.export_file_format != "geojson" and settings.export_file_format != "gpkg": + raise ValueError("settings.export_file_format must be 'geojson' or 'gpkg'") + # To do: check export_path + if type(settings.seed_point_snap_distance) is not int and settings.seed_point_snap_distance != 'auto': + raise TypeError("settings.seed_point_snap_distance must be 'auto' or an integer") + if type(settings.seed_point_snap_distance) is int and settings.seed_point_snap_distance <= 0: + raise ValueError("settings.seed_point_snap_distance must be a positive integer") + + setting_was_auto = {'crs_calculations': False, 'crs_viz': False} + # Ask whether constants._CRS_CALCULATIONS was 'auto' ahead of network + # construction, as 'auto' is resolved inside `prepare_nodes_edges()`. + if constants._CRS_CALCULATIONS == 'auto': + setting_was_auto['crs_calculations'] = True + if settings.viz["crs"] == 'auto': + setting_was_auto['crs_viz'] = True + return setting_was_auto + + +def _gbn_validate_parameters( + city_query, + ordering, + seed_point_type, + seed_point_grid_spacing, + seed_point_linking, + existing_network_spacing, + export_data, + city_id, + export_plots, + import_files, + seed_point_tags, + ): + """ Check if user parameter input is valid. If not, raise an exception or + warning. + + Parameters + ---------- + Same as `growbikenet.growbikenet()` + Additionally: + constants._PRESET_TAGS : dict + Dictionary of preset seed point tags. + + Returns + ------- + True + """ + + if type(city_query) is not str: + raise TypeError("city_query must be a string") + if type(ordering) is not str: + raise TypeError("ordering must be a string") + if ordering not in ["betweenness", "closeness", "random"]: + raise ValueError( + "ordering must be either 'betweenness', 'closeness', or 'random'" + ) + if seed_point_type not in ['auto', 'grid_square', 'grid_triangle', 'rail', 'school', 'park', 'file', 'tags']: + raise ValueError("seed_point_type must be 'auto' or 'grid_square' or 'grid_triangle' or 'rail' or 'school' or 'park' or 'file' or 'tags'") + if type(seed_point_grid_spacing) is not int and seed_point_grid_spacing != 'auto': + raise TypeError("seed_point_grid_spacing must be 'auto' or an integer") + if type(seed_point_grid_spacing) is int and seed_point_grid_spacing <= 0: + raise ValueError("seed_point_grid_spacing must be a positive integer") + if type(import_files) is not dict: + raise TypeError("import_files must be a dictionary") + # Prepare special case import_files. Turn it into a defaultdict where missing keys are None. + import_files = defaultdict(lambda: None, import_files) + if seed_point_type == 'file' and import_files['seed_points'] is None: + raise ValueError("With seed_point_type 'file', a seed_points file must be provided") + if seed_point_type == 'tags' and seed_point_tags is None: + raise ValueError("With seed_point_type 'tags', seed_point_tags must be provided") + if seed_point_linking not in ['auto', 'triangulate_delaunay', 'quadrangulate']: + raise ValueError("seed_point_linking must be 'auto' or 'triangulate_delaunay' or 'quadrangulate'") + if seed_point_linking == 'quadrangulate' and (seed_point_type != 'grid_square' or existing_network_spacing is not None): + raise ValueError("With seed_point_linking 'quadrangulate', seed_point_type must be set to 'grid_square' and existing_network_spacing must be set to None") + if type(existing_network_spacing) is not int and existing_network_spacing is not None and existing_network_spacing != 'auto': + raise TypeError("existing_network_spacing must be None or 'auto' or a positive integer") + if type(existing_network_spacing) is int and existing_network_spacing <= 0: + raise ValueError("existing_network_spacing must be None or a positive integer") + if type(existing_network_spacing) is int and seed_point_grid_spacing is int and existing_network_spacing >= seed_point_grid_spacing: + warnings.warn("existing_network_spacing is recommended to be smaller than seed_point_grid_spacing, ideally around a third, to ensure that the existing bicycle network is built first.") + if type(export_data) is not bool: + raise TypeError("export_data must be a boolean") + if city_id is not None and type(city_id) is not str: + raise TypeError("city_id must be None or a string") + if type(city_id) is str and ( + len(city_id) < 1 or len(slugify(city_id)) < 1 + ): + raise ValueError( + "city_id must contain at least one non-special character" + ) + if type(export_plots) is not bool: + raise TypeError("export_plots must be a boolean") + + # Import files + for filename in ['city_boundary','growable_network','bike_network','seed_points','point_data','trip_data']: + if type(import_files[filename]) is str and not os.path.isfile(settings.import_path+import_files[filename]): + raise FileNotFoundError(filename+" not found") + + if seed_point_tags is not None and type(seed_point_tags) is not dict: + raise TypeError("seed_point_tags must be None or a dictionary") + + return import_files + + +def _gbn_resolve_auto_parameters( + seed_point_type, + seed_point_grid_spacing, + seed_point_linking, + existing_network_spacing, + g, + import_files, + ): + """Resolve auto parameters their inconsistencies, and settings. + + Parameters + ---------- + Several parameters from `growbikenet.growbikenet()`. + + Additionally: + g : networkx.classes.multigraph.MultiGraph + Street network. + + Returns + ------- + Several parameters from `growbikenet.growbikenet()`. + """ + phi = orientation_order(g) + + if import_files['seed_points']: + seed_point_type = 'file' + seed_point_linking = 'triangulate_delaunay' + + if seed_point_type == 'auto': + if phi>constants._PHI_LIMITS[1] and existing_network_spacing is None: # Case grid. For example, Barcelona, Manhattan. Incompatible with existing_network_spacing not None. + seed_point_type = 'grid_square' + if seed_point_linking == 'auto': + seed_point_linking = 'quadrangulate' + elif (phi>constants._PHI_LIMITS[1] and existing_network_spacing is not None) or (phi<=constants._PHI_LIMITS[1] and phi>constants._PHI_LIMITS[0]): # Case contains some grid elements. For example, Prague, Budapest. Or, it should have been quadrangulation but existing_network_spacing is not None. + seed_point_type = 'grid_square' + if seed_point_linking == 'auto': + seed_point_linking = 'triangulate_delaunay' + elif phi<=constants._PHI_LIMITS[0]: # Case negligible grid elements. For example, Berlin, London + seed_point_type = 'grid_triangle' + if seed_point_linking == 'auto': + seed_point_linking = 'triangulate_delaunay' + elif seed_point_linking == 'quadrangulate': # Quadrangulation incompatible with seed_point_type grid_triangle from auto + seed_point_linking = 'triangulate_delaunay' + warnings.warn("seed_point_linking 'quadrangulate' is incompatible with automatically selected seed_point_type 'grid_triangle'. Changing seed_point_linking to 'triangulate_delaunay'.") + else: + if seed_point_linking == 'auto': + if seed_point_type != 'grid_square': # Everything is triangulated, but the grid could also be quadrangulated + seed_point_linking = 'triangulate_delaunay' + else: # Case 'grid_square' + if phi>constants._PHI_LIMITS[1] and existing_network_spacing is None: # Case grid. For example, Barcelona, Manhattan. We can quadrangulate, except when existing_network_spacing is not None - in that case, just triangulate again. + seed_point_linking = 'quadrangulate' + else: + seed_point_linking = 'triangulate_delaunay' + + if seed_point_grid_spacing == 'auto': + # These values ensure that any point in the city is always within b=500m of the network (if seed points snap perfectly). + # In comments, general equations for arbitrary buffer distance b + if seed_point_type == 'grid_square' and seed_point_linking == 'triangulate_delaunay': + seed_point_grid_spacing = constants.GRID_SPACING_TRIANGULATE # a=2b/(2-sqrt(2)) + elif seed_point_type == 'grid_square' and seed_point_linking == 'quadrangulate': + seed_point_grid_spacing = constants.GRID_SPACING_QUADRANGULATE # a=2b + elif seed_point_type == 'grid_triangle': + seed_point_grid_spacing = constants.GRID_SPACING_TRIANGLE # h/2=b=a*sqrt(3)/4 -> a=4b/sqrt(3) + else: + seed_point_grid_spacing = constants.GRID_SPACING_TRIANGULATE + + if settings.seed_point_snap_distance == 'auto': + settings.seed_point_snap_distance = int(np.ceil(seed_point_grid_spacing*constants._SEED_POINT_SNAP_DISTANCE_FACTOR)) + + if existing_network_spacing == 'auto': + existing_network_spacing = int(np.ceil(seed_point_grid_spacing*constants._EXISTING_NETWORK_SPACING_FACTOR)) + + # Set import data balance to 0 or 1 if only one data set is imported + if import_files['point_data'] is not None and import_files['trip_data'] is None: + settings.import_data_trip_point_balance = 0 + elif import_files['trip_data'] is not None and import_files['point_data'] is None: + settings.import_data_trip_point_balance = 1 + + return seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing + + +def _gbn_print_header(city_query, ordering, seed_point_type, existing_network_spacing): + """Print header. + """ + if not settings.silent: + print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"=") + print("RUNNING GROWBIKENET FOR CITY: " + city_query) + print(ordering + " | " + seed_point_type + " | " + ("from existing bike network " if existing_network_spacing else "from scratch")) + print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-"+"╮") + + +def _gbn_print_footer(export_data, export_plots, endtime, starttime): + """Print footer. + """ + if not settings.silent: + print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-"+"╯") + if export_data: + print("Data exported to "+settings.export_path['results']) + if export_plots: + print("Plots exported to "+settings.export_path['plots']) + if export_data or export_plots: + print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-") + print("FINISHED IN " + str(datetime.timedelta(seconds = round(endtime - starttime)))) + print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"=") + + +def _gbn_import_data_files(import_files): + """Import data files. + """ + num_data_files = int(bool(import_files['point_data'])) + int(bool(import_files['trip_data'])) + point_data = None + trip_data = None + if num_data_files: + progress_bar = initialize_progress_bar("Importing data files", num_data_files, "file") + if import_files['point_data']: + point_data = gpd.read_file(settings.import_path+import_files['point_data']) + progress_bar.update(1) + if import_files['trip_data']: + trip_data = pd.read_csv(settings.import_path+import_files['trip_data']) + progress_bar.update(1) + progress_bar.close() + return num_data_files, point_data, trip_data + + +def _gbn_acquire_network(import_files, existing_network_spacing, city_query): + """Import or download networks, prepare them, and create boundary. + """ + if import_files['growable_network'] is not None: + ### Import and preprocess data from file + city_boundary_exists = True + progress_bar = initialize_progress_bar("Importing network data", 1+int(bool(existing_network_spacing)), "network") + nodes, edges, g_undir, city_boundary_gdf = import_network(import_files['growable_network']) + city_boundary_geometry = city_boundary_gdf.geometry[0] + progress_bar.update(1) + else: + ### Download and preprocess data from OSM + city_boundary_exists = True + progress_bar = initialize_progress_bar("Downloading OSM data", 1+int(bool(existing_network_spacing)), "network") + # Get city boundary + if import_files['city_boundary']: + city_boundary_shp = gpd.read_file(settings.import_path+import_files['city_boundary']) + city_boundary_gdf = city_boundary_shp.iloc[[0]] + else: + city_boundary_gdf = ox.geocoder.geocode_to_gdf(city_query) + city_boundary_geometry = city_boundary_gdf.geometry[0] + # Fetch growable network data from osmnx + # Due to retain_all=False, this fetches the largest connected component + nodes, edges, g_undir = download_network(city_query, network_type=constants.GROWABLE_NETWORK_TYPE, custom_filter=constants.GROWABLE_NETWORK_CUSTOM_FILTER, retain_all=False, city_boundary_geometry=city_boundary_geometry) + progress_bar.update(1) + + # Update g_undir: Add the existing bike network + if existing_network_spacing is not None: + nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered = update_with_existing_bike_network(city_query, g_undir, import_files=import_files, city_boundary_geometry=city_boundary_geometry) + progress_bar.update(1) + else: + nodes_exnw = edges_exnw = g_undir_exnw = nodes_exnw_filtered = None + + # Update g_undir: Add bearings + ox.bearing.add_edge_bearings(g_undir) + + progress_bar.close() + + return city_boundary_exists, nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered, city_boundary_geometry, city_boundary_gdf + + +def _gbn_create_seed_points(existing_network_spacing, seed_point_type, g_undir, edges, nodes, seed_point_grid_spacing, import_files, city_query, seed_point_tags, city_boundary_geometry): + """Create seed points. + """ + progress_bar = initialize_progress_bar("Creating seed points", 3+int(bool(existing_network_spacing))) + seed_network = nx.Graph() # This is only relevant for some methods + if seed_point_type == 'grid_square' or seed_point_type == 'grid_triangle': + # Bearings work on unprojected graph + principal_bearing = get_principal_bearing(g_undir) + + # But this is on the projected edges now + seed_points, seed_network = _get_grid_seed_points( + edges, seed_point_grid_spacing, principal_bearing, seed_point_type + ) # The seed_network is only relevant for quadrangulation + elif seed_point_type in constants._PRESET_TAGS: + seed_point_tags = constants._PRESET_TAGS[seed_point_type] + elif seed_point_type == 'file': + seed_points = gpd.read_file(settings.import_path+import_files['seed_points']) + seed_points = _prepare_seed_points(seed_points) + + if seed_point_type == 'tags' or seed_point_type in constants._PRESET_TAGS: + seed_points = _get_tags_seed_points(city_query, tags=seed_point_tags, city_boundary_geometry=city_boundary_geometry) + progress_bar.update(1) + return seed_points, seed_network, progress_bar + + +def _gbn_snap_filter_seed_points(progress_bar, seed_points, seed_network, nodes, seed_point_linking, existing_network_spacing, nodes_exnw_filtered): + """Snap and filter seed points. + """ + seed_points_snapped = snap_points_to_osm_nodes(seed_points, nodes) + if seed_point_linking == 'quadrangulate': # Map geometry to osmid + mapping = {row.geometry_generated: row.osmid for row in seed_points_snapped.itertuples()} + nx.relabel_nodes(seed_network, mapping, copy=False) + progress_bar.update(1) + seed_points_snapped_filtered = filter_points_distant_from_osm_nodes(seed_points_snapped, settings.seed_point_snap_distance) + if seed_point_linking == 'quadrangulate': # Remove all filtered out nodes + filtered_nodes = set(seed_points_snapped.osmid) - set(seed_points_snapped_filtered.osmid) + seed_network.remove_nodes_from(filtered_nodes) + seed_network = seed_network.subgraph(sorted(nx.connected_components(seed_network), key=len, reverse=True)[0]) # Keep only the largest connected component (the network might have fallen apart) + progress_bar.update(1) + + if existing_network_spacing is not None: + seed_points_snapped_filtered = _update_seed_points_with_existing_bike_network(seed_points_snapped_filtered, nodes_exnw_filtered, existing_network_spacing) + progress_bar.update(1) + progress_bar.close() + + # Abort if less than 3 seed points. Triangulation needs at least 3. + if len(seed_points_snapped_filtered) < 3: + raise RuntimeError("Found less than 3 seed points, but more are needed.") + return seed_points_snapped_filtered + + +def _gbn_angulate_seed_points(seed_point_linking, seed_points_snapped_filtered, seed_network): + """Triangulate or quadrangulate seed points or seed point network. + """ + if seed_point_linking != 'quadrangulate': + # Triangulation and metrics (betweenness, closeness) are calculated for the unrouted, abstract network for which egde lengths are taken from the routed network. + progress_bar = initialize_progress_bar("Triangulation", 1) + # Create unrouted network with delaunay triangulation edges + grown_bikenet_edges_abstract = _create_delaunay_edges(seed_points_snapped_filtered) + else: # Build the same dataframe structure for the abstract network from the seed_network.edges + progress_bar = initialize_progress_bar("Quadrangulation", 1) + grown_bikenet_edges_abstract = pd.DataFrame({ + 'pair': seed_network.edges, + 'source': [e[0] for e in seed_network.edges], + 'target': [e[1] for e in seed_network.edges] + }) # Afterwards, all steps are identical + progress_bar.update(1) + progress_bar.close() + return grown_bikenet_edges_abstract + + +def _gbn_reset_auto_settings(setting_was_auto): + """Reset settings and constants to auto. + """ + if setting_was_auto['crs_calculations']: + constants._CRS_CALCULATIONS = 'auto' + if setting_was_auto['crs_viz']: + settings.viz["crs"] = 'auto' + + +def _gbn_route(g_undir, edges, grown_bikenet_edges_abstract, seed_points_snapped_filtered, num_data_files, point_data, trip_data): + """Weigh edges, route, make graph object, add point and trip data. + """ + progress_bar = initialize_progress_bar("Routing", 3) + + # Add pbi and weight before calculating shortest paths + g_undir = assign_pbi_to_edges(g_undir) + edges = bike_infra_mapping_gdf(g_undir, edges) + g_undir = weigh_edges(g_undir, constants._ROUTING_PENALTY) + + # Map each unrouted edge to a merged geometry of corresponding OSMnx edges (routed on g_undir) + grown_bikenet_edges_abstract = route_nodepairs(grown_bikenet_edges_abstract, edges, g_undir) + progress_bar.update(1) + grown_bikenet_edges = create_gdf_with_geoms(grown_bikenet_edges_abstract, edges) + progress_bar.update(1) + + # Add distances between source and target from geometry + grown_bikenet_edges["dist"] = grown_bikenet_edges["geometry"].length + + node_lon = seed_points_snapped_filtered.geometry.x # Needed for add_trip_data_to_net() + node_lat = seed_points_snapped_filtered.geometry.y # Needed for add_trip_data_to_net() + edge_list = grown_bikenet_edges["pair"] + print(edge_list) + sys.exit() + dist_list = grown_bikenet_edges["dist"] + dist_dict = dict(zip(edge_list, dist_list)) + geom_dict = dict(zip(edge_list, grown_bikenet_edges["geometry"].tolist())) + # Add point data to edges + if point_data is not None: + grown_bikenet_edges = add_point_data_to_net(point_data, grown_bikenet_edges) + num_points_list = grown_bikenet_edges["num_points"] + num_points_dict = dict(zip(edge_list, num_points_list)) + + # Make graph object from edge list + B = nx.Graph() # B like bike network + B.add_nodes_from(seed_points_snapped_filtered.index) + nx.set_node_attributes(B, node_lon, "x") # Needed for add_trip_data_to_net() + nx.set_node_attributes(B, node_lat, "y") # Needed for add_trip_data_to_net() + B.add_edges_from(edge_list) + nx.set_edge_attributes(B, dist_dict, "distance") + nx.set_edge_attributes(B, geom_dict, "geometry") + if point_data is not None: + nx.set_edge_attributes(B, num_points_dict, "num_points") + B.graph["crs"] = constants._CRS_CALCULATIONS # Needed for add_trip_data_to_net() + B = B.subgraph(sorted(nx.connected_components(B), key=len, reverse=True)[0]) # Keep only the largest connected component (the network might have fallen apart) + seed_points_snapped_filtered = seed_points_snapped_filtered[seed_points_snapped_filtered.index.isin(B.nodes)] # Remove seed points from disconnected components + + if num_data_files: + # Add trip data to edges + if trip_data is not None: + B = add_trip_data_to_net(trip_data, B) + + # Compute weighted distances + if trip_data is not None: + dist_weighted_by_trips_dict = _get_weighted_distances(B, "num_trips") # d_trip in [2]_ + if point_data is not None: + dist_weighted_by_points_dict = _get_weighted_distances(B, "num_points") # d_crash in [2]_ + + # Combine them + dist_weighted_dict = {} + if trip_data is not None and point_data is not None: + for k in dist_weighted_by_trips_dict: + dist_weighted_dict[k] = settings.import_data_trip_point_balance*dist_weighted_by_trips_dict[k] + (1-settings.import_data_trip_point_balance)*dist_weighted_by_points_dict[k] # d_{W} in [2]_ + elif trip_data is not None and point_data is None: + dist_weighted_dict = dist_weighted_by_trips_dict + elif trip_data is None and point_data is not None: + dist_weighted_dict = dist_weighted_by_points_dict + + metric_weight = "distance_weighted" + nx.set_edge_attributes(B, dist_weighted_dict, metric_weight) + else: + metric_weight = "distance" + + progress_bar.update(1) + progress_bar.close() + + return B, metric_weight + + +def _gbn_reroute(edges_ordered, edges, g_undir, grown_bikenet_edges_abstract): + """Reroute all edges, set pbi iteratively. + + The point of rerouting is to go through all abstract edges one by one in + their determined ordering. In each step, reroute with pbi penalties, and + then set the routed path to pbi. That way newly routed paths assume that + all previously routed paths are already pbi, and go more likely through + those previous paths, preventing "nile delta" artefacts to some extent, and + making the resulting grown network shorter / more effective. + + Parameters + ---------- + edges_ordered : geopandas.geodataframe.GeoDataFrame + Geodataframe of all edges in street network ordered by the `ordering` + method, in projected CRS `constants._CRS_CALCULATIONS`. + edges : geopandas.geodataframe.GeoDataFrame + Edges of the growable network, in projected CRS + `constants._CRS_CALCULATIONS`. + g_undir : networkx.classes.multigraph.MultiGraph + NetworkX graph of growable network, undirected. + grown_bikenet_edges_abstract : pandas.DataFrame + Dataframe of abstract edges. + + Returns + ------- + edges_reordered : geopandas.geodataframe.GeoDataFrame + Geodataframe of all edges in street network ordered by the `ordering` + method, and reordered to account for pbi dynamically, in projected CRS + `constants._CRS_CALCULATIONS`. If `settings.reroute` is set to False, + returns the input `edges_ordered`. + """ + if not settings.reroute: # Do nothing + return edges_ordered + else: + edges_reordered = edges_ordered.copy() + + for edge in tqdm( + edges_ordered.itertuples(index=True), + desc=("{:<"+str(constants._PROGRESS_BAR_DESC_LENGTH)+"}").format("Rerouting"), + leave=True, + unit="edge", + total=len(list(edges_ordered.itertuples())), + bar_format='{l_bar}{bar:'+str(constants._PROGRESS_BAR_LENGTH-7)+'}{r_bar}', + disable=settings.silent, + ): + i = grown_bikenet_edges_abstract.index[(grown_bikenet_edges_abstract['source']==edge.source) & (grown_bikenet_edges_abstract['target']==edge.target)] # grown_bikenet_edges_abstract was never reordered like the edges. Need to match. + grown_bikenet_edges_abstract_onerow = grown_bikenet_edges_abstract.iloc[i].drop(columns=['path_nodes', 'path_edges']) + g_undir = set_path_to_pbi(edge.source, edge.target, edges, g_undir) + g_undir = weigh_edges(g_undir, constants._ROUTING_PENALTY) + grown_bikenet_edges_abstract_onerow = route_nodepairs(grown_bikenet_edges_abstract_onerow, edges, g_undir) + grown_bikenet_edge = create_gdf_with_geoms(grown_bikenet_edges_abstract_onerow, edges) + if not grown_bikenet_edge.empty: # To do: How can it be empty? See _get_weighted_distances() + edges_reordered.loc[edge.Index, "geometry"] = grown_bikenet_edge['geometry'].iloc[0] + + return edges_reordered + + +def _gbn_compute_edge_metrics(ordering, B, metric_weight): + """Compute edge metrics. + """ + progress_bar = initialize_progress_bar("Computing edge metrics", 2) + + # The ordering=="random" case has no edge attributes and is handled in _order_df + if ordering == "betweenness": + # Add betweenness attributes to edges + bc_values = nx.edge_betweenness_centrality( + B, weight=metric_weight, normalized=True + ) + nx.set_edge_attributes(B, bc_values, name="betweenness") + elif ordering == "closeness": + # Add closeness attributes to nodes and edges + cc_values_nodes = nx.closeness_centrality(B, distance=metric_weight) + nx.set_node_attributes(B, cc_values_nodes, name="closeness") + cc_values = node_to_edge_attributes(cc_values_nodes, B.edges) + nx.set_edge_attributes(B, cc_values, name="closeness") + progress_bar.update(1) + + # Export attributes to gdfs: + # Create dataframe and add method as edge attribute + edges_ordered = df_from_graph(B, ordering) + + # Order edges by specified method + edges_ordered = _order_df(edges_ordered, ordering) + + edges_ordered = gpd.GeoDataFrame(edges_ordered, crs=constants._CRS_CALCULATIONS, geometry="geometry") + progress_bar.update(1) + progress_bar.close() + + return edges_ordered + + +def import_network(growable_network): + """Import and project a street network from gpkg file. + + Parameters + ---------- + growable_network : str + The street network is loaded from this file. Must be a gpkg file + in unprojected CRS EPSG:4326 with layers nodes and edges, with the + structure that a OSMnx street network ``g`` has after saving its + undirected version via ``ox.io.save_graph_geopackage()``. For example: + + >>> g = ox.graph_from_place("Barcelona", network_type='drive') + >>> g = nx.MultiGraph(ox.convert.to_digraph(g)) + >>> ox.io.save_graph_geopackage(g, "Barcelona_streets.gpkg") + import_path : str, default `settings.import_path` + Path to import files. + + Returns + ------- + nodes : geopandas.geodataframe.GeoDataFrame + Extracted OSM nodes, projected. + edges : geopandas.geodataframe.GeoDataFrame + Extracted OSM edges, projected. + g_undir : networkx.classes.multigraph.MultiGraph + Extracted networkX graph, undirected. + city_boundary_gdf : geopandas.geodataframe.GeoDataFrame + Convex hull of the street network. + + Notes + ----- + For all edges between a pair of nodes u and v there must be one edge with + key 0. + """ + + nodes = gpd.read_file(settings.import_path+growable_network, layer='nodes') + edges = gpd.read_file(settings.import_path+growable_network, layer='edges') + + # Set indices as required by osmnx.convert.graph_from_gdfs + # See: https://osmnx.readthedocs.io/en/stable/user-reference.html#osmnx.utils_graph.graph_from_gdfs + nodes = nodes.set_index(['osmid']) + edges = edges.set_index(['u', 'v', 'key']) + + g = ox.convert.graph_from_gdfs(nodes, edges) + g_undir = g.to_undirected().copy() # convert to undirected (dropping OSMnx keys!) + + city_boundary_gdf = gpd.GeoDataFrame(gpd.GeoSeries(nodes.union_all().convex_hull), geometry=0, crs=nodes.crs) # We do this before the projection of nodes below + # To do: To be super-correct, the hull should be buffered by settings.seed_point_snap_distance (in degrees due to being unprojected) + + nodes, edges = prepare_nodes_edges(nodes, edges) + + return nodes, edges, g_undir, city_boundary_gdf + + +def _gbn_postprocess_edges(existing_network_spacing, edges_exnw, edges_ordered): + """Postprocess edges: Add bike net on top, remove overlaps, add length + metrics, reorder, reproject. + """ + # Add existing bike network on top, https://stackoverflow.com/a/43408736 + if existing_network_spacing: + existing_bikenet = gpd.GeoDataFrame({c: None for c in edges_ordered.columns}, index=[-1], crs=constants._CRS_CALCULATIONS) + existing_bikenet.loc[-1, 'geometry'] = gpd.GeoSeries(edges_exnw.geometry).union_all() + edges_ordered.loc[-1] = existing_bikenet.loc[-1] + edges_ordered.index = edges_ordered.index+1 + edges_ordered.sort_index(inplace=True) + edges_ordered.crs = constants._CRS_CALCULATIONS + + # Remove edge overlaps + if not settings.allow_edge_overlaps: + edges_ordered = _remove_edge_overlaps(edges_ordered) # Can take a while, could be sped up. + + # Add lengths and cumulative lengths, rounded to integer meters + edges_ordered['length'] = edges_ordered.geometry.length + edges_ordered['length_cumulative'] = edges_ordered.geometry.length.cumsum() + edges_ordered = edges_ordered.astype({'length': int, 'length_cumulative': int}) + + if constants.REORDER: + edges_ordered['ordering'] = edges_ordered.index + + # Back to unprojected (potentially). No more calculations after here. + edges_ordered.to_crs(epsg=settings.crs_result, inplace=True) + + return edges_ordered + + +def _gbn_prepare_export(export_data, export_plots, city_id, city_query, existing_network_spacing, seed_point_type, ordering): + """Prepare export: Create folder and filename for exported data + """ + if export_data or export_plots: + os.makedirs(settings.export_path['results'], exist_ok=True) + if city_id is None: + city_string = slugify(city_query) + else: + city_string = slugify(city_id) + if existing_network_spacing: + exnw_string = "from_bikenw" + else: + exnw_string = "from_scratch" + if seed_point_type == "file": + seed_point_string = slugify(settings.seed_point_type_name) + else: + seed_point_string = seed_point_type + export_data_filename = city_string + "-growbikenet-" + ordering + "-" + seed_point_string + "-" + exnw_string + "." + settings.export_file_format + export_strings = { + "export_data_filename": export_data_filename, + "city_string": city_string, + "exnw_string": exnw_string, + "seed_point_string": seed_point_string, + } + return export_strings + + +def _gbn_export_data_to_file(export_data, seed_points_snapped_filtered, city_boundary_exists, city_boundary_gdf, existing_network_spacing, edges_ordered, export_strings): + """Export data. + """ + + if export_data: + if len(edges_ordered)<1+bool(existing_network_spacing is not None): + warnings.warn("Not enough space to grow links. No data was exported.") + else: + seed_points_snapped_filtered.drop(["osmid"], axis=1, inplace=True) + seed_points_snapped_filtered.to_crs(epsg=settings.crs_result, inplace=True) + if city_boundary_exists: + city_boundary_gdf.to_crs(epsg=settings.crs_result, inplace=True) + if settings.export_file_format == "geojson": # To do: Simplify ugly code duplications and make this an IO function + progress_bar = initialize_progress_bar("Exporting data", 2+int(bool(existing_network_spacing))+int(city_boundary_exists), "file") + if settings.crs_result == '4326': # Export with RFC7946="YES" + if existing_network_spacing: + existing_network = edges_ordered.iloc[[0]] + edges_ordered = edges_ordered.iloc[1:].astype({'source': int, 'target': int}) + existing_network.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-existing_bike_network.geojson", driver="GeoJSON", RFC7946="YES") + progress_bar.update(1) + edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON", RFC7946="YES") + progress_bar.update(1) + else: + edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON", RFC7946="YES") + progress_bar.update(1) + seed_points_snapped_filtered.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-seed_points-"+export_strings['seed_point_string']+"-"+export_strings['exnw_string']+".geojson", driver="GeoJSON", RFC7946="YES") + progress_bar.update(1) + if city_boundary_exists: + city_boundary_gdf.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-city_boundary.geojson", driver="GeoJSON", RFC7946="YES") + progress_bar.update(1) + else: + if existing_network_spacing: + existing_network = edges_ordered.iloc[[0]] + edges_ordered = edges_ordered.iloc[1:].astype({'source': int, 'target': int}) + existing_network.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-existing_bike_network.geojson", driver="GeoJSON") + progress_bar.update(1) + edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON") + progress_bar.update(1) + else: + edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON") + progress_bar.update(1) + seed_points_snapped_filtered.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-seed_points-"+export_strings['seed_point_string']+"-"+export_strings['exnw_string']+".geojson", driver="GeoJSON") + progress_bar.update(1) + if city_boundary_exists: + city_boundary_gdf.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-city_boundary.geojson", driver="GeoJSON") + progress_bar.update(1) + elif settings.export_file_format == "gpkg": + progress_bar = initialize_progress_bar("Exporting data", 1, "file") + f = settings.export_path['results']+export_strings['export_data_filename'] + if os.path.exists(f): + os.remove(f) # mode="w" does not work for to_file with gpkg. It always appends. Therefore, existing file needs to be deleted. + if existing_network_spacing: + edges_ordered.iloc[[0]].to_file(f, driver="GPKG", layer="Existing bike network") + edges_ordered.iloc[1:].to_file(f, driver="GPKG", layer="Grown bike network") + else: + edges_ordered.to_file(f, driver="GPKG", layer="Grown bike network") + seed_points_snapped_filtered.to_file(f, driver="GPKG", layer="Seed points") + if city_boundary_exists: city_boundary_gdf.to_file(f, driver="GPKG", layer="City boundary") + progress_bar.update(1) + progress_bar.close() + + +def _gbn_export_plots_to_file(export_plots, ordering, edges_ordered, seed_points_snapped_filtered, existing_network_spacing): + """Export plots. + """ + if export_plots and len(edges_ordered)>=1+bool(existing_network_spacing is not None): + os.makedirs(settings.export_path['plots']+"ordering_"+ordering+"/", exist_ok=True) + + if settings.viz["crs"] == "auto": # Make it local azimuthal by default + network_center = edges_ordered.to_crs(constants._CRS_CALCULATIONS).dissolve().centroid.to_crs('4326') # Calculate centroid in projected CRS, then go back to unprojected CRS for lat lon + settings.viz["crs"] = f"+proj=aeqd +R=6371000 +units=m +lat_0={network_center.y[0]} +lon_0={network_center.x[0]}" # The first coordinate x is the latitude + + generate_plots( + edges_ordered, + seed_points_snapped_filtered, + ordering, + bool(existing_network_spacing), + ) diff --git a/growbikenet/growbikenet.py b/growbikenet/growbikenet.py index 1252b3c..24f2784 100644 --- a/growbikenet/growbikenet.py +++ b/growbikenet/growbikenet.py @@ -1,31 +1,27 @@ -import numpy as np -import pandas as pd - -from . import settings +"""Main procedure for running `growbikenet`.""" -pd.set_option('display.max_columns', None) # for debugging +import numpy as np import time - -from growbikenet.functions import ( - _acquire_network, - _angulate_seed_points, - _compute_edge_metrics, - _create_seed_points, - _import_data_files, - _postprocess_edges, - _prepare_export, - _print_footer, - _print_header, - _reroute, - _reset_auto_settings, - _resolve_auto_parameters, - _route, - _snap_filter_seed_points, - _validate_parameters, - _validate_settings, - export_data_to_file, - export_plots_to_file, - orientation_order, +from . import settings +from growbikenet._procedure import ( + _gbn_acquire_network, + _gbn_angulate_seed_points, + _gbn_compute_edge_metrics, + _gbn_create_seed_points, + _gbn_export_data_to_file, + _gbn_export_plots_to_file, + _gbn_import_data_files, + _gbn_postprocess_edges, + _gbn_prepare_export, + _gbn_print_footer, + _gbn_print_header, + _gbn_reroute, + _gbn_reset_auto_settings, + _gbn_resolve_auto_parameters, + _gbn_route, + _gbn_snap_filter_seed_points, + _gbn_validate_parameters, + _gbn_validate_settings, ) @@ -209,55 +205,55 @@ def growbikenet( # Setup starttime = time.time() np.random.seed(settings.random_seed) # Set random number generator seed for reproducibility - setting_was_auto = _validate_settings() - import_files = _validate_parameters(city_query, ordering, seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing, export_data, city_id, export_plots, import_files, seed_point_tags) - _print_header(city_query, ordering, seed_point_type, existing_network_spacing) + setting_was_auto = _gbn_validate_settings() + import_files = _gbn_validate_parameters(city_query, ordering, seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing, export_data, city_id, export_plots, import_files, seed_point_tags) + _gbn_print_header(city_query, ordering, seed_point_type, existing_network_spacing) ### Import data files - num_data_files, point_data, trip_data = _import_data_files(import_files) + num_data_files, point_data, trip_data = _gbn_import_data_files(import_files) ### Download/Import network - city_boundary_exists, nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered, city_boundary_geometry, city_boundary_gdf = _acquire_network(import_files, existing_network_spacing, city_query) + city_boundary_exists, nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered, city_boundary_geometry, city_boundary_gdf = _gbn_acquire_network(import_files, existing_network_spacing, city_query) # Now that the graph is ready, resolve auto parameters - seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing = _resolve_auto_parameters(seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing, orientation_order(g_undir), import_files) + seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing = _gbn_resolve_auto_parameters(seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing, g_undir, import_files) # At this point no parameter should be on 'auto' any longer and # inconsistencies should be resolved. ### Create seed points - seed_points, seed_network, progress_bar = _create_seed_points(existing_network_spacing, seed_point_type, g_undir, edges, nodes, seed_point_grid_spacing, import_files, city_query, seed_point_tags, city_boundary_geometry) + seed_points, seed_network, progress_bar = _gbn_create_seed_points(existing_network_spacing, seed_point_type, g_undir, edges, nodes, seed_point_grid_spacing, import_files, city_query, seed_point_tags, city_boundary_geometry) # Snap and filter seed points to OSM nodes - seed_points_snapped_filtered = _snap_filter_seed_points(progress_bar, seed_points, seed_network, nodes, seed_point_linking, existing_network_spacing, nodes_exnw_filtered) + seed_points_snapped_filtered = _gbn_snap_filter_seed_points(progress_bar, seed_points, seed_network, nodes, seed_point_linking, existing_network_spacing, nodes_exnw_filtered) ### *angulate - grown_bikenet_edges_abstract = _angulate_seed_points(seed_point_linking, seed_points_snapped_filtered, seed_network) + grown_bikenet_edges_abstract = _gbn_angulate_seed_points(seed_point_linking, seed_points_snapped_filtered, seed_network) ### Routing: Get routed geometry (LineString) for each abstract edge (row) - B, metric_weight = _route(g_undir, edges, grown_bikenet_edges_abstract, seed_points_snapped_filtered, num_data_files, point_data, trip_data) + B, metric_weight = _gbn_route(g_undir, edges, grown_bikenet_edges_abstract, seed_points_snapped_filtered, num_data_files, point_data, trip_data) ### Compute edge metrics - edges_ordered = _compute_edge_metrics(ordering, B, metric_weight) + edges_ordered = _gbn_compute_edge_metrics(ordering, B, metric_weight) ### Reroute - edges_ordered = _reroute(edges_ordered, edges, g_undir, grown_bikenet_edges_abstract) + edges_ordered = _gbn_reroute(edges_ordered, edges, g_undir, grown_bikenet_edges_abstract) # Postprocess edges, like removing overlaps - edges_ordered = _postprocess_edges(existing_network_spacing, edges_exnw, edges_ordered) + edges_ordered = _gbn_postprocess_edges(existing_network_spacing, edges_exnw, edges_ordered) # Generate export data filenames - export_strings = _prepare_export(export_data, export_plots, city_id, city_query, existing_network_spacing, seed_point_type, ordering) + export_strings = _gbn_prepare_export(export_data, export_plots, city_id, city_query, existing_network_spacing, seed_point_type, ordering) ### Export data - export_data_to_file(export_data, seed_points_snapped_filtered, city_boundary_exists, city_boundary_gdf, existing_network_spacing, edges_ordered, export_strings) + _gbn_export_data_to_file(export_data, seed_points_snapped_filtered, city_boundary_exists, city_boundary_gdf, existing_network_spacing, edges_ordered, export_strings) ### Export plots - export_plots_to_file(export_plots, ordering, edges_ordered, seed_points_snapped_filtered, existing_network_spacing) + _gbn_export_plots_to_file(export_plots, ordering, edges_ordered, seed_points_snapped_filtered, existing_network_spacing) # Cleanup, finalize - _reset_auto_settings(setting_was_auto) + _gbn_reset_auto_settings(setting_was_auto) endtime = time.time() - _print_footer(export_data, export_plots, endtime, starttime) + _gbn_print_footer(export_data, export_plots, endtime, starttime) return edges_ordered diff --git a/growbikenet/functions.py b/growbikenet/utils.py similarity index 59% rename from growbikenet/functions.py rename to growbikenet/utils.py index 6d0b80d..2329d58 100644 --- a/growbikenet/functions.py +++ b/growbikenet/utils.py @@ -3,7 +3,6 @@ import os import re import warnings -from collections import defaultdict import numpy as np import pandas as pd @@ -12,7 +11,6 @@ pd.set_option('display.max_columns', None) # for debugging import datetime -import warnings import geopandas as gpd import networkx as nx @@ -26,123 +24,6 @@ from shapely.strtree import STRtree from tqdm import tqdm -from growbikenet.visualization import generate_plots - - -def _validate_settings(): - """ Check if user settings input is valid. If not, raise an exception or - warning. - - Returns - ------- - setting_was_auto : dict - Dictionary remembering which setting or constant was set to auto, so - it can be reset to auto in the end. - """ - - if type(constants._CRS_CALCULATIONS) is not str: - raise TypeError("constants._CRS_CALCULATIONS must be a string") - if settings.export_file_format != "geojson" and settings.export_file_format != "gpkg": - raise ValueError("settings.export_file_format must be 'geojson' or 'gpkg'") - # To do: check export_path - if type(settings.seed_point_snap_distance) is not int and settings.seed_point_snap_distance != 'auto': - raise TypeError("settings.seed_point_snap_distance must be 'auto' or an integer") - if type(settings.seed_point_snap_distance) is int and settings.seed_point_snap_distance <= 0: - raise ValueError("settings.seed_point_snap_distance must be a positive integer") - - setting_was_auto = {'crs_calculations': False, 'crs_viz': False} - # Ask whether constants._CRS_CALCULATIONS was 'auto' ahead of network - # construction, as 'auto' is resolved inside `prepare_nodes_edges()`. - if constants._CRS_CALCULATIONS == 'auto': - setting_was_auto['crs_calculations'] = True - if settings.viz["crs"] == 'auto': - setting_was_auto['crs_viz'] = True - return setting_was_auto - - -def _validate_parameters( - city_query, - ordering, - seed_point_type, - seed_point_grid_spacing, - seed_point_linking, - existing_network_spacing, - export_data, - city_id, - export_plots, - import_files, - seed_point_tags, - ): - """ Check if user parameter input is valid. If not, raise an exception or - warning. - - Parameters - ---------- - Same as `growbikenet.growbikenet()` - Additionally: - constants._PRESET_TAGS : dict - Dictionary of preset seed point tags. - - Returns - ------- - True - """ - - if type(city_query) is not str: - raise TypeError("city_query must be a string") - if type(ordering) is not str: - raise TypeError("ordering must be a string") - if ordering not in ["betweenness", "closeness", "random"]: - raise ValueError( - "ordering must be either 'betweenness', 'closeness', or 'random'" - ) - if seed_point_type not in ['auto', 'grid_square', 'grid_triangle', 'rail', 'school', 'park', 'file', 'tags']: - raise ValueError("seed_point_type must be 'auto' or 'grid_square' or 'grid_triangle' or 'rail' or 'school' or 'park' or 'file' or 'tags'") - if type(seed_point_grid_spacing) is not int and seed_point_grid_spacing != 'auto': - raise TypeError("seed_point_grid_spacing must be 'auto' or an integer") - if type(seed_point_grid_spacing) is int and seed_point_grid_spacing <= 0: - raise ValueError("seed_point_grid_spacing must be a positive integer") - if type(import_files) is not dict: - raise TypeError("import_files must be a dictionary") - # Prepare special case import_files. Turn it into a defaultdict where missing keys are None. - import_files = defaultdict(lambda: None, import_files) - if seed_point_type == 'file' and import_files['seed_points'] is None: - raise ValueError("With seed_point_type 'file', a seed_points file must be provided") - if seed_point_type == 'tags' and seed_point_tags is None: - raise ValueError("With seed_point_type 'tags', seed_point_tags must be provided") - if seed_point_linking not in ['auto', 'triangulate_delaunay', 'quadrangulate']: - raise ValueError("seed_point_linking must be 'auto' or 'triangulate_delaunay' or 'quadrangulate'") - if seed_point_linking == 'quadrangulate' and (seed_point_type != 'grid_square' or existing_network_spacing is not None): - raise ValueError("With seed_point_linking 'quadrangulate', seed_point_type must be set to 'grid_square' and existing_network_spacing must be set to None") - if type(existing_network_spacing) is not int and existing_network_spacing is not None and existing_network_spacing != 'auto': - raise TypeError("existing_network_spacing must be None or 'auto' or a positive integer") - if type(existing_network_spacing) is int and existing_network_spacing <= 0: - raise ValueError("existing_network_spacing must be None or a positive integer") - if type(existing_network_spacing) is int and seed_point_grid_spacing is int and existing_network_spacing >= seed_point_grid_spacing: - warnings.warn("existing_network_spacing is recommended to be smaller than seed_point_grid_spacing, ideally around a third, to ensure that the existing bicycle network is built first.") - if type(export_data) is not bool: - raise TypeError("export_data must be a boolean") - if city_id is not None and type(city_id) is not str: - raise TypeError("city_id must be None or a string") - if type(city_id) is str and ( - len(city_id) < 1 or len(slugify(city_id)) < 1 - ): - raise ValueError( - "city_id must contain at least one non-special character" - ) - if type(export_plots) is not bool: - raise TypeError("export_plots must be a boolean") - - # Import files - for filename in ['city_boundary','growable_network','bike_network','seed_points','point_data','trip_data']: - if type(import_files[filename]) is str and not os.path.isfile(settings.import_path+import_files[filename]): - raise FileNotFoundError(filename+" not found") - - if seed_point_tags is not None and type(seed_point_tags) is not dict: - raise TypeError("seed_point_tags must be None or a dictionary") - - return import_files - def slugify(s): """Slugify a string. @@ -177,109 +58,6 @@ def slugify(s): return s -def _resolve_auto_parameters( - seed_point_type, - seed_point_grid_spacing, - seed_point_linking, - existing_network_spacing, - phi, - import_files, - ): - """Resolve auto parameters their inconsistencies, and settings. - - Parameters - ---------- - Several parameters from `growbikenet.growbikenet()`. - - Additionally: - phi : float - Weighted orientation order. - - Returns - ------- - Several parameters from `growbikenet.growbikenet()`. - """ - - if import_files['seed_points']: - seed_point_type = 'file' - seed_point_linking = 'triangulate_delaunay' - - if seed_point_type == 'auto': - if phi>constants._PHI_LIMITS[1] and existing_network_spacing is None: # Case grid. For example, Barcelona, Manhattan. Incompatible with existing_network_spacing not None. - seed_point_type = 'grid_square' - if seed_point_linking == 'auto': - seed_point_linking = 'quadrangulate' - elif (phi>constants._PHI_LIMITS[1] and existing_network_spacing is not None) or (phi<=constants._PHI_LIMITS[1] and phi>constants._PHI_LIMITS[0]): # Case contains some grid elements. For example, Prague, Budapest. Or, it should have been quadrangulation but existing_network_spacing is not None. - seed_point_type = 'grid_square' - if seed_point_linking == 'auto': - seed_point_linking = 'triangulate_delaunay' - elif phi<=constants._PHI_LIMITS[0]: # Case negligible grid elements. For example, Berlin, London - seed_point_type = 'grid_triangle' - if seed_point_linking == 'auto': - seed_point_linking = 'triangulate_delaunay' - elif seed_point_linking == 'quadrangulate': # Quadrangulation incompatible with seed_point_type grid_triangle from auto - seed_point_linking = 'triangulate_delaunay' - warnings.warn("seed_point_linking 'quadrangulate' is incompatible with automatically selected seed_point_type 'grid_triangle'. Changing seed_point_linking to 'triangulate_delaunay'.") - else: - if seed_point_linking == 'auto': - if seed_point_type != 'grid_square': # Everything is triangulated, but the grid could also be quadrangulated - seed_point_linking = 'triangulate_delaunay' - else: # Case 'grid_square' - if phi>constants._PHI_LIMITS[1] and existing_network_spacing is None: # Case grid. For example, Barcelona, Manhattan. We can quadrangulate, except when existing_network_spacing is not None - in that case, just triangulate again. - seed_point_linking = 'quadrangulate' - else: - seed_point_linking = 'triangulate_delaunay' - - if seed_point_grid_spacing == 'auto': - # These values ensure that any point in the city is always within b=500m of the network (if seed points snap perfectly). - # In comments, general equations for arbitrary buffer distance b - if seed_point_type == 'grid_square' and seed_point_linking == 'triangulate_delaunay': - seed_point_grid_spacing = constants.GRID_SPACING_TRIANGULATE # a=2b/(2-sqrt(2)) - elif seed_point_type == 'grid_square' and seed_point_linking == 'quadrangulate': - seed_point_grid_spacing = constants.GRID_SPACING_QUADRANGULATE # a=2b - elif seed_point_type == 'grid_triangle': - seed_point_grid_spacing = constants.GRID_SPACING_TRIANGLE # h/2=b=a*sqrt(3)/4 -> a=4b/sqrt(3) - else: - seed_point_grid_spacing = constants.GRID_SPACING_TRIANGULATE - - if settings.seed_point_snap_distance == 'auto': - settings.seed_point_snap_distance = int(np.ceil(seed_point_grid_spacing*constants._SEED_POINT_SNAP_DISTANCE_FACTOR)) - - if existing_network_spacing == 'auto': - existing_network_spacing = int(np.ceil(seed_point_grid_spacing*constants._EXISTING_NETWORK_SPACING_FACTOR)) - - # Set import data balance to 0 or 1 if only one data set is imported - if import_files['point_data'] is not None and import_files['trip_data'] is None: - settings.import_data_trip_point_balance = 0 - elif import_files['trip_data'] is not None and import_files['point_data'] is None: - settings.import_data_trip_point_balance = 1 - - return seed_point_type, seed_point_grid_spacing, seed_point_linking, existing_network_spacing - - -def _print_header(city_query, ordering, seed_point_type, existing_network_spacing): - """Print header. - """ - if not settings.silent: - print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"=") - print("RUNNING GROWBIKENET FOR CITY: " + city_query) - print(ordering + " | " + seed_point_type + " | " + ("from existing bike network " if existing_network_spacing else "from scratch")) - print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-"+"╮") - -def _print_footer(export_data, export_plots, endtime, starttime): - """Print footer. - """ - if not settings.silent: - print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-"+"╯") - if export_data: - print("Data exported to "+settings.export_path['results']) - if export_plots: - print("Plots exported to "+settings.export_path['plots']) - if export_data or export_plots: - print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"-") - print("FINISHED IN " + str(datetime.timedelta(seconds = round(endtime - starttime)))) - print((constants._PROGRESS_BAR_DESC_LENGTH+constants._PROGRESS_BAR_LENGTH)*"=") - def initialize_progress_bar(desc_string, total=1, unit="step"): """Initialize tqdm progress bar. """ @@ -291,311 +69,6 @@ def initialize_progress_bar(desc_string, total=1, unit="step"): disable=settings.silent, ) -def _import_data_files(import_files): - """Import data files. - """ - num_data_files = int(bool(import_files['point_data'])) + int(bool(import_files['trip_data'])) - point_data = None - trip_data = None - if num_data_files: - progress_bar = initialize_progress_bar("Importing data files", num_data_files, "file") - if import_files['point_data']: - point_data = gpd.read_file(settings.import_path+import_files['point_data']) - progress_bar.update(1) - if import_files['trip_data']: - trip_data = pd.read_csv(settings.import_path+import_files['trip_data']) - progress_bar.update(1) - progress_bar.close() - return num_data_files, point_data, trip_data - - -def _acquire_network(import_files, existing_network_spacing, city_query): - """Import or download networks, prepare them, and create boundary. - """ - if import_files['growable_network'] is not None: - ### Import and preprocess data from file - city_boundary_exists = True - progress_bar = initialize_progress_bar("Importing network data", 1+int(bool(existing_network_spacing)), "network") - nodes, edges, g_undir, city_boundary_gdf = import_network(import_files['growable_network']) - city_boundary_geometry = city_boundary_gdf.geometry[0] - progress_bar.update(1) - else: - ### Download and preprocess data from OSM - city_boundary_exists = True - progress_bar = initialize_progress_bar("Downloading OSM data", 1+int(bool(existing_network_spacing)), "network") - # Get city boundary - if import_files['city_boundary']: - city_boundary_shp = gpd.read_file(settings.import_path+import_files['city_boundary']) - city_boundary_gdf = city_boundary_shp.iloc[[0]] - else: - city_boundary_gdf = ox.geocoder.geocode_to_gdf(city_query) - city_boundary_geometry = city_boundary_gdf.geometry[0] - # Fetch growable network data from osmnx - # Due to retain_all=False, this fetches the largest connected component - nodes, edges, g_undir = download_network(city_query, network_type=constants.GROWABLE_NETWORK_TYPE, custom_filter=constants.GROWABLE_NETWORK_CUSTOM_FILTER, retain_all=False, city_boundary_geometry=city_boundary_geometry) - progress_bar.update(1) - - # Update g_undir: Add the existing bike network - if existing_network_spacing is not None: - nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered = update_with_existing_bike_network(city_query, g_undir, import_files=import_files, city_boundary_geometry=city_boundary_geometry) - progress_bar.update(1) - else: - nodes_exnw = edges_exnw = g_undir_exnw = nodes_exnw_filtered = None - - # Update g_undir: Add bearings - ox.bearing.add_edge_bearings(g_undir) - - progress_bar.close() - - return city_boundary_exists, nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered, city_boundary_geometry, city_boundary_gdf - - -def _create_seed_points(existing_network_spacing, seed_point_type, g_undir, edges, nodes, seed_point_grid_spacing, import_files, city_query, seed_point_tags, city_boundary_geometry): - """Create seed points. - """ - progress_bar = initialize_progress_bar("Creating seed points", 3+int(bool(existing_network_spacing))) - seed_network = nx.Graph() # This is only relevant for some methods - if seed_point_type == 'grid_square' or seed_point_type == 'grid_triangle': - # Bearings work on unprojected graph - principal_bearing = get_principal_bearing(g_undir) - - # But this is on the projected edges now - seed_points, seed_network = _get_grid_seed_points( - edges, seed_point_grid_spacing, principal_bearing, seed_point_type - ) # The seed_network is only relevant for quadrangulation - elif seed_point_type in constants._PRESET_TAGS: - seed_point_tags = constants._PRESET_TAGS[seed_point_type] - elif seed_point_type == 'file': - seed_points = gpd.read_file(settings.import_path+import_files['seed_points']) - seed_points = _prepare_seed_points(seed_points) - - if seed_point_type == 'tags' or seed_point_type in constants._PRESET_TAGS: - seed_points = _get_tags_seed_points(city_query, tags=seed_point_tags, city_boundary_geometry=city_boundary_geometry) - progress_bar.update(1) - return seed_points, seed_network, progress_bar - - -def _snap_filter_seed_points(progress_bar, seed_points, seed_network, nodes, seed_point_linking, existing_network_spacing, nodes_exnw_filtered): - """Snap and filter seed points. - """ - seed_points_snapped = snap_points_to_osm_nodes(seed_points, nodes) - if seed_point_linking == 'quadrangulate': # Map geometry to osmid - mapping = {row.geometry_generated: row.osmid for row in seed_points_snapped.itertuples()} - nx.relabel_nodes(seed_network, mapping, copy=False) - progress_bar.update(1) - seed_points_snapped_filtered = filter_points_distant_from_osm_nodes(seed_points_snapped, settings.seed_point_snap_distance) - if seed_point_linking == 'quadrangulate': # Remove all filtered out nodes - filtered_nodes = set(seed_points_snapped.osmid) - set(seed_points_snapped_filtered.osmid) - seed_network.remove_nodes_from(filtered_nodes) - seed_network = seed_network.subgraph(sorted(nx.connected_components(seed_network), key=len, reverse=True)[0]) # Keep only the largest connected component (the network might have fallen apart) - progress_bar.update(1) - - if existing_network_spacing is not None: - seed_points_snapped_filtered = _update_seed_points_with_existing_bike_network(seed_points_snapped_filtered, nodes_exnw_filtered, existing_network_spacing) - progress_bar.update(1) - progress_bar.close() - - # Abort if less than 3 seed points. Triangulation needs at least 3. - if len(seed_points_snapped_filtered) < 3: - raise RuntimeError("Found less than 3 seed points, but more are needed.") - return seed_points_snapped_filtered - - -def _angulate_seed_points(seed_point_linking, seed_points_snapped_filtered, seed_network): - """Triangulate or quadrangulate seed points or seed point network. - """ - if seed_point_linking != 'quadrangulate': - # Triangulation and metrics (betweenness, closeness) are calculated for the unrouted, abstract network for which egde lengths are taken from the routed network. - progress_bar = initialize_progress_bar("Triangulation", 1) - # Create unrouted network with delaunay triangulation edges - grown_bikenet_edges_abstract = _create_delaunay_edges(seed_points_snapped_filtered) - else: # Build the same dataframe structure for the abstract network from the seed_network.edges - progress_bar = initialize_progress_bar("Quadrangulation", 1) - grown_bikenet_edges_abstract = pd.DataFrame({ - 'pair': seed_network.edges, - 'source': [e[0] for e in seed_network.edges], - 'target': [e[1] for e in seed_network.edges] - }) # Afterwards, all steps are identical - progress_bar.update(1) - progress_bar.close() - return grown_bikenet_edges_abstract - - -def _reset_auto_settings(setting_was_auto): - """Reset settings and constants to auto. - """ - if setting_was_auto['crs_calculations']: - constants._CRS_CALCULATIONS = 'auto' - if setting_was_auto['crs_viz']: - settings.viz["crs"] = 'auto' - -def _route(g_undir, edges, grown_bikenet_edges_abstract, seed_points_snapped_filtered, num_data_files, point_data, trip_data): - """Weigh edges, route, make graph object, add point and trip data. - """ - progress_bar = initialize_progress_bar("Routing", 3) - - # Add pbi and weight before calculating shortest paths - g_undir = assign_pbi_to_edges(g_undir) - edges = bike_infra_mapping_gdf(g_undir, edges) - g_undir = weigh_edges(g_undir, constants._ROUTING_PENALTY) - - # Map each unrouted edge to a merged geometry of corresponding OSMnx edges (routed on g_undir) - grown_bikenet_edges_abstract = route_nodepairs(grown_bikenet_edges_abstract, edges, g_undir) - progress_bar.update(1) - grown_bikenet_edges = create_gdf_with_geoms(grown_bikenet_edges_abstract, edges) - progress_bar.update(1) - - # Add distances between source and target from geometry - grown_bikenet_edges["dist"] = grown_bikenet_edges["geometry"].length - - node_lon = seed_points_snapped_filtered.geometry.x # Needed for add_trip_data_to_net() - node_lat = seed_points_snapped_filtered.geometry.y # Needed for add_trip_data_to_net() - edge_list = grown_bikenet_edges["pair"] - dist_list = grown_bikenet_edges["dist"] - dist_dict = dict(zip(edge_list, dist_list)) - geom_dict = dict(zip(edge_list, grown_bikenet_edges["geometry"].tolist())) - # Add point data to edges - if point_data is not None: - grown_bikenet_edges = add_point_data_to_net(point_data, grown_bikenet_edges) - num_points_list = grown_bikenet_edges["num_points"] - num_points_dict = dict(zip(edge_list, num_points_list)) - - # Make graph object from edge list - B = nx.Graph() # B like bike network - B.add_nodes_from(seed_points_snapped_filtered.index) - nx.set_node_attributes(B, node_lon, "x") # Needed for add_trip_data_to_net() - nx.set_node_attributes(B, node_lat, "y") # Needed for add_trip_data_to_net() - B.add_edges_from(edge_list) - nx.set_edge_attributes(B, dist_dict, "distance") - nx.set_edge_attributes(B, geom_dict, "geometry") - if point_data is not None: - nx.set_edge_attributes(B, num_points_dict, "num_points") - B.graph["crs"] = constants._CRS_CALCULATIONS # Needed for add_trip_data_to_net() - B = B.subgraph(sorted(nx.connected_components(B), key=len, reverse=True)[0]) # Keep only the largest connected component (the network might have fallen apart) - seed_points_snapped_filtered = seed_points_snapped_filtered[seed_points_snapped_filtered.index.isin(B.nodes)] # Remove seed points from disconnected components - - if num_data_files: - # Add trip data to edges - if trip_data is not None: - B = add_trip_data_to_net(trip_data, B) - - # Compute weighted distances - if trip_data is not None: - dist_weighted_by_trips_dict = _get_weighted_distances(B, "num_trips") # d_trip in [2]_ - if point_data is not None: - dist_weighted_by_points_dict = _get_weighted_distances(B, "num_points") # d_crash in [2]_ - - # Combine them - dist_weighted_dict = {} - if trip_data is not None and point_data is not None: - for k in dist_weighted_by_trips_dict: - dist_weighted_dict[k] = settings.import_data_trip_point_balance*dist_weighted_by_trips_dict[k] + (1-settings.import_data_trip_point_balance)*dist_weighted_by_points_dict[k] # d_{W} in [2]_ - elif trip_data is not None and point_data is None: - dist_weighted_dict = dist_weighted_by_trips_dict - elif trip_data is None and point_data is not None: - dist_weighted_dict = dist_weighted_by_points_dict - - metric_weight = "distance_weighted" - nx.set_edge_attributes(B, dist_weighted_dict, metric_weight) - else: - metric_weight = "distance" - - progress_bar.update(1) - progress_bar.close() - - return B, metric_weight - - -def _reroute(edges_ordered, edges, g_undir, grown_bikenet_edges_abstract): - """Reroute all edges, set pbi iteratively. - - The point of rerouting is to go through all abstract edges one by one in - their determined ordering. In each step, reroute with pbi penalties, and - then set the routed path to pbi. That way newly routed paths assume that - all previously routed paths are already pbi, and go more likely through - those previous paths, preventing "nile delta" artefacts to some extent, and - making the resulting grown network shorter / more effective. - - Parameters - ---------- - edges_ordered : geopandas.geodataframe.GeoDataFrame - Geodataframe of all edges in street network ordered by the `ordering` - method, in projected CRS `constants._CRS_CALCULATIONS`. - edges : geopandas.geodataframe.GeoDataFrame - Edges of the growable network, in projected CRS - `constants._CRS_CALCULATIONS`. - g_undir : networkx.classes.multigraph.MultiGraph - NetworkX graph of growable network, undirected. - grown_bikenet_edges_abstract : pandas.DataFrame - Dataframe of abstract edges. - - Returns - ------- - edges_reordered : geopandas.geodataframe.GeoDataFrame - Geodataframe of all edges in street network ordered by the `ordering` - method, and reordered to account for pbi dynamically, in projected CRS - `constants._CRS_CALCULATIONS`. If `settings.reroute` is set to False, - returns the input `edges_ordered`. - """ - if not settings.reroute: # Do nothing - return edges_ordered - else: - edges_reordered = edges_ordered.copy() - - for edge in tqdm( - edges_ordered.itertuples(index=True), - desc=("{:<"+str(constants._PROGRESS_BAR_DESC_LENGTH)+"}").format("Rerouting"), - leave=True, - unit="edge", - total=len(list(edges_ordered.itertuples())), - bar_format='{l_bar}{bar:'+str(constants._PROGRESS_BAR_LENGTH-7)+'}{r_bar}', - disable=settings.silent, - ): - i = grown_bikenet_edges_abstract.index[(grown_bikenet_edges_abstract['source']==edge.source) & (grown_bikenet_edges_abstract['target']==edge.target)] # grown_bikenet_edges_abstract was never reordered like the edges. Need to match. - grown_bikenet_edges_abstract_onerow = grown_bikenet_edges_abstract.iloc[i].drop(columns=['path_nodes', 'path_edges']) - g_undir = set_path_to_pbi(edge.source, edge.target, edges, g_undir) - g_undir = weigh_edges(g_undir, constants._ROUTING_PENALTY) - grown_bikenet_edges_abstract_onerow = route_nodepairs(grown_bikenet_edges_abstract_onerow, edges, g_undir) - grown_bikenet_edge = create_gdf_with_geoms(grown_bikenet_edges_abstract_onerow, edges) - if not grown_bikenet_edge.empty: # To do: How can it be empty? See _get_weighted_distances() - edges_reordered.loc[edge.Index, "geometry"] = grown_bikenet_edge['geometry'].iloc[0] - - return edges_reordered - - -def _compute_edge_metrics(ordering, B, metric_weight): - """Compute edge metrics. - """ - progress_bar = initialize_progress_bar("Computing edge metrics", 2) - - # The ordering=="random" case has no edge attributes and is handled in _order_df - if ordering == "betweenness": - # Add betweenness attributes to edges - bc_values = nx.edge_betweenness_centrality( - B, weight=metric_weight, normalized=True - ) - nx.set_edge_attributes(B, bc_values, name="betweenness") - elif ordering == "closeness": - # Add closeness attributes to nodes and edges - cc_values_nodes = nx.closeness_centrality(B, distance=metric_weight) - nx.set_node_attributes(B, cc_values_nodes, name="closeness") - cc_values = node_to_edge_attributes(cc_values_nodes, B.edges) - nx.set_edge_attributes(B, cc_values, name="closeness") - progress_bar.update(1) - - # Export attributes to gdfs: - # Create dataframe and add method as edge attribute - edges_ordered = df_from_graph(B, ordering) - - # Order edges by specified method - edges_ordered = _order_df(edges_ordered, ordering) - - edges_ordered = gpd.GeoDataFrame(edges_ordered, crs=constants._CRS_CALCULATIONS, geometry="geometry") - progress_bar.update(1) - progress_bar.close() - - return edges_ordered - def add_trip_data_to_net(trips, A, matching_distance=settings.import_trip_data_snap_distance): """Match trip data to network edges. @@ -628,9 +101,7 @@ def add_trip_data_to_net(trips, A, matching_distance=settings.import_trip_data_s of all trips where both origins and destinations could be matched to the closest network nodes within `matching_distance`. """ - graph_with_data = A.copy() - transformer = Transformer.from_crs("EPSG:4326", constants._CRS_CALCULATIONS, always_xy=True) # If column "num" is not provided assume 1 trip per origin-destination (OD) pair @@ -639,7 +110,6 @@ def add_trip_data_to_net(trips, A, matching_distance=settings.import_trip_data_s else: nums = [1] * len(trips) - # Project trip data trips_projected = trips.copy() trips_projected['o_lon'], trips_projected['o_lat'] = transformer.transform(trips['o_lon'], trips['o_lat']) @@ -649,12 +119,10 @@ def add_trip_data_to_net(trips, A, matching_distance=settings.import_trip_data_s o_nodes, o_distances = ox.distance.nearest_nodes(graph_with_data, trips_projected['o_lon'], trips_projected['o_lat'], return_dist=True) d_nodes, d_distances = ox.distance.nearest_nodes(graph_with_data, trips_projected['d_lon'], trips_projected['d_lat'], return_dist=True) - # Add number of trips to graph trip_dict = dict(zip(graph_with_data.edges, [0] * len(graph_with_data.edges))) for o_node, o_distance, d_node, d_distance, num in zip(o_nodes, o_distances, d_nodes, d_distances, nums): - # If: # - origin and destination are snapped to the same node, or # - origin is >500m away from the snapped node, or @@ -679,7 +147,6 @@ def add_trip_data_to_net(trips, A, matching_distance=settings.import_trip_data_s trip_dict[edge_id] += num nx.set_edge_attributes(graph_with_data, trip_dict, "num_trips") - return graph_with_data @@ -707,9 +174,7 @@ def add_point_data_to_net(points, edges, matching_distance=settings.import_point ``num_points`` populated with the summed up ``num`` values of all points, matched to the closest links if within `matching_distance`. """ - edges_with_data = edges.copy() - points_projected = points.to_crs(constants._CRS_CALCULATIONS) # If column "num" is not provided assume 1 event per point @@ -729,7 +194,6 @@ def add_point_data_to_net(points, edges, matching_distance=settings.import_point rtree = STRtree(edges_geoms) for point in points_geoms.items(): # point = (id, POINT(x, y)) - # Query the tree for all the edges within the matching distance of the point edges_within_matching_distance = rtree.query( point[1], @@ -800,7 +264,6 @@ def import_network(growable_network): For all edges between a pair of nodes u and v there must be one edge with key 0. """ - nodes = gpd.read_file(settings.import_path+growable_network, layer='nodes') edges = gpd.read_file(settings.import_path+growable_network, layer='edges') @@ -816,7 +279,6 @@ def import_network(growable_network): # To do: To be super-correct, the hull should be buffered by settings.seed_point_snap_distance (in degrees due to being unprojected) nodes, edges = prepare_nodes_edges(nodes, edges) - return nodes, edges, g_undir, city_boundary_gdf @@ -831,6 +293,7 @@ def _resolve_crs_calculations(gdf): if constants._CRS_CALCULATIONS == 'auto': constants._CRS_CALCULATIONS = gdf.estimate_utm_crs() + def prepare_nodes_edges(nodes, edges): """Project and prepare nodes and edges for further use. @@ -907,7 +370,6 @@ def download_network(city_query, network_type='drive', custom_filter=None, retai g_undir : networkx.classes.multigraph.MultiGraph Extracted networkX graph, undirected. """ - # Fetch street network data from osmnx if city_boundary_geometry is None: g = ox.graph_from_place( @@ -947,7 +409,6 @@ def nx_to_nodes_edges(G): node_geometry=True, fill_edge_geometry=True ) - nodes, edges = prepare_nodes_edges(nodes, edges) return nodes, edges @@ -1002,7 +463,6 @@ def _get_existing_network_seed_points(nodes_exnw, existing_network_spacing): Seed points, already part of the network, in the same projected CRS as edges. """ - seed_points_exnw = gpd.GeoDataFrame() if len(nodes_exnw) == 0: return gpd.GeoDataFrame() @@ -1093,7 +553,6 @@ def update_with_existing_bike_network(city_query, g_undir, import_files, city_bo edges_exnw = edges_exnw.iloc[edges_exnw.index.get_level_values('u').isin(valid_node_osmids)] edges_exnw = edges_exnw.iloc[edges_exnw.index.get_level_values('v').isin(valid_node_osmids)] nodes, edges = nx_to_nodes_edges(g_undir) - return nodes, edges, g_undir, nodes_exnw, edges_exnw, g_undir_exnw, nodes_exnw_filtered @@ -1118,7 +577,6 @@ def filter_network_by_component_length(g_undir): g_undir_filtered : networkx.classes.multigraph.MultiGraph Filtered street networkX graph, undirected. """ - g_undir_filtered = nx.MultiGraph() components_by_length = [g_undir.subgraph(c).copy() for c in sorted(nx.connected_components(g_undir), key=lambda c: sum([l[-1] for l in g_undir.subgraph(c).copy().edges.data('length')]), reverse=True)] for c in components_by_length: # Create the union of long enough components. Probably there is a way to do this faster/vectorized. @@ -1141,6 +599,7 @@ def filter_network_by_component_length(g_undir): else: nodes_filtered = gpd.GeoDataFrame(columns = ['y', 'x', 'timestamp', 'street_count', 'geometry', 'osmid'], geometry='geometry', crs=constants._CRS_CALCULATIONS).set_index('osmid', drop=False) edges_filtered = gpd.GeoDataFrame(columns = ['highway', 'osmid', 'length', 'oneway', 'from', 'to', 'geometry', 'u', 'v', 'key'], geometry='geometry', crs=constants._CRS_CALCULATIONS).set_index(['u','v','key']) + return nodes_filtered, edges_filtered, g_undir_filtered @@ -1167,7 +626,6 @@ def _update_seed_points_with_existing_bike_network(seed_points_snapped, nodes_ex Snapped seed points incorporating both street grid and existing bike network. """ - # If the existing bicycle network is used, create extra seed points on it. They are by construction already snapped. seed_points_exnw = _get_existing_network_seed_points(nodes_exnw, existing_network_spacing) if len(seed_points_exnw) == 0: # Nothing happens @@ -1194,6 +652,7 @@ def _update_seed_points_with_existing_bike_network(seed_points_snapped, nodes_ex seed_points_snapped.set_index("osmid", drop=False, inplace=True) else: seed_points_snapped = seed_points_exnw + return seed_points_snapped @@ -1223,7 +682,6 @@ def _get_grid_seed_points(edges, seed_point_spacing, principal_bearing, seed_poi If `seed_point_type` is 'grid_triangle', empty network because the seed points will be triangulated. """ - # Rotate edges counter to the principal bearing edges_temp = edges.copy() edges_temp.geometry = edges_temp.geometry.rotate(principal_bearing, origin=(0, 0)) @@ -1330,7 +788,6 @@ def _get_tags_seed_points(city_query, tags, city_boundary_geometry=None): ---------- .. [3] https://osmnx.readthedocs.io/en/stable/user-reference.html#osmnx.features.features_from_place """ - if city_boundary_geometry: seed_points = ox.features_from_polygon( city_boundary_geometry, tags @@ -1361,7 +818,6 @@ def get_principal_bearing(G): principal_bearing : float The principal bearing, precise to 5 degrees. """ - bearings = {} # weight bearings by length (meters) city_bearings = [] @@ -1375,7 +831,6 @@ def get_principal_bearing(G): bins = np.arange(constants._BEARING_BINS + 1) * 360 / constants._BEARING_BINS count = _count_and_merge(constants._BEARING_BINS, bearings) principal_bearing = bins[np.where(count == max(count))][0] - return principal_bearing @@ -1421,7 +876,8 @@ def _count_and_merge(n, bearings): bins = np.arange(n + 1) * 360 / n count, _ = np.histogram(bearings, bins=bins) - # move the last bin to the front, so eg 0.01° and 359.99° will be binned together + # Move the last bin to the front, so eg 0.01° and 359.99° will be binned + # together count = np.roll(count, 1) bearings_merged = count[::2] + count[1::2] return bearings_merged @@ -1466,11 +922,8 @@ def snap_points_to_osm_nodes(points, nodes): points = points.reset_index(drop=True) nodes_subset = nodes_subset.reset_index(drop=True) - points_snapped = points.merge(nodes_subset, on="osmid", how="left") - points_snapped.set_geometry("geometry_osm") - return points_snapped @@ -1512,7 +965,6 @@ def filter_points_distant_from_osm_nodes(points_snapped, snap_distance=settings. gdf = gdf.set_index("osmid", drop=False) points_snapped_filtered = gdf.copy() - return points_snapped_filtered @@ -1552,12 +1004,10 @@ def _create_delaunay_edges(nodes_gdf): # Compute triangulation tri = Delaunay(coords) - edges_set = set() - # Each triangle has 3 edges + edges_set = set() for simplex in tri.simplices: i, j, k = simplex - edges_set.add(tuple(sorted((i, j)))) edges_set.add(tuple(sorted((j, k)))) edges_set.add(tuple(sorted((i, k)))) @@ -1565,7 +1015,6 @@ def _create_delaunay_edges(nodes_gdf): pairs = [] sources = [] targets = [] - for i, j in edges_set: pairs.append((osmids[i], osmids[j])) sources.append(osmids[i]) @@ -1578,7 +1027,6 @@ def _create_delaunay_edges(nodes_gdf): "target": targets, } ) - return df @@ -1616,9 +1064,9 @@ def _remove_edge_overlaps(edges_in): edges_out.drop(index=row.Index, inplace=True) edges_out.drop_duplicates(inplace=True) # How can duplicates happen?? edges_out.reset_index(drop=True, inplace=True) - return edges_out + def df_from_graph(A, method): """Create a dataframe from an input graph. @@ -1635,7 +1083,6 @@ def df_from_graph(A, method): Dataframe with source and target information for each edge, as well as edge attributes as columns. """ - if method != "random": attrs = { edge: { @@ -1843,6 +1290,7 @@ def orientation_order(g_undir): phi = 1 - ((Hw-Hg)/(Hmax-Hg))**2 return phi + def _get_weighted_distances(B, num_types): """Get weighted distances by edge attribute num_types. @@ -1870,7 +1318,6 @@ def _get_weighted_distances(B, num_types): ---------- .. [1] P. Folco, L. Gauvin, M. Tizzoni, M. Szell, `Data-driven micromobility network planning for demand and safety`, Environment and planning B: Urban analytics and city science 50(8), 2087-2102 (2023) """ - num_types_dict = nx.get_edge_attributes(B, num_types) dist_dict = nx.get_edge_attributes(B, "distance") num_types_per_km_dict = {} @@ -1914,24 +1361,26 @@ def assign_pbi_to_edges(g): if g.edges[edge].get(tag) in osm.PBI_DICT_SUB[tag][0] and g.edges[edge].get(subtag) in osm.PBI_DICT_SUB[tag][1][subtag] and g.edges[edge].get("access") != 'private': assigned_pbi = 1 g.edges[edge]["pbi"] = assigned_pbi + return g def bike_infra_mapping_gdf(g, edges_gdf): - """ - add binary edge attribute pbi to edges_gdf + """Add binary edge attribute pbi to edges_gdf. Parameters ---------- g : networkx.MultiDiGraph - simplified graph representing the street network, with added binary edge attribute "pbi" - edges_gdf: geopandas.GeoDataFrame - edges representing the street network + Simplified graph representing the street network, with added binary + edge attribute "pbi". + edges_gdf : geopandas.GeoDataFrame + Edges representing the street network. Returns ------- - edges_gdf: geopandas.GeoDataFrame - edges representing the street network with added binary attribute "pbi" + edges_gdf : geopandas.GeoDataFrame + Edges representing the street network with added binary attribute + "pbi". """ # Build dict of edge attribute attr_dict = { @@ -1941,23 +1390,27 @@ def bike_infra_mapping_gdf(g, edges_gdf): # Map to GeoDataFrame edges_gdf['pbi'] = edges_gdf.index.map(attr_dict) + return edges_gdf + def weigh_edges(G, penalty): - """ - adds weight parameter to all edges in G, which is calculated by multiplying the length of the edge with the corresponding penalty value + """Add weight parameter to all edges in G, which is calculated by + multiplying the length of the edge with the corresponding penalty value. Parameters ---------- - G: networkx.Graph - undirected simple graph representing the street network - penalty: dictionary - dictionary of penalty values, dependent on if edge has bike infrastructure or not + G : networkx.Graph + Undirected simple graph representing the street network. + penalty : dictionary + Dictionary of penalty values, dependent on if edge has bike + infrastructure or not. Returns ------- - G: networkx.Graph - undirected simple graph representing the street network with weighted edges + G : networkx.Graph + Undirected simple graph representing the street network with weighted + edges. """ for edge in G.edges: # compute edge weight @@ -1966,138 +1419,5 @@ def weigh_edges(G, penalty): edge_weight = edge_length * penalty[edge_pbi] # add as attribute G.edges[edge]["weight"] = edge_weight - return G - -def _postprocess_edges(existing_network_spacing, edges_exnw, edges_ordered): - """Postprocess edges: Add bike net on top, remove overlaps, add length - metrics, reorder, reproject. - """ - # Add existing bike network on top, https://stackoverflow.com/a/43408736 - if existing_network_spacing: - existing_bikenet = gpd.GeoDataFrame({c: None for c in edges_ordered.columns}, index=[-1], crs=constants._CRS_CALCULATIONS) - existing_bikenet.loc[-1, 'geometry'] = gpd.GeoSeries(edges_exnw.geometry).union_all() - edges_ordered.loc[-1] = existing_bikenet.loc[-1] - edges_ordered.index = edges_ordered.index+1 - edges_ordered.sort_index(inplace=True) - edges_ordered.crs = constants._CRS_CALCULATIONS - - # Remove edge overlaps - if not settings.allow_edge_overlaps: - edges_ordered = _remove_edge_overlaps(edges_ordered) # Can take a while, could be sped up. - - # Add lengths and cumulative lengths, rounded to integer meters - edges_ordered['length'] = edges_ordered.geometry.length - edges_ordered['length_cumulative'] = edges_ordered.geometry.length.cumsum() - edges_ordered = edges_ordered.astype({'length': int, 'length_cumulative': int}) - - if constants.REORDER: - edges_ordered['ordering'] = edges_ordered.index - - # Back to unprojected (potentially). No more calculations after here. - edges_ordered.to_crs(epsg=settings.crs_result, inplace=True) - - return edges_ordered - -def _prepare_export(export_data, export_plots, city_id, city_query, existing_network_spacing, seed_point_type, ordering): - """Prepare export: Create folder and filename for exported data - """ - if export_data or export_plots: - os.makedirs(settings.export_path['results'], exist_ok=True) - if city_id is None: - city_string = slugify(city_query) - else: - city_string = slugify(city_id) - if existing_network_spacing: - exnw_string = "from_bikenw" - else: - exnw_string = "from_scratch" - if seed_point_type == "file": - seed_point_string = slugify(settings.seed_point_type_name) - else: - seed_point_string = seed_point_type - export_data_filename = city_string + "-growbikenet-" + ordering + "-" + seed_point_string + "-" + exnw_string + "." + settings.export_file_format - export_strings = { - "export_data_filename": export_data_filename, - "city_string": city_string, - "exnw_string": exnw_string, - "seed_point_string": seed_point_string, - } - return export_strings -def export_data_to_file(export_data, seed_points_snapped_filtered, city_boundary_exists, city_boundary_gdf, existing_network_spacing, edges_ordered, export_strings): - """Export data. - """ - - if export_data: - if len(edges_ordered)<1+bool(existing_network_spacing is not None): - warnings.warn("Not enough space to grow links. No data was exported.") - else: - seed_points_snapped_filtered.drop(["osmid"], axis=1, inplace=True) - seed_points_snapped_filtered.to_crs(epsg=settings.crs_result, inplace=True) - if city_boundary_exists: - city_boundary_gdf.to_crs(epsg=settings.crs_result, inplace=True) - if settings.export_file_format == "geojson": # To do: Simplify ugly code duplications and make this an IO function - progress_bar = initialize_progress_bar("Exporting data", 2+int(bool(existing_network_spacing))+int(city_boundary_exists), "file") - if settings.crs_result == '4326': # Export with RFC7946="YES" - if existing_network_spacing: - existing_network = edges_ordered.iloc[[0]] - edges_ordered = edges_ordered.iloc[1:].astype({'source': int, 'target': int}) - existing_network.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-existing_bike_network.geojson", driver="GeoJSON", RFC7946="YES") - progress_bar.update(1) - edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON", RFC7946="YES") - progress_bar.update(1) - else: - edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON", RFC7946="YES") - progress_bar.update(1) - seed_points_snapped_filtered.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-seed_points-"+export_strings['seed_point_string']+"-"+export_strings['exnw_string']+".geojson", driver="GeoJSON", RFC7946="YES") - progress_bar.update(1) - if city_boundary_exists: - city_boundary_gdf.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-city_boundary.geojson", driver="GeoJSON", RFC7946="YES") - progress_bar.update(1) - else: - if existing_network_spacing: - existing_network = edges_ordered.iloc[[0]] - edges_ordered = edges_ordered.iloc[1:].astype({'source': int, 'target': int}) - existing_network.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-existing_bike_network.geojson", driver="GeoJSON") - progress_bar.update(1) - edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON") - progress_bar.update(1) - else: - edges_ordered.to_file(settings.export_path['results']+export_strings['export_data_filename'], driver="GeoJSON") - progress_bar.update(1) - seed_points_snapped_filtered.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-seed_points-"+export_strings['seed_point_string']+"-"+export_strings['exnw_string']+".geojson", driver="GeoJSON") - progress_bar.update(1) - if city_boundary_exists: - city_boundary_gdf.to_file(settings.export_path['results']+export_strings['city_string']+"-growbikenet-city_boundary.geojson", driver="GeoJSON") - progress_bar.update(1) - elif settings.export_file_format == "gpkg": - progress_bar = initialize_progress_bar("Exporting data", 1, "file") - f = settings.export_path['results']+export_strings['export_data_filename'] - if os.path.exists(f): - os.remove(f) # mode="w" does not work for to_file with gpkg. It always appends. Therefore, existing file needs to be deleted. - if existing_network_spacing: - edges_ordered.iloc[[0]].to_file(f, driver="GPKG", layer="Existing bike network") - edges_ordered.iloc[1:].to_file(f, driver="GPKG", layer="Grown bike network") - else: - edges_ordered.to_file(f, driver="GPKG", layer="Grown bike network") - seed_points_snapped_filtered.to_file(f, driver="GPKG", layer="Seed points") - if city_boundary_exists: city_boundary_gdf.to_file(f, driver="GPKG", layer="City boundary") - progress_bar.update(1) - progress_bar.close() - -def export_plots_to_file(export_plots, ordering, edges_ordered, seed_points_snapped_filtered, existing_network_spacing): - """Export plots. - """ - if export_plots and len(edges_ordered)>=1+bool(existing_network_spacing is not None): - os.makedirs(settings.export_path['plots']+"ordering_"+ordering+"/", exist_ok=True) - - if settings.viz["crs"] == "auto": # Make it local azimuthal by default - network_center = edges_ordered.to_crs(constants._CRS_CALCULATIONS).dissolve().centroid.to_crs('4326') # Calculate centroid in projected CRS, then go back to unprojected CRS for lat lon - settings.viz["crs"] = f"+proj=aeqd +R=6371000 +units=m +lat_0={network_center.y[0]} +lon_0={network_center.x[0]}" # The first coordinate x is the latitude - - generate_plots( - edges_ordered, - seed_points_snapped_filtered, - ordering, - bool(existing_network_spacing), - ) \ No newline at end of file + return G From 4abac55c43ad683751fc45fa4cd2b655ebe24bfd Mon Sep 17 00:00:00 2001 From: Michael Szell Date: Mon, 12 Oct 2026 07:45:30 +0200 Subject: [PATCH 2/5] Fix tests --- growbikenet/_procedure.py | 4 +--- tests/{test_functions.py => test_utils.py} | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) rename tests/{test_functions.py => test_utils.py} (99%) diff --git a/growbikenet/_procedure.py b/growbikenet/_procedure.py index 0d44e2c..3be5096 100644 --- a/growbikenet/_procedure.py +++ b/growbikenet/_procedure.py @@ -27,7 +27,7 @@ assign_pbi_to_edges, bike_infra_mapping_gdf, create_gdf_with_geoms, - df_from_graph + df_from_graph, download_network, filter_network_by_component_length, filter_points_distant_from_osm_nodes, @@ -440,8 +440,6 @@ def _gbn_route(g_undir, edges, grown_bikenet_edges_abstract, seed_points_snapped node_lon = seed_points_snapped_filtered.geometry.x # Needed for add_trip_data_to_net() node_lat = seed_points_snapped_filtered.geometry.y # Needed for add_trip_data_to_net() edge_list = grown_bikenet_edges["pair"] - print(edge_list) - sys.exit() dist_list = grown_bikenet_edges["dist"] dist_dict = dict(zip(edge_list, dist_list)) geom_dict = dict(zip(edge_list, grown_bikenet_edges["geometry"].tolist())) diff --git a/tests/test_functions.py b/tests/test_utils.py similarity index 99% rename from tests/test_functions.py rename to tests/test_utils.py index 3b2d4f2..7c82ec9 100644 --- a/tests/test_functions.py +++ b/tests/test_utils.py @@ -10,7 +10,7 @@ from shapely.geometry import LineString, MultiLineString, Point from growbikenet import constants, settings -from growbikenet.functions import ( +from growbikenet.utils import ( _get_grid_seed_points, _get_weighted_distances, _order_df, From c084d57be9d3f34f0d695cabba79653ab5bbdf56 Mon Sep 17 00:00:00 2001 From: Michael Szell Date: Mon, 12 Oct 2026 07:46:37 +0200 Subject: [PATCH 3/5] Add modules and docstring to init --- growbikenet/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/growbikenet/__init__.py b/growbikenet/__init__.py index c9e03b7..b224c9f 100644 --- a/growbikenet/__init__.py +++ b/growbikenet/__init__.py @@ -1,11 +1,24 @@ +""" +GrowBikeNet is a Python package to grow an urban bicycle network from scratch +or from an existing bicycle network. + +Full documentation at: https://docs.bikenetkit.org/GrowBikeNet/ + +If you use GrowBikeNet, please cite: https://doi.org/10.1038/s41598-022-10783-y +""" + import contextlib from importlib.metadata import PackageNotFoundError, version from growbikenet.growbikenet import growbikenet +from . import constants +from . import growbikenet +from . import osm +from . import settings +from . import utils +from . import visualization -from . import functions, visualization - -__author__ = "Michael Szell, Manuel Knepper, Anastassia Vybornova" +__author__ = "GrowBikeNet contributors" __author_email__ = "michael@szell.net" with contextlib.suppress(PackageNotFoundError): From d7bc038b77e199747e00c781f2751ea5c44bb67b Mon Sep 17 00:00:00 2001 From: Michael Szell Date: Mon, 12 Oct 2026 07:52:08 +0200 Subject: [PATCH 4/5] Update test workflow and docs with refactor --- .github/workflows/test.yml | 2 +- docs/source/reference_developer.rst | 20 +++++++++++++++++--- docs/source/reference_user.rst | 7 +++++++ examples/batchexport_onecity.py | 2 +- growbikenet/osm.py | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8755ce2..5e752eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,7 @@ jobs: os: [ubuntu-latest] submodule: - { name: "growbikenet", pytest_args: "tests/test_growbikenet.py" } - - { name: "functions", pytest_args: "tests/test_functions.py" } + - { name: "utils", pytest_args: "tests/test_utils.py" } - { name: "visualization", pytest_args: "tests/test_visualization.py --mpl" } fail-fast: false diff --git a/docs/source/reference_developer.rst b/docs/source/reference_developer.rst index 8d6adcf..0f78c24 100644 --- a/docs/source/reference_developer.rst +++ b/docs/source/reference_developer.rst @@ -16,10 +16,17 @@ growbikenet.constants :members: :private-members: -growbikenet.functions ---------------------- +growbikenet.osm +--------------- + +.. automodule:: growbikenet.osm + :members: + :private-members: + +growbikenet.utils +----------------- -.. automodule:: growbikenet.functions +.. automodule:: growbikenet.utils :members: :private-members: @@ -34,5 +41,12 @@ growbikenet.visualization ------------------------- .. automodule:: growbikenet.visualization + :members: + :private-members: + +growbikenet._procedure +---------------------- + +.. automodule:: growbikenet._procedure :members: :private-members: \ No newline at end of file diff --git a/docs/source/reference_user.rst b/docs/source/reference_user.rst index a53410d..9fe0592 100644 --- a/docs/source/reference_user.rst +++ b/docs/source/reference_user.rst @@ -12,6 +12,13 @@ growbikenet.growbikenet :members: :no-index: +growbikenet.osm +--------------- + +.. automodule:: growbikenet.osm + :members: + :no-index: + growbikenet.settings -------------------- diff --git a/examples/batchexport_onecity.py b/examples/batchexport_onecity.py index 7a0b780..9de195b 100644 --- a/examples/batchexport_onecity.py +++ b/examples/batchexport_onecity.py @@ -41,7 +41,7 @@ import growbikenet as gbn from growbikenet import settings -from growbikenet.functions import slugify +from growbikenet.utils import slugify city_query = "Barcelona" city_id = "barcelona_es" diff --git a/growbikenet/osm.py b/growbikenet/osm.py index b7a6b37..e18151e 100644 --- a/growbikenet/osm.py +++ b/growbikenet/osm.py @@ -1,4 +1,4 @@ -"""Global constants related to OpenStreetMap(OSM). +"""Global constants for `growbikenet` related to OpenStreetMap (OSM). GROWABLE_NETWORK_CUSTOM_FILTER : list[str] or None Custom filter for all infrastructure elements that are considered as From 1d5e35b23aa9f9a24ea5c7a2d471a03e7a6714bd Mon Sep 17 00:00:00 2001 From: Michael Szell Date: Mon, 12 Oct 2026 07:54:23 +0200 Subject: [PATCH 5/5] Name cache test consistently --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e752eb..64139e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: run: shell: bash -elo pipefail {0} - name: Cache for ${{ matrix.python-version }} on ${{ matrix.os }} + name: cache (${{ matrix.python-version }} on ${{ matrix.os }}) runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7