From d015f69a5242d63ff8dfda763b804af2833772a4 Mon Sep 17 00:00:00 2001 From: Andres Chamorro Date: Mon, 18 Nov 2024 16:15:03 -0500 Subject: [PATCH] run pre-commit locally --- notebooks/MP_SCRIPTS/zonal_urbanization.py | 57 +++++++++++++--------- notebooks/src/country_zonal.py | 3 +- notebooks/src/global_zonal.py | 3 +- notebooks/src/h3_helper.py | 2 +- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/notebooks/MP_SCRIPTS/zonal_urbanization.py b/notebooks/MP_SCRIPTS/zonal_urbanization.py index 31a517df..73265df7 100644 --- a/notebooks/MP_SCRIPTS/zonal_urbanization.py +++ b/notebooks/MP_SCRIPTS/zonal_urbanization.py @@ -22,7 +22,6 @@ AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN") if __name__ == "__main__": - verbose = True run_urban = False run_urban_pop = True @@ -36,11 +35,15 @@ unq_urban = [11, 12, 13, 21, 22, 23, 30] ghsl_folder = "/home/public/Data/GLOBAL/GHSL/" - ghs_smod = os.path.join(ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif") - ghs_pop = os.path.join(ghsl_folder, "Pop", "GHS_POP_E2020_GLOBE_R2023A_54009_100_V1_0.tif") - - #h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False) - #if verbose: + ghs_smod = os.path.join( + ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif" + ) + ghs_pop = os.path.join( + ghsl_folder, "Pop", "GHS_POP_E2020_GLOBE_R2023A_54009_100_V1_0.tif" + ) + + # h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False) + # if verbose: ghs_smod = os.path.join( ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif" @@ -64,17 +67,21 @@ urban_args = [] for h3_1_key, cur_gdf in h3_1_list.items(): if run_urban_pop: - # Set up mp arguments for urban population - pop_filename = 'GHS_POP_2020_Urban_Breakdown.csv' - pop_out_s3_key = f'Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}' + # Set up mp arguments for urban population + pop_filename = "GHS_POP_2020_Urban_Breakdown.csv" + pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}" pop_full_path = os.path.join("s3://", AWS_S3_BUCKET, pop_out_s3_key) - urban_pop_args.append([cur_gdf, "shape_id", ghs_pop, ghs_smod, pop_full_path, unq_urban]) + urban_pop_args.append( + [cur_gdf, "shape_id", ghs_pop, ghs_smod, pop_full_path, unq_urban] + ) if run_urban: # set up mp arguments for urban summary - urban_filename = 'GHS_SMOD_2020.csv' - urban_out_s3_key = f'Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}' + urban_filename = "GHS_SMOD_2020.csv" + urban_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}" urban_full_path = os.path.join("s3://", AWS_S3_BUCKET, urban_out_s3_key) - urban_args.append([cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path]) + urban_args.append( + [cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path] + ) # Set up mp arguments for urban population pop_filename = "GHS_POP_2020_Urban_Breakdown.csv" pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}" @@ -91,15 +98,13 @@ urban_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}" urban_full_path = os.path.join("s3://", AWS_S3_BUCKET, urban_out_s3_key) urban_args.append([cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path]) - - - + if run_urban: tPrint(f"Running calculations on urban: {len(urban_args)} processes") # Run multi processing on urban if multiprocess: - with multiprocessing.Pool(processes=min([70,len(urban_args)])) as pool: - results = pool.starmap(global_zonal.zonal_stats_categories, urban_args) + with multiprocessing.Pool(processes=min([70, len(urban_args)])) as pool: + results = pool.starmap(global_zonal.zonal_stats_categories, urban_args) tPrint(f"Finished urban calculations: {len(results)}") for combo in results: out_file = list(combo.keys())[0] @@ -128,11 +133,15 @@ tPrint(f"Finished {out_file}") if run_urban_pop: - tPrint(f"Running calculations on urban population: {len(urban_pop_args)} processes") + tPrint( + f"Running calculations on urban population: {len(urban_pop_args)} processes" + ) # Run multi processing on urban_pop_calculations if multiprocess: - with multiprocessing.Pool(processes=min([40,len(urban_pop_args)])) as pool: - results = pool.starmap(global_zonal.zonal_stats_categorical, urban_pop_args) + with multiprocessing.Pool(processes=min([40, len(urban_pop_args)])) as pool: + results = pool.starmap( + global_zonal.zonal_stats_categorical, urban_pop_args + ) tPrint(f"Finished multiprocessing urban pop calculations: {len(results)}") for combo in results: out_file = list(combo.keys())[0] @@ -147,7 +156,9 @@ ) else: for a in urban_pop_args: - combo = global_zonal.zonal_stats_categorical(*a, verbose=verbose, minVal=0) + combo = global_zonal.zonal_stats_categorical( + *a, verbose=verbose, minVal=0 + ) out_file = list(combo.keys())[0] tPrint(f"Completed {out_file}") res = combo[out_file] @@ -158,5 +169,5 @@ "secret": AWS_SECRET_ACCESS_KEY, "token": AWS_SESSION_TOKEN, }, - ) + ) tPrint("Finished") diff --git a/notebooks/src/country_zonal.py b/notebooks/src/country_zonal.py index 6814ac51..fa5606d9 100644 --- a/notebooks/src/country_zonal.py +++ b/notebooks/src/country_zonal.py @@ -9,6 +9,7 @@ import geopandas as gpd import GOSTRocks.ntlMisc as ntl import GOSTRocks.rasterMisc as rMisc +import h3_helper import matplotlib import matplotlib.patches as mpatches import matplotlib.pyplot as plt @@ -24,8 +25,6 @@ from shapely.ops import unary_union from tqdm import tqdm -import h3_helper - def calculate_value( in_shp, diff --git a/notebooks/src/global_zonal.py b/notebooks/src/global_zonal.py index 31aceede..f1448660 100644 --- a/notebooks/src/global_zonal.py +++ b/notebooks/src/global_zonal.py @@ -5,6 +5,7 @@ import geopandas as gpd import GOSTrocks.ntlMisc as ntl import GOSTrocks.rasterMisc as rMisc +import h3_helper import numpy as np import pandas as pd from GOSTrocks.misc import tPrint @@ -14,8 +15,6 @@ from shapely.ops import unary_union from tqdm import tqdm -import h3_helper - def get_global_table_from_s3( variable, diff --git a/notebooks/src/h3_helper.py b/notebooks/src/h3_helper.py index 7494bae7..abc5a6fd 100644 --- a/notebooks/src/h3_helper.py +++ b/notebooks/src/h3_helper.py @@ -22,6 +22,7 @@ from shapely.ops import unary_union from tqdm import tqdm + def generate_h3_gdf(in_gdf, h3_level=7): """Generate a GeoDataFrame of h3 grid cells from an input geodataframe @@ -325,4 +326,3 @@ def static_map_h3(sub, map_epsg=3857, legend_loc="upper right"): ) # zorder=-10, 'EPSG:4326' ax = ax.set_axis_off() return ax -