run pre-commit locally

worldbank · Nov 18, 2024 · d015f69 · d015f69
1 parent 2ac5c2d
commit d015f69
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 28 deletions.
diff --git a/notebooks/MP_SCRIPTS/zonal_urbanization.py b/notebooks/MP_SCRIPTS/zonal_urbanization.py
@@ -22,7 +22,6 @@
 AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN")
 
 if __name__ == "__main__":
-
     verbose = True
     run_urban = False
     run_urban_pop = True
@@ -36,11 +35,15 @@
     unq_urban = [11, 12, 13, 21, 22, 23, 30]
     ghsl_folder = "/home/public/Data/GLOBAL/GHSL/"
 
-    ghs_smod = os.path.join(ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif")
-    ghs_pop = os.path.join(ghsl_folder, "Pop", "GHS_POP_E2020_GLOBE_R2023A_54009_100_V1_0.tif")
-
-    #h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False)
-    #if verbose:
+    ghs_smod = os.path.join(
+        ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif"
+    )
+    ghs_pop = os.path.join(
+        ghsl_folder, "Pop", "GHS_POP_E2020_GLOBE_R2023A_54009_100_V1_0.tif"
+    )
+
+    # h3_0_list = h3_helper.generate_lvl0_lists(h3_level, return_gdf=True, buffer0=False)
+    # if verbose:
 
     ghs_smod = os.path.join(
         ghsl_folder, "SMOD", "GHS_SMOD_E2020_GLOBE_R2023A_54009_1000_V1_0.tif"
@@ -64,17 +67,21 @@
     urban_args = []
     for h3_1_key, cur_gdf in h3_1_list.items():
         if run_urban_pop:
-            # Set up mp arguments for urban population        
-            pop_filename = 'GHS_POP_2020_Urban_Breakdown.csv'
-            pop_out_s3_key = f'Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}'
+            # Set up mp arguments for urban population
+            pop_filename = "GHS_POP_2020_Urban_Breakdown.csv"
+            pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}"
             pop_full_path = os.path.join("s3://", AWS_S3_BUCKET, pop_out_s3_key)
-            urban_pop_args.append([cur_gdf, "shape_id", ghs_pop, ghs_smod, pop_full_path, unq_urban])
+            urban_pop_args.append(
+                [cur_gdf, "shape_id", ghs_pop, ghs_smod, pop_full_path, unq_urban]
+            )
         if run_urban:
             # set up mp arguments for urban summary
-            urban_filename = 'GHS_SMOD_2020.csv'
-            urban_out_s3_key = f'Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}'
+            urban_filename = "GHS_SMOD_2020.csv"
+            urban_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}"
             urban_full_path = os.path.join("s3://", AWS_S3_BUCKET, urban_out_s3_key)
-            urban_args.append([cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path])
+            urban_args.append(
+                [cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path]
+            )
         # Set up mp arguments for urban population
         pop_filename = "GHS_POP_2020_Urban_Breakdown.csv"
         pop_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix_pop}/{h3_1_key}/{pop_filename}"
@@ -91,15 +98,13 @@
         urban_out_s3_key = f"Space2Stats/h3_stats_data/GLOBAL/{data_prefix}/{h3_1_key}/{urban_filename}"
         urban_full_path = os.path.join("s3://", AWS_S3_BUCKET, urban_out_s3_key)
         urban_args.append([cur_gdf, "shape_id", ghs_smod, unq_urban, urban_full_path])
-
-
-
+
     if run_urban:
         tPrint(f"Running calculations on urban: {len(urban_args)} processes")
         # Run multi processing on urban
         if multiprocess:
-            with multiprocessing.Pool(processes=min([70,len(urban_args)])) as pool:
-                results = pool.starmap(global_zonal.zonal_stats_categories, urban_args)  
+            with multiprocessing.Pool(processes=min([70, len(urban_args)])) as pool:
+                results = pool.starmap(global_zonal.zonal_stats_categories, urban_args)
             tPrint(f"Finished urban calculations: {len(results)}")
             for combo in results:
                 out_file = list(combo.keys())[0]
@@ -128,11 +133,15 @@
                 tPrint(f"Finished {out_file}")
 
     if run_urban_pop:
-        tPrint(f"Running calculations on urban population: {len(urban_pop_args)} processes")
+        tPrint(
+            f"Running calculations on urban population: {len(urban_pop_args)} processes"
+        )
         # Run multi processing on urban_pop_calculations
         if multiprocess:
-            with multiprocessing.Pool(processes=min([40,len(urban_pop_args)])) as pool:
-                results = pool.starmap(global_zonal.zonal_stats_categorical, urban_pop_args)    
+            with multiprocessing.Pool(processes=min([40, len(urban_pop_args)])) as pool:
+                results = pool.starmap(
+                    global_zonal.zonal_stats_categorical, urban_pop_args
+                )
             tPrint(f"Finished multiprocessing urban pop calculations: {len(results)}")
             for combo in results:
                 out_file = list(combo.keys())[0]
@@ -147,7 +156,9 @@
                 )
         else:
             for a in urban_pop_args:
-                combo = global_zonal.zonal_stats_categorical(*a, verbose=verbose, minVal=0)
+                combo = global_zonal.zonal_stats_categorical(
+                    *a, verbose=verbose, minVal=0
+                )
                 out_file = list(combo.keys())[0]
                 tPrint(f"Completed {out_file}")
                 res = combo[out_file]
@@ -158,5 +169,5 @@
                         "secret": AWS_SECRET_ACCESS_KEY,
                         "token": AWS_SESSION_TOKEN,
                     },
-                )        
+                )
     tPrint("Finished")
diff --git a/notebooks/src/country_zonal.py b/notebooks/src/country_zonal.py
@@ -9,6 +9,7 @@
 import geopandas as gpd
 import GOSTRocks.ntlMisc as ntl
 import GOSTRocks.rasterMisc as rMisc
+import h3_helper
 import matplotlib
 import matplotlib.patches as mpatches
 import matplotlib.pyplot as plt
@@ -24,8 +25,6 @@
 from shapely.ops import unary_union
 from tqdm import tqdm
 
-import h3_helper
-
 
 def calculate_value(
     in_shp,

diff --git a/notebooks/src/global_zonal.py b/notebooks/src/global_zonal.py
@@ -5,6 +5,7 @@
 import geopandas as gpd
 import GOSTrocks.ntlMisc as ntl
 import GOSTrocks.rasterMisc as rMisc
+import h3_helper
 import numpy as np
 import pandas as pd
 from GOSTrocks.misc import tPrint
@@ -14,8 +15,6 @@
 from shapely.ops import unary_union
 from tqdm import tqdm
 
-import h3_helper
-
 
 def get_global_table_from_s3(
     variable,

diff --git a/notebooks/src/h3_helper.py b/notebooks/src/h3_helper.py
@@ -22,6 +22,7 @@
 from shapely.ops import unary_union
 from tqdm import tqdm
 
+
 def generate_h3_gdf(in_gdf, h3_level=7):
     """Generate a GeoDataFrame of h3 grid cells from an input geodataframe
 
@@ -325,4 +326,3 @@ def static_map_h3(sub, map_epsg=3857, legend_loc="upper right"):
     )  # zorder=-10, 'EPSG:4326'
     ax = ax.set_axis_off()
     return ax
-