Updated units field in netcdf, adding shift for GloFAS and EFAS water…

… demand historic data usage

src/lisfloodutilities/water-demand-historic/config_Europe_EFAS_1arcmin.cfg

@@ -1,12 +1,14 @@
 year_start = 1979
 year_end = 2021
+shift_hours_units_start = 12
 templatemap_path = /eos/jeodpp/data/projects/WEFE/shared/area_European_01min.nc
 output_folder = /eos/jeodpp/data/projects/WEFE/shared/lisflood-water-demand-historic-Europe-1arcmin
 ancillary_data_folder = ancillary_data
 aquastat_folder = /eos/jeodpp/data/projects/WEFE/shared/FAO_AQUASTAT
 eia_folder = /eos/jeodpp/data/projects/WEFE/shared/EIA_electricity_database
 gcam_folder = /eos/jeodpp/data/projects/WEFE/shared/GCAM_V5.4_output
 ghsl_folder = /eos/jeodpp/data/projects/WEFE/shared/GHS-POP
+ghsl_folder_factor = 144
 glw_folder = /eos/jeodpp/data/projects/WEFE/shared/Gridded_Livestock_of_the_World_2010_GLW_3
 huang_folder = /eos/jeodpp/data/projects/WEFE/shared/huang_2018_water_use
 htap_folder = /eos/jeodpp/data/projects/WEFE/shared/OMI-HTAP

src/lisfloodutilities/water-demand-historic/config_global_0p1deg.cfg

@@ -1,12 +1,14 @@
 year_start = 1979
 year_end = 2021
+shift_hours_units_start = 24
 templatemap_path = /eos/jeodpp/data/projects/WEFE/shared/areaOrigin.nc
 output_folder = /eos/jeodpp/data/projects/WEFE/shared/lisflood-water-demand-historic_global_0p1deg
 ancillary_data_folder = ancillary_data
 aquastat_folder = /eos/jeodpp/data/projects/WEFE/shared/FAO_AQUASTAT
 eia_folder = /eos/jeodpp/data/projects/WEFE/shared/EIA_electricity_database
 gcam_folder = /eos/jeodpp/data/projects/WEFE/shared/GCAM_V5.4_output
 ghsl_folder = /eos/jeodpp/data/projects/WEFE/shared/GHS-POP
+ghsl_folder_factor = 144
 glw_folder = /eos/jeodpp/data/projects/WEFE/shared/Gridded_Livestock_of_the_World_2010_GLW_3
 huang_folder = /eos/jeodpp/data/projects/WEFE/shared/huang_2018_water_use
 htap_folder = /eos/jeodpp/data/projects/WEFE/shared/OMI-HTAP

src/lisfloodutilities/water-demand-historic/config_global_3arcmin.cfg

@@ -1,5 +1,6 @@
 year_start = 1975
 year_end = 2023
+shift_hours_units_start = 24
 templatemap_path = /media/sf_VMSharedFolder/WaterDemandMapsDataset2024/template_Global_03min.nc
 output_folder = /media/sf_VMSharedFolder/WaterDemandMapsDataset2024/vmtest_lisflood-water-demand-historic_global_3arcmin
 ancillary_data_folder = ancillary_data

src/lisfloodutilities/water-demand-historic/step1_population_density.py

@@ -189,7 +189,7 @@ def main():
     ncfile.variables['lat'].long_name = 'latitude'
 
     ncfile.createVariable('time', 'f8', 'time')
-    ncfile.variables['time'].units = 'days since 1979-01-02 00:00:00'
+    ncfile.variables['time'].units = 'days since {}'.format(pd.to_datetime(datetime(int(config['year_start']), 1, int(1+config['shift_hours_units_start']/24), int(config['shift_hours_units_start']%24))))
     ncfile.variables['time'].long_name = 'time'
     ncfile.variables['time'].calendar = 'proleptic_gregorian'
 
@@ -208,8 +208,7 @@ def main():
         for month in np.arange(1,13):
 
             index = (year-config['year_start'])*12+month-1
-
-            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(1979, 1, 1))).total_seconds()/86400    
+            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(config['year_start'], 1, 1))).total_seconds()/86400    
             ncfile.variables[varname][index,:,:] = data
 
     print("Time elapsed is "+str(time.time()-t0)+" sec")

src/lisfloodutilities/water-demand-historic/step2_domestic_demand.py

@@ -459,7 +459,7 @@ def main():
     ncfile.variables['lat'].long_name = 'latitude'
 
     ncfile.createVariable('time', 'f8', 'time')
-    ncfile.variables['time'].units = 'days since 1979-01-02 00:00:00'
+    ncfile.variables['time'].units = 'days since {}'.format(pd.to_datetime(datetime(int(config['year_start']), 1, int(1+config['shift_hours_units_start']/24), int(config['shift_hours_units_start']%24))))
     ncfile.variables['time'].long_name = 'time'
     ncfile.variables['time'].calendar = 'proleptic_gregorian'
 
@@ -479,7 +479,7 @@ def main():
 
             index = (year-config['year_start'])*12+month-1
 
-            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(1979, 1, 1))).total_seconds()/86400    
+            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(config['year_start'], 1, 1))).total_seconds()/86400    
             ncfile.variables[varname][index,:,:] = data
 
             print("Time elapsed is "+str(time.time()-t0)+" sec")

src/lisfloodutilities/water-demand-historic/step3b_industrial_demand.py

@@ -186,7 +186,7 @@ def main():
         ncfile.variables['lat'].long_name = 'latitude'
 
         ncfile.createVariable('time', 'f8', 'time')
-        ncfile.variables['time'].units = 'days since 1979-01-02 00:00:00'
+        ncfile.variables['time'].units = 'days since {}'.format(pd.to_datetime(datetime(int(config['year_start']), 1, int(1+config['shift_hours_units_start']/24), int(config['shift_hours_units_start']%24))))
         ncfile.variables['time'].long_name = 'time'
         ncfile.variables['time'].calendar = 'proleptic_gregorian'
 
@@ -331,7 +331,7 @@ def main():
                 data = np.round(data,decimals=2)
                 data = data[row_upper:row_upper+len(template_lat),col_left:col_left+len(template_lon)] # Subset to template map area
                 index = (year-config['year_start'])*12+month-1
-                ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(1979, 1, 1))).total_seconds()/86400    
+                ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(config['year_start'], 1, 1))).total_seconds()/86400    
                 ncfile.variables[varname][index,:,:] = data
 
             print("Time elapsed is "+str(time.time()-t1)+" sec")

src/lisfloodutilities/water-demand-historic/step4_livestock_demand.py

@@ -403,7 +403,7 @@ def main():
     ncfile.variables['lat'].long_name = 'latitude'
 
     ncfile.createVariable('time', 'f8', 'time')
-    ncfile.variables['time'].units = 'days since 1979-01-02 00:00:00'
+    ncfile.variables['time'].units = 'days since {}'.format(pd.to_datetime(datetime(int(config['year_start']), 1, int(1+config['shift_hours_units_start']/24), int(config['shift_hours_units_start']%24))))
     ncfile.variables['time'].long_name = 'time'
     ncfile.variables['time'].calendar = 'proleptic_gregorian'
 
@@ -482,7 +482,7 @@ def main():
             data = np.round(data,decimals=2)
             data = data[row_upper:row_upper+len(template_lat),col_left:col_left+len(template_lon)] # Subset to template map area
             index = (year-config['year_start'])*12+month-1
-            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(1979, 1, 1))).total_seconds()/86400    
+            ncfile.variables['time'][index] = (pd.to_datetime(datetime(year,month,1))-pd.to_datetime(datetime(config['year_start'], 1, 1))).total_seconds()/86400    
             ncfile.variables[varname][index,:,:] = data
 
         print("Time elapsed is "+str(time.time()-t1)+" sec")

src/lisfloodutilities/water-demand-historic/tools.py

@@ -97,9 +97,9 @@ def load_config(filepath):
     df = pd.read_csv(filepath,header=None,index_col=False)
     config = {}
     for ii in np.arange(len(df)): 
-        string = df.iloc[ii,0].replace(" ","")
-        varname = string.rpartition('=')[0]
-        varcontents = string.rpartition('=')[2]
+        string = df.iloc[ii,0]
+        varname = string.rpartition('=')[0].strip()
+        varcontents = string.rpartition('=')[2].strip()
         try:
             varcontents = float(varcontents)
         except: