From cafe6d9e5ee8e3882e08f972bd6936bfb7c017c0 Mon Sep 17 00:00:00 2001 From: arulalant Date: Thu, 4 May 2017 12:01:01 +0530 Subject: [PATCH 01/14] upadted out fcst file name of regional grib2 files --- .../ncum_regional_post/ncum_regional_post_um2grb2_setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bsubScripts/ncum_regional_post/ncum_regional_post_um2grb2_setup.cfg b/bsubScripts/ncum_regional_post/ncum_regional_post_um2grb2_setup.cfg index ddb965f..ad5dd36 100644 --- a/bsubScripts/ncum_regional_post/ncum_regional_post_um2grb2_setup.cfg +++ b/bsubScripts/ncum_regional_post/ncum_regional_post_um2grb2_setup.cfg @@ -198,7 +198,7 @@ anlOutGrib2FilesNameStructure = ('anal', '_', '*HH*', 'z', '*YYYYMMDD*', '.grb2' # Hycom needs suffix as Z in analysis files. ## Defining forecast grib2 fileName structure. Must be in single line. -fcstOutGrib2FilesNameStructure = ('ncum_reg_fcst', '_', '*HH*', 'h', '*YYYYMMDD*', '.grb2') +fcstOutGrib2FilesNameStructure = ('umreg_prg', '_', '*HHH*', 'hr', '_', '*YYYYMMDD*', '_', '*ZZ*', 'Z', '.grib2') # Hycom needs suffix as h in forecast files. From 4147b3b0ac6fcfc3250c2ceca57dedeb0d040678 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:31:55 +0530 Subject: [PATCH 02/14] update gribapi --- bsubScripts/umtid_bashrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bsubScripts/umtid_bashrc b/bsubScripts/umtid_bashrc index 62dbd4e..4f086ee 100644 --- a/bsubScripts/umtid_bashrc +++ b/bsubScripts/umtid_bashrc @@ -13,7 +13,7 @@ PS1="[`hostname -s`]["'$PWD]> ' # load the required modules to run UMRider module load GNU-UVCDAT-IRIS/2.4.0_1.10.0d0 module load GNU-qt/4.8.4 -module load GNU-gribapi/1.13.1 +module load GNU-gribapi/1.21.0 module load GNU-wgrib2/2.0.4 module load cnvgrib/1.4.1 module load GNU-grads/grads-2.1.a2 From 87a794b2da4b9d99387c4e8231f30ecec979291b Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:32:37 +0530 Subject: [PATCH 03/14] updated variable with height --- g2utils/ncum_load_rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/g2utils/ncum_load_rules.py b/g2utils/ncum_load_rules.py index 4d4990d..00cd6d5 100644 --- a/g2utils/ncum_load_rules.py +++ b/g2utils/ncum_load_rules.py @@ -42,8 +42,8 @@ # duplicate STASH but cell_methods are different, variables comes here. # STASH : {cell_method1 : (standard_name, long_name, unit, height), # cell_method2 : (standard_name, long_name, unit, height)} -'m01s03i236': {'maximum': (None, 'air_temperature_maximum', 'K', None), - 'minimum': (None, 'air_temperature_minimum', 'K', None)} +'m01s03i236': {'maximum': (None, 'air_temperature_maximum', 'K', 1.5), + 'minimum': (None, 'air_temperature_minimum', 'K', 1.5)} } G2Param_vs_cf = { From d824964533a0c9b36a1f7c24a99001e5cf660d59 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:33:24 +0530 Subject: [PATCH 04/14] added 2 new variables --- others/ncmrwfIRIS/_grib_cf_map.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/others/ncmrwfIRIS/_grib_cf_map.py b/others/ncmrwfIRIS/_grib_cf_map.py index bf6a4be..84e5cbb 100644 --- a/others/ncmrwfIRIS/_grib_cf_map.py +++ b/others/ncmrwfIRIS/_grib_cf_map.py @@ -159,6 +159,7 @@ def __new__(cls, edition, discipline, category, number, # G2Param(grib version, discipline, parameter category, parameter no): CFName('standard_name', 'long_name', 'units') # paramid G2Param(2, 0, 2, 9): CFName('upward_air_velocity', None, 'm s-1'), # 500032 + G2Param(2, 0, 2, 8): CFName(None, 'upward_air_velocity_in_pascal', 'Pa s-1'), G2Param(2, 0, 6, 23): CFName(None, 'cloud_ice_mixing_ratio', 'kg kg-1'), # 260118 G2Param(2, 0, 3, 10): CFName(None, 'density', 'kg m-3'), # 3089 G2Param(2, 0, 19, 0): CFName('visibility_in_air', None, 'm'), # 3020 @@ -308,6 +309,7 @@ def __new__(cls, edition, discipline, category, number, CFName('dew_point_temperature', None, 'K'): G2Param(2, 0, 0, 6), CFName('geopotential', None, 'm2 s-2'): G2Param(2, 0, 3, 4), CFName('geopotential_height', None, 'm'): G2Param(2, 0, 3, 5), + CFName(None, 'surface_geopotential_height', 'm'): G2Param(2, 0, 3, 5), # this is orography, but some model requires orography should be written as in gpm. CFName('geopotential_height_anomaly', None, 'm'): G2Param(2, 0, 3, 9), CFName('high_type_cloud_area_fraction', None, '%'): G2Param(2, 0, 6, 5), CFName('humidity_mixing_ratio', None, 'kg kg-1'): G2Param(2, 0, 1, 2), @@ -360,6 +362,7 @@ def __new__(cls, edition, discipline, category, number, # CFName('standard_name', 'long_name', 'units'): G2Param(grib version, discipline, parameter category, parameter no) # paramid CFName('upward_air_velocity', None, 'm s-1'): G2Param(2, 0, 2, 9), # 500032 + CFName(None, 'upward_air_velocity_in_pascal', 'Pa s-1'): G2Param(2, 0, 2, 8), CFName(None, 'cloud_ice_mixing_ratio', 'kg kg-1'): G2Param(2, 0, 6, 23), # 260118 CFName(None, 'density', 'kg m-3'): G2Param(2, 0, 3, 10), # 3089 CFName('visibility_in_air', None, 'm'): G2Param(2, 0, 19, 0), # 3020 From 039470f827f045e1c9ba91d92cf867208afefe5e Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:34:56 +0530 Subject: [PATCH 05/14] updates in variables --- others/ncmrwfIRIS/_grib_cf_map_tigge.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/others/ncmrwfIRIS/_grib_cf_map_tigge.py b/others/ncmrwfIRIS/_grib_cf_map_tigge.py index 8329815..0d1175e 100644 --- a/others/ncmrwfIRIS/_grib_cf_map_tigge.py +++ b/others/ncmrwfIRIS/_grib_cf_map_tigge.py @@ -151,6 +151,7 @@ def __new__(cls, edition, discipline, category, number, # G2Param(grib version, discipline, parameter category, parameter no): CFName('standard_name', 'long_name', 'units') # paramid G2Param(2, 0, 2, 9): CFName('upward_air_velocity', None, 'm s-1'), # 500032 + G2Param(2, 0, 2, 8): CFName(None, 'upward_air_velocity_in_pascal', 'Pa s-1'), G2Param(2, 0, 6, 23): CFName(None, 'cloud_ice_mixing_ratio', 'kg kg-1'), # 260118 G2Param(2, 0, 3, 10): CFName(None, 'density', 'kg m-3'), # 3089 G2Param(2, 0, 19, 0): CFName('visibility_in_air', None, 'm'), # 3020 @@ -177,8 +178,8 @@ def __new__(cls, edition, discipline, category, number, G2Param(2, 3, 1, 198): CFName('atmosphere_mass_content_of_dust_dry_aerosol_particles', None, 'kg m-2'), # NCMRWF Local - G2Param(2, 0, 0, 4): CFName(None, 'air_temperature_maximum', 'K'), # WMO deprecated - G2Param(2, 0, 0, 5): CFName(None, 'air_temperature_minimum', 'K'), # WMO deprecated +# G2Param(2, 0, 0, 0): CFName(None, 'air_temperature_maximum', 'K'), # TIGGE +# G2Param(2, 0, 0, 0): CFName(None, 'air_temperature_minimum', 'K'), # TIGGE G2Param(2, 0, 1, 69): CFName('atmosphere_cloud_liquid_water_content', None, 'kg m-2'), G2Param(2, 0, 1, 70): CFName('atmosphere_cloud_ice_content', None, 'kg m-2'), @@ -281,6 +282,7 @@ def __new__(cls, edition, discipline, category, number, CFName('dew_point_temperature', None, 'K'): G2Param(2, 0, 0, 6), CFName('geopotential', None, 'm2 s-2'): G2Param(2, 0, 3, 4), CFName('geopotential_height', None, 'm'): G2Param(2, 0, 3, 5), + CFName(None, 'surface_geopotential_height', 'm'): G2Param(2, 0, 3, 5), # this is orography, but some model requires orography should be written as in gpm. CFName('geopotential_height_anomaly', None, 'm'): G2Param(2, 0, 3, 9), CFName('high_type_cloud_area_fraction', None, '%'): G2Param(2, 0, 6, 5), CFName('humidity_mixing_ratio', None, 'kg kg-1'): G2Param(2, 0, 1, 2), @@ -314,8 +316,7 @@ def __new__(cls, edition, discipline, category, number, CFName('surface_upwelling_longwave_flux_in_air', None, 'W m-2'): G2Param(2, 0, 5, 4), CFName('surface_roughness_length', None, 'm'): G2Param(2, 2, 0, 1), CFName('surface_runoff_flux', None, 'kg m-2 s-1'): G2Param(2, 2, 0, 34), -# CFName('surface_temperature', None, 'K'): G2Param(2, 0, 0, 17), #WRF-Noah requirments - CFName('surface_temperature', None, 'K'): G2Param(2, 0, 0, 0), #INCOIS-OSF, HYCOM requirements + CFName('surface_temperature', None, 'K'): G2Param(2, 0, 0, 17), #TIGGE # OSF required it should be just TMP in grib2 file. So lets write as temperature itself. CFName('surface_upward_latent_heat_flux', None, 'W m-2 s'): G2Param(2, 0, 0, 10),# TIGGE CFName(None, 'time_integrated_surface_upward_latent_heat_flux', 'W m-2 s'): G2Param(2, 0, 0, 10), # TIGGE @@ -330,6 +331,7 @@ def __new__(cls, edition, discipline, category, number, # CFName('standard_name', 'long_name', 'units'): G2Param(grib version, discipline, parameter category, parameter no) # paramid CFName('upward_air_velocity', None, 'm s-1'): G2Param(2, 0, 2, 9), # 500032 + CFName(None, 'upward_air_velocity_in_pascal', 'Pa s-1'): G2Param(2, 0, 2, 8), CFName(None, 'cloud_ice_mixing_ratio', 'kg kg-1'): G2Param(2, 0, 6, 23), # 260118 CFName(None, 'density', 'kg m-3'): G2Param(2, 0, 3, 10), # 3089 CFName('visibility_in_air', None, 'm'): G2Param(2, 0, 19, 0), # 3020 @@ -365,8 +367,8 @@ def __new__(cls, edition, discipline, category, number, CFName('atmosphere_convective_inhibition_wrt_surface', None, 'J kg-1'): G2Param(2, 0, 7, 7), ## end the ncum load rules cf names - CFName(None, 'air_temperature_maximum', 'K'): G2Param(2, 0, 0, 4), # WMO deprecated - CFName(None, 'air_temperature_minimum', 'K'): G2Param(2, 0, 0, 5), # WMO deprecated + CFName(None, 'air_temperature_maximum', 'K'): G2Param(2, 0, 0, 0), # TIGGE + CFName(None, 'air_temperature_minimum', 'K'): G2Param(2, 0, 0, 0), # TIGGE CFName('atmosphere_cloud_liquid_water_content', None, 'kg m-2'): G2Param(2, 0, 1, 69), CFName('atmosphere_cloud_ice_content', None, 'kg m-2'): G2Param(2, 0, 1, 70), From 88dfb3f7a1751bf40888a0f15bc7911b1058245a Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:36:30 +0530 Subject: [PATCH 06/14] many updated reg tigge_check pass --- g2utils/um2grb2tigge.py | 61 ++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/g2utils/um2grb2tigge.py b/g2utils/um2grb2tigge.py index 2e2b98c..727777a 100644 --- a/g2utils/um2grb2tigge.py +++ b/g2utils/um2grb2tigge.py @@ -268,7 +268,7 @@ ('surface_upward_sensible_heat_flux', 'm01s03i217'), ('toa_outgoing_longwave_flux', 'm01s02i205')] -# TIGGE's totoal time cummulated variables +# TIGGE's total time cummulated variables _total_cummulativeVars_ = ['precipitation_amount', 'surface_net_downward_shortwave_flux', 'surface_net_downward_longwave_flux', @@ -359,6 +359,7 @@ ('land_binary_mask', 'm01s00i030'): ('land_sea_mask', 'lsm', None), # Proportion ('air_pressure_at_sea_level', 'm01s16i222'): ('mean_sea_level_pressure', 'msl', 'Pa'), ('surface_altitude', 'm01s00i033'): ('orography', 'orog', None), # 'gpm' +('orography', 'm01s00i033'): ('orography', 'orog', None), # 'gpm' # required to work. ('air_temperature', 'm01s03i236'): ('surface_air_temperature', '2t', 'K'), ('air_temperature_maximum', 'm01s03i236'): ('surface_air_maximum_temperature', 'mx2t6', 'K'), ('air_temperature_minimum', 'm01s03i236'): ('surface_air_minimum_temperature', 'mn2t6', 'K'), @@ -849,12 +850,11 @@ def regridAnlFcstFiles(arg): regdCube.data = numpy.ma.array(regdCube.data, dtype=numpy.int) # end of if (varName, varSTASH) in [('land_binary_mask', 'm01s00i030')]: - if (varName, varSTASH) in [('surface_altitude', 'm01s00i033')]: regdCube.standard_name = None regdCube.long_name = 'orography' # end of if (varName, varSTASH) in [('surface_altitude', 'm01s00i033')]: - + if exmode == 'mask': # For the above set of variables we shouldnot convert into # masked array. Otherwise its full data goes as nan. @@ -905,19 +905,14 @@ def regridAnlFcstFiles(arg): # get all other dimensions # generate list of tuples contain index and coordinate dim_coords = [(coord, i) for i,coord in enumerate(list(regdCube.dim_coords))] - aux_factories = regdCube.aux_factories - t = regdCube.coords('time')[0] - fp = regdCube.coords('forecast_period')[0] - ft = regdCube.coords('forecast_reference_time')[0] + aux_coords = regdCube.aux_coords # create ensemble packed cubes regdData = iris.cube.Cube(regdCube.data, regdCube.standard_name, regdCube.long_name, regdCube.var_name, unit, regdCube.attributes, regdCube.cell_methods, dim_coords) - # add all time coordinates - regdData.add_aux_coord(fp) - regdData.add_aux_coord(ft) - regdData.add_aux_coord(t) + # add all aux coordinates + for axc in aux_coords: regdData.add_aux_coord(axc) print regdData # make memory free @@ -998,7 +993,8 @@ def save_tigge_tweaked_messages(cubeList): for cube in cubeList: for cube, grib_message in iris.fileformats.grib.as_pairs(cube): #save_pairs_from_cube(cube): # - print "Tweaking begin ", cube.standard_name, cube.long_name + cstash = str(cube.attributes.get('STASH', 'None')) + print "Tweaking begin ", cube.standard_name, cube.long_name, cstash # post process the GRIB2 message, prior to saving gribapi.grib_set_long(grib_message, "centre", 29) # RMC of India gribapi.grib_set_long(grib_message, "subCentre", 0) # No subcentre @@ -1062,12 +1058,17 @@ def save_tigge_tweaked_messages(cubeList): # but mean while lets fix by setting typeOfTimeIncrement=2. # http://www.cosmo-model.org/content/model/documentation/grib/pdtemplate_4.11.htm gribapi.grib_set(grib_message, "typeOfTimeIncrement", 2) - # end of if cube.coord("realization"): - - # end of if cube.coords('depth_below_land_surface'): + # end of if cube.coord("realization"): + if cube.standard_name or cube.long_name: if cube.standard_name: loc_longname = None + if (cube.standard_name, cstash) == ('air_temperature', 'm01s03i236'): + # we have to explicitly re-set the type of first fixed + # surfcae as (103) and scale factor, scale value of 2m temperature + gribapi.grib_set(grib_message, "typeOfFirstFixedSurface", 103) + gribapi.grib_set(grib_message, "scaleFactorOfFirstFixedSurface", 0) + gribapi.grib_set(grib_message, "scaledValueOfFirstFixedSurface", 2) if cube.standard_name.startswith('air_pressure_at_sea_level'): # we have to explicitly re-set the type of first fixed # surfcae as Mean sea level (101) @@ -1094,6 +1095,14 @@ def save_tigge_tweaked_messages(cubeList): # end of if cube.standard_name: if cube.long_name: + if ((cube.long_name, cstash) in [('air_temperature_maximum', 'm01s03i236'), \ + ('air_temperature_minimum', 'm01s03i236')]): + # we have to explicitly re-set the type of first fixed + # surfcae as (103) and scale factor, scale value of 2m temperature + gribapi.grib_set(grib_message, "typeOfFirstFixedSurface", 103) + gribapi.grib_set(grib_message, "scaleFactorOfFirstFixedSurface", 0) + gribapi.grib_set(grib_message, "scaledValueOfFirstFixedSurface", 2) + gribapi.grib_set(grib_message, "timeIncrementBetweenSuccessiveFields", 0) aod_name = _aod_pseudo_level_var_.keys()[0] if cube.long_name.startswith(aod_name): # we have to explicitly re-set the type of first surfcae @@ -1132,6 +1141,10 @@ def save_tigge_tweaked_messages(cubeList): if (cube.standard_name in _total_cummulativeVars_ or \ cube.long_name in _total_cummulativeVars_): + gribapi.grib_set(grib_message, "typeOfFirstFixedSurface", 1) + gribapi.grib_set(grib_message, "scaleFactorOfFirstFixedSurface", 255) + gribapi.grib_set(grib_message, "scaledValueOfFirstFixedSurface", -1) + gribapi.grib_set(grib_message, "timeIncrementBetweenSuccessiveFields", 0) gribapi.grib_set(grib_message, "typeOfStatisticalProcessing", 1) # end of if cube.standard_name or ...: @@ -1149,7 +1162,7 @@ def save_tigge_tweaked_messages(cubeList): createDirWhileParallelRacing(outgdir) outgpath = os.path.join(outgdir, outgname) # Join the outpath & outfilename print "lets save into", outgpath - cstash = str(cube.attributes.get('STASH', 'None')) + if (cube.standard_name, cstash) in _accumulationVars_: iris.fileformats.netcdf.save(cube, outgpath+'.nc') # save nc file else: @@ -2010,6 +2023,11 @@ def packEnsembles(arg, **kwarg): regdCube.data = numpy.ma.array(regdCube.data, dtype=numpy.int) # end of if (varName, varSTASH) in [('land_binary_mask', 'm01s00i030')]: + if (varName, varSTASH) in [('surface_altitude', 'm01s00i033')]: + regdCube.standard_name = None + regdCube.long_name = 'orography' + # end of if (varName, varSTASH) in [('surface_altitude', 'm01s00i033')]: + if exmode == 'mask': # For the above set of variables we shouldnot convert into # masked array. Otherwise its full data goes as nan. @@ -2090,21 +2108,14 @@ def packEnsembles(arg, **kwarg): dim_coords.insert(0, enscoord) # generate list of tuples contain index and coordinate dim_coords = [(coord, i) for i,coord in enumerate(dim_coords)] - # get all other dimensions - aux_factories = regdCube.aux_factories - t = regdCube.coords('time')[0] - fp = regdCube.coords('forecast_period')[0] - ft = regdCube.coords('forecast_reference_time')[0] - # create ensemble packed cubes + # set all other dimensions ensembleData = iris.cube.Cube(ensembleData, regdCube.standard_name, regdCube.long_name, regdCube.var_name, unit, regdCube.attributes, regdCube.cell_methods, dim_coords) # add all time coordinates + for axc in regdCube.aux_coords: ensembleData.add_aux_coord(axc) print "setting aux_coords to", ensembleData.shape, varName, fhr - ensembleData.add_aux_coord(fp) - ensembleData.add_aux_coord(ft) - ensembleData.add_aux_coord(t) # create cell method for ensembles cm = iris.coords.CellMethod('realization', ('realization',), intervals=('1',), comments=(' ENS',)) From c9c60700f18b50cd16eef7cd978b0a29ab747df3 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:40:59 +0530 Subject: [PATCH 07/14] NCUM-TIGGE resolution which pass through tigge_check tool --- data/sample_global_0p36x0p45.grib2 | Bin 0 -> 179 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/sample_global_0p36x0p45.grib2 diff --git a/data/sample_global_0p36x0p45.grib2 b/data/sample_global_0p36x0p45.grib2 new file mode 100644 index 0000000000000000000000000000000000000000..5296d0a315c745dca6fd0589dff3685fb932d771 GIT binary patch literal 179 zcmZ<{@^t$DpMi-10yYCFQAP$?1_l-eM)rrS{2*awAoc(fY_bbLTt)_lMDw}-;Q**W z0Vw(vBnJ{?jrD2(u?<>*4ADFv0|vIc7Z_OU96*9fEFhAZf$4t&kOmsXAOs>HI)M%W YTF(lzh=HMj3oHk;nhhky%5Dw>0GWR$%>V!Z literal 0 HcmV?d00001 From e33e5664bebd5e71bb9d44b17815d42af8957128 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:41:38 +0530 Subject: [PATCH 08/14] sample grid which works for hysplit model input --- data/sample_global_0p5_hysplit.grib2 | Bin 0 -> 179 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/sample_global_0p5_hysplit.grib2 diff --git a/data/sample_global_0p5_hysplit.grib2 b/data/sample_global_0p5_hysplit.grib2 new file mode 100644 index 0000000000000000000000000000000000000000..c28f90bd2a5bb96676e3370b02c84ce93ec2d91a GIT binary patch literal 179 zcmZ<{@^t$DpMi-10yYCFQAP$?1_l-eM)rrSk|1G_qz5wt1M`oaATA>VL!$ZI|8M|Q za1kh)36cW|w#IrjfY=7CPzD40LIn_Z017Ix00|&q`kw$KfkrVfq=N`Xh;cxcC@ah& V28ISMkQ@-OvHb^ Date: Tue, 6 Jun 2017 12:42:18 +0530 Subject: [PATCH 09/14] update --- g2scripts/tigge_um2grb2_fcst_00Z.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/g2scripts/tigge_um2grb2_fcst_00Z.py b/g2scripts/tigge_um2grb2_fcst_00Z.py index 12835c7..6b399a5 100644 --- a/g2scripts/tigge_um2grb2_fcst_00Z.py +++ b/g2scripts/tigge_um2grb2_fcst_00Z.py @@ -25,7 +25,7 @@ if loadg2utils == 'system': # Load g2utils from system python which has installed through setup.py - from g2utils.um2grb2 import convertFcstFiles + from g2utils.um2grb2tigge import convertFcstFiles print "INFO : imported g2utils.um2grb2 from system python" elif loadg2utils == 'local': # Load g2utils from previous directory for the operational purpose, From 320e9986e040f7aa0c643f690a86736b77de289f Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:44:35 +0530 Subject: [PATCH 10/14] update for tigge and hysplit --- g2utils/um2grb2.py | 128 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/g2utils/um2grb2.py b/g2utils/um2grb2.py index 408aca9..842b3bb 100644 --- a/g2utils/um2grb2.py +++ b/g2utils/um2grb2.py @@ -88,7 +88,8 @@ g2ctl = "/gpfs2/home/umtid/Softwares/grib2ctl/g2ctl.pl" grib2ctl = "/gpfs2/home/umtid/Softwares/grib2ctl/grib2ctl.pl" gribmap = "/gpfs1/home/Libs/GNU/GRADS/grads-2.0.2.oga.1/Contents/gribmap" -cnvgrib = "/gpfs1/home/Libs/INTEL/CNVGRIB/CNVGRIB-1.4.1/cnvgrib-1.4.1/cnvgrib" +#cnvgrib = "/gpfs1/home/Libs/INTEL/CNVGRIB/CNVGRIB-1.4.1/cnvgrib-1.4.1/cnvgrib" +cnvgrib = "/gpfs2/home/umtid/Softwares/cnvgrib/CNVGRIB-1.4.1/cnvgrib-1.4.1/cnvgrib" wgrib2 = "/gpfs1/home/Libs/GNU/WGRIB2/v2.0.4/wgrib2/wgrib2" # other global variables @@ -160,6 +161,7 @@ ('x_wind', 'm01s15i201'), ('y_wind', 'm01s15i202'), ('upward_air_velocity', 'm01s15i242'), +('upward_air_velocity_in_pascal', 'm01s15i242'), ('air_temperature', 'm01s16i203'), ('relative_humidity', 'm01s16i256'), ('specific_humidity', 'm01s30i205')], @@ -257,7 +259,10 @@ ('soil_temperature', 'm01s08i225'), # the below one is for orography which presents only in analysis file. # so we must keep this as the last one in the ordered variables! -('surface_altitude', 'm01s00i033')], +('surface_altitude', 'm01s00i033'), +('surface_geopotential_height', 'm01s00i033') # this is orography, but +# some model requires orography has to be written in gpm unit. +], } @@ -421,7 +426,13 @@ ('atmosphere_mass_content_of_dust_dry_aerosol_particles', 'm01s30i403'), ('atmosphere_cloud_liquid_water_content', 'm01s30i405'), ('atmosphere_cloud_ice_content', 'm01s30i406'), - ] + ], + +('surface_geopotential_height', 'm01s00i033'): [('surface_altitude', 'm01s00i033')], + +('upward_air_velocity_in_pascal', 'm01s15i242'): [ + ('upward_air_velocity', 'm01s15i242'), + ('air_temperature', 'm01s16i203')] } @@ -2268,12 +2279,13 @@ def regridAnlFcstFiles(arg): clength = regdCube.shape[0] # subtract from previously cummulated to make it as hourly accumulated, # instead of writing as hourly cummulated. - for ci in range(clength-1, 1, -1): regdCube[ci].data -= regdCube[ci-1].data + for ci in range(clength-1, 1, -1): regdCube.data[ci] -= regdCube.data[ci-1] # removing cummulative time informations regdCube.remove_coord('forecast_period') regdCube.remove_coord('time') + # here the snowfall_amount extract from 0 to 9 timestep snowvarCon = iris.Constraint(name='snowfall_amount') snowSTASHCon = iris.AttributeConstraint(STASH='m01s00i023') snowVar = cubes.extract(snowvarCon & snowSTASHCon)[0] @@ -2827,7 +2839,7 @@ def doShuffleVarsInOrder(fpath): standard_name='precipitation_amount', removeSTASH=True) # lets fix -ve precipitation as 0.0 - precipitation_amount.data[precipitation_amount.data < 0.0] = 0.0 + precipitation_amount.data[precipitation_amount.data <= 0.0] = 0.0 # store the 'precipitation_amount' into orderedVars orderedVars.insert(idx, precipitation_amount) @@ -2895,6 +2907,73 @@ def doShuffleVarsInOrder(fpath): orderedVars.insert(idx, atmosphere_precipitable_water_content) # end of if ('atmosphere_precipitable_water_content', 'None') in _convertVars_: + + if ('upward_air_velocity_in_pascal', 'm01s15i242') in _convertVars_: + + # find the index in _convertVars_ + idx = _convertVars_.index(('upward_air_velocity_in_pascal', 'm01s15i242')) + # adjust the current index by subtract 1, because in previous insertion + # causes order index increased by 1. + idx = idx-1 if (idx and oidx is None) else idx + oidx = idx + # store the upward_air_velocity data into temporary variable + upward_air_velocity = [var for var in orderedVars + if var.standard_name == 'upward_air_velocity'] + if not upward_air_velocity: + raise ValueError("Can not calculate upward_air_velocity_in_pascal, because unable to load upward_air_velocity") + # store the air_temperature data into temporary variable + air_temperature = [var for var in orderedVars + if var.standard_name == 'air_temperature'] + if not air_temperature: + raise ValueError("Can not calculate upward_air_velocity_in_pascal, because unable to load air_temperature") + + # calculate 'upward_air_velocity_in_pascal' by subtracting [1] - [2] - [3] -[4] + # upward_air_velocity_in_pascal = (upward_air_velocity / air_temperature) * -3.4162608 + # For eg : (VVEL(850)/Temp(850))*-3.4162608 + # ref : https://www.ncl.ucar.edu/Document/Functions/Contributed/omega_to_w.shtml + # Formula given by Dr.Sumit, Project Scientist-D, NCMRWF + upward_air_velocity = upward_air_velocity[0] + air_temperature = air_temperature[0] + print "upward_air_velocity=", upward_air_velocity.data.min(), upward_air_velocity.data.max() + print "air_temperature=", air_temperature.data.min(), air_temperature.data.max() + pressure = upward_air_velocity.coords('pressure')[0].points + upward_air_velocity_in_pascal = (upward_air_velocity / air_temperature) * -0.034162608 + for idx, p in enumerate(pressure): upward_air_velocity_in_pascal.data[idx] *= float(int(p)) + print "pressure=", pressure + print "upward_air_velocity_in_pascal", upward_air_velocity_in_pascal.data.min(), upward_air_velocity_in_pascal.data.max() + # Lets make sure that standard_name is None for this variable. + upward_air_velocity_in_pascal.standard_name = None + upward_air_velocity_in_pascal.long_name = 'upward_air_velocity_in_pascal' + upward_air_velocity_in_pascal.attributes = upward_air_velocity[0].attributes + + upward_air_velocity_in_pascal.units = Unit('Pa s-1') + + # lets store the 'upward_air_velocity_in_pascal' into orderedVars + orderedVars.insert(idx, upward_air_velocity_in_pascal) + # end of if ('upward_air_velocity_in_pascal', 'm01s15i242') in _convertVars_: + + if ('surface_geopotential_height', 'm01s00i033') in _convertVars_: + + # find the index in _convertVars_ + idx = _convertVars_.index(('surface_geopotential_height', 'm01s00i033')) + # adjust the current index by subtract 1, because in previous insertion + # causes order index increased by 1. + idx = idx-1 if (idx and oidx is None) else idx + oidx = idx + # store the surface_altitude data into temporary variable + surface_altitude = [var for var in orderedVars + if var.standard_name == 'surface_altitude'] + if not surface_altitude: + raise ValueError("Can not calculate surface_geopotential_height, because unable to load surface_altitude") + surface_geopotential_height = surface_altitude[0] + surface_geopotential_height.standard_name = None + surface_geopotential_height.long_name = 'surface_geopotential_height' + # remove older orography variable (to avoid duplicate) + orderedVars.remove(surface_altitude[0]) + # lets store the 'surface_geopotential_height' into orderedVars + orderedVars.insert(idx, surface_geopotential_height) + # end of if ('surface_geopotential_height', 'm01s00i033') in _convertVars_: + # remove temporary variables from ordered vars list for dvar, dSTASH in _removeVars_: for ovar in orderedVars: @@ -3328,9 +3407,13 @@ def _checkOutFilesStatus(path, ftype, date, utc, overwrite): if os.path.isfile(fpath): print "Out File already exists", fpath, if overwrite: - os.remove(fpath) - status = 'FilesRemoved' - print ", but overwrite option is True. So removed it!" + try: + os.remove(fpath) + except Exception, e: + print "Got error while removing file", e + finally: + status = 'FilesRemoved' + print ", but overwrite option is True. So removed it!" else: status = 'FilesExist' else: @@ -3342,19 +3425,32 @@ def _checkOutFilesStatus(path, ftype, date, utc, overwrite): status = 'PartialFilesExist' break # end of for ext in ['', '.ctl', '.idx']: + + for ext in [_preExtension_, '_Ordered']: + fpath = os.path.join(path, fname) + if os.path.isfile(fpath) and ext in fpath: + try: + os.remove(fpath) + except Exception, e: + print "Got error while removing file", e + finally: + status = 'IntermediateFilesExist' + print "removed intermediate file" + # end of for ext in [_preExtension_, '_Ordered']: # end of for fhr in fhrs: - ifiles1 = [fname for fname in os.listdir(path) if _preExtension_ in fname] - ifiles2 = [fname for fname in os.listdir(path) if '_Ordered' in fname] - ifiles = ifiles1 + ifiles2 + ifiles = [fname for fname in os.listdir(path) if fname.endswith('.nc')] if ifiles: print "Intermediate files are exists in the outdirectory.", path - for ifile in ifiles: + for ifile in ifiles: if outFileNameStructure[0] in ifile and utc in ifile and _preExtension_ in ifile: - os.remove(os.path.join(path, ifile)) - status = 'IntermediateFilesExist' - print "removed intermediate nc file" - # end of for ncfile in ncfiles: + try: + os.remove(os.path.join(path, ifile)) + except Exception, e: + print "Got error while removing file", e + finally: + status = 'IntermediateFilesExist' + print "removed intermediate nc file" # end of if ncfiles: if status in ['PartialFilesExist', 'IntermediateFilesExist']: # partial files exist, so make overwrite option as True and do From 9c0608b0dfa3c99bdac1f4454fbb01e622a24659 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:49:28 +0530 Subject: [PATCH 11/14] tigge_check cmd --- ...tigge_create_tarball_g2files_put_into_ftp-arulalan.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp-arulalan.py b/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp-arulalan.py index 982e116..cb15205 100755 --- a/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp-arulalan.py +++ b/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp-arulalan.py @@ -18,6 +18,7 @@ pbzip2 = '/gpfs1/home/Libs/GNU/ZIPUTIL/pbzip2' pigz = '/gpfs1/home/Libs/GNU/ZIPUTIL/pigz' +tigge_check = '/gpfs2/home/prasanna/SOFTWARE/GNU/grib_api-1.21.0-path/bin/tigge_check' filesCount = {'ttr': 41, 'lsm': 41, 'orog': 41, '10v': 41, 'tcc': 41, 'gh': 369, 'skt': 41, 'tp': 41, 'msl': 41, 'mx2t6': 40, '2d': 41, '10u': 41, 'mn2t6': 40, 'sshf': 41, 'slhf': 41, 'ssr': 41, '2t': 41, 'sp': 41, 'st': 41, 'q': 328, 'u': 328, 't': 328, 'str': 41, 'v': 328, 'sd': 41} @@ -41,6 +42,14 @@ def createTarBalls(path, today, member): cdir = os.getcwd() os.chdir(inpath) + for tgf in os.listdir('.'): + cmd = tigge_check + '-v -w %s/*' % tgf + tigge_check_val = os.system(cmd) # it should return 0 on pass + if tigge_check_val != 0 : + print "Error : While checking via tigge_check cmd got error!" + sys.exit(0) + # end of for tgf in os.listdir('.'): + tDay = datetime.datetime.strptime(today, "%Y%m%d") lag1 = datetime.timedelta(days=1) yDay = (tDay - lag1).strftime('%Y%m%d') From be95268d43b5dc6b0d0e836dd7dcc40ed60dc063 Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:49:59 +0530 Subject: [PATCH 12/14] tigge_check cmd --- .../tigge_create_tarball_g2files_put_into_ftp.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp.py b/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp.py index b8120e4..4caa279 100755 --- a/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp.py +++ b/bsubScripts/ncum_global_tigge/tigge_create_tarball_g2files_put_into_ftp.py @@ -18,6 +18,7 @@ pbzip2 = '/gpfs1/home/Libs/GNU/ZIPUTIL/pbzip2' pigz = '/gpfs1/home/Libs/GNU/ZIPUTIL/pigz' +tigge_check = '/gpfs2/home/prasanna/SOFTWARE/GNU/grib_api-1.21.0-path/bin/tigge_check' filesCount = {'ttr': 41, 'lsm': 41, 'orog': 41, '10v': 41, 'tcc': 41, 'gh': 369, 'skt': 41, 'tp': 41, 'msl': 41, 'mx2t6': 40, '2d': 41, '10u': 41, 'mn2t6': 40, 'sshf': 41, 'slhf': 41, 'ssr': 41, '2t': 41, 'sp': 41, 'st': 41, 'q': 328, 'u': 328, 't': 328, 'str': 41, 'v': 328, 'sd': 41} @@ -41,7 +42,15 @@ def createTarBalls(path, today, member): cdir = os.getcwd() os.chdir(inpath) - + + for tgf in os.listdir('.'): + cmd = tigge_check + '-v -w %s/*' % tgf + tigge_check_val = os.system(cmd) # it should return 0 on pass + if tigge_check_val != 0 : + print "Error : While checking via tigge_check cmd got error!" + sys.exit(0) + # end of for tgf in os.listdir('.'): + tDay = datetime.datetime.strptime(today, "%Y%m%d") lag1 = datetime.timedelta(days=1) yDay = (tDay - lag1).strftime('%Y%m%d') From 2c7fd66e2f32b71559de75b9d3f1f02083e6ad4d Mon Sep 17 00:00:00 2001 From: arulalant Date: Tue, 6 Jun 2017 12:52:27 +0530 Subject: [PATCH 13/14] added umglc.pp0 into UMInShortFcstFiles to access orography at all cycles of analysis --- bsubScripts/ncum_global_2p5/ncum_global_2p5_um2grb2_setup.cfg | 2 +- .../ncum_global_imd_mfi_input_um2grb2_setup.cfg | 2 +- bsubScripts/ncum_global_post/ncum_global_post_um2grb2_setup.cfg | 2 +- .../ncum_india_reg_from_global_um2grb2_setup.cfg | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bsubScripts/ncum_global_2p5/ncum_global_2p5_um2grb2_setup.cfg b/bsubScripts/ncum_global_2p5/ncum_global_2p5_um2grb2_setup.cfg index cc0b7b1..fb58355 100644 --- a/bsubScripts/ncum_global_2p5/ncum_global_2p5_um2grb2_setup.cfg +++ b/bsubScripts/ncum_global_2p5/ncum_global_2p5_um2grb2_setup.cfg @@ -49,7 +49,7 @@ UMInAnlFiles = ['qwqg00.pp0'] ## UMInShortFcstFiles takes list of partial filenames which has short forecast ## (kind of analysis) fieldsfile/pp files of UM model produced for 00UTC, 06UTC, 12UTC, 18UTC. ## None option will take hardcoded proper infile names. -UMInShortFcstFiles = ['umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] +UMInShortFcstFiles = ['umglc.pp0', 'umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] ## UMInLongFcstFiles takes list of partial filenames which has long forecast ## fieldsfile/pp file of UM model based on reference time at 00UTC, 12UTC. diff --git a/bsubScripts/ncum_global_imd_mfi_input/ncum_global_imd_mfi_input_um2grb2_setup.cfg b/bsubScripts/ncum_global_imd_mfi_input/ncum_global_imd_mfi_input_um2grb2_setup.cfg index a67c5e4..1620246 100644 --- a/bsubScripts/ncum_global_imd_mfi_input/ncum_global_imd_mfi_input_um2grb2_setup.cfg +++ b/bsubScripts/ncum_global_imd_mfi_input/ncum_global_imd_mfi_input_um2grb2_setup.cfg @@ -49,7 +49,7 @@ UMInAnlFiles = ['qwqg00.pp0'] ## UMInShortFcstFiles takes list of partial filenames which has short forecast ## (kind of analysis) fieldsfile/pp files of UM model produced for 00UTC, 06UTC, 12UTC, 18UTC. ## None option will take hardcoded proper infile names. -UMInShortFcstFiles = ['umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] +UMInShortFcstFiles = ['umglc.pp0', 'umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] ## UMInLongFcstFiles takes list of partial filenames which has long forecast ## fieldsfile/pp file of UM model based on reference time at 00UTC, 12UTC. diff --git a/bsubScripts/ncum_global_post/ncum_global_post_um2grb2_setup.cfg b/bsubScripts/ncum_global_post/ncum_global_post_um2grb2_setup.cfg index b5d67d9..7e86b6d 100644 --- a/bsubScripts/ncum_global_post/ncum_global_post_um2grb2_setup.cfg +++ b/bsubScripts/ncum_global_post/ncum_global_post_um2grb2_setup.cfg @@ -49,7 +49,7 @@ UMInAnlFiles = ['qwqg00.pp0'] ## UMInShortFcstFiles takes list of partial filenames which has short forecast ## (kind of analysis) fieldsfile/pp files of UM model produced for 00UTC, 06UTC, 12UTC, 18UTC. ## None option will take hardcoded proper infile names. -UMInShortFcstFiles = ['umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] +UMInShortFcstFiles = ['umglc.pp0', 'umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] ## UMInLongFcstFiles takes list of partial filenames which has long forecast ## fieldsfile/pp file of UM model based on reference time at 00UTC, 12UTC. diff --git a/bsubScripts/ncum_india_reg_from_global/ncum_india_reg_from_global_um2grb2_setup.cfg b/bsubScripts/ncum_india_reg_from_global/ncum_india_reg_from_global_um2grb2_setup.cfg index ec85bbf..da3b98b 100644 --- a/bsubScripts/ncum_india_reg_from_global/ncum_india_reg_from_global_um2grb2_setup.cfg +++ b/bsubScripts/ncum_india_reg_from_global/ncum_india_reg_from_global_um2grb2_setup.cfg @@ -49,7 +49,7 @@ UMInAnlFiles = ['qwqg00.pp0'] ## UMInShortFcstFiles takes list of partial filenames which has short forecast ## (kind of analysis) fieldsfile/pp files of UM model produced for 00UTC, 06UTC, 12UTC, 18UTC. ## None option will take hardcoded proper infile names. -UMInShortFcstFiles = ['umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] +UMInShortFcstFiles = ['umglc.pp0', 'umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] ## UMInLongFcstFiles takes list of partial filenames which has long forecast ## fieldsfile/pp file of UM model based on reference time at 00UTC, 12UTC. From f3b6576e3fcce4ca9cd5e1ff460fd898c4881e8f Mon Sep 17 00:00:00 2001 From: arulalant Date: Mon, 12 Jun 2017 11:45:08 +0530 Subject: [PATCH 14/14] hysplit model input setup files --- .../ncum_global_hysplit_input/REAMDE.md | 25 ++ .../ncum_global_hysplit_input_anl_00Z.bash | 37 +++ .../ncum_global_hysplit_input_anl_06Z.bash | 38 +++ .../ncum_global_hysplit_input_anl_12Z.bash | 38 +++ .../ncum_global_hysplit_input_anl_18Z.bash | 38 +++ .../ncum_global_hysplit_input_fcst_00Z.bash | 39 +++ ..._global_hysplit_input_um2grb2_anl_vars.cfg | 31 ++ ...global_hysplit_input_um2grb2_fcst_vars.cfg | 31 ++ ...cum_global_hysplit_input_um2grb2_setup.cfg | 292 ++++++++++++++++++ 9 files changed, 569 insertions(+) create mode 100644 bsubScripts/ncum_global_hysplit_input/REAMDE.md create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_00Z.bash create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_06Z.bash create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_12Z.bash create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_18Z.bash create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_fcst_00Z.bash create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_anl_vars.cfg create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_fcst_vars.cfg create mode 100644 bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_setup.cfg diff --git a/bsubScripts/ncum_global_hysplit_input/REAMDE.md b/bsubScripts/ncum_global_hysplit_input/REAMDE.md new file mode 100644 index 0000000..a31f1ba --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/REAMDE.md @@ -0,0 +1,25 @@ +This will produce analysis 00utc files. + +$ bsub < ncum_global_hysplit_input_anl_00Z.bash + +This will produce analysis 06utc files. + +$ bsub < ncum_global_hysplit_input_anl_06Z.bash + +This will produce analysis 12utc files. + +$ bsub < ncum_global_hysplit_input_anl_12Z.bash + +This will produce analysis 18utc files. + +$ bsub < ncum_global_hysplit_input_anl_18Z.bash + +This will produce forecast 00utc files. + +$ bsub < ncum_global_hysplit_input_fcst_00Z.bash + +This will produce forecast-fluxes 00utc files. + +$ bsub < ncum_global_hysplit_input_flux_00Z.bash + +Arulalan.T diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_00Z.bash b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_00Z.bash new file mode 100644 index 0000000..bb91e08 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_00Z.bash @@ -0,0 +1,37 @@ +#!/bin/bash +# +#BSUB -a poe # set parallel operating environment +#BSUB -J u2g2hysplit # job name +#BSUB -W 06:00 # wall-clock time (hrs:mins) +#BSUB -n 6 # number of tasks in job +#BSUB -x # exclusive mode +#BSUB -q ultra # queue +#BSUB -e /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.00hr.err.%J.hybrid # error file name in which %J is replaced by the job ID +#BSUB -o /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.00hr.out.%J.hybrid # output file name in which %J is replaced by the job ID + +# find out the directory of this bash script after submitted to bsub +DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )" + +# get the absolute path of the local table +localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/" +localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )" +localTable=$localTable_absolute_dir/ncmr_grib2_local_table + +# get the absolute path of the script for analysis 00utc +g2scripts_relative_dir="$DIR/../../g2scripts/" +g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )" +g2script=$g2scripts_absolute_dir/um2grb2_anl_00Z.py + +# export the configure paths to needed variables +export UMRIDER_SETUP=$DIR/ncum_global_hysplit_input_um2grb2_setup.cfg +export UMRIDER_VARS=$DIR/ncum_global_hysplit_input_um2grb2_anl_vars.cfg +export GRIB2TABLE=$localTable + +echo "export UMRIDER_SETUP="$UMRIDER_SETUP +echo "export UMRIDER_VARS="$UMRIDER_VARS +echo "export GRIB2TABLE="$GRIB2TABLE + +# sourcing umtid_bashrc to load module python-uvcdat-iris! +source "$DIR/../umtid_bashrc" +# execute the script +python $g2script diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_06Z.bash b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_06Z.bash new file mode 100644 index 0000000..5009910 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_06Z.bash @@ -0,0 +1,38 @@ +#!/bin/bash +# +#BSUB -a poe # set parallel operating environment +#BSUB -J u2g2hysplit # job name +#BSUB -W 06:00 # wall-clock time (hrs:mins) +#BSUB -n 4 # number of tasks in job +#BSUB -x # exclusive mode +#BSUB -q ultra # queue +#BSUB -e /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.06hr.err.%J.hybrid # error file name in which %J is replaced by the job ID +#BSUB -o /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.06hr.out.%J.hybrid # output file name in which %J is replaced by the job ID + +# find out the directory of this bash script after submitted to bsub +DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )" + +# get the absolute path of the local table +localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/" +localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )" +localTable=$localTable_absolute_dir/ncmr_grib2_local_table + +# get the absolute path of the script for analysis 06utc +g2scripts_relative_dir="$DIR/../../g2scripts/" +g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )" +g2script=$g2scripts_absolute_dir/um2grb2_anl_06Z.py + +# export the configure paths to needed variables +export UMRIDER_SETUP=$DIR/ncum_global_hysplit_input_um2grb2_setup.cfg +export UMRIDER_VARS=$DIR/ncum_global_hysplit_input_um2grb2_anl_vars.cfg +export GRIB2TABLE=$localTable + +echo "export UMRIDER_SETUP="$UMRIDER_SETUP +echo "export UMRIDER_VARS="$UMRIDER_VARS +echo "export GRIB2TABLE="$GRIB2TABLE + +# sourcing umtid_bashrc to load module python-uvcdat-iris! +source "$DIR/../umtid_bashrc" +# execute the script +python $g2script + diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_12Z.bash b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_12Z.bash new file mode 100644 index 0000000..9f71eae --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_12Z.bash @@ -0,0 +1,38 @@ +#!/bin/bash +# +#BSUB -a poe # set parallel operating environment +#BSUB -J u2g2hysplit # job name +#BSUB -W 06:00 # wall-clock time (hrs:mins) +#BSUB -n 4 # number of tasks in job +#BSUB -x # exclusive mode +#BSUB -q ultra # queue +#BSUB -e /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.12hr.err.%J.hybrid # error file name in which %J is replaced by the job ID +#BSUB -o /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.12hr.out.%J.hybrid # output file name in which %J is replaced by the job ID + +# find out the directory of this bash script after submitted to bsub +DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )" + +# get the absolute path of the local table +localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/" +localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )" +localTable=$localTable_absolute_dir/ncmr_grib2_local_table + +# get the absolute path of the script for analysis 12utc +g2scripts_relative_dir="$DIR/../../g2scripts/" +g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )" +g2script=$g2scripts_absolute_dir/um2grb2_anl_12Z.py + +# export the configure paths to needed variables +export UMRIDER_SETUP=$DIR/ncum_global_hysplit_input_um2grb2_setup.cfg +export UMRIDER_VARS=$DIR/ncum_global_hysplit_input_um2grb2_anl_vars.cfg +export GRIB2TABLE=$localTable + +echo "export UMRIDER_SETUP="$UMRIDER_SETUP +echo "export UMRIDER_VARS="$UMRIDER_VARS +echo "export GRIB2TABLE="$GRIB2TABLE + +# sourcing umtid_bashrc to load module python-uvcdat-iris! +source "$DIR/../umtid_bashrc" +# execute the script +python $g2script + diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_18Z.bash b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_18Z.bash new file mode 100644 index 0000000..614b153 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_anl_18Z.bash @@ -0,0 +1,38 @@ +#!/bin/bash +# +#BSUB -a poe # set parallel operating environment +#BSUB -J u2g2hysplit # job name +#BSUB -W 06:00 # wall-clock time (hrs:mins) +#BSUB -n 4 # number of tasks in job +#BSUB -x # exclusive mode +#BSUB -q ultra # queue +#BSUB -e /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.18hr.err.%J.hybrid # error file name in which %J is replaced by the job ID +#BSUB -o /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.anl.18hr.out.%J.hybrid # output file name in which %J is replaced by the job ID + +# find out the directory of this bash script after submitted to bsub +DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )" + +# get the absolute path of the local table +localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/" +localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )" +localTable=$localTable_absolute_dir/ncmr_grib2_local_table + +# get the absolute path of the script for analysis 18utc +g2scripts_relative_dir="$DIR/../../g2scripts/" +g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )" +g2script=$g2scripts_absolute_dir/um2grb2_anl_18Z.py + +# export the configure paths to needed variables +export UMRIDER_SETUP=$DIR/ncum_global_hysplit_input_um2grb2_setup.cfg +export UMRIDER_VARS=$DIR/ncum_global_hysplit_input_um2grb2_anl_vars.cfg +export GRIB2TABLE=$localTable + +echo "export UMRIDER_SETUP="$UMRIDER_SETUP +echo "export UMRIDER_VARS="$UMRIDER_VARS +echo "export GRIB2TABLE="$GRIB2TABLE + +# sourcing umtid_bashrc to load module python-uvcdat-iris! +source "$DIR/../umtid_bashrc" +# execute the script +python $g2script + diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_fcst_00Z.bash b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_fcst_00Z.bash new file mode 100644 index 0000000..b214677 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_fcst_00Z.bash @@ -0,0 +1,39 @@ +#!/bin/bash +# +#BSUB -a poe # set parallel operating environment +#BSUB -J u2g2hysplit # job name +#BSUB -W 06:00 # wall-clock time (hrs:mins) +#BSUB -n 16 # number of tasks in job (max task in one node) +#BSUB -x # exclusive mode +#BSUB -R span[ptile=16] # task per node +#BSUB -q ultra # queue +#BSUB -e /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.fcst.00hr.err.%J.hybrid # error file name in which %J is replaced by the job ID +#BSUB -o /gpfs3/home/umfcst/UMRiderLogs/hysplit/bsub/um2grb2.fcst.00hr.out.%J.hybrid # output file name in which %J is replaced by the job ID + +# find out the directory of this bash script after submitted to bsub +DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )" + +# get the absolute path of the local table +localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/" +localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )" +localTable=$localTable_absolute_dir/ncmr_grib2_local_table + +# get the absolute path of the script for forecast 00utc +g2scripts_relative_dir="$DIR/../../g2scripts/" +g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )" +g2script=$g2scripts_absolute_dir/um2grb2_fcst_00Z.py + +# export the configure paths to needed variables +export UMRIDER_SETUP=$DIR/ncum_global_hysplit_input_um2grb2_setup.cfg +export UMRIDER_VARS=$DIR/ncum_global_hysplit_input_um2grb2_fcst_vars.cfg +export GRIB2TABLE=$localTable + +echo "export UMRIDER_SETUP="$UMRIDER_SETUP +echo "export UMRIDER_VARS="$UMRIDER_VARS +echo "export GRIB2TABLE="$GRIB2TABLE + +# sourcing umtid_bashrc to load module python-uvcdat-iris! +source "$DIR/../umtid_bashrc" +# execute the script +python $g2script + diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_anl_vars.cfg b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_anl_vars.cfg new file mode 100644 index 0000000..affd701 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_anl_vars.cfg @@ -0,0 +1,31 @@ +############################################################################## +## vars configure file: Used for the purpose of um2grb2 conversion of only ## +## needed NCUM model out variables. um2grb2 python parallel scripts will ## +## create analysis and forecast files, by conveting to gri2 file only for ## +## for the following cf_standard_name and varSTASH coded vars. ## +## ## +## NCUM GLOBAL MODEL OSF INPUT PREPARATION PROCESSING VARS CONFIGURE FILE ## +## ## +## Author : Arulalan ## +## Updated : 25-May-2017 ## +############################################################################## + +##### BEGIN OF UMRIDER VARS CONFIGURE FOR um2grb2 SCRIPTS ###### +## Pressure Level Variable names & STASH codes +('x_wind', 'm01s15i201') +('y_wind', 'm01s15i202') +('geopotential_height', 'm01s16i202') +# Instead of ('surface_altitude', 'm01s00i033') i.e. orography, model needs to written as below. +('surface_geopotential_height', 'm01s00i033') +('relative_humidity', 'm01s16i256') +('specific_humidity', 'm01s30i205') +('air_temperature', 'm01s16i203') +('upward_air_velocity_in_pascal', 'm01s15i242') +## Non Pressure Level Variable names & STASH codes +('x_wind', 'm01s03i225') +('y_wind', 'm01s03i226') +('air_temperature', 'm01s03i236') +('specific_humidity', 'm01s03i237') +('surface_air_pressure', 'm01s00i409') +('atmosphere_boundary_layer_thickness', 'm01s00i025') +##### END OF UMRIDER VARS CONFIGURE um2grb2 SCRIPTS ###### diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_fcst_vars.cfg b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_fcst_vars.cfg new file mode 100644 index 0000000..29c7c28 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_fcst_vars.cfg @@ -0,0 +1,31 @@ +############################################################################## +## vars configure file: Used for the purpose of um2grb2 conversion of only ## +## needed NCUM model out variables. um2grb2 python parallel scripts will ## +## create analysis and forecast files, by conveting to gri2 file only for ## +## for the following cf_standard_name and varSTASH coded vars. ## +## ## +## NCUM GLOBAL MODEL OSF INPUT PREPARATION PROCESSING VARS CONFIGURE FILE ## +## ## +## Author : Arulalan ## +## Updated : 25-May-2017 ## +############################################################################## + +##### BEGIN OF UMRIDER VARS CONFIGURE FOR um2grb2 SCRIPTS ###### +## Pressure Level Variable names & STASH codes +('x_wind', 'm01s15i201') +('y_wind', 'm01s15i202') +('geopotential_height', 'm01s16i202') +# Instead of ('surface_altitude', 'm01s00i033') i.e. orography, model needs to written as below. +##('surface_geopotential_height', 'm01s00i033') # not avilable for fcst. +('relative_humidity', 'm01s16i256') +('specific_humidity', 'm01s30i205') +('air_temperature', 'm01s16i203') +('upward_air_velocity_in_pascal', 'm01s15i242') +## Non Pressure Level Variable names & STASH codes +('x_wind', 'm01s03i225') +('y_wind', 'm01s03i226') +('air_temperature', 'm01s03i236') +('specific_humidity', 'm01s03i237') +('surface_air_pressure', 'm01s00i409') +('atmosphere_boundary_layer_thickness', 'm01s00i025') +##### END OF UMRIDER VARS CONFIGURE um2grb2 SCRIPTS ###### diff --git a/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_setup.cfg b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_setup.cfg new file mode 100644 index 0000000..0ab6996 --- /dev/null +++ b/bsubScripts/ncum_global_hysplit_input/ncum_global_hysplit_input_um2grb2_setup.cfg @@ -0,0 +1,292 @@ +############################################################################## +## setup configure file: Used to setup indata path, outdata path, temporary ## +## path to run the um2grb2 python parallel scripts which will create ## +## analysis and forecast files. ## +## ## +## NCUM GLOBAL MODEL OSF INPUT PREPARATION PROCESSING SETUP CONFIGURE FILE ## +## ## +## Author : Arulalan ## +## Updated : 09-Feb-2016 ## +############################################################################## + +###################### User Defined Arguments Begin ########################## + +## By default startdate takes argument as YYYYMMDD (which means it assume today's date) +## But user can specify the different startdate by follwing the same format. +## for eg, startdate = 20151209 # then it will execute the scripts for 09-Dec-2015. +## startdate = YYYYMMDD # then it will execute the scripts for today's date. +## Note : However UMRIDER_STARTDATE environment variable will override this startdate option. + +startdate = YYYYMMDD + +## By default enddate is None. User can specify different enddate (but > startdate) +## i.e. for only specfied date / one date, user needs to control only in the +## startdate and enddate must be None. If user want to execute the um2grb2 +## conversion for the range of dates, then user need to set the lower startdate +## end higher enddate. enddate could be even YYYYMMDD, but make sure that +## startdate is lower than enddate. +## For eg : startdate = 20151209 and enddate = 20160114 , then um2grb2 +## conversion program executes from 09-dec-2015 to 14-jan-2016. +## Note : However UMRIDER_ENDDATE environment variable will override this enddate option. + +enddate = None + +############################################################################## +########### CAUTION DO NOT TOUCH BELOW CONFIGURE SETUP LINES ################ +############################################################################## + +########### BEGIN OF UMRIDER SETUP CONFIGURE FOR um2grb2 SCRIPTS ############# + +## UM model type takes either 'global' or 'regional' or 'ensemble'. By default +## it takes 'global' as argument. +UMtype = global + +## UMInAnlFiles takes list of absolute filenames which has correct analysis +## fieldsfile/pp file of UM model valid at 00UTC. +## None option will take hardcoded proper infile names. +UMInAnlFiles = ['qwqg00.pp0'] + +## UMInShortFcstFiles takes list of partial filenames which has short forecast +## (kind of analysis) fieldsfile/pp files of UM model produced for 00UTC, 06UTC, 12UTC, 18UTC. +## None option will take hardcoded proper infile names. +UMInShortFcstFiles = ['umglc.pp0', 'umglca_pb', 'umglca_pd', 'umglca_pe', 'umglca_pf', 'umglca_pi'] + +## UMInLongFcstFiles takes list of partial filenames which has long forecast +## fieldsfile/pp file of UM model based on reference time at 00UTC, 12UTC. +## None option will take hardcoded proper infile names. +UMInLongFcstFiles = ['umglaa_pb','umglaa_pd', 'umglaa_pe', 'umglaa_pf', 'umglaa_pi'] + +## model pp filedsfiles path +inPath = /gpfs3/home/umfcst/NCUM/fcst/*YYYYMMDD*/*ZZ*/ + +## model grib2 files path +outPath = /gpfs3/home/umfcst/ShortJobs/NCUM_HYSPLIT/*YYYYMMDD*/ + +## working directory (used to create temporary log files) +tmpPath = /gpfs3/home/umfcst/UMRiderLogs/hysplit/ + + +## analysis step/interval hours. By default it takes 6 hour which mean +## um2grb2 produce 6 hourly instantaneous and/or 6 hourly average and/or +## 6 hourly accumulation values analysis files. +## If user specfied as 3 then it will extract only 3 houlry instantaneous fields. +## By default model produced 3 hourly average/accumulation. +## Note: the average and accumulation supports only for 6 hours!!! +anl_step_hour = 6 + +## anl_aavars_reference_time takes either 'analysis' or 'shortforecast'. When some +## variables are taken from previous cycle short-forecast (average/accumulation) +## vars, the reference time need to be set as either current 'analysis' reference +## cycle (utc) or previous cycle's 'shortforecast' reference time. +## 'shortforecast' gives exactly based on which utc that variable has processed, +## whereas 'analysis' shift reference time utc as actual analysis utc time. +## Note : This option applicable only to average/accumulation vars in analysis grib2 files. +## For OSF, incois required analysis reference time should be based on analysis, not shortforecast! +anl_aavars_reference_time = shortforecast + +## anl_aavars_time_bounds takes either 'True' or 'False'. By default, True keeps the +## analysis time bounds, reference time bounds and False removes it (so that +## it become instantaneous instead of average/accumulation vars). +## False will be applicable only if anl_aavars_reference_time arg passed as 'analysis'. +## Note : This option applicable only to average/accumulation vars in analysis grib2 files. +## For OSF, we keeps the time bounds on till date. +anl_aavars_time_bounds = True + +## long forecast step/interval hours . By default it +## takes 6 hour which mean um2grb2 produce 6 hourly instantaneous and/or +## 6 hourly average and/or 6 hourly accumulation values. +## If user specfied as 3 or 24 then it will extract only 3 or 24 houlry +## instantaneous fields and for calculate average/accumulation for 24 hourly. +## By default model produced 3 hourly average/accumulation. +## Note: the average and accumulation supports only for either 3 or 6 or 24 hours!!! +fcst_step_hour = 3 + +## long forecast start hour. By default it +## takes 6 hour which mean um2grb2 produce grib2 files from 06-th hour forecasts. +## If user wants from different hours, then they can specify it ! +## It should be multiples of 'fcst_step_hour' (see above option)! +start_long_fcst_hour = 3 # for hycom model, we need to feed 3 hourly data. + +## maximum long forecast hours at 00utc cycle produced by NCUM model +## for 10days forecast 240hour (by default 240 hours) +end_long_fcst_hour_at_00z = 120 + +## maximum long forecast hours at 12utc cycle produced by NCUM model +## for 5days forecast 120hour (by default 120 hours) +end_long_fcst_hour_at_12z = 120 + +## latitude is required latitude which user wants to extract from the model +## global data. By default it takes None (i.e. extract model global latitudes) +## User can specify their required latitude in tuple +## For eg : latitude = (-30, 30) will extract only latitudes from 30S to 30N. +latitude = None + +## longitude is required longitude which user wants to extract from the model +## global data. By default it takes None (i.e. extract model global longitudes) +## User can specify their required longitude in tuple +## For eg : longitude = (60, 100) will extract only longitude from 60E to 100E. +## Note : Model requires longitude should specfied based on (0 to 360), and +## not by (-180 to 180.) In future I may fix it, if user wish to specify +## longitude by within range of (-180, 180) +longitude = None + +## targetGridResolution is resolution in degree (1 degree = 100km approx) +## if targetGridResolution is set to None, then model resolution will be +## kept in the grib2 file. This must be a number or None. +## WARNING : targetGridResolution option will be disabled if targetGridFile enabled. +targetGridResolution = None + +## targetGridFile should be path of sample grib2 / pp / ff / nc file, which +## contains atleast one variable with latitude, longitude informations. Model +## variables will be converted to this target grid file resolution including +## spatial regrid, start-end of latitude-and-longitude. +targetGridFile = ../../data/sample_global_0p5_hysplit.grib2 + +## extraPolateMethod takes either 'auto' or 'linear' or 'mask' or iris supported +## interpolation method. 'linear' means all variables will be linearly extrapolated +## over masked regions also. 'mask' means all masked variables of model outputs, +## will not be extrapolated over masked regions. +## 'auto' will take care properly the neccessary variabels will be extrapolated +## over masked regions and remaining variables will be masked over mask regions. +## 'auto' options is suggested one! +extraPolateMethod = linear + +## pressureLevels is required pressure levels slice / extract only particular +## set of pressure levels from model pressure levels. User can specify either +## one or more levels. By default it takes None, i.e. it will extract all the +## model pressure levels. +## For eg 1 : pressureLevels = [850] -> extract 850 hPa only +## For eg 2 : pressureLevels = [850, 500, 200] -> extract only 850, 500 & 200 hPa levels only. +## Note 1 : These pressure slice levels applicable to all the pressure level variables. +## Note 2 : At the moment pressure levels interpolation is not supported! +pressureLevels = [100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 950, 1000] + +## If some variable has fully masked (for eg: Incoming Shortwave flux during +## night time) then this option value will be set to that variable. By default +## it takes None, which wont do anything. If it has set to 0 as value, then +## those fully masked vars will be filled with this value. +fillFullyMaskedVars = None + +## soilFirstSecondFixedSurfaceUnit takes either 'cm' or 'mm'. By default it +## takes 'cm' argument (suggested for general purpose/ WRF-Noah supported). +## For soil moisture/ soil temeperature variables depth below land surface, +## units are initially set to either 'cm' (centimeter) or 'mm' (millimeter), +## and finally converted to 'm' (meter) in wgrib2. But anyhow if grib2 files +## will be ready by some other utility other than wgrib2, then this first & +## second fixed surface units plays matter. So suggested units is 'cm'. +soilFirstSecondFixedSurfaceUnit = cm + +## Load g2utils from 'system' python which has installed through setup.py (OR) +## Load g2utils from 'local' previous directory for the operational purpose, +## where normal user don't have write permission to change the g2utils! +## So loadg2utils argument should be either system (default) or local. +loadg2utils = local + +## If overwriteFiles option is 'True' then existing output final files (if any) +## will be deleted from outPath and re-creating freshly. +## If overwriteFiles option is 'False' and all output final files are already +## exists in the outPath, then program will be exited without re-creating the +## output files. +## If partially created files exist (like few hours outfiles only exist or +## intermediate nc files) then by default make overwriteFiles option as True +## (though False as passed to overwriteFiles option). +overwriteFiles = True + +## anlOutGrib2FilesNameStructure takes list of string naming arguments to +## construct out file names. um2grb2 will just concatinate the arguments, +## by replacing 3 predefined naming structure ('*HHH*', '*YYYYMMDD*', '*ZZ*') +## with its corresponding values/numbers inplace of it. +## '*H*' - forecast_hours +## '*YYYYMMDD*' - forecast reference date +## '*Z*' - forecast reference utc time (optional) +## If user wants to 3 digit filled hours, then they need to specify as 3 times +## '*HHH*'. If they specify 2 digit filled hours (say '*HH*' only), but forecast +## hours have 3 digit, then by default it will assume as 3 digits but for +## single digit hour, it will fill 0 as prefix to make it as 2 digit. +## same option for utc '*Z*'. +## Note : * will not be included in the name of the final out grib2 files. +## eg1 : ('um_ana', '_', '*HHH*', 'hr', '_', '*YYYYMMDD*', '_', '*ZZ*', 'Z', '.grib2') +## this will produce analysis files as 'um_ana_006hr_20160208_12Z.grib2' +## eg2 : ('fcs', '_', '*HH*', 'h', '_z', '*YYYYMMDD*', '.grb2') this will +## produce forecast files as 'fcs_06h_z20160208.grb2' +## eg3 : ('prg', '*D*', '00z', '*%d%m%y*', '.grb2') will produce grib2 files +## as 'prg100z080216.grb2' +## eg4 : ('prg', '*D*', '00z', '*%d%m%y*', '_', '*pXp*' '.grb2') will produce grib2 files +## as 'prg100z080216_0p17x0p17.grb2' in case of targetGridResolution = None (i.e modelResolution) +## or as 'prg100z080216_2p5x2p5.grb2' in case of targetGridResolution = 2.5 + +## Defining analysis grib2 fileName structure. Must be in single line. +anlOutGrib2FilesNameStructure = ('ncum_anal', '_', '*ZZ*', 'z', '*YYYYMMDD*', '.grb2') + +## Defining forecast grib2 fileName structure. Must be in single line. +fcstOutGrib2FilesNameStructure = ('ncum_fcst', '_', '*HH*', 'h', '*YYYYMMDD*', '.grb2') + +## If createCtlIdxFiles is True then um2grb2 module will create grads control +## files and its index files for each and every grib2 files by using g2ctl.pl +createGrib2CtlIdxFiles = False + +## If convertGrib2FilestoGrib1Files is True then using 'cnvgrib -g21' command +## line um2grb2 module will convert grib2 file to grib1 files. +## CAUTION : it may produce invalid variables names, grib1 param code for few +## variables which are produced by this um2grib2 conversion tool!!! +convertGrib2FilestoGrib1Files = True + +## If grib1FilesNameSuffix is '.grib1', then grib1 files will endswith '.grib1' (default). +## otherwise will whatever string assigned will be added at the end of grib1 +## file names. None will add nothing to grib1 file names at the end of it. +grib1FilesNameSuffix = '.grb' + +## If removeGrib2FilesAfterGrib1FilesCreated is True, then grib2 files will be +## deleted and kept only grib1 files. By default False. +removeGrib2FilesAfterGrib1FilesCreated = True + +## If createCtlIdxFiles is True then um2grb2 module will create grads control +## files and its index files for each and every grib1 files by using grib2ctl.pl +createGrib1CtlIdxFiles = False + +## This debug option should be either True or False. This will just print +## extra informations like varibles details, shape, execution process, etc., +debug = False + +## setGrib2TableParameters option takes list of tuples which may contain +## WMO-Grib2 table parameters and its value. Which means, the grib2 table +## parameter options will be overwritten as per user's setting in this option. +## eg1 : setGrib2TableParameters = [('centre', 28), ('subCentre', 0)] +## The above two options will be set to out grib2 files. +## eg2 : setGrib2TableParameters = [('shapeOfTheEarth', 0)] +## The above option will be set to out grib2 files. +## CATUION : User must be aware on what are they setting in this option and +## its causes in out grib2 files! By default this option takes None. +setGrib2TableParameters = None + +## After successfully created the grib2 (final ordered variabels) file, +## wgrib2 commnad will be executed with the 'wgrib2Arguments' options. +## pygrib/IRIS/UMRider is able to write grib2 file with "grid_simple" packing +## algorithm, whereas wgrib2 able to convert packing from "grid_simple" to +## "grid_complex_spatial_differencing" by setting -set_grib_type complex2 +## option in it. The second type packing reduces file size 1/3 compare to first +## type packing. And further can be reduced the file size, by passing +## -set_bin_prec 12 (compatible same as ECMWF) which reduces the floating points +## precision (which further reduces the file size 1/5 th of original of first +## packing). By default wgrib2Arguments takes "-set_grib_type complex2 -grib_out" +## as argument. User can override this option by including extra wgrib2 arguments +## Or None (wgrib2 will not be executed). +## -grib_out is important argument (to be compress, set precsion, etc) +## http://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/speed.html +## http://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_bin_prec.html +## +wgrib2Arguments = -set_bin_prec 12 -set_grib_type simple -grib_out + +## This callBackScript option takes any user defined script (any script)! +## User should provide absolute or relative path of their script and make sure +## that script is self executable with shebang and executable permission! +## After successfully created out grib2 files, this callBackScript will be +## executed with possibly command line keyword arguments as follows +## KWargs : (date, outpath, oftype, utc) +## where '--date' -> out files processed date, +## '--outpath' -> out files path, +## '--oftype' -> 'analysis' or 'forecast' +## '--utc' -> UTC cycle value in string ('00' or '06' or '12' or '18') +callBackScript = None + +##### END OF UMRIDER SETUP CONFIGURE FOR um2grb2 SCRIPTS ######