diff --git a/download_observations/get_AODN_AltData.sh b/download_observations/get_AODN_AltData.sh index 49b3b0e46..5d8188bc5 100755 --- a/download_observations/get_AODN_AltData.sh +++ b/download_observations/get_AODN_AltData.sh @@ -4,12 +4,13 @@ # # VERSION AND LAST UPDATE: # v1.0 04/04/2022 +# v1.1 07/11/2022 # # PURPOSE: # Script to download AODN altimeter data. See the available files at # http://thredds.aodn.org.au/thredds/catalog/IMOS/SRS/Surface-Waves/Wave-Wind-Altimetry-DM00/catalog.html # Altimeters: -# JASON-3 JASON-2 CRYOSAT-2 JASON-1 HY-2 SARAL SENTINEL-3A ENVISAT ERS-1 ERS-2 GEOSAT GFO TOPEX +# JASON-3 JASON-2 CRYOSAT-2 JASON-1 HY-2 SARAL SENTINEL-3A SENTINEL-3B ENVISAT ERS-1 ERS-2 GEOSAT GFO TOPEX # Satellite data from Integrated Marine Observing System (IMOS), Australian Ocean Data Network (AODN) # https://portal.aodn.org.au/ # Altimeter @@ -36,6 +37,7 @@ # # AUTHOR and DATE: # 04/04/2022: Ricardo M. Campos, first version. +# 07/11/2022: Ricardo M. Campos, correct lat2 format for the Southern H. # # PERSON OF CONTACT: # Ricardo M Campos: ricardo.campos@noaa.gov @@ -55,13 +57,13 @@ for lon in `seq -f "%03g" 0 20 340`; do fi for adlon in `seq -f "%03g" 0 19`; do lon2=(`expr $lon + $adlon`) - test -f $DIR/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc + test -f $DIR/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc TE=$? if [ "$TE" -eq 1 ]; then - wget -l1 -H -t1 -nd -N -np -erobots=off --tries=3 $fname/${s}/${lat}${h}_${lon}E/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc -O $DIR/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc + wget -l1 -H -t1 -nd -N -np -erobots=off --tries=3 $fname/${s}/${lat}${h}_${lon}E/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc -O $DIR/IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc wait $! sleep 1 - echo IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc >> listDownloaded_${s}.txt + echo IMOS_SRS-Surface-Waves_MW_${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc >> listDownloaded_${s}.txt find $DIR -empty -type f -delete fi done diff --git a/download_observations/get_AODN_ScatData.sh b/download_observations/get_AODN_ScatData.sh index 443bd1e32..56c29c9c6 100755 --- a/download_observations/get_AODN_ScatData.sh +++ b/download_observations/get_AODN_ScatData.sh @@ -55,13 +55,13 @@ for lon in `seq -f "%03g" 0 20 340`; do fi for adlon in `seq -f "%03g" 0 19`; do lon2=(`expr $lon + $adlon`) - test -f $DIR/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc + test -f $DIR/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc TE=$? if [ "$TE" -eq 1 ]; then - wget -l1 -H -t1 -nd -N -np -erobots=off --tries=3 $fname/${s}/${lat}${h}_${lon}E/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc -O $DIR/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc + wget -l1 -H -t1 -nd -N -np -erobots=off --tries=3 $fname/${s}/${lat}${h}_${lon}E/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc -O $DIR/IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc wait $! sleep 1 - echo IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" $lat2)"${h}-"$(printf "%03d" $lon2)"E-DM00.nc >> listDownloaded_${s}.txt + echo IMOS_SRS-Surface-Waves_M_Wind-${s}_FV02_"$(printf "%03d" ${lat2/#-})"${h}-"$(printf "%03d" $lon2)"E-DM00.nc >> listDownloaded_${s}.txt find $DIR -empty -type f -delete fi done diff --git a/validation/gridSatGlobal_Altimeter.py b/validation/gridSatGlobal_Altimeter.py index 9de483035..c00083b09 100755 --- a/validation/gridSatGlobal_Altimeter.py +++ b/validation/gridSatGlobal_Altimeter.py @@ -6,27 +6,37 @@ VERSION AND LAST UPDATE: v1.0 04/04/2022 + v1.1 07/18/2022 PURPOSE: Script to take altimeter tracks and collocate into regular - lat/lon grid (gridInfo.nc, generated by preprGridMask.py) - and hourly time interval (for different time intervals, edit atime array) + lat/lon grid (gridInfo.nc, generated by preprGridMask.py) + and hourly time interval (for different time intervals, edit atime array) + A total of 14 satellite missions are listed below. The period of each + altimeter can be verified at: + https://www.sciencedirect.com/science/article/pii/S0273117721000594 + https://ars.els-cdn.com/content/image/1-s2.0-S0273117721000594-gr1_lrg.jpg USAGE: - It runs one satellite mission, entered as argument (only the ID, sys.argv), - see the sdname for the list of altimeters available. + This program processes one satellite mission per run, entered as + argument (only the ID, sys.argv), see the sdname for the list of + altimeters available. Altimeters must have been previously downloaded (see get_AODN_AltData.sh) Path where altimeter data is saved must be informed and edited (see dirs below) Check the pre-selected parameters below for the altimeter collocation and date interval (datemin and datemax) - Example (from linux/terminal command line): + Example (from linux terminal command line): An example for JASON3 (first in the sdname list) is nohup python3 gridSatGlobal_Altimeter.py 0 >> nohup_sat0.out 2>&1 & OUTPUT: netcdf file AltimeterGridded_*.nc containing the collocated altimeter data into lat/lon grid points given by gridInfo.nc + hsk: significant wave height, Ku or Ka altimeter band. + hsc: significant wave height, C altimeter band. + wnd: 10-meter wind speed. + 'cal' means calibrated by IMOS-AODN. DEPENDENCIES: See dependencies.py and the imports below. @@ -36,6 +46,7 @@ AUTHOR and DATE: 04/04/2022: Ricardo M. Campos, first version. + 07/18/2022: Ricardo M. Campos, SENTINEL-3B included. Longitude standard checked. PERSON OF CONTACT: Ricardo M Campos: ricardo.campos@noaa.gov @@ -71,15 +82,15 @@ # Satellite missions available at AODN dataset, pick one as this code runs one satellite at a time! s=np.int(sys.argv[1]) # argument satellite ID for satellite mission selection. s=0 is JASON3, s=1 is JASON2 etc. See list below. -sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX']) -sname=np.array(['JASON-3','JASON-2','CRYOSAT-2','JASON-1','HY-2','SARAL','SENTINEL-3A','ENVISAT','ERS-1','ERS-2','GEOSAT','GFO','TOPEX']) +sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX','SENTINEL3B']) +sname=np.array(['JASON-3','JASON-2','CRYOSAT-2','JASON-1','HY-2','SARAL','SENTINEL-3A','ENVISAT','ERS-1','ERS-2','GEOSAT','GFO','TOPEX','SENTINEL-3B']) # Quality Control parameters max_swh_rms = 1.5 # Max RMS of the band significant wave height max_sig0_rms = 0.8 # Max RMS of the backscatter coefficient max_swh_qc = 2.0 # Max SWH Ku band quality control hsmax=20.; wspmax=60. -min_swh_numval = np.array([17,17,17,17,17,17,17,17,17,17,-inf,3,7]) +min_swh_numval = np.array([17,17,17,17,17,17,17,17,17,17,-inf,3,7,17]) # weight function for pyresample def wf(pdist): diff --git a/validation/modelBuoy_collocation.py b/validation/modelBuoy_collocation.py index 48aa28994..6df98b40f 100755 --- a/validation/modelBuoy_collocation.py +++ b/validation/modelBuoy_collocation.py @@ -16,8 +16,6 @@ Additional information can be collocated as well, such as water depth, distance to the nearest coast, ocean names, forecast zones, and cyclone information. - -USAGE: This code is designed for ww3 hindcasts or forecast with consecutive cycles (overlapped time). The ww3 file(s) are not entered directly, but it is informed through a list, ww3list.txt (or any other @@ -28,23 +26,36 @@ (any value greated than zero), the program assumes it is a forecast data structure, i.e., the list contains consecutive cycles (each file is one cycle) interpreted as another dimension. - WW3 netcdf results for point output tables (tab) is utilized. - It uses two public buoy databases, NDBC and Copernicus, - which (at least one) must have been previously downloaded. See - get_buoydata_copernicus.sh and retrieve_ndbc_nc.py + +USAGE: + WW3 netcdf results with point output tables (tab) is utilized. + For the observations, it uses two public buoy databases, + NDBC and Copernicus, which (at least one) must have been previously + downloaded. See get_buoydata_copernicus.sh and retrieve_ndbc_nc.py Users must edit ndbcp and copernp paths below. Users should confirm the buoys' names at the "f=nc.Dataset" lines below. - Python code can be run directly, at least one argument is needed, with - the list of ww3 files (or one single file). - In addition to ww3list.txt and forecast data-shape, users and enter + Python code can be run directly. At least one argument is needed, with + the list name of ww3 files (or one single file). + The second argument (optional) is an integer with any value greated than + zero for the program to assume it is a forecast data. + In addition to ww3list.txt and forecast data-shape, users can enter two extra arguments, gridIndo and CycloneMap, generated by - prepGridMask.py and procyclmap.py, where the information for the buoy's + prepGridMask.py and procyclmap.py, where the information of buoy's position (nearest grid point) will be extracted and included in the output netcdf file. + Example (from linux terminal command line): + multiple forecast data files: + nohup python3 modelBuoy_collocation.py ww3list.gfs-d36.GSE1.5.txt 2 gridInfo.nc CycloneMap_2021.nc >> nohup_modelBuoy_collocation.out 2>&1 & + multiple hindcast data files: + nohup python3 modelBuoy_collocation.py ww3list.gfs-d36.GSE1.5.txt 0 gridInfo.nc CycloneMap_2021.nc >> nohup_modelBuoy_collocation.out 2>&1 & OUTPUT: netcdf file WW3.Buoy*.nc containing matchups of buoy and ww3 data, for the stations (lat/lon) where both data sources are available. + for the examples above, the program will select the tag gfs-d36.GSE1.5 to + name the output file as WW3.Buoy.gfs-d36.GSE1.5_2021092400to2021102400.nc + containing WW3.Buoy, the tag to identify the simulation, and start + and final date. DEPENDENCIES: See dependencies.py and the imports below. @@ -58,7 +69,7 @@ with water depth, distance to coast, ocean names, forecast areas, and cyclone information. 06/30/2022: Ricardo M. Campos, including option of using a forecast - data structure, where each file name on the ww3list is taken as a + data structure, where each file name in the ww3list is taken as a forecast cycle. PERSON OF CONTACT: diff --git a/validation/modelSat_collocation.py b/validation/modelSat_collocation.py index 2ce381281..c3a88246d 100755 --- a/validation/modelSat_collocation.py +++ b/validation/modelSat_collocation.py @@ -7,6 +7,7 @@ VERSION AND LAST UPDATE: v1.0 05/10/2022 v1.1 06/30/2022 + v1.2 07/20/2022 PURPOSE: Collocation/pairing ww3 field output results with altimeters. @@ -15,8 +16,6 @@ Additional information is collocated as well, such as water depth, distance to the nearest coast, ocean names, forecast zones, satellite names, and cyclone information. - -USAGE: This code is designed for ww3 hindcasts or forecast with consecutive cycles (overlapped time). The ww3 file(s) are not entered directly, but it is informed through a list, ww3list.txt (or any other @@ -24,12 +23,13 @@ in the list, ww3 results will be appended, depending on the data structure selected (hindcast or forecast). The default is hindcast, so arrays are directly appended in time. By entering a second argument - (any value greated than zero), the program assumes it is a forecast data + (any integer greated than zero), the program assumes it is a forecast data structure, i.e., the list contains consecutive cycles (each file is one cycle). In this case, another output variable is included in the final netcdf file, 'cycle', with the time of the nowcast/cycle. By - calculating time-cycle, you can obtain the forecast range (lead time) - in seconds. + calculating time-cycle, you can obtain the forecast lead time in seconds. + +USAGE: Four information must be entered by the user: - GridInfo netcdf file (generated by prepGridMask.py), see and edit gridinfo variable below, with path+name. @@ -45,12 +45,20 @@ must be the same as GridInfo, Cyclone map, and gridded satellite data, so it is recommended to run the following codes to prepare that: prepGridMask.py, procyclmap.py, and gridSatGlobal_Altimeter.py. + Note that GridInfo and CycloneMap information (names and paths) are fixed + and must be editted in the code below. + Example (from linux terminal command line): + nohup python3 modelSat_collocation.py ww3list.gfs-d36.GSE1.5.txt 2 >> nohup_modelSat_collocation.out 2>&1 & OUTPUT: netcdf file WW3.Altimeter_*.nc containing the matchups of WAVEWATCHIII and altimeter data for the same positions, plus the water depth, distance to the nearest coast, ocean names, forecast zones, satellite names, and cyclone information. + for the example above, the program will select the tag gfs-d36.GSE1.5 to + name the output file as WW3.Altimeter.gfs-d36.GSE1.5_2021092400to2021102400.nc + containing WW3.Altimeter, the tag to identify the simulation, and start + and final date. DEPENDENCIES: See dependencies.py and the imports below. @@ -64,6 +72,7 @@ 06/30/2022: Ricardo M. Campos, including option of using a forecast data structure, where each file name on the ww3list is taken as a forecast cycle. + 07/20/2022: Ricardo M. Campos, fix longitude standards among data. PERSON OF CONTACT: Ricardo M Campos: ricardo.campos@noaa.gov @@ -84,9 +93,9 @@ fnetcdf="NETCDF4" # Grid Info File -gridinfo='/work/noaa/marine/ricardo.campos/work/analysis/1preproc/mask/gridInfo_GEFSv12.nc' +gridinfo='/work2/noaa/marine/ricardo.campos/GFSv17dev/validation/2collocation/modelsat/gridInfo.nc' # Cyclone Map file (or files). You can enter CycloneMap_*.nc and all the files/years in that directory will be read. -cyclonemap='/work/noaa/marine/ricardo.campos/work/analysis/1preproc/cyclonemap/CycloneMap_2016.nc' +cyclonemap='/work2/noaa/marine/ricardo.campos/GFSv17dev/validation/1preproc/cyclonemap/CycloneMap_2021.nc' wlist=[]; ftag=''; forecastds=0 if len(sys.argv) >= 2: @@ -96,7 +105,7 @@ print(' Tag '+ftag) if len(sys.argv) >= 3: forecastds=np.int(sys.argv[2]) - if forecast>0: + if forecastds>0: print(' Forecast-type data structure') forecastds=np.int(forecastds+1) @@ -119,6 +128,11 @@ if np.array_equal(clat,mlat)==True & np.array_equal(clon,mlon)==True: print(" CycloneMap Ok.") + ind=np.where(mlon>180.) + if size(ind)>0: + mlon[mlon>180.] = mlon[mlon>180.]-360. + clon[clon>180.] = clon[clon>180.]-360. + del ind else: sys.exit(' Error: Cyclone grid and Mask grid are different.') @@ -133,7 +147,7 @@ except: sys.exit(' Could not open the list with satellite information AltimeterGridded_*.nc, resulted from gridSatGlobal_Altimeter.py') else: - sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX']) + sdname=np.array(['JASON3','JASON2','CRYOSAT2','JASON1','HY2','SARAL','SENTINEL3A','ENVISAT','ERS1','ERS2','GEOSAT','GFO','TOPEX','SENTINEL3B']) slat=[];slon=[];swnd=[];shs=[];stime=[];sid=[] for i in range(0,size(slist)): f=nc.Dataset(slist[i]) @@ -143,7 +157,13 @@ shs=np.append(shs,np.array(f.variables['hskcal'][:])) stime=np.append(stime,np.array(f.variables['stime'][:])) f.close(); del f - sid=np.append(sid,np.zeros(stime.shape[0],'int')+np.int(np.where(np.str(slist[i]).split('_')[1].split('.')[0] == sdname)[0][0])) + if np.str(slist[i]).split('_')[1].split('.')[0] in sdname: + sid=np.append(sid,np.zeros(stime.shape[0],'int')+np.int(np.where(np.str(slist[i]).split('_')[1].split('.')[0] == sdname)[0][0])) + else: + sys.exit(' Error: Problem identifying satellite mission from the file name: '+slist[i]) + +if size( np.where(slon>180.) )>0: + slon[slon>180.] = slon[slon>180.]-360. print(" Satellite Data Ok.") @@ -178,6 +198,9 @@ else: print(" Ok "+np.str(wlist[i])) wlon=np.array(f.variables['longitude'][:]); wlat=np.array(f.variables['latitude'][:]) + if size( np.where(wlon>180.) )>0: + wlon[wlon>180.] = wlon[wlon>180.]-360. + if np.array_equal(wlat,mlat)==False | np.array_equal(wlon,mlon)==False: sys.exit(' Error: Cyclone grid and Mask grid are different.') diff --git a/validation/mvalstats.py b/validation/mvalstats.py index 0ffd5442e..4b1ab58a9 100755 --- a/validation/mvalstats.py +++ b/validation/mvalstats.py @@ -21,7 +21,8 @@ functions smrstat metrics - Explanation of each function is contained in the headers + Explanation of each function is contained in the headers, including + examples. OUTPUT: summary statistics values; and error metrics diff --git a/validation/prepGridMask.py b/validation/prepGridMask.py index 5d4739e48..8beca00f7 100755 --- a/validation/prepGridMask.py +++ b/validation/prepGridMask.py @@ -9,8 +9,8 @@ v1.1 05/03/2022 PURPOSE: - Create a grid mask to identify coastal points and open/deep water - points, based on water depth and distance to the coast. + This program creates a grid mask to identify coastal points and + open/deep water points, based on water depth and distance to the coast. This is useful for model validation against satellite data, where coastal areas should be excluded, as well as to run specific assessments comparing deep water with coastal areas. @@ -40,6 +40,9 @@ mindfc), as well as the prefix name for the outputs (figures and netcdf), gridn + Example (from linux terminal command line): + nohup python3 prepGridMask.py 2 >> nohup_prepGridMask.out 2>&1 & + OUTPUT: netcdf file gridInfo_*.nc containing: mask info identifying land, ocean points, and coastal points. diff --git a/validation/pvalstats.py b/validation/pvalstats.py index c11e5dfda..fb779f531 100755 --- a/validation/pvalstats.py +++ b/validation/pvalstats.py @@ -27,7 +27,8 @@ combinerrors pdf - Explanation of each function is contained in the headers + Explanation of each function is contained in the headers, including + examples. OUTPUT: png figures saved in the local directory or in the path given through tag/prefix.