updated concat script to be more automatic.

.github/workflows/gpcp.yml

@@ -26,4 +26,5 @@ jobs:
               python gpcp.py -y $yr >> update_log.txt
               python gpcp.py -y $yr -t monthly >> update_log.txt
 
-              bash gpcp_concat.sh -y $yr
+              bash gpcp_concat.sh -y $yr -v 1-3 -f day
+              bash gpcp_concat.sh -y $yr -v 3-2 -f mon

README.md

@@ -53,9 +53,9 @@ where `<frequency>` is `mon` or `day` and
 `<version>` is `v1-2`,`v3-2` for daily data and `v2-3`, `v3-2` for monthly.
 
 Files are in netcdf4 format and have been concatenated into yearly files:
-- v1.2 daily: gpcp_v01r03_daily_YYYY.nc
+- v1.3 daily: gpcp_v1-3_day_YYYY.nc
 - v3.2 daily: gpcp_L3_V3.2_day_YYYY.nc
-- v2.3 monthly: gpcp_v02r03_monthly_YYYY.nc
+- v2.3 monthly: gpcp_v2-3_mon_YYYY.nc
 - v3.2 monthly: gpcp_L3_V3.2_mon_YYYY.nc
 
 If preliminary files are present (currently applies only to v2.3 monthly)

gpcp_concat.sh

@@ -20,16 +20,42 @@
 #
 #Date created: 07-05-2024
 
-# The year to concatenate:
-if [ "$1" == "-y" ]; then
-    yr=$2
-    echo "The year is $yr"
-else
-    echo "Usage: $0 -y <year>"
+# Initialize variables
+yr=""
+ver=""
+freq=""
+i=""
+
+# Parse command-line arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -y) yr="$2"; shift ;;
+        -v) ver="$2"; shift ;;
+        -f) freq="$2"; shift ;;
+        *) echo "Usage: $0 -y <year> -v <version> -f <frequency>"
+           echo "<version> can be 1-3, 2-3, or 3-2"
+           echo "<frequency> can be day or mon"
+           exit 1 ;;
+    esac
+    shift
+done
+
+# Check if all required arguments are provided
+if [[ -z "$yr" || -z "$ver" || -z "$freq" ]]; then
+    echo "Usage: $0 -y <year> -v <version> -f <frequency>"
+    echo "<version> can be 1-3, 2-3, or 3-2"
+    echo "<frequency> can be day or mon"
+    exit 1
 fi
 
-root_dir="/g/data/ia39/aus-ref-clim-data-nci/gpcp/data/day/v1-3/"
-outdir="/g/data/ia39/aus-ref-clim-data-nci/gpcp/data/day_concat/"
+# Output the values
+echo "The year is $yr"
+echo "The version is $ver"
+echo "The frequency is $freq"
+
+
+root_dir="/g/data/ia39/aus-ref-clim-data-nci/gpcp/data/$freq/v$ver/tmp/$yr/"
+outdir="/g/data/ia39/aus-ref-clim-data-nci/gpcp/data/$freq/v$ver/"
 
 if [ -d "$outdir" ]; then
     echo "Directory $outdir exists."
@@ -38,8 +64,8 @@ else
     mkdir -p "$outdir" || { echo "Failed to create directory $outdir" >&2; exit 1; }
 fi
 
-f_in=$root_dir/$yr/gpcp_v01r03_daily_d$yr*.nc
-f_out=$outdir/gpcp_v01r03_daily_$yr.nc
+f_in=$root_dir/*$yr*.nc*
+f_out=$outdir/gpcp_v${ver}_${freq}_${yr}.nc
 
 echo "Concatenating $yr"
 
@@ -50,17 +76,25 @@ else
     echo "File doesn't exist, proceeding"
 fi
 
+if [[ "$freq" == "mon" ]]; then
+    i=12
+else
+    i=31  # or some other default value if freq is not "mon"
+fi
+
+
 # Concatenate all files from a day together, save as a tmp.nc file
 cdo --silent --no_history -L -s -f nc4c -z zip_4 cat $f_in $outdir/tmp.nc
 # Re-chunk the tmp.nc file
 echo "Concatenating complete, now re-chunking...."
-ncks --cnk_dmn time,31 --cnk_dmn lat,600 --cnk_dmn lon,600 $outdir/tmp.nc $f_out
+ncks --cnk_dmn time,$i --cnk_dmn lat,600 --cnk_dmn lon,600 $outdir/tmp.nc $f_out
+#ncks --cnk_dmn time,31 --cnk_dmn lat,600 --cnk_dmn lon,600 $outdir/tmp.nc $f_out
 rm $outdir/tmp.nc
 # rewrite history attribute
 hist="downloaded original files from 
-    https://www.ncei.noaa.gov/data/global-precipitation-climatology-project-gpcp-{tstep}/access/
+    https://measures.gesdisc.eosdis.nasa.gov/data/GPCP/GPCPMON.2.3/
     Using cdo to concatenate files, and nco to modify chunks: 
     cdo --silent --no_warnings --no_history -L -s -f nc4c -z zip_4 cat $f_in $outdir/tmp.nc
-    ncks --cnk_dmn time,48 --cnk_dmn lat,600 --cnk_dmn lon,600 tmp.nc $f_out"
+    ncks --cnk_dmn time,$i --cnk_dmn lat,600 --cnk_dmn lon,600 tmp.nc $f_out"
 # Add what we've done into the history attribute in the file. 
 ncatted -h -O -a history,global,o,c,"$hist" ${f_out}