generalise support for cice filenames (#22)

* generalise support for cice filenames

---------

Co-authored-by: minghangli-uni <minghang.li1@anu.edu.au>
Co-authored-by: minghang.li <minghangl1101@gmail.com>

.github/workflows/ci.yml

@@ -34,17 +34,18 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install nco
+    - name: Install sys dependencies
       run: |
         sudo apt-get update
         sudo apt-get -y install nco
+        sudo apt-get -y install ncal
     - name: Install python dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r test/test_requirements.txt
     - name: Test with pytest
       run: |
-        python -m pytest -m "not broken"
+        python -m pytest -m "not broken" -v -s
     - name: Upload coverage reports to Codecov with GitHub Action
       uses: codecov/codecov-action@v3
       env:

payu_config/archive_scripts/archive_cice_restarts.sh

@@ -3,18 +3,16 @@
 # SPDX-License-Identifier: Apache-2.0
 # clean up cice_restarts.sh
 
-if [ -f archive/output*/INPUT/iced.1900-01-01-10800.nc ]
-then
-rm archive/output*/INPUT/iced.1900-01-01-10800.nc
-return 0
+if [ -f archive/output*/INPUT/iced.1900-01-01-10800.nc ]; then
+    rm archive/output*/INPUT/iced.1900-01-01-10800.nc
+    return 0
 fi
 
 latest_o=$(ls -drv archive/output*[0-9] | head -1)
 
 #initial restart was copied from the previous run
 ic_restart=$(ls -dv $latest_o/access-om3.cice.r.* | head -1)
 
-if [ -f $latest_o/access-om3.cice.r.* ]
-then
-rm $latest_o/access-om3.cice.r.*
+if [ -f $latest_o/access-om3.cice.r.* ]; then
+    rm $latest_o/access-om3.cice.r.*
 fi

payu_config/archive_scripts/concat_ice_daily.sh

@@ -51,12 +51,22 @@ if ! command -v -- "ncrcat" > /dev/null 2>&1; then
     module load nco
 fi
 
-for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do
-   #if the 1st and the 28th of that month exists, then assume its a whole month and concatenate
-   if [ -f $f ] && [ -f ${f/-01.nc/-28.nc} ]; then 
+for f in $out_dir/access-om3.cice*.????-??-01.nc ; do
+   # extract the year and month from existing files
+   year_month=$(echo "$f" | sed -E "s/.*\.([0-9]{4}-[0-9]{2})-[0-9]{2}\.nc/\1/")
+   year=$(echo "$year_month" | cut -d- -f1)
+   month=$(echo "$year_month" | cut -d- -f2)
+
+   # calculate the expected end day for the given year and month
+   end_day=$(cal $month $year | awk "NF {end_day=\$NF}; END {print end_day}")
+   end_day_file=${f/-01.nc/-$end_day.nc}
+
+   output_f=${f/-01.nc/.nc} #remove day in date string
+
+   if [ -f $output_f ]; then
+      echo WARN: $output_f exists, skipping concatenation daily sea ice files
+   elif [ -f $f ] && [ -f $end_day_file ] ; then
 
-      output_f=${f/-01.nc/.nc} #remove day in date string
-
       #concat daily files for this month
       echo LOG: concatenating daily sea ice files in $out_dir
       echo doing ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f
@@ -65,5 +75,7 @@ for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do
       if [[ $? == 0 ]]; then 
          rm ${f/-01.nc/-??.nc} #delete individual dailys on success
       fi
+   else
+      echo "LOG: skipping concatenating daily sea ice files (incomplete month)"
    fi
 done

payu_config/setup_scripts/setup_cice_restarts.sh

@@ -1,13 +1,12 @@
 #!/bin/bash
 # patch for https://github.com/open-mpi/ompi/issues/12141
-if ! [ -f work/access-om3.cice.r.* ]
-then 
-# no restart files yet, use initial conditions
-IC=$(readlink work/INPUT/iced.1900-01-01-10800.nc)
-rm work/INPUT/iced.1900-01-01-10800.nc
-cp $IC work/INPUT/iced.1900-01-01-10800.nc
+if ! [ -f work/access-om3.cice.r.* ]; then
+    # no restart files yet, use initial conditions
+    IC=$(readlink work/INPUT/iced.1900-01-01-10800.nc)
+    rm work/INPUT/iced.1900-01-01-10800.nc
+    cp $IC work/INPUT/iced.1900-01-01-10800.nc
 else
-# change restart symlink to hardlink
-RESTART=$(echo work/access-om3.cice.r.*)
-ln -f $(readlink $RESTART) $RESTART
+    # change restart symlink to hardlink
+    RESTART=$(echo work/access-om3.cice.r.*)
+    ln -f $(readlink $RESTART) $RESTART
 fi

test/test_payu_conf/test_concat_ice_daily.py

@@ -21,8 +21,7 @@ def assert_f_not_exists(p):
         raise AssertionError("File exists and should not: %s" % str(p))
 
 
-@pytest.fixture
-def daily_files(request, tmp_path):
+def daily_files(dir_name, hist_base, ndays, tmp_path):
     """
     Make 365 days of fake data, and then write it into 365 files
 
@@ -31,12 +30,8 @@ def daily_files(request, tmp_path):
 
     """
 
-    if request.param[0] == "Default":
+    if dir_name == "Default":
         dir_name = "archive/output000"
-    else:
-        dir_name = str(request.param[0])
-
-    ndays = request.param[1]
 
     nx = 30
     ny = 50
@@ -58,7 +53,7 @@ def daily_files(request, tmp_path):
     # ds.time.encoding['dtype'] = 'float'
 
     out_dir = str(tmp_path) + "/" + dir_name + "/"
-    paths = [f"{out_dir}access-om3.cice.h.{str(t.values)[0:10]}.nc" for t in ds.time]
+    paths = [f"{out_dir}{hist_base}.{str(t.values)[0:10]}.nc" for t in ds.time]
     datasets = [ds.sel(time=slice(t, t)) for t in ds.time]
 
     makedirs(out_dir)
@@ -67,23 +62,30 @@ def daily_files(request, tmp_path):
     return paths
 
 
+@pytest.fixture(
+    params=["access-om3.cice.h", "access-om3.cice", "access-om3.cice.1day.mean"]
+)
+def hist_base(request):
+    return str(request.param)
+
+
 @pytest.mark.parametrize(
-    "daily_files, use_dir, nmonths",
+    "hist_dir, ndays, use_dir, nmonths",
     [
-        (("archive/output000", 365), False, 12),
-        (("archive/output999", 31), False, 1),
-        (("archive/output9999", 31), False, 1),
-        (("archive/output574", 365), True, 12),
+        ("Default", 365, False, 12),
+        ("archive/output999", 31, False, 1),
+        ("archive/output9999", 31, False, 1),
+        ("archive/output574", 365, True, 12),
     ],
-    indirect=["daily_files"],
 )  # run this test with a several folder names and lengths, provide the directory as an argument sometimes
-def test_true_case(daily_files, use_dir, nmonths, tmp_path):
+def test_true_case(hist_dir, ndays, use_dir, nmonths, hist_base, tmp_path):
     """
     Run the script to convert the daily data into monthly files, and check the monthly files and the daily files dont exist.
     """
 
+    daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path)
     chdir(tmp_path)
-    output_dir = Path(daily_files[0]).parents[0]
+    output_dir = Path(daily_paths[0]).parents[0]
 
     if not use_dir:  # default path
         run([run_str])
@@ -94,13 +96,13 @@ def test_true_case(daily_files, use_dir, nmonths, tmp_path):
                 run_str,
                 "-d",
                 output_dir,
-            ]
+            ],
         )
         expected_months = pd.date_range("2010-01-01", freq="ME", periods=nmonths + 1)
 
     # valid output filenames
     monthly_paths = [
-        f"{output_dir}/access-om3.cice.h.{str(t)[0:7]}.nc" for t in expected_months
+        f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months
     ]
 
     for p in monthly_paths[0:nmonths]:
@@ -109,30 +111,58 @@ def test_true_case(daily_files, use_dir, nmonths, tmp_path):
     for p in monthly_paths[nmonths]:
         assert_f_not_exists(p)
 
-    for p in daily_files:
+    for p in daily_paths:
         assert_f_not_exists(p)
 
 
-@pytest.mark.parametrize(
-    "daily_files", [("Default", 1), ("Default", 27)], indirect=True
-)
-def test_no_concat_case(daily_files, tmp_path):
+@pytest.mark.parametrize("hist_dir, ndays", [("Default", 1), ("Default", 30)])
+def test_incomplete_month(hist_dir, ndays, hist_base, tmp_path):
     """
     Run the script to convert the daily data into monthly files, with less than 28 days data, and check no things happen.
     """
 
+    daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path)
+
     chdir(tmp_path)
-    output_dir = Path(daily_files[0]).parents[0]
+    output_dir = Path(daily_paths[0]).parents[0]
 
     run([run_str])
     expected_months = pd.date_range("2010-01-01", freq="ME", periods=1)
 
     monthly_paths = [
-        f"{output_dir}/access-om3.cice.h.{str(t)[0:7]}.nc" for t in expected_months
+        f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months
     ]
 
-    for p in daily_files:
+    for p in daily_paths:
         assert_file_exists(p)
 
     for p in monthly_paths:
         assert_f_not_exists(p)
+
+
+@pytest.mark.parametrize("hist_dir, ndays", [("Default", 31), ("Default", 27)])
+def test_no_override(hist_dir, ndays, hist_base, tmp_path):
+    """
+    Run the script to convert the daily data into monthly files, but the output filename already exists, and check nothing happens.
+    """
+
+    daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path)
+
+    chdir(tmp_path)
+    output_dir = Path(daily_paths[0]).parents[0]
+
+    expected_months = pd.date_range("2010-01-01", freq="ME", periods=1)
+
+    monthly_paths = [
+        f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months
+    ]
+    for p in monthly_paths:
+        Path(p).touch()
+
+    run([run_str])
+
+    for p in daily_paths:
+        assert_file_exists(p)
+
+    for p in monthly_paths:
+        assert_file_exists(p)