OM3 user scripts (#13)

Creation of users scripts for OM3, see COSIMA/access-om3#182

There are three processes here:

1. Per COSIMA/access-om3#81 initial conditions file for CICE are converted from symlink to files/hardlinks
2. CICE daily output in concatenated into one file per month
3. An intake esm datastore for the run is generated

---------

Co-authored-by: Dougie Squire <42455466+dougiesquire@users.noreply.github.com>
Co-authored-by: dougiesquire <dougiesquire@gmail.com>

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+# This workflow will install dependencies, run tests and the black-formatter
+
+name: om3_scripts
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Code formatting
+      uses: psf/black@stable
+      with:
+        options: "--check --verbose --diff"
+
+  test:
+    needs: formatting
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install nco
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install nco
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r test/test_requirements.txt
+    - name: Test with pytest
+      run: |
+        python -m pytest -m "not broken"
+    - name: Upload coverage reports to Codecov with GitHub Action
+      uses: codecov/codecov-action@v3
+      env:
+        CODEOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        entry: black
+        language: system
+        types: [python]

README.md

@@ -13,3 +13,5 @@ When contributing scripts to this repo, please follow these guidelines where it
 - [x] Include documentation with your script. For example, a README explaining what the script is for, how to use it, what it can/can't do etc.
 - [x] Any outputs from your script should include metadata that allows someone in the future to track down the exact version of the script used to create that output. Any easy way to do this is to include as metadata the url and commit hash of the script on GitHub. See existing scripts for examples of how to do this.
 - [x] Consider including information about the dependencies or environment required to run your script, e.g. a conda-lock file describing your conda environment.
+
+Python scripts are formatted using the [black formatter](https://github.com/psf/black). This is enforced using a github action running on pull requests. You will need to have _pre-commit_ and _black_ installed in your python environment. It is included in the _conda/analysis_ enviornments in _hh5_, otherwise it can be installed through pip or conda. Then run `pre-commit install` once to ensure your commits have black run on them before committing.

data_stream_xml_generation/generate_xml_datm.py

@@ -48,7 +48,9 @@
 # Add metadata
 metadata = SubElement(root, "metadata")
 SubElement(metadata, "File_type").text = "DATM xml file provides forcing data"
-SubElement(metadata, "date_generated").text = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+SubElement(metadata, "date_generated").text = datetime.now().strftime(
+    "%Y-%m-%d %H:%M:%S"
+)
 SubElement(metadata, "history").text = metadata_info
 
 # Define the stream info names and corresponding var names
@@ -96,15 +98,17 @@
     datafiles = SubElement(stream_info, "datafiles")
     datavars = SubElement(stream_info, "datavars")
 
-    if stream_name in ([
-        "CORE_IAF_JRA55do.PRSN",
-        "CORE_IAF_JRA55do.PRRN",
-        "CORE_IAF_JRA55do.LWDN",
-        "CORE_IAF_JRA55do.SWDN",
-    ]) and (year_first != year_last): 
-        SubElement(
-            stream_info, "offset"
-        ).text = "-5400"  # shift back 1.5hr to match RYF
+    if stream_name in (
+        [
+            "CORE_IAF_JRA55do.PRSN",
+            "CORE_IAF_JRA55do.PRRN",
+            "CORE_IAF_JRA55do.LWDN",
+            "CORE_IAF_JRA55do.SWDN",
+        ]
+    ) and (year_first != year_last):
+        SubElement(stream_info, "offset").text = (
+            "-5400"  # shift back 1.5hr to match RYF
+        )
     else:
         SubElement(stream_info, "offset").text = "0"
 

data_stream_xml_generation/generate_xml_drof.py

@@ -46,7 +46,9 @@
 # Add metadata
 metadata = SubElement(root, "metadata")
 SubElement(metadata, "File_type").text = "DROF xml file provides river runoff data"
-SubElement(metadata, "date_generated").text = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+SubElement(metadata, "date_generated").text = datetime.now().strftime(
+    "%Y-%m-%d %H:%M:%S"
+)
 SubElement(metadata, "history").text = metadata_info
 
 # Define the stream info names and corresponding var names
@@ -77,11 +79,11 @@
     datavars = SubElement(stream_info, "datavars")
 
     if year_first == year_last:
-        SubElement(stream_info, "offset").text = "0" #RYF starts at midnight
-    else:    
-        SubElement(
-        stream_info, "offset"
-    ).text = "-43200"  # shift backwards from noon to midnight to match RYF
+        SubElement(stream_info, "offset").text = "0"  # RYF starts at midnight
+    else:
+        SubElement(stream_info, "offset").text = (
+            "-43200"  # shift backwards from noon to midnight to match RYF
+        )
 
     var_element = SubElement(datavars, "var")
     var_element.text = f"{var_prefix} {var_suffix}"

payu_config/README.md

@@ -0,0 +1,15 @@
+This directory contains Payu setup and archive userscripts for ACCESS-OM3.
+
+To use, add these to config.yaml:
+
+```yaml
+userscripts:
+    setup: /usr/bin/bash /g/data/vk83/apps/om3-scripts/payu_config/setup.sh
+    archive: /usr/bin/bash /g/data/vk83/apps/om3-scripts/payu_config/archive.sh
+
+modules:
+    use: 
+        - /g/data/hh5/public/modules
+    load: 
+        - conda/analysis
+```

payu_config/archive.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/bash -i
+
+source $(dirname "$0")/archive_scripts/archive_cice_restarts.sh
+source $(dirname "$0")/archive_scripts/concat_ice_daily.sh
+python3 $(dirname "$0")/archive_scripts/build_intake_ds.py

payu_config/archive_scripts/archive_cice_restarts.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/bash -i
+# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
+# SPDX-License-Identifier: Apache-2.0
+# clean up cice_restarts.sh
+
+if [ -f archive/output*/input/iced.1900-01-01-10800.nc ] 
+then
+rm archive/output*/input/iced.1900-01-01-10800.nc
+return 0
+fi
+
+latest_o=$(ls -drv archive/output*[0-9] | head -1)
+
+#initial restart was copied from the previous run
+ic_restart=$(ls -dv $latest_o/access-om3.cice.r.* | head -1)
+
+if [ -f $latest_o/access-om3.cice.r.* ]
+then
+rm $latest_o/access-om3.cice.r.*
+fi

payu_config/archive_scripts/build_intake_ds.py

@@ -0,0 +1,72 @@
+#!python3
+# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
+# SPDX-License-Identifier: Apache-2.0
+# modules:
+# use:
+#     - /g/data/hh5/public/modules
+# load:
+#     - conda/analysis
+
+from access_nri_intake.source import builders
+import os
+import sys
+from pathlib import Path
+from warnings import warn
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+
+path_root = Path(__file__).parents[2]
+sys.path.append(str(path_root))
+
+from scripts_common import get_provenance_metadata, md5sum
+
+METADATA_FILENAME = "metadata.yaml"
+UUID_FIELD = "experiment_uuid"
+ARCHIVE_PATH = "archive"
+
+
+def description():
+
+    # Get experiment uuid
+    # follows https://github.com/payu-org/payu/blob/ef55e93fe23fcde19024479c0dc4112dcdf6603f/payu/metadata.py#L90
+    metadata_filename = Path(METADATA_FILENAME)
+    if metadata_filename.exists():
+        metadata = CommentedMap()
+        metadata = YAML().load(metadata_filename)
+        uuid = metadata.get(UUID_FIELD, None)
+    else:
+        warn(f"{METADATA_FILENAME} not found in archive folder")
+        uuid = False
+
+    # Check git status of this .py file
+    this_file = os.path.normpath(__file__)
+
+    runcmd = f"python3 {os.path.basename(this_file)}"
+
+    # Get string "Created using $file: $command"
+    provenance = get_provenance_metadata(this_file, runcmd)
+
+    if uuid:
+        description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})"
+    else:
+        description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})"
+
+    return description
+
+
+if __name__ == "__main__":
+
+    builder = builders.AccessOm3Builder(path=ARCHIVE_PATH)
+
+    print("LOG: Building intake-esm datastore")
+
+    builder.build()
+
+    # Log invalid assets
+    builder.invalid_assets
+
+    builder.save(
+        name="intake_esm_ds",
+        description=description(),
+        directory=ARCHIVE_PATH,
+    )

payu_config/archive_scripts/concat_ice_daily.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/bash -i
+# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
+# SPDX-License-Identifier: Apache-2.0
+# 
+# Concatenate sea-ice daily output from access-om3
+# this was written assuming it would be used as a payu "userscript" at the "archive" stage, but alternatively a path to an "archive" directory can be provided
+# Script inspired from https://github.com/COSIMA/1deg_jra55_ryf/blob/master/sync_data.sh#L87-L108
+#
+# This script uses "ncrcat". Load this through either 'module use /g/data/vk83/modules; module load payu' or 'module load nco'.
+
+shopt -s extglob
+
+Help()
+{
+   # Display Help
+   echo "Concatenante daily history output from the (sea) ice model to a single file"
+   echo
+   echo "Syntax: scriptTemplate [-h|d DIRECTORY]"
+   echo "options:"
+   echo "h     Print this Help."
+   echo "d     Process "name" directory rather than latest output in archive folder."
+   echo
+}
+
+# Get the options
+while getopts ":hd:" option; do
+   case $option in
+      h) # display Help
+         Help
+         exit;;
+      d) # Enter a directory
+         out_dir=$OPTARG
+         if [ ! -d $out_dir ]; then 
+            echo $out_dir Does not exist
+            exit
+         fi;;  
+     \?) # Invalid option
+         echo "Error: Invalid option"
+         exit;;
+   esac
+done
+
+#If no directory option provided , then use latest
+if [ -z $out_dir ]; then 
+    #latest output dir only
+    out_dir=$(ls -drv archive/output*[0-9] | head -1) 
+fi
+
+if ! command -v -- "ncrcat" > /dev/null 2>&1; then
+    echo "ncrcat not available, trying module load nco"
+    module load nco
+fi
+
+for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do
+   #if the 1st and the 28th of that month exists, then assume its a whole month and concatenate
+   if [ -f $f ] && [ -f ${f/-01.nc/-28.nc} ]; then 
+
+      output_f=${f/-01.nc/.nc} #remove day in date string
+
+      #concat daily files for this month
+      echo LOG: concatenating daily sea ice files in $out_dir
+      echo doing ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f
+      ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f
+
+      if [[ $? == 0 ]]; then 
+         rm ${f/-01.nc/-??.nc} #delete individual dailys on success
+      fi
+   fi
+done

payu_config/setup.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/bash -i
+
+source $(dirname "$0")/setup_scripts/setup_cice_restarts.sh

payu_config/setup_scripts/setup_cice_restarts.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# patch for https://github.com/open-mpi/ompi/issues/12141
+if ! [ -f work/access-om3.cice.r.* ]
+then 
+# no restart files yet, use initial conditions
+IC=$(readlink work/input/iced.1900-01-01-10800.nc)
+rm work/input/iced.1900-01-01-10800.nc
+cp $IC work/input/iced.1900-01-01-10800.nc
+else
+# change restart symlink to hardlink
+RESTART=$(echo work/access-om3.cice.r.*)
+ln -f $(readlink $RESTART) $RESTART
+fi