Add capability to do unit conversations to capgen (#504)

This PR adds automatic conversions for supported unit and type transforms.
Also, included is a new ccpp variable property to indicate a scheme variables vertical orientation vertical, top_at_one.
As default, top_at_one is set to .false.(GFS ordering convention).
Adding top_at_one = .true. to a variable in a schemes metadata file will trigger automatic array flipping:

Addresses #329 and #403

Conversions supported:
https://github.com/NCAR/ccpp-framework/blob/main/scripts/conversion_tools/unit_conversion.py

---------

Co-authored-by: dustinswales <dswales@ucar.edu>
Co-authored-by: Grant Firl <grantf@ucar.edu>
Co-authored-by: Dom Heinzeller <dom.heinzeller@icloud.com>

.github/workflows/prebuild.yaml

@@ -0,0 +1,47 @@
+name: ccpp-prebuild
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unit-tests:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: ccpp-prebuild unit tests
+      run: |
+        export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools
+        cd test_prebuild
+        python3 test_metadata_parser.py
+        python3 test_mkstatic.py
+    - name: ccpp-prebuild blocked data tests
+      run: |
+        cd test_prebuild/test_blocked_data
+        python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build
+        cd build
+        cmake ..
+        make
+        ./test_blocked_data.x

CMakeLists.txt

@@ -7,7 +7,7 @@ project(ccpp_framework
 #------------------------------------------------------------------------------
 # Set package definitions
 set(PACKAGE "ccpp-framework")
-set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber")
+set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Dustin Swales" "Courtney Peverley")
 string(TIMESTAMP YEAR "%Y")
 
 #------------------------------------------------------------------------------

scripts/ccpp_datafile.py

@@ -654,7 +654,7 @@ def _new_var_entry(parent, var, full_entry=True):
                           "diagnostic_name", "diagnostic_name_fixed",
                           "kind", "persistence", "polymorphic", "protected",
                           "state_variable", "type", "units", "molar_mass",
-                          "advected"])
+                          "advected", "top_at_one"])
         prop_list.extend(Var.constituent_property_names())
     # end if
     ventry = ET.SubElement(parent, "var")

scripts/metavar.py

@@ -204,6 +204,8 @@ class Var:
                                      default_in='.true.'),
                     VariableProperty('polymorphic', bool, optional_in=True,
                                      default_in=False),
+                    VariableProperty('top_at_one', bool, optional_in=True,
+                                     default_in=False),
                     VariableProperty('target', bool, optional_in=True,
                                      default_in=False)]
 
@@ -376,15 +378,17 @@ def compatible(self, other, run_env):
         sunits = self.get_prop_value('units')
         sstd_name = self.get_prop_value('standard_name')
         sloc_name = self.get_prop_value('local_name')
+        stopp = self.get_prop_value('top_at_one')
         sdims = self.get_dimensions()
         otype = other.get_prop_value('type')
         okind = other.get_prop_value('kind')
         ounits = other.get_prop_value('units')
         ostd_name = other.get_prop_value('standard_name')
         oloc_name = other.get_prop_value('local_name')
+        otopp = other.get_prop_value('top_at_one')
         odims = other.get_dimensions()
-        compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name,
-                              ostd_name, otype, okind, ounits, odims, oloc_name,
+        compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name, stopp,
+                              ostd_name, otype, okind, ounits, odims, oloc_name, otopp,
                               run_env,
                               v1_context=self.context, v2_context=other.context)
         if (not compat) and (run_env.logger is not None):

scripts/suite_objects.py

@@ -20,6 +20,7 @@
 from parse_tools import init_log, set_log_to_null
 from var_props import is_horizontal_dimension, find_horizontal_dimension
 from var_props import find_vertical_dimension
+from var_props import VarCompatObj
 
 # pylint: disable=too-many-lines
 
@@ -816,7 +817,7 @@ def find_variable(self, standard_name=None, source_var=None,
         # end if
         return found_var
 
-    def match_variable(self, var, vstdname=None, vdims=None):
+    def match_variable(self, var, run_env):
         """Try to find a source for <var> in this SuiteObject's dictionary
         tree. Several items are returned:
         found_var: True if a match was found
@@ -825,21 +826,19 @@ def match_variable(self, var, vstdname=None, vdims=None):
         missing_vert: Vertical dim in parent but not in <var>
         perm: Permutation (XXgoldyXX: Not yet implemented)
         """
-        if vstdname is None:
-            vstdname = var.get_prop_value('standard_name')
-        # end if
-        if vdims is None:
-            vdims = var.get_dimensions()
-        # end if
+        vstdname = var.get_prop_value('standard_name')
+        vdims    = var.get_dimensions()
         if (not vdims) and self.run_phase():
             vmatch = VarDictionary.loop_var_match(vstdname)
         else:
             vmatch = None
         # end if
+
         found_var = False
         missing_vert = None
         new_vdims = list()
         var_vdim = var.has_vertical_dimension(dims=vdims)
+        compat_obj = None
         # Does this variable exist in the calling tree?
         dict_var = self.find_variable(source_var=var, any_scope=True)
         if dict_var is None:
@@ -872,6 +871,10 @@ def match_variable(self, var, vstdname=None, vdims=None):
                 new_vdims = list()
                 new_dict_dims = dict_dims
                 match = True
+            # Create compatability object, containing any necessary forward/reverse 
+            # transforms from <var> and <dict_var>
+            compat_obj = var.compatible(dict_var, run_env)
+
             # end if
             # Add the variable to the parent call tree
             if dict_dims == new_dict_dims:
@@ -894,7 +897,7 @@ def match_variable(self, var, vstdname=None, vdims=None):
                 # end if
             # end if
         # end if
-        return found_var, var_vdim, new_vdims, missing_vert
+        return found_var, var_vdim, new_vdims, missing_vert, compat_obj
 
     def in_process_split(self):
         """Find out if we are in a process-split region"""
@@ -1077,6 +1080,8 @@ def __init__(self, scheme_xml, context, parent, run_env):
         self.__lib = scheme_xml.get('lib', None)
         self.__has_vertical_dimension = False
         self.__group = None
+        self.__forward_transforms = list()
+        self.__reverse_transforms = list()
         super().__init__(name, context, parent, run_env, active_call_list=True)
 
     def update_group_call_list_variable(self, var):
@@ -1144,8 +1149,8 @@ def analyze(self, phase, group, scheme_library, suite_vars, level):
             def_val = var.get_prop_value('default_value')
             vdims = var.get_dimensions()
             vintent = var.get_prop_value('intent')
-            args = self.match_variable(var, vstdname=vstdname, vdims=vdims)
-            found, vert_dim, new_dims, missing_vert = args
+            args = self.match_variable(var, self.run_env)
+            found, vert_dim, new_dims, missing_vert, compat_obj = args
             if found:
                 if not self.has_vertical_dim:
                     self.__has_vertical_dimension = vert_dim is not None
@@ -1200,6 +1205,12 @@ def analyze(self, phase, group, scheme_library, suite_vars, level):
                                                   vstdname))
                 # end if
             # end if
+            # Are there any forward/reverse transforms for this variable?
+            if compat_obj is not None and (compat_obj.has_vert_transforms or
+                                           compat_obj.has_unit_transforms or
+                                           compat_obj.has_kind_transforms):
+                self.add_var_transform(var, compat_obj, vert_dim)
+
         # end for
         if self.needs_vertical is not None:
             self.parent.add_part(self, replace=True) # Should add a vloop
@@ -1211,6 +1222,81 @@ def analyze(self, phase, group, scheme_library, suite_vars, level):
         # end if
         return scheme_mods
 
+    def add_var_transform(self, var, compat_obj, vert_dim):
+        """Register any variable transformation needed by <var> for this Scheme.
+        For any transformation identified in <compat_obj>, create dummy variable
+        from <var> to perform the transformation. Determine the indices needed
+        for the transform and save for use during write stage"""
+
+        # Add dummy variable (<var>_local) needed for transformation.
+        dummy = var.clone(var.get_prop_value('local_name')+'_local')
+        self.__group.manage_variable(dummy)
+
+        # Create indices (default) for transform.
+        lindices   = [':']*var.get_rank()
+        rindices   = [':']*var.get_rank()
+
+        # If needed, modify vertical dimension for vertical orientation flipping
+        _, vdim    = find_vertical_dimension(var.get_dimensions())
+        vdim_name  = vert_dim.split(':')[-1]
+        group_vvar = self.__group.call_list.find_variable(vdim_name)
+        vname      = group_vvar.get_prop_value('local_name')
+        lindices[vdim] = '1:'+vname
+        rindices[vdim] = '1:'+vname
+        if compat_obj.has_vert_transforms:
+            rindices[vdim] = vname+':1:-1'
+
+        # If needed, modify horizontal dimension for loop substitution.
+        # NOT YET IMPLEMENTED
+        #hdim = find_horizontal_dimension(var.get_dimensions())
+        #if compat_obj.has_dim_transforms:
+
+        #
+        # Register any reverse (pre-Scheme) transforms.
+        #
+        if (var.get_prop_value('intent') != 'out'):
+            self.__reverse_transforms.append([dummy.get_prop_value('local_name'),
+                                              var.get_prop_value('local_name'),
+                                              rindices, lindices, compat_obj])
+
+        #
+        # Register any forward (post-Scheme) transforms.
+        #
+        if (var.get_prop_value('intent') != 'in'):
+            self.__forward_transforms.append([var.get_prop_value('local_name'),
+                                              dummy.get_prop_value('local_name'),
+                                              lindices, rindices, compat_obj])
+
+    def write_var_transform(self, var, dummy, rindices, lindices, compat_obj,
+                            outfile, indent, forward):
+        """Write variable transformation needed to call this Scheme in <outfile>.
+        <var> is the varaible that needs transformation before and after calling Scheme.
+        <dummy> is the local variable needed for the transformation..
+        <lindices> are the LHS indices of <dummy> for reverse transforms (before Scheme).
+        <rindices> are the RHS indices of <var>   for reverse transforms (before Scheme).
+        <lindices> are the LHS indices of <var>   for forward transforms (after  Scheme).
+        <rindices> are the RHS indices of <dummy> for forward transforms (after  Scheme).
+        """
+        #
+        # Write reverse (pre-Scheme) transform.
+        #
+        if not forward:
+            # dummy(lindices) = var(rindices)
+            stmt = compat_obj.reverse_transform(lvar_lname=dummy,
+                                                rvar_lname=var,
+                                                lvar_indices=lindices,
+                                                rvar_indices=rindices)
+        #
+        # Write forward (post-Scheme) transform.
+        #
+        else:
+            # var(lindices) = dummy(rindices)
+            stmt = compat_obj.forward_transform(lvar_lname=var,
+                                                rvar_lname=dummy,
+                                                lvar_indices=rindices,
+                                                rvar_indices=lindices)
+        outfile.write(stmt, indent+1)
+
     def write(self, outfile, errcode, indent):
         # Unused arguments are for consistent write interface
         # pylint: disable=unused-argument
@@ -1222,9 +1308,18 @@ def write(self, outfile, errcode, indent):
         my_args = self.call_list.call_string(cldicts=cldicts,
                                              is_func_call=True,
                                              subname=self.subroutine_name)
-        stmt = 'call {}({})'
+
         outfile.write('if ({} == 0) then'.format(errcode), indent)
+        # Write any reverse (pre-Scheme) transforms.
+        for (dummy, var, rindices, lindices, compat_obj) in self.__reverse_transforms:
+            tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, False)
+        # Write the scheme call.
+        stmt = 'call {}({})'
         outfile.write(stmt.format(self.subroutine_name, my_args), indent+1)
+        # Write any forward (post-Scheme) transforms.
+        for (var, dummy, lindices, rindices, compat_obj) in self.__forward_transforms:
+            tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, True)
+        #
         outfile.write('end if', indent)
 
     def schemes(self):