add some changes to vcs

configs/components/oifs/oifs.yaml

@@ -1415,8 +1415,8 @@ prepcompute_recipe:
    - "modify_files"
    - "copy_files_to_work"
    - "report_missing_files"
+   - "add_vcs_info"
    # see https://github.com/esm-tools/esm_tools/discussions/774
-   # - "add_vcs_info"
    #- "check_vcs_info_against_last_run"
    - "database_entry"
 # this does not work as expected, solution is work in progress

configs/esm_software/esm_runscripts/esm_plugins.yaml

@@ -25,7 +25,7 @@ core:
                         - "initialize_batch_system"
                         - "initialize_coupler"
                         - "set_logfile"
-                        #- "add_vcs_info"
+                        - "add_vcs_info"
                         #- "check_vcs_info_against_last_run"
                         - "check_config_for_warnings_errors"
 

configs/esm_software/esm_runscripts/esm_runscripts.yaml

@@ -88,7 +88,7 @@ choose_job_type:
                         - "modify_files"
                         - "copy_files_to_work"
                         - "report_missing_files"
-                        #- "add_vcs_info"
+                        - "add_vcs_info"
                         #- "check_vcs_info_against_last_run"
                         - "_write_finalized_config"
                         - "database_entry"

configs/setups/awiesm/awiesm.yaml

@@ -70,7 +70,7 @@ general:
                 - "modify_files"
                 - "copy_files_to_work"
                 - "report_missing_files"
-                #- "add_vcs_info"
+                - "add_vcs_info"
                 #- "check_vcs_info_against_last_run"
                 - "_write_finalized_config"
                 - "database_entry"

src/esm_runscripts/helpers.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from datetime import datetime
 
@@ -8,7 +9,6 @@
 import esm_parser
 import esm_plugin_manager
 import esm_tools
-from loguru import logger
 from esm_profile import print_profile_summary
 
 
@@ -22,7 +22,6 @@ def print_datetime(config):
 
 
 def evaluate(config, job_type, recipe_name):
-
     # Check for a user defined compute recipe in the setup section of the
     # general section. If nothing is found, recipe_steps should evaluate to
     # None and the default is used
@@ -334,7 +333,9 @@ def is_git_repo(path):
     except git.exc.InvalidGitRepositoryError:
         return False
     except git.exc.NoSuchPathError:
-        logger.error("You specified a non-existant directory")
+        logger.error(
+            f"esm_runscripts.helpers.is_git_repo points at a non-existing path: {path}"
+        )
         # Is False the right answer here? I am not 100% sure...
         return False
 
@@ -369,7 +370,7 @@ def get_git_hash(path, allow_dirty=True):
         raise GitDirtyError(
             f"Your repo at {path} is dirty, thus the git hash is not meaningful!"
         )
-    return repo.git.rev_parse(repo.head, short=True)
+    return repo.git.rev_parse(repo.head, short=False)
 
 
 def get_git_branch(path):
@@ -445,3 +446,45 @@ def get_all_git_info(path):
         "diffs": get_git_diffs(path, add_colors=False),
     }
     return git_info
+
+
+class CachedFile:
+    """
+    Represents a file that might already have saved information somewhere. Can be used to check if a cache is valid
+    """
+
+    def __init__(self, path):
+        self.path = path
+
+    def is_younger_than(self, other, check_by="mtime"):
+        if not os.path.exists(self.path):
+            return False
+        if check_by == "mtime":
+            logger.debug("Checking modification times:")
+            logger.debug(f"{self.path}: {os.path.getmtime(self.path)}")
+            logger.debug(f"{other}: {os.path.getmtime(other)}")
+            am_I_younger = os.path.getmtime(self.path) < os.path.getmtime(other)
+            if am_I_younger:
+                logger.debug(f"This file {self.path} is younger than {other}")
+            return am_I_younger
+        else:
+            raise ValueError("Invalid check_by value. Only 'mtime' is supported.")
+
+    def is_older_than(self, other, check_by="mtime"):
+        if not os.path.exists(self.path):
+            return False
+        if check_by == "mtime":
+            logger.debug("Checking modification times:")
+            logger.debug(f"{self.path}: {os.path.getmtime(self.path)}")
+            logger.debug(f"{other}: {os.path.getmtime(other)}")
+            return os.path.getmtime(self.path) > os.path.getmtime(other)
+        else:
+            raise ValueError("Invalid check_by value. Only 'mtime' is supported.")
+
+    def read(self):
+        with open(self.path, "r") as f:
+            return f.read()
+
+    def load_cache(self):
+        data = self.read()
+        return yaml.safe_load(data)

src/esm_runscripts/prepare.py

@@ -5,12 +5,12 @@
 
 import questionary
 import yaml
+from loguru import logger
 
 import esm_parser
 import esm_utilities
 from esm_calendar import Calendar, Date
 from esm_plugin_manager import install_missing_plugins
-from loguru import logger
 
 from . import batch_system, helpers
 
@@ -93,7 +93,6 @@ def check_model_lresume(config):
                     model, user_lresume, config, [], []
                 )
             if isinstance(user_lresume, str):
-
                 if user_lresume == "0" or user_lresume.upper() == "FALSE":
                     user_lresume = False
                 elif user_lresume == "1" or user_lresume.upper() == "TRUE":
@@ -355,7 +354,6 @@ def set_leapyear(config):
 
 
 def set_overall_calendar(config):
-
     # set the overall calendar
     if config["general"]["leapyear"]:
         config["general"]["calendar"] = Calendar(1)
@@ -365,7 +363,6 @@ def set_overall_calendar(config):
 
 
 def find_last_prepared_run(config):
-
     calendar = config["general"]["calendar"]
     current_date = Date(config["general"]["current_date"], calendar)
     initial_date = Date(config["general"]["initial_date"], calendar)
@@ -409,7 +406,6 @@ def find_last_prepared_run(config):
 
 
 def set_most_dates(config):
-
     calendar = config["general"]["calendar"]
     if isinstance(config["general"]["current_date"], Date):
         current_date = config["general"]["current_date"]
@@ -694,21 +690,41 @@ def add_vcs_info(config):
     config : dict
         The experiment configuration
     """
+    vcs_logger = logger.add(
+        f"{config['general']['experiment_log_dir']}/pauls_vcs_info.log"
+    )
     exp_vcs_info_file = f"{config['general']['thisrun_log_dir']}/{config['general']['expid']}_vcs_info.yaml"
-    logger.debug("Experiment information is being stored for usage under:")
-    logger.debug(f">>> {exp_vcs_info_file}")
+    esm_vcs_info_file_cache = helpers.CachedFile(
+        f"{config['general']['experiment_log_dir']}/{config['general']['expid']}_vcs_info.yaml"
+    )
+    logger.critical("Experiment information is being stored for usage under:")
+    logger.critical(f">>> {exp_vcs_info_file}")
+    if os.path.exists(esm_vcs_info_file_cache.path):
+        logger.critical(">>> CACHE FILE EXISTS!")
+        logger.critical(
+            f">>> Last modification time of cache: {os.path.getmtime(esm_vcs_info_file_cache.path)}"
+        )
+    else:
+        logger.critical(">>> CACHE FILE DOES NOT EXIST!")
+        logger.critical(f"I was looking for: {esm_vcs_info_file_cache.path}")
     vcs_versions = {}
     all_models = config.get("general", {}).get("models", [])
     for model in all_models:
-        logger.debug(f"Locating {model}")
+        logger.critical(f"Locating {model}")
         try:
             model_dir = config[model]["model_dir"]
         except KeyError:
             # XIOS does not seem to define model_dir? Jan? What?
             vcs_versions[model] = f"Unable to locate model_dir for {model}."
             continue
         if helpers.is_git_repo(model_dir):
-            vcs_versions[model] = helpers.get_all_git_info(model_dir)
+            if esm_vcs_info_file_cache.is_older_than(model_dir):
+                logger.critical(f"Using cached VCS info for {model}")
+                cached_info = esm_vcs_info_file_cache.load_cache()
+                vcs_versions[model] = cached_info[model]
+            else:
+                logger.critical(f"Getting get info for {model_dir}")
+                vcs_versions[model] = helpers.get_all_git_info(model_dir)
         else:
             vcs_versions[model] = "Not a git-controlled model!"
 
@@ -720,10 +736,15 @@ def add_vcs_info(config):
     else:
         # FIXME(PG): This should absolutely never happen. The error message could use a better wording though...
         esm_parser.user_error(
-            "esm_tools doesn't know where it's own install location is. Something is very seriously wrong."
+            "VCS Info",
+            "esm_tools doesn't know where it's own install location is. Something is very seriously wrong.",
         )
-    with open(exp_vcs_info_file, "w") as f:
-        yaml.dump(vcs_versions, f)
+
+    esm_parser.yaml_dump(vcs_versions, exp_vcs_info_file)
+
+    logger.debug(exp_vcs_info_file)
+    logger.remove(vcs_logger)
+
     return config
 
 
@@ -793,7 +814,6 @@ def finalize_config(config):
 
 
 def add_submission_info(config):
-
     bs = batch_system.batch_system(config, config["computer"]["batch_system"])
 
     submitted = bs.check_if_submitted()
@@ -808,7 +828,6 @@ def add_submission_info(config):
 
 
 def initialize_batch_system(config):
-
     config["general"]["batch"] = batch_system.batch_system(
         config, config["computer"]["batch_system"]
     )