Merge pull request #69 from natefoo/stateless-update

Drop most of the usage of configstate

gravity/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 
-__version__ = "0.13.4"
+__version__ = "1.0.0"

gravity/commands/cmd_instances.py

@@ -9,7 +9,7 @@ def cli(ctx):
     """List all known instances."""
     with config_manager.config_manager(state_dir=ctx.parent.state_dir) as cm:
         configs = cm.get_registered_configs()
-        instances = cm.get_registered_instances()
+        instances = cm.get_registered_instance_names()
         if instances:
             click.echo("%-24s  %-10s  %-10s  %s" % ("INSTANCE NAME", "TYPE", "SERVER", "NAME"))
             # not the most efficient...

gravity/config_manager.py

@@ -5,14 +5,15 @@
 import hashlib
 import logging
 import os
+import shutil
 import xml.etree.ElementTree as elementtree
 from os import pardir
 from os.path import abspath, dirname, exists, expanduser, isabs, join
 from typing import Union
 
+import packaging.version
 from yaml import safe_load
 
-from gravity import __version__
 from gravity.settings import Settings
 from gravity.io import debug, error, exception, info, warn
 from gravity.state import (
@@ -44,6 +45,7 @@ def __init__(self, state_dir=None, python_exe=None):
             state_dir = DEFAULT_STATE_DIR
         self.state_dir = abspath(expanduser(state_dir))
         debug(f"Gravity state dir: {self.state_dir}")
+        self.__configs = {}
         self.config_state_path = join(self.state_dir, "configstate.yaml")
         self.python_exe = python_exe
         try:
@@ -58,18 +60,38 @@ def __copy_config(self, old_path):
             state.set_name(self.config_state_path)
         # copies on __exit__
 
+    def __backup_config(self, backup_ext):
+        backup_path = f"{self.config_state_path}.{backup_ext}"
+        info(f"Previous Gravity config state saved in: {backup_path}")
+        shutil.copy(self.config_state_path, backup_path)
+
     def __convert_config(self):
-        config_state_json = join(self.state_dir, "configstate.json")
-        if exists(config_state_json) and not exists(self.config_state_path):
-            warn(f"Converting {config_state_json} to {self.config_state_path}")
-            json_state = GravityState.open(config_state_json)
-            self.__copy_config(config_state_json)
-            assert exists(self.config_state_path), f"Conversion failed ({self.config_state_path} does not exist)"
-            yaml_state = GravityState.open(self.config_state_path)
-            assert json_state == yaml_state, f"Converted config differs from previous config, remove {self.config_state_path} to retry"
-            os.unlink(config_state_json)
+        # the gravity version has been included in the configstate since 0.10.0, but it was previously a configfile
+        # attrib, which doesn't really make sense, and 1.0.0 removes persisted configfile attribs anyway
+        state_version = self.state.get("gravity_version")
+        debug(f"Gravity state version: {state_version}")
+        if not state_version or (packaging.version.parse(state_version) < packaging.version.parse("1.0.0")):
+            # this hardcoded versioning suffices for now, might have to get fancier in the future
+            with self.state as state:
+                self.__convert_config_1_0(state)
+
+    def __convert_config_1_0(self, state):
+        info("Converting Gravity config state to 1.0 format, this will only occur once")
+        self.__backup_config("pre-1.0")
+        for config_file, config in state.config_files.items():
+            try:
+                config.galaxy_root = config.attribs["galaxy_root"]
+            except KeyError:
+                warn(f"Unable to read 'galaxy_root' from attribs: {config.attribs}")
+            for key in list(config.keys()):
+                if key not in ConfigFile.persist_keys:
+                    del config[key]
+            state.config_files[config_file] = config
 
     def get_config(self, conf, defaults=None):
+        if conf in self.__configs:
+            return self.__configs[conf]
+
         defaults = defaults or {}
         server_section = self.galaxy_server_config_section
         with open(conf) as config_fh:
@@ -101,18 +123,17 @@ def get_config(self, conf, defaults=None):
         config.attribs["celery"] = gravity_config.celery.dict()
         config.attribs["handlers"] = gravity_config.handlers
         # Store gravity version, in case we need to convert old setting
-        config.attribs['gravity_version'] = __version__
         webapp_service_names = []
 
         # shortcut for galaxy configs in the standard locations -- explicit arg ?
-        config.attribs["galaxy_root"] = app_config.get("root") or gravity_config.galaxy_root
-        if config.attribs["galaxy_root"] is None:
+        config["galaxy_root"] = app_config.get("root") or gravity_config.galaxy_root
+        if config["galaxy_root"] is None:
             if os.environ.get("GALAXY_ROOT_DIR"):
-                config.attribs["galaxy_root"] = abspath(os.environ["GALAXY_ROOT_DIR"])
+                config["galaxy_root"] = abspath(os.environ["GALAXY_ROOT_DIR"])
             elif exists(join(dirname(conf), pardir, "lib", "galaxy")):
-                config.attribs["galaxy_root"] = abspath(join(dirname(conf), pardir))
+                config["galaxy_root"] = abspath(join(dirname(conf), pardir))
             elif conf.endswith(join('galaxy', 'config', 'sample', 'galaxy.yml.sample')):
-                config.attribs["galaxy_root"] = abspath(join(dirname(conf), pardir, pardir, pardir, pardir))
+                config["galaxy_root"] = abspath(join(dirname(conf), pardir, pardir, pardir, pardir))
             else:
                 exception(f"Cannot locate Galaxy root directory: set $GALAXY_ROOT_DIR or `root' in the `galaxy' section of {conf}")
 
@@ -136,7 +157,7 @@ def get_config(self, conf, defaults=None):
             job_config = app_config.get("job_config_file", DEFAULT_JOB_CONFIG_FILE)
             if not isabs(job_config):
                 # FIXME: relative to root
-                job_config = abspath(join(config.attribs["galaxy_root"], job_config))
+                job_config = abspath(join(config["galaxy_root"], job_config))
                 if not exists(job_config):
                     job_config = None
         if config.config_type == "galaxy" and job_config:
@@ -155,6 +176,7 @@ def get_config(self, conf, defaults=None):
         # see how this is determined.
         self.create_handler_services(gravity_config, config)
         self.create_gxit_services(gravity_config, app_config, config)
+        self.__configs[conf] = config
         return config
 
     def create_handler_services(self, gravity_config: Settings, config):
@@ -238,101 +260,7 @@ def _deregister_config_file(self, key):
         ensure that it was previously registered.
         """
         with self.state as state:
-            if "remove_configs" not in state:
-                state.remove_configs = {}
-            state.remove_configs[key] = state.config_files.pop(key)
-
-    def _purge_config_file(self, key):
-        """Forget a previously deregister config file.  The caller should
-        ensure that it was previously deregistered.
-        """
-        with self.state as state:
-            del state.remove_configs[key]
-            if not state.remove_configs:
-                del state["remove_configs"]
-
-    def determine_config_changes(self):
-        """The magic: Determine what has changed since the last time.
-
-        Caller should pass the returned config to register_config_changes to persist.
-        """
-        # 'update' here is synonymous with 'add or update'
-        instances = set()
-        new_configs = {}
-        meta_changes = {"changed_instances": set(), "remove_instances": [], "remove_configs": self.get_remove_configs()}
-        for config_file, stored_config in self.get_registered_configs().items():
-            new_config = stored_config
-            try:
-                ini_config = self.get_config(config_file, defaults=stored_config.defaults)
-            except OSError as exc:
-                warn("Unable to read %s (hint: use `rename` or `remove` to fix): %s", config_file, exc)
-                new_configs[config_file] = stored_config
-                instances.add(stored_config["instance_name"])
-                continue
-            if ini_config["instance_name"] is not None:
-                # instance name is explicitly set in the config
-                instance_name = ini_config["instance_name"]
-                if ini_config["instance_name"] != stored_config["instance_name"]:
-                    # instance name has changed
-                    # (removal of old instance will happen later if no other config references it)
-                    new_config["update_instance_name"] = instance_name
-                meta_changes["changed_instances"].add(instance_name)
-            else:
-                # instance name is dynamically generated
-                instance_name = stored_config["instance_name"]
-            if ini_config["attribs"] != stored_config["attribs"]:
-                new_config["update_attribs"] = ini_config["attribs"]
-                meta_changes["changed_instances"].add(instance_name)
-            # make sure this instance isn't removed
-            instances.add(instance_name)
-            services = []
-            for service in ini_config["services"]:
-                for stored_service in stored_config["services"]:
-                    if service.full_match(stored_service):
-                        # service is configured and has no changes
-                        break
-                else:
-                    # instance has a new service or service has config change
-                    if "update_services" not in new_config:
-                        new_config["update_services"] = []
-                    new_config["update_services"].append(service)
-                    meta_changes["changed_instances"].add(instance_name)
-                # make sure this service isn't removed
-                services.append(service)
-            for service in stored_config["services"]:
-                if service not in services:
-                    if "remove_services" not in new_config:
-                        new_config["remove_services"] = []
-                    new_config["remove_services"].append(service)
-                    meta_changes["changed_instances"].add(instance_name)
-            new_configs[config_file] = new_config
-        # once finished processing all configs, find any instances which have been deleted
-        for instance_name in self.get_registered_instances(include_removed=True):
-            if instance_name not in instances:
-                meta_changes["remove_instances"].append(instance_name)
-        return new_configs, meta_changes
-
-    def register_config_changes(self, configs, meta_changes):
-        """Persist config changes to the JSON state file. When a config
-        changes, a process manager may perform certain actions based on these
-        changes. This method can be called once the actions are complete.
-        """
-        for config_file in meta_changes["remove_configs"].keys():
-            self._purge_config_file(config_file)
-        for config_file, config in configs.items():
-            if "update_attribs" in config:
-                config["attribs"] = config.pop("update_attribs")
-            if "update_instance_name" in config:
-                config["instance_name"] = config.pop("update_instance_name")
-            if "update_services" in config or "remove_services" in config:
-                remove = config.pop("remove_services", [])
-                services = config.pop("update_services", [])
-                # need to prevent old service defs from overwriting new ones
-                for service in config["services"]:
-                    if service not in remove and service not in services:
-                        services.append(service)
-                config["services"] = services
-            self._register_config_file(config_file, config)
+            del state.config_files[key]
 
     @property
     def state(self):
@@ -351,34 +279,21 @@ def single_instance(self):
 
     def get_registered_configs(self, instances=None):
         """Return the persisted values of all config files registered with the config manager."""
+        rval = {}
         configs = self.state.config_files
-        if instances is not None:
-            for config_file, config in list(configs.items()):
-                if config["instance_name"] not in instances:
-                    configs.pop(config_file)
-        return configs
-
-    def get_remove_configs(self):
-        """Return the persisted values of all config files pending removal by the process manager."""
-        return self.state.get("remove_configs", {})
+        for config_file, config in list(configs.items()):
+            if (instances is not None and config["instance_name"] in instances) or instances is None:
+                rval[config_file] = self.get_config(config_file)
+        return rval
 
     def get_registered_config(self, config_file):
         """Return the persisted value of the named config file."""
-        return self.state.config_files.get(config_file, None)
-
-    def get_registered_instances(self, include_removed=False):
-        """Return the persisted names of all instances across all registered configs."""
-        rval = []
-        configs = list(self.state.config_files.values())
-        if include_removed:
-            configs.extend(list(self.get_remove_configs().values()))
-        for config in configs:
-            if config["instance_name"] not in rval:
-                rval.append(config["instance_name"])
-        return rval
+        if config_file in self.state.config_files:
+            return self.get_config(config_file)
+        return None
 
     def get_instance_config(self, instance_name):
-        for config in list(self.state.config_files.values()):
+        for config in list(self.get_registered_configs().values()):
             if config["instance_name"] == instance_name:
                 return config
         exception(f'Instance "{instance_name}" unknown, known instance(s) are {", ".join(self.get_registered_instance_names())}.')
@@ -391,7 +306,7 @@ def get_instance_services(self, instance_name):
 
     def get_registered_services(self):
         rval = []
-        for config_file, config in self.state.config_files.items():
+        for config_file, config in self.get_registered_configs.items():
             for service in config["services"]:
                 service["config_file"] = config_file
                 service["instance_name"] = config["instance_name"]
@@ -430,12 +345,9 @@ def add(self, config_files, galaxy_root=None):
                 exception(f"Cannot add {config_file}: File is unknown type")
             if conf["instance_name"] is None:
                 conf["instance_name"] = conf["config_type"] + "-" + hashlib.md5(os.urandom(32)).hexdigest()[:12]
-            conf_data = {
-                "config_type": conf["config_type"],
-                "instance_name": conf["instance_name"],
-                "attribs": conf["attribs"],
-                "services": [],  # services will be populated by the update method
-            }
+            conf_data = {}
+            for key in ConfigFile.persist_keys:
+                conf_data[key] = conf[key]
             self._register_config_file(config_file, conf_data)
             info("Registered %s config: %s", conf["config_type"], config_file)
 

gravity/process_manager/__init__.py

@@ -58,7 +58,7 @@ def start(self, instance_names):
         """ """
 
     @abstractmethod
-    def _process_config_changes(self, configs, meta_changes):
+    def _process_config(self, config_file, config, **kwargs):
         """ """
 
     @abstractmethod
@@ -118,7 +118,7 @@ def shutdown(self, instance_names):
         """ """
 
     def get_instance_names(self, instance_names):
-        registered_instance_names = self.config_manager.get_registered_instances()
+        registered_instance_names = self.config_manager.get_registered_instance_names()
         unknown_instance_names = []
         if instance_names:
             _instance_names = []