Add ability to include gravity config from a separate file and document

README.rst

@@ -130,6 +130,9 @@ unset are shown)::
     # Configuration for Gunicorn.
     gunicorn:
 
+      # Enable Galaxy gunicorn server.
+      # enable: true
+
       # The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST.
       # bind: localhost:8080
 
@@ -154,6 +157,12 @@ unset are shown)::
     # Configuration for Celery Processes.
     celery:
 
+      # Enable Celery distributed task queue.
+      # enable: true
+
+      # Enable Celery Beat periodic task runner.
+      # enable_beat: true
+
       # Number of Celery Workers to start.
       # concurrency: 2
 
@@ -228,6 +237,39 @@ unset are shown)::
     # handlers: {}
 
 
+As a convenience for cases where you may want to have different Gravity configurations but a single Galaxy
+configuration (e.g. your Galaxy server is split across multiple hosts), the Gravity configuration can be stored in a
+separate file and included into the Galaxy configuration. For example, on a deployment where the web (gunicorn) and job
+handler processes run on different hosts, one might have:
+
+In ``galaxy.yml``::
+
+    gravity: !include gravity.yml
+    galaxy:
+      database_connection: postgresql://...
+      ...
+
+In ``gravity.yml`` on the web host::
+
+    log_dir: /var/log/galaxy
+    gunicorn:
+      bind: localhost:8888
+    celery:
+      enable: false
+      enable_beat: false
+
+In ``gravity.yml`` on the job handler host::
+
+    log_dir: /var/log/galaxy
+    gunicorn:
+      enable: false
+    celery:
+      enable: true
+      enable_beat: true
+    handlers:
+      handler:
+        processes: 2
+
 Galaxy Job Handlers
 -------------------
 

gravity/config_manager.py

@@ -20,7 +20,7 @@
     GravityState,
     service_for_service_type,
 )
-from gravity.util import recursive_update
+from gravity.util import recursive_update, yaml_safe_load_with_include
 
 log = logging.getLogger(__name__)
 
@@ -73,7 +73,7 @@ def get_config(self, conf, defaults=None):
         defaults = defaults or {}
         server_section = self.galaxy_server_config_section
         with open(conf) as config_fh:
-            config_dict = safe_load(config_fh)
+            config_dict = yaml_safe_load_with_include(config_fh)
         _gravity_config = config_dict.get(self.gravity_config_section) or {}
         gravity_config = Settings(**recursive_update(defaults, _gravity_config))
         if gravity_config.log_dir is None:

gravity/util/__init__.py

@@ -11,6 +11,24 @@
 from gravity.settings import Settings
 
 
+class SafeLoaderWithInclude(yaml.SafeLoader):
+    def __init__(self, stream):
+        self.__config_dir = os.path.dirname(stream.name)
+        super().__init__(stream)
+
+    def include(self, node):
+        included = os.path.join(self.__config_dir, self.construct_scalar(node))
+        with open(included) as fh:
+            return yaml.load(fh, SafeLoaderWithInclude)
+
+
+SafeLoaderWithInclude.add_constructor('!include', SafeLoaderWithInclude.include)
+
+
+def yaml_safe_load_with_include(stream):
+    return yaml.load(stream, SafeLoaderWithInclude)
+
+
 class AttributeDict(dict):
     yaml_tag = "tag:yaml.org,2002:map"