Improve Ordering of Operations

aomi/model/__init__.py

@@ -14,6 +14,13 @@
 from aomi.validation import sanitize_mount, check_obj
 
 
+def absent_sort(resource):
+    """Used to sort resources in a way where things that
+    are being removed are prioritized over things that
+    are being added or modified"""
+    return resource.present
+
+
 def py_resources():
     """Discovers all aomi Vault resource models"""
     aomi_mods = [m for
@@ -76,7 +83,7 @@ def ensure_backend(resource, backend, backends, opt):
         if backend == LogBackend:
             new_mount = backend(resource, opt)
         else:
-            new_mount = backend(resource.mount, resource.backend, opt)
+            new_mount = backend(resource, opt)
 
         backends.append(new_mount)
         return new_mount
@@ -256,7 +263,11 @@ def filtered(self):
     def read(self, client):
         """Read from Vault while handling non surprising errors."""
         log("Reading from %s" % self, self.opt)
-        return client.read(self.path)
+        try:
+            return client.read(self.path)
+        except hvac.exceptions.InvalidRequest as vault_exception:
+            if str(vault_exception).startswith('no handler for route'):
+                return None
 
     @wrap_vault("writing")
     def write(self, client):
@@ -267,7 +278,11 @@ def write(self, client):
     def delete(self, client):
         """Delete from Vault while handling non-surprising errors."""
         log("Deleting %s" % self, self.opt)
-        client.delete(self.path)
+        try:
+            client.delete(self.path)
+        except hvac.exceptions.InvalidPath as vault_exception:
+            if str(vault_exception).startswith('no handler for route'):
+                return None
 
 
 class Secret(Resource):
@@ -386,23 +401,32 @@ def sync(self, vault_client):
         has the effect of updating every resource which is
         in the context."""
         active_mounts = []
-        for mount in self.mounts():
-            if not mount.existing:
-                mount.sync(vault_client)
         for auth in self.auths():
-            if not auth.existing:
-                auth.sync(vault_client)
-        for blog in self.logs():
-            if not blog.existing:
-                blog.sync(vault_client)
-        for resource in self.resources():
-            if isinstance(resource, (Secret, Mount)) and resource.present:
-                active_mount = find_backend(resource.mount, active_mounts)
-                if not active_mount:
-                    actual_mount = find_backend(resource.mount, self._mounts)
-                    if actual_mount:
-                        active_mounts.append(actual_mount)
-
+            auth.sync(vault_client)
+        for audit_log in self.logs():
+            audit_log.sync(vault_client)
+
+        # Handle mounts only on the first pass. This allows us to
+        # ensure that everything is in order prior to actually
+        # provisioning secrets. Note we handle removals before
+        # anything else, allowing us to address mount conflicts.
+        mounts = [x for x in self.resources()
+                  if isinstance(x, (Secret, Mount))]
+
+        s_resources = sorted(mounts, key=absent_sort)
+
+        for resource in s_resources:
+            active_mount = find_backend(resource.mount, active_mounts)
+            if not active_mount:
+                actual_mount = find_backend(resource.mount, self._mounts)
+                active_mounts.append(actual_mount)
+                actual_mount.sync(vault_client)
+
+        # Now handle everything else. If "best practices" are being
+        # adhered to then every generic mountpoint should exist by now
+        not_mounts = [x for x in self.resources()
+                      if not isinstance(x, (Mount))]
+        for resource in not_mounts:
             resource.sync(vault_client)
 
         for mount in self.mounts():
@@ -444,6 +468,7 @@ class Mount(Resource):
     required_fields = ['path']
     config_key = 'mounts'
     backend = 'generic'
+    secret_format = 'mount point'
 
     def __init__(self, obj, opt):
         super(Mount, self).__init__(obj, opt)
@@ -478,22 +503,30 @@ def __str__(self):
 
         return "%s %s" % (self.backend, self.path)
 
-    def __init__(self, path, backend, opt):
-        self.path = sanitize_mount(path)
-        self.backend = backend
+    def __init__(self, resource, opt):
+        self.path = sanitize_mount(resource.mount)
+        self.backend = resource.backend
         self.existing = False
+        self.present = resource.present
         self.opt = opt
 
     def sync(self, vault_client):
         """Synchronizes the local and remote Vault resources. Has the net
         effect of adding backend if needed"""
-        if not self.existing:
+        if not self.existing and self.present:
             self.actually_mount(vault_client)
             log("Mounting %s backend on %s" %
                 (self.backend, self.path), self.opt)
-        else:
+        elif self.existing and self.present:
             log("%s backend already mounted on %s" %
                 (self.backend, self.path), self.opt)
+        elif self.existing and not self.present:
+            self.unmount(vault_client)
+            log("Unmounting %s backend on %s" %
+                (self.backend, self.path), self.opt)
+        elif not self.existing and not self.present:
+            log("%s backend already unmounted on %s" %
+                (self.backend, self.path), self.opt)
 
     def fetch(self, backends):
         """Updates local resource with context on whether this
@@ -502,8 +535,6 @@ def fetch(self, backends):
 
     def unmount(self, client):
         """Unmounts a backend within Vault"""
-        log("Unmounting %s backend from %s" %
-            (self.backend, self.path), self.opt)
         getattr(client, self.unmount_fun)(mount_point=self.path)
 
     def actually_mount(self, client):

docs/aws.md

@@ -9,7 +9,7 @@ The aomi tool is able to write to the AWS backend of Vault.
 
 By specifying an appropriately populated `aws_file` you can create [AWS secret backends](https://www.vaultproject.io/docs/secrets/aws/index.html) in Vault. The `aws_file` must point to a valid file, and the base of the AWS credentials will be set by the `mount`.
 
-The AWS file contains the `access_key_id`, and `secret_access_key`. The `region`, and a list of AWS roles that will be loaded by Vault are in the `Secretfile`. Note that you may specify either an inline `policy` _or_ a native AWS `arn`. The `name` of each role will be used to compute the final path for accessing credentials. The policy files are simply JSON IAM Access representations. The following example would create an AWS Vault secret backend at `foo/bar/baz` based on the account and policy information defined in `.secrets/aws.yml`. While `lease` and `lease_max` are provided in this example, they are not strictly required. Note that you can also specify a `state` as either `present` (the default) or `absent`.
+The AWS file contains the `access_key_id`, and `secret_access_key`. The `region`, and a list of AWS roles that will be loaded by Vault are in the `Secretfile`. Note that you may specify either an inline `policy` _or_ a native AWS `arn`. The `name` of each role will be used to compute the final path for accessing credentials. The policy files are simply JSON IAM Access representations. The following example would create an AWS Vault secret backend at `foo/bar/baz` based on the account and policy information defined in `.secrets/aws.yml`. While `lease` and `lease_max` are provided in this example, they are not strictly required. Note that you can specify the `state` as either `absent` or `present` for each individual role.
 
 Note that a previous version had `lease`, `lease_max`, `region`, and the `roles` section located in the `aws_file` itself - this behavior is now considered deprecated. The _only_ thing which should be present in the AWS yaml is the actual secrets.
 

docs/secretfile.md

@@ -43,6 +43,8 @@ The Secretfile is interpreted as a YAML file. Prior to parsing, aomi will render
 * Vault policies and audit logs are also configurable. These do not have any secrets associated with them.
 * You can define some metadata which is limited to GPG/Keybase information, used for cold storage of secrets.
 
+It is possible to be explict about the presence of a Vault construct on the server. Every entry should support the `state` value, which can be set to either `present` (the default) or `absent`.
+
 # Tagging
 
 Every entry which will affect Vault may be "tagged". Any and all tags must be referenced in order for the resource to be processed. Untagged resources will only be processed if tags are not specified on the command line. The following example shows two sets of static files, each tied to a different tag. This is one way of having a single `Secretfile` which can be used to populate multiple environments.