Skip to content

Commit

Permalink
Significantly improves storage engines including (1) converting comma…
Browse files Browse the repository at this point in the history
…nd line flags to ENV variables (fixes runtheops#15), (2) a way to generate YAML files for branches of the SSM tree (closes runtheops#11), (3) the ability to ignore SecureString keys if they are not necessary (closes runtheops#13), (4) support for the SSM StringList type and more timely type coercion so e.g. YAML integers and SSM strings match, and (5) the introduction of metadata in the YAML files to permit compatibility checking (more general fix for runtheops#15 with support for new features)
  • Loading branch information
claytondaley committed May 1, 2019
1 parent e33935d commit d01c7dc
Show file tree
Hide file tree
Showing 5 changed files with 532 additions and 64 deletions.
57 changes: 40 additions & 17 deletions ssm-diff
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,27 @@
from __future__ import print_function

import argparse
import logging
import os
import sys

from states import *

root = logging.getLogger()
root.setLevel(logging.INFO)

handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)s - %(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)


def configure_endpoints(args):
# configure() returns a DiffBase class (whose constructor may be wrapped in `partial` to pre-configure it)
diff_class = DiffBase.get_plugin(args.engine).configure(args)
return storage.ParameterStore(args.profile, diff_class, paths=args.path), storage.YAMLFile(args.filename, paths=args.path)
return storage.ParameterStore(args.profile, diff_class, paths=args.paths, no_secure=args.no_secure), \
storage.YAMLFile(args.filename, paths=args.paths, no_secure=args.no_secure, root_path=args.yaml_root)


def init(args):
Expand Down Expand Up @@ -39,18 +51,12 @@ def apply(args):
def plan(args):
"""Print a representation of the changes that would be applied to SSM Parameter Store if applied (per config in args)"""
remote, local = configure_endpoints(args)
diff = remote.dry_run(local.get())

if diff.differ:
print(DiffBase.describe_diff(diff.plan))
else:
print("Remote state is up to date.")
print(DiffBase.describe_diff(remote.dry_run(local.get())))


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-f', help='local state yml file', action='store', dest='filename', default='parameters.yml')
parser.add_argument('--path', '-p', action='append', help='filter SSM path')
parser.add_argument('-f', help='local state yml file', action='store', dest='filename')
parser.add_argument('--engine', '-e', help='diff engine to use when interacting with SSM', action='store', dest='engine', default='DiffResolver')
parser.add_argument('--profile', help='AWS profile name', action='store', dest='profile')
subparsers = parser.add_subparsers(dest='func', help='commands')
Expand All @@ -70,12 +76,29 @@ if __name__ == "__main__":
parser_apply.set_defaults(func=apply)

args = parser.parse_args()
args.path = args.path if args.path else ['/']

if args.filename == 'parameters.yml':
if not args.profile:
if 'AWS_PROFILE' in os.environ:
args.filename = os.environ['AWS_PROFILE'] + '.yml'
else:
args.filename = args.profile + '.yml'

args.no_secure = os.environ.get('SSM_NO_SECURE', 'false').lower() in ['true', '1']
args.yaml_root = os.environ.get('SSM_YAML_ROOT', '/')
args.paths = os.environ.get('SSM_PATHS', None)
if args.paths is not None:
args.paths = args.paths.split(';:')
else:
# this defaults to '/'
args.paths = args.yaml_root

# root filename
if args.filename is not None:
filename = args.filename
elif args.profile:
filename = args.profile
elif 'AWS_PROFILE' in os.environ:
filename = os.environ['AWS_PROFILE']
else:
filename = 'parameters'

# remove extension (will be restored by storage classes)
if filename[-4:] == '.yml':
filename = filename[:-4]
args.filename = filename

args.func(args)
26 changes: 15 additions & 11 deletions states/engine.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import collections
import logging
import re
from functools import partial

from termcolor import colored
Expand Down Expand Up @@ -38,24 +39,24 @@ def configure(cls, args):
@classmethod
def _flatten(cls, d, current_path='', sep='/'):
"""Convert a nested dict structure into a "flattened" dict i.e. {"full/path": "value", ...}"""
items = []
for k in d:
items = {}
for k, v in d.items():
new = current_path + sep + k if current_path else k
if isinstance(d[k], collections.MutableMapping):
items.extend(cls._flatten(d[k], new, sep=sep).items())
if isinstance(v, collections.MutableMapping):
items.update(cls._flatten(v, new, sep=sep).items())
else:
items.append((sep + new, d[k]))
return dict(items)
items[sep + new] = v
return items

@classmethod
def _unflatten(cls, d, sep='/'):
"""Converts a "flattened" dict i.e. {"full/path": "value", ...} into a nested dict structure"""
output = {}
for k in d:
for k, v in d.items():
add(
obj=output,
path=k,
value=d[k],
value=v,
sep=sep,
)
return output
Expand All @@ -66,15 +67,18 @@ def describe_diff(cls, plan):
description = ""
for k, v in plan['add'].items():
# { key: new_value }
description += colored("+", 'green'), "{} = {}".format(k, v) + '\n'
description += colored("+", 'green') + "{} = {}".format(k, v) + '\n'

for k in plan['delete']:
# { key: old_value }
description += colored("-", 'red'), k + '\n'
description += colored("-", 'red') + k + '\n'

for k, v in plan['change'].items():
# { key: {'old': value, 'new': value} }
description += colored("~", 'yellow'), "{}:\n\t< {}\n\t> {}".format(k, v['old'], v['new']) + '\n'
description += colored("~", 'yellow') + "{}:\n\t< {}\n\t> {}".format(k, v['old'], v['new']) + '\n'

if description == "":
description = "No Changes Detected"

return description

Expand Down
29 changes: 18 additions & 11 deletions states/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,31 @@ def add(obj, path, value, sep='/'):
"""Add value to the `obj` dict at the specified path"""
parts = path.strip(sep).split(sep)
last = len(parts) - 1
current = obj
for index, part in enumerate(parts):
if index == last:
obj[part] = value
current[part] = value
else:
obj = obj.setdefault(part, {})
current = current.setdefault(part, {})
# convenience return, object is mutated
return obj


def search(state, path):
result = state
"""Get value in `state` at the specified path, returning {} if the key is absent"""
if path.strip("/") == '':
return state
for p in path.strip("/").split("/"):
if result.clone(p):
result = result[p]
else:
result = {}
break
output = {}
add(output, path, result)
return output
if p not in state:
return {}
state = state[p]
return state


def filter(state, path):
if path.strip("/") == '':
return state
return add({}, path, search(state, path))


def merge(a, b):
Expand Down
Loading

0 comments on commit d01c7dc

Please sign in to comment.