diff --git a/LICENSE b/LICENSE index e17aa362b..d89fe75a8 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 Cloud Posse, LLC + Copyright 2020-2021 Cloud Posse, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index f4e6c09e7..316c6557c 100644 --- a/README.md +++ b/README.md @@ -224,15 +224,15 @@ __NOTE:__ For the example, we import all the CLI modules, but they could be incl ```hcl imports = [ - "git::https://git@github.com/cloudposse/atmos@modules/utils?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/shell?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/kubeconfig?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/terraform?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/helmfile?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/helm?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/workflow?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/istio?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/vendor?ref=master" + "git::https://git@github.com/cloudposse/atmos@modules/utils?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/shell?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/kubeconfig?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/terraform?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/helmfile?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/helm?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/workflow?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/istio?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/vendor?ref=master" ] ``` @@ -558,7 +558,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyright -Copyright © 2017-2020 [Cloud Posse, LLC](https://cpco.io/copyright) +Copyright © 2017-2021 [Cloud Posse, LLC](https://cpco.io/copyright) diff --git a/README.yaml b/README.yaml index acbcf8b2b..d77df772b 100644 --- a/README.yaml +++ b/README.yaml @@ -198,15 +198,15 @@ introduction: |- ```hcl imports = [ - "git::https://git@github.com/cloudposse/atmos@modules/utils?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/shell?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/kubeconfig?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/terraform?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/helmfile?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/helm?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/workflow?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/istio?ref=master", - "git::https://git@github.com/cloudposse/atmos@modules/vendor?ref=master" + "git::https://git@github.com/cloudposse/atmos@modules/utils?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/shell?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/kubeconfig?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/terraform?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/helmfile?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/helm?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/workflow?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/istio?ref=master", + "git::https://git@github.com/cloudposse/atmos@modules/vendor?ref=master" ] ``` diff --git a/example/stacks/globals-2.yaml b/example/stacks/globals-2.yaml new file mode 100644 index 000000000..386eb6755 --- /dev/null +++ b/example/stacks/globals-2.yaml @@ -0,0 +1,16 @@ +vars: {} + +terraform: + vars: + test-map: + g: g1 + atr-2: + atr1: 1 + atr2: 2 + list2: + - 1 + - 2 + - 3 + +helmfile: + vars: {} diff --git a/example/stacks/globals.yaml b/example/stacks/globals.yaml index 778f679ff..b33cc771c 100644 --- a/example/stacks/globals.yaml +++ b/example/stacks/globals.yaml @@ -3,9 +3,20 @@ vars: terraform: vars: + test-map: + a: a1 + b: b1 + c: c1 + atr: + atr1: 1 + atr2: 2 + list: + - 1 + - 2 + - 3 helmfile: - vars: + vars: {} components: terraform: diff --git a/example/stacks/ue2-dev.yaml b/example/stacks/ue2-dev.yaml index 5a968429d..c096b158b 100644 --- a/example/stacks/ue2-dev.yaml +++ b/example/stacks/ue2-dev.yaml @@ -6,9 +6,18 @@ vars: terraform: vars: + test-map: + a: a3 + d: d3 + e: e3 + atr: + atr2: 2-3 + atr4: 4 + list: + - 1b helmfile: - vars: + vars: {} components: terraform: @@ -17,6 +26,16 @@ components: cidr_block: "10.100.0.0/18" eks: vars: + test-map: + a: a4 + d: d4 + e: e4 + f: f4 + atr: + atr2: 2-4 + atr5: 5 + list: + - 1c helmfile: nginx-ingress: diff --git a/example/stacks/ue2-globals.yaml b/example/stacks/ue2-globals.yaml index 18fae356f..5d467474f 100644 --- a/example/stacks/ue2-globals.yaml +++ b/example/stacks/ue2-globals.yaml @@ -1,5 +1,6 @@ import: - globals + - globals-2 vars: region: us-east-2 @@ -7,18 +8,17 @@ vars: terraform: vars: - backend_type: s3 # s3, remote, vault, etc. - backend: - s3: - encrypt: true - bucket: "eg-ue2-root-tfstate" - key: "terraform.tfstate" - dynamodb_table: "eg-ue2-root-tfstate-lock" - role_arn: "arn:aws:iam::xxxxxxxx:role/eg-gbl-root-terraform" - acl: "bucket-owner-full-control" - region: "us-east-2" - remote: - vault: + test-map: + b: b2 + atr: + atr1: 1-1 + atr3: 3 + list: + - 1a + list: + - 4 + - 5 + - 6 helmfile: - vars: + vars: {} diff --git a/example/stacks/ue2-prod.yaml b/example/stacks/ue2-prod.yaml index 64f66afba..415a77dbd 100644 --- a/example/stacks/ue2-prod.yaml +++ b/example/stacks/ue2-prod.yaml @@ -5,10 +5,10 @@ vars: stage: prod terraform: - vars: + vars: {} helmfile: - vars: + vars: {} components: terraform: @@ -17,7 +17,7 @@ components: vars: cidr_block: "10.102.0.0/18" eks: - vars: + vars: {} helmfile: nginx-ingress: diff --git a/example/stacks/ue2-staging.yaml b/example/stacks/ue2-staging.yaml index c637007eb..7810f5d78 100644 --- a/example/stacks/ue2-staging.yaml +++ b/example/stacks/ue2-staging.yaml @@ -5,10 +5,10 @@ vars: stage: staging terraform: - vars: + vars: {} helmfile: - vars: + vars: {} components: terraform: @@ -16,7 +16,7 @@ components: vars: cidr_block: "10.104.0.0/18" eks: - vars: + vars: {} helmfile: nginx-ingress: diff --git a/modules/utils/stack-config.variant b/modules/utils/stack-config.variant index 29896ea66..5f7de597f 100644 --- a/modules/utils/stack-config.variant +++ b/modules/utils/stack-config.variant @@ -1,6 +1,38 @@ #!/usr/bin/env variant # vim: filetype=hcl +function "import" { + params = [config-dir, path] + + result = flatten( + concat( + [ + for k, imports in yamldecode(file(format("%s/%s.yaml", config-dir, path))): [ + for imported in imports: [ + import(config-dir, imported) + ] + ] if k == "import" + ], + [ + format("%s/%s.yaml", config-dir, path) + ] + ) + ) +} + +job "stack-config-echo" { + private = true + + option "opt1" { + type = string + } + + exec { + command = "echo" + args = [opt.opt1] + } +} + job "stack config" { concurrency = 1 description = "Generate stack config in YAML or JSON format" @@ -25,7 +57,7 @@ job "stack config" { option "config-type" { type = string - description = "Config type (`vars`, `backend`, `command`)" + description = "Config type (`vars`, `backend-type`, `backend`, `command`, `all`)" default = "vars" } @@ -35,335 +67,85 @@ job "stack config" { description = "Output format (`yaml`, `json`)" } - config "stack-config" { - source file { - path = "${opt.config-dir}/${opt.stack}.yaml" - } - } - - # 1st level of imports - variable "import-paths-1" { - value = try(conf.stack-config.import, null) != null ? conf.stack-config.import : [] - } - - variable "imports-1" { - value = merge( - [ - for p in var.import-paths-1 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 2nd level of imports - variable "import-paths-2" { - value = try(var.imports-1.import, null) != null ? var.imports-1.import : [] - } - - variable "imports-2" { - value = merge( - [ - for p in var.import-paths-2 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 3rd level of imports - variable "import-paths-3" { - value = try(var.imports-2.import, null) != null ? var.imports-2.import : [] - } - - variable "imports-3" { - value = merge( - [ - for p in var.import-paths-3 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 4th level of imports - variable "import-paths-4" { - value = try(var.imports-3.import, null) != null ? var.imports-3.import : [] - } - - variable "imports-4" { - value = merge( - [ - for p in var.import-paths-4 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 5th level of imports - variable "import-paths-5" { - value = try(var.imports-4.import, null) != null ? var.imports-4.import : [] - } - - variable "imports-5" { - value = merge( - [ - for p in var.import-paths-5 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 6th level of imports - variable "import-paths-6" { - value = try(var.imports-5.import, null) != null ? var.imports-5.import : [] - } - - variable "imports-6" { - value = merge( - [ - for p in var.import-paths-6 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) - } - - # 7th level of imports - variable "import-paths-7" { - value = try(var.imports-6.import, null) != null ? var.imports-6.import : [] - } - - variable "imports-7" { - value = merge( - [ - for p in var.import-paths-7 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v - } - ] - ...) + variable "files" { + value = import(opt.config-dir, opt.stack) } - # 8th level of imports - variable "import-paths-8" { - value = try(var.imports-7.import, null) != null ? var.imports-7.import : [] + config "all" { + source file { + paths = var.files + } } - variable "imports-8" { - value = merge( - [ - for p in var.import-paths-8 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v + # Deep-merge all variables in this order: global-scoped, component-type-scoped, and component-scoped + config "vars" { + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all["vars"], {})) } - ] - ...) - } - - # 9th level of imports - variable "import-paths-9" { - value = try(var.imports-8.import, null) != null ? var.imports-8.import : [] - } - - variable "imports-9" { - value = merge( - [ - for p in var.import-paths-9 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v + } + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all[opt.component-type]["vars"], {})) } - ] - ...) - } - - # 10th level of imports - variable "import-paths-10" { - value = try(var.imports-9.import, null) != null ? var.imports-9.import : [] - } - - variable "imports-10" { - value = merge( - [ - for p in var.import-paths-10 : { - for k, v in yamldecode(file(format("%s/%s.yaml", opt.config-dir, p))) : k => v + } + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all.components[opt.component-type][param.component]["vars"], {})) } - ] - ...) - } - - variable "vars" { - value = merge( - # Merge the global vars in the order of imports - try(var.imports-10["vars"], {}), - try(var.imports-9["vars"], {}), - try(var.imports-8["vars"], {}), - try(var.imports-7["vars"], {}), - try(var.imports-6["vars"], {}), - try(var.imports-5["vars"], {}), - try(var.imports-4["vars"], {}), - try(var.imports-3["vars"], {}), - try(var.imports-2["vars"], {}), - try(var.imports-1["vars"], {}), - try(conf.stack-config["vars"], {}), - - # Merge the component type vars (terraform, helmfile) in the order of imports - try(var.imports-10[opt.component-type]["vars"], {}), - try(var.imports-9[opt.component-type]["vars"], {}), - try(var.imports-8[opt.component-type]["vars"], {}), - try(var.imports-7[opt.component-type]["vars"], {}), - try(var.imports-6[opt.component-type]["vars"], {}), - try(var.imports-5[opt.component-type]["vars"], {}), - try(var.imports-4[opt.component-type]["vars"], {}), - try(var.imports-3[opt.component-type]["vars"], {}), - try(var.imports-2[opt.component-type]["vars"], {}), - try(var.imports-1[opt.component-type]["vars"], {}), - try(conf.stack-config[opt.component-type]["vars"], {}), - - # Merge the component vars in the order of imports - try(var.imports-10.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-9.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-8.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-7.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-6.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-5.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-4.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-3.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-2.components[opt.component-type][param.component]["vars"], {}), - try(var.imports-1.components[opt.component-type][param.component]["vars"], {}), - try(conf.stack-config.components[opt.component-type][param.component]["vars"], {}) - ) + } } variable "command" { - value = { command = coalesce( - try(var.imports-10["command"], null), - try(var.imports-9["command"], null), - try(var.imports-8["command"], null), - try(var.imports-7["command"], null), - try(var.imports-6["command"], null), - try(var.imports-5["command"], null), - try(var.imports-4["command"], null), - try(var.imports-3["command"], null), - try(var.imports-2["command"], null), - try(var.imports-1["command"], null), - try(conf.stack-config["command"], null), - - try(var.imports-10[opt.component-type]["command"], null), - try(var.imports-9[opt.component-type]["command"], null), - try(var.imports-8[opt.component-type]["command"], null), - try(var.imports-7[opt.component-type]["command"], null), - try(var.imports-6[opt.component-type]["command"], null), - try(var.imports-5[opt.component-type]["command"], null), - try(var.imports-4[opt.component-type]["command"], null), - try(var.imports-3[opt.component-type]["command"], null), - try(var.imports-2[opt.component-type]["command"], null), - try(var.imports-1[opt.component-type]["command"], null), - try(conf.stack-config[opt.component-type]["command"], null), - - try(var.imports-10.components[opt.component-type][param.component]["command"], null), - try(var.imports-9.components[opt.component-type][param.component]["command"], null), - try(var.imports-8.components[opt.component-type][param.component]["command"], null), - try(var.imports-7.components[opt.component-type][param.component]["command"], null), - try(var.imports-6.components[opt.component-type][param.component]["command"], null), - try(var.imports-5.components[opt.component-type][param.component]["command"], null), - try(var.imports-4.components[opt.component-type][param.component]["command"], null), - try(var.imports-3.components[opt.component-type][param.component]["command"], null), - try(var.imports-2.components[opt.component-type][param.component]["command"], null), - try(var.imports-1.components[opt.component-type][param.component]["command"], null), - try(conf.stack-config.components[opt.component-type][param.component]["command"], null), - - opt.component-type - ) - } + value = coalesce( + try(conf.all["command"], null), + try(conf.all[opt.component-type]["command"], null), + try(conf.all.components[opt.component-type][param.component]["command"], null), + opt.component-type + ) } variable "backend-type" { value = coalesce( - try(var.imports-10["backend_type"], null), - try(var.imports-9["backend_type"], null), - try(var.imports-8["backend_type"], null), - try(var.imports-7["backend_type"], null), - try(var.imports-6["backend_type"], null), - try(var.imports-5["backend_type"], null), - try(var.imports-4["backend_type"], null), - try(var.imports-3["backend_type"], null), - try(var.imports-2["backend_type"], null), - try(var.imports-1["backend_type"], null), - try(conf.stack-config["backend_type"], null), - - try(var.imports-10[opt.component-type]["backend_type"], null), - try(var.imports-9[opt.component-type]["backend_type"], null), - try(var.imports-8[opt.component-type]["backend_type"], null), - try(var.imports-7[opt.component-type]["backend_type"], null), - try(var.imports-6[opt.component-type]["backend_type"], null), - try(var.imports-5[opt.component-type]["backend_type"], null), - try(var.imports-4[opt.component-type]["backend_type"], null), - try(var.imports-3[opt.component-type]["backend_type"], null), - try(var.imports-2[opt.component-type]["backend_type"], null), - try(var.imports-1[opt.component-type]["backend_type"], null), - try(conf.stack-config[opt.component-type]["backend_type"], null), - - try(var.imports-10.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-9.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-8.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-7.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-6.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-5.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-4.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-3.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-2.components[opt.component-type][param.component]["backend_type"], null), - try(var.imports-1.components[opt.component-type][param.component]["backend_type"], null), - try(conf.stack-config.components[opt.component-type][param.component]["backend_type"], null), - + try(conf.all["backend_type"], null), + try(conf.all[opt.component-type]["backend_type"], null), + try(conf.all.components[opt.component-type][param.component]["backend_type"], null), "s3" ) } - variable "backend" { - value = merge( - try(var.imports-10["backend"][var.backend-type], {}), - try(var.imports-9["backend"][var.backend-type], {}), - try(var.imports-8["backend"][var.backend-type], {}), - try(var.imports-7["backend"][var.backend-type], {}), - try(var.imports-6["backend"][var.backend-type], {}), - try(var.imports-5["backend"][var.backend-type], {}), - try(var.imports-4["backend"][var.backend-type], {}), - try(var.imports-3["backend"][var.backend-type], {}), - try(var.imports-2["backend"][var.backend-type], {}), - try(var.imports-1["backend"][var.backend-type], {}), - try(conf.stack-config["backend"][var.backend-type], {}), - - try(var.imports-10[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-9[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-8[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-7[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-6[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-5[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-4[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-3[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-2[opt.component-type]["backend"][var.backend-type], {}), - try(var.imports-1[opt.component-type]["backend"][var.backend-type], {}), - try(conf.stack-config[opt.component-type]["backend"][var.backend-type], {}), - - try(var.imports-10.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-9.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-8.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-7.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-6.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-5.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-4.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-3.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-2.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(var.imports-1.components[opt.component-type][param.component]["backend"][var.backend-type], {}), - try(conf.stack-config.components[opt.component-type][param.component]["backend"][var.backend-type], {}) - ) + # Deep-merge all backend attributes in this order: global-scoped, component-type-scoped, and component-scoped + config "backend" { + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all["backend"][var.backend-type], {})) + } + } + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all[opt.component-type]["backend"][var.backend-type], {})) + } + } + source job { + name = "stack-config-echo" + args = { + opt1 = yamlencode(try(conf.all.components[opt.component-type][param.component]["backend"][var.backend-type], {})) + } + } } variable "output" { value = { - vars = var.vars, - command = var.command, - backend = { "${var.backend-type}" = var.backend } + vars = conf.vars, + command = { command = var.command }, + backend = { "${var.backend-type}" = conf.backend }, + backend-type = var.backend-type, + all = conf.all } }