Support for merging multiple config files from variable (#43)

To better support the use-case of dynamically generating configs from other configs and variables, the following changes are made:

- Added `paths` to `file` config source. Either the existing `path` or `paths` is required.
   - All the files listed in `paths` are merged using `mergo` with `mergo.WithOverride` and `mergo.WithOverwriteWithEmptyValue`
- config source's `paths` and `path` can now refer to `var`s. `variant` internally builds a DAG of `var`s, `conf`s, and `sec`s to make it possible.
- Added `function` block to define user-functions, with support for recursive call. Note that user-functions are visible within a variant command that defines it.
  - This means two things:
    - You can't `import` user-functions
    - User-functions defined in the parent variant command are not visible to imported variant commands.
- Added new example for config-depends-on-var use-case at `examples/advanced/dynamic-config-inheritance`
- Added new example for importing variant command with local user-function at `examples/advanced/userfunc-local-scope`

Resolves #42

.github/workflows/release.yml

@@ -2,8 +2,6 @@ name: goreleaser
 
 on:
   push:
-    branches:
-    - "!*"
     tags:
     - "v*"
 

examples/advanced/dynamic-config-inheritance/config/globals.yaml

@@ -0,0 +1,5 @@
+vars:
+  namespace: eg
+
+override_with_empty: [1]
+override_with_ints: [1]

examples/advanced/dynamic-config-inheritance/config/ue2-globals.yaml

@@ -0,0 +1,9 @@
+import:
+- globals
+
+vars:
+  region: us-east-2
+  environment: ue2
+
+override_with_empty: []
+override_with_ints: [2]

examples/advanced/dynamic-config-inheritance/config/ue2-prod.yaml

@@ -0,0 +1,5 @@
+import:
+- ue2-globals
+
+vars:
+  stage: prod

examples/advanced/dynamic-config-inheritance/config/ue3-prod.yaml

@@ -0,0 +1,5 @@
+import:
+- ue3-globals
+
+vars:
+  stage: prod

examples/advanced/dynamic-config-inheritance/main.variant

@@ -0,0 +1,48 @@
+option "config-dir" {
+  type = string
+}
+
+job "stack config" {
+  concurrency = 1
+  description = "Generate stack config in YAML format"
+
+  option "stack" {
+    type        = string
+    description = "Stack"
+    short       = "s"
+  }
+
+  variable "configs" {
+    value = flatten(concat([
+      # 1st level of imports
+      for k1, imports1 in yamldecode(file(format("%s/%s.yaml", opt.config-dir, opt.stack))): [
+        for import1 in imports1: concat([
+          # 1st level import's imports = 2nd level of imports
+          for k2, imports2 in yamldecode(file(format("%s/%s.yaml", opt.config-dir, import1))): [
+            for import2 in imports2: [
+              # 2nd level import's imports = 3rd level of imports
+              format("%s/%s.yaml", opt.config-dir, import2)
+            ]
+          ] if k2 == "import"
+        ], [
+          format("%s/%s.yaml", opt.config-dir, import1)
+        ])
+      ] if k1 == "import"
+    ], [
+      format("%s/%s.yaml", opt.config-dir, opt.stack)
+    ]))
+  }
+
+  config "all" {
+    source file {
+      paths = var.configs
+    }
+  }
+
+  exec {
+    command = "echo"
+    args = [
+      jsonencode(conf.all)
+    ]
+  }
+}

examples/advanced/dynamic-config-inheritance/main_test.variant

@@ -0,0 +1,165 @@
+test "stack config" {
+  case "ue2-prod" {
+    stack = "ue2-prod"
+    exitstatus = 0
+    err = ""
+    namespace = "eg"
+    region = "us-east-2"
+    environment = "ue2"
+    stage = "prod"
+    override_with_empty = []
+    override_with_ints = [2]
+  }
+
+  // missing stack config
+  case "ue1-prod" {
+    stack = "ue1-prod"
+    exitstatus = 1
+    err = trimspace(<<EOS
+job "stack config": ${abspath("main.variant")}:18,43-50: Invalid function argument; Invalid value for "path" parameter: no file exists at config/ue1-prod.yaml; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource.
+EOS
+    )
+    namespace = "UNDEFINED"
+    region = "UNDEFINED"
+    environment = "UNDEFINED"
+    stage = "UNDEFINED"
+    override_with_empty = "UNDEFINED"
+    override_with_ints = "UNDEFINED"
+  }
+
+  // missing stack config import
+  case "ue3-prod" {
+    stack = "ue3-prod"
+    exitstatus = 1
+    err = trimspace(<<EOS
+job "stack config": ${abspath("main.variant")}:21,47-54: Invalid function argument; Invalid value for "path" parameter: no file exists at config/ue3-globals.yaml; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource.
+EOS
+    )
+    namespace = "UNDEFINED"
+    region = "UNDEFINED"
+    environment = "UNDEFINED"
+    stage = "UNDEFINED"
+    override_with_empty = "UNDEFINED"
+    override_with_ints = "UNDEFINED"
+  }
+
+  run "stack config" {
+    config-dir = "config"
+    stack = case.stack
+  }
+
+  assert "error" {
+    condition = run.err == case.err
+  }
+
+  assert "namespace" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.namespace, "UNDEFINED") == case.namespace) || !run.res.set
+  }
+
+  assert "region" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.region, "UNDEFINED") == case.region) || !run.res.set
+  }
+
+  assert "environment" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.environment, "UNDEFINED") == case.environment) || !run.res.set
+  }
+
+  assert "stage" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.stage, "UNDEFINED") == case.stage) || !run.res.set
+  }
+
+  assert "override_with_empty" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).override_with_empty, "UNDEFINED") == case.override_with_empty) || !run.res.set
+  }
+
+  assert "override_with_ints" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).override_with_ints, "UNDEFINED") == case.override_with_ints) || !run.res.set
+  }
+
+  assert "exitstatus" {
+    condition = (run.res.set && run.res.exitstatus == case.exitstatus) || !run.res.set
+  }
+}
+
+test "userfunc stack config" {
+  case "ue2-prod" {
+    stack = "ue2-prod"
+    exitstatus = 0
+    err = ""
+    namespace = "eg"
+    region = "us-east-2"
+    environment = "ue2"
+    stage = "prod"
+    override_with_empty = []
+    override_with_ints = [2]
+  }
+
+  // missing stack config
+  case "ue1-prod" {
+    stack = "ue1-prod"
+    exitstatus = 1
+    err = trimspace(<<EOS
+job "userfunc stack config": ${abspath("userfunc.variant")}:25,13-20: Error in function call; Call to function "import" failed: ${abspath("userfunc.variant")}:4,39-46: Invalid function argument; Invalid value for "path" parameter: no file exists at config/ue1-prod.yaml; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource..
+EOS
+    )
+    namespace = "UNDEFINED"
+    region = "UNDEFINED"
+    environment = "UNDEFINED"
+    stage = "UNDEFINED"
+    override_with_empty = "UNDEFINED"
+    override_with_ints = "UNDEFINED"
+  }
+
+  // missing stack config import
+  case "ue3-prod" {
+    stack = "ue3-prod"
+    exitstatus = 1
+    err = trimspace(<<EOS
+job "userfunc stack config": ${abspath("userfunc.variant")}:25,13-20: Error in function call; Call to function "import" failed: ${abspath("userfunc.variant")}:6,9-16: Error in function call; Call to function "import" failed: ${abspath("userfunc.variant")}:4,39-46: Invalid function argument; Invalid value for "path" parameter: no file exists at config/ue3-globals.yaml; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource...
+EOS
+    )
+    namespace = "UNDEFINED"
+    region = "UNDEFINED"
+    environment = "UNDEFINED"
+    stage = "UNDEFINED"
+    override_with_empty = "UNDEFINED"
+    override_with_ints = "UNDEFINED"
+  }
+
+  run "userfunc stack config" {
+    config-dir = "config"
+    stack = case.stack
+  }
+
+  assert "error" {
+    condition = run.err == case.err
+  }
+
+  assert "namespace" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.namespace, "UNDEFINED") == case.namespace) || !run.res.set
+  }
+
+  assert "region" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.region, "UNDEFINED") == case.region) || !run.res.set
+  }
+
+  assert "environment" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.environment, "UNDEFINED") == case.environment) || !run.res.set
+  }
+
+  assert "stage" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).vars.stage, "UNDEFINED") == case.stage) || !run.res.set
+  }
+
+  assert "override_with_empty" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).override_with_empty, "UNDEFINED") == case.override_with_empty) || !run.res.set
+  }
+
+  assert "override_with_ints" {
+    condition = (run.res.set && try(jsondecode(run.res.stdout).override_with_ints, "UNDEFINED") == case.override_with_ints) || !run.res.set
+  }
+
+  assert "exitstatus" {
+    condition = (run.res.set && run.res.exitstatus == case.exitstatus) || !run.res.set
+  }
+}

examples/advanced/dynamic-config-inheritance/userfunc.variant

@@ -0,0 +1,40 @@
+function "import" {
+  params = [config-dir, path]
+  result = flatten(concat([
+    for k, imports in yamldecode(file(format("%s/%s.yaml", config-dir, path))): [
+      for imported in imports: [
+        import(config-dir, imported)
+      ]
+    ] if k == "import"
+  ], [
+    format("%s/%s.yaml", config-dir, path)
+  ]))
+}
+
+job "userfunc stack config" {
+  concurrency = 1
+  description = "Generate stack config in YAML format"
+
+  option "stack" {
+    type        = string
+    description = "Stack"
+    short       = "s"
+  }
+
+  variable "files" {
+    value = import(opt.config-dir, opt.stack)
+  }
+
+  config "all" {
+    source file {
+      paths = var.files
+    }
+  }
+
+  exec {
+    command = "echo"
+    args = [
+      jsonencode(conf.all)
+    ]
+  }
+}

examples/advanced/userfunc-local-scope/config/globals.yaml

@@ -0,0 +1,5 @@
+vars:
+  namespace: eg
+
+override_with_empty: [1]
+override_with_ints: [1]

examples/advanced/userfunc-local-scope/config/ue2-globals.yaml

@@ -0,0 +1,9 @@
+import:
+- globals
+
+vars:
+  region: us-east-2
+  environment: ue2
+
+override_with_empty: []
+override_with_ints: [2]

examples/advanced/userfunc-local-scope/config/ue2-prod.yaml

@@ -0,0 +1,5 @@
+import:
+- ue2-globals
+
+vars:
+  stage: prod

examples/advanced/userfunc-local-scope/config/ue3-prod.yaml

@@ -0,0 +1,5 @@
+import:
+- ue3-globals
+
+vars:
+  stage: prod

examples/advanced/userfunc-local-scope/main.variant

@@ -0,0 +1,5 @@
+import = "../dynamic-config-inheritance"
+
+job "nested" {
+  import = "../dynamic-config-inheritance"
+}