From 7fc899f6aee13e60b0b07fa4fc1cdf45c2530bbd Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 21 Oct 2024 16:11:52 -0500 Subject: [PATCH 1/6] Document helmfile --- .github/workflows/test.yml | 56 +++++------ examples/demo-helmfile/atmos.yaml | 12 +-- examples/demo-vendoring/atmos.yaml | 11 ++- website/docs/core-concepts/deploy/deploy.mdx | 2 - .../stacks/define-components.mdx | 95 +++++++++++++++++-- website/docs/core-concepts/stacks/stacks.mdx | 44 +++++---- .../core-concepts/vendor/vendor-manifest.mdx | 27 ++++++ website/docs/core-concepts/vendor/vendor.mdx | 55 +++++------ website/docs/quick-start/install-atmos.mdx | 26 ++--- 9 files changed, 219 insertions(+), 109 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f06ab784f..d2a5b1c09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,10 +10,10 @@ on: - main - release/v* paths-ignore: - - '.github/**' - - 'docs/**' - - 'examples/**' - - 'test/**' + - ".github/**" + - "docs/**" + - "examples/**" + - "test/**" workflow_dispatch: @@ -94,29 +94,29 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@v4 - - - uses: hadolint/hadolint-action@v3.1.0 - id: hadolint - with: - dockerfile: Dockerfile - failure-threshold: warning - format: sarif - output-file: hadolint.sarif - # https://github.com/hadolint/hadolint?tab=readme-ov-file#rules - # DL3008 Pin versions in apt-get install - ignore: DL3008 - - - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v3 - if: always() - with: - # Path to SARIF file relative to the root of the repository - sarif_file: hadolint.sarif - # Optional category for the results (used to differentiate multiple results for one commit) - category: hadolint - wait-for-processing: true + - name: Check out code + uses: actions/checkout@v4 + + - uses: hadolint/hadolint-action@v3.1.0 + id: hadolint + with: + dockerfile: Dockerfile + failure-threshold: warning + format: sarif + output-file: hadolint.sarif + # https://github.com/hadolint/hadolint?tab=readme-ov-file#rules + # DL3008 Pin versions in apt-get install + ignore: DL3008 + + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + # Path to SARIF file relative to the root of the repository + sarif_file: hadolint.sarif + # Optional category for the results (used to differentiate multiple results for one commit) + category: hadolint + wait-for-processing: true # run localstack demo tests localstack: @@ -254,7 +254,7 @@ jobs: # - demo-mock-architecture # - demo-stack-templating # - demo-multi-cloud - # - demo-vendoring + - demo-vendoring timeout-minutes: 20 steps: diff --git a/examples/demo-helmfile/atmos.yaml b/examples/demo-helmfile/atmos.yaml index 97dc1b202..538b70d66 100644 --- a/examples/demo-helmfile/atmos.yaml +++ b/examples/demo-helmfile/atmos.yaml @@ -7,9 +7,8 @@ schemas: components: helmfile: base_path: "components/helmfile" + use_eks: false kubeconfig_path: "{{ .Env.KUBECONFIG }}" - helm_aws_profile_pattern: default - cluster_name_pattern: 'demo' stacks: base_path: "stacks" @@ -31,10 +30,11 @@ commands: description: "Run all tests" steps: - atmos validate stacks - # FIXME: `atmos helmfile apply` assumes EKS - #- atmos helmfile apply demo -s dev - - atmos helmfile generate varfile demo -s dev - - helmfile -f components/helmfile/nginx/helmfile.yaml apply --values dev-demo.helmfile.vars.yaml + - atmos helmfile apply demo -s dev + + # This is equivalent to the following commands: + #- atmos helmfile generate varfile demo -s dev + #- helmfile -f components/helmfile/nginx/helmfile.yaml apply --values dev-demo.helmfile.vars.yaml # Use Nested Custom Commands to provide easier interface for Docker Compose - name: "k3s" diff --git a/examples/demo-vendoring/atmos.yaml b/examples/demo-vendoring/atmos.yaml index 555d07515..1c7d3ad24 100644 --- a/examples/demo-vendoring/atmos.yaml +++ b/examples/demo-vendoring/atmos.yaml @@ -7,7 +7,7 @@ components: deploy_run_init: true init_run_reconfigure: true auto_generate_backend_file: false - + stacks: base_path: "stacks" included_paths: @@ -19,3 +19,12 @@ stacks: logs: file: "/dev/stderr" level: Info + +# Custom CLI commands + +# No arguments or flags are required +commands: +- name: "test" + description: "Run all tests" + steps: + - atmos vendor pull diff --git a/website/docs/core-concepts/deploy/deploy.mdx b/website/docs/core-concepts/deploy/deploy.mdx index 6a431373a..b29177e23 100644 --- a/website/docs/core-concepts/deploy/deploy.mdx +++ b/website/docs/core-concepts/deploy/deploy.mdx @@ -11,7 +11,6 @@ import Intro from '@site/src/components/Intro'; Once you're done developing your components and configuring them with stacks, you can deploy them with a single command or in a CI/CD pipeline. - In Atmos, when we talk about "Deployment," it refers to taking the [fully rendered and deep-merged configuration](/core-concepts/describe) of a [stack](/core-concepts/stacks) and provisioning an instance of one of the components. We call this a "component instance," and it's simply a component that has been deployed in a specific stack. ### Deployment in Atmos @@ -40,7 +39,6 @@ All configurations in Atmos are defined in YAML. If you can write a Terraform mo - [Spacelift](/integrations/spacelift): Our Spacelift integrations support dependency order application. - [Atlantis](/integrations/atlantis): By customizing the template generated for Atlantis, similar dependency handling can probably be achieved, although we do not have any documentation on this. - ### Automate Cold Starts Atmos supports [workflows](/core-concepts/workflows), which provide a convenient way to automate deployments, especially for cold starts. A cold start is when you go from zero to a full deployment, typically occurring on day zero in the life cycle of your resources. diff --git a/website/docs/core-concepts/stacks/define-components.mdx b/website/docs/core-concepts/stacks/define-components.mdx index 0379e5984..3ed4f5567 100644 --- a/website/docs/core-concepts/stacks/define-components.mdx +++ b/website/docs/core-concepts/stacks/define-components.mdx @@ -23,11 +23,14 @@ To configure a Component in a [Stack](/core-concepts/stacks), you define the com that consists of the resources defined in the `.tf` files in a working directory (e.g. [components/terraform/infra/vpc](https://github.com/cloudposse/atmos/tree/main/examples/tests/components/terraform/infra/vpc)) -- **Component Configuration** provides configuration (variables and other settings) for a type of component (e.g. a Terraform component) +- **Component Configuration** provides configuration (variables and other settings) for a type of component (e.g. a Terraform component) and is defined in one or more YAML stack config files (which are called [Atmos stacks](/core-concepts/stacks)) ::: -The schema of an Atmos Component in an Atmos Stack is as follows: + +### Terraform Schema + +The schema of an Atmos Terraform Component in an Atmos Stack is as follows: ```yaml components: @@ -39,7 +42,7 @@ components: metadata: # Components can be of type "real" (default) or "abstract" type: real - # This is the directory path of the component. + # This is the directory path of the component. # In this example, we're referencing a component in the `components/terraform/stable/example` folder. component: stable/example @@ -59,10 +62,9 @@ components: nodes: 10 ``` -### Component Attributes +#### Terraform Attributes
-
`vars` (optional)
The `vars` section is a free-form map. Use [component validation](/core-concepts/validate) to enforce policies.
@@ -134,15 +136,94 @@ components:
+ + +### Helmfile Schema + +The schema of an Atmos Helmfile Component in an Atmos Stack is as follows: + +```yaml +components: + helmfile: + # the slug of the component + example: + + # configuration specific to atmos + metadata: + # Components can be of type "real" (default) or "abstract" + type: real + + # This is the directory path of the component. + # In this example, we're referencing a component in the `components/terraform/stable/example` folder. + component: stable/example + + # We can leverage multiple inheritance to sequentially deep merge multiple configurations + inherits: + - example-defaults + + # Define the terraform variables, which will get deep-merged and exported to a `.tfvars` file by atmos. + vars: + enabled: true + name: superduper + nodes: 10 +``` + +#### Helmfile Attributes + +
+ +
`vars` (optional)
+
The `vars` section is a free-form map. Use [component validation](/core-concepts/validate) to enforce policies.
+ +
`vars.namespace` (optional)
+
+ This is an *optional* [`terraform-null-label`](https://github.com/cloudposse/terraform-null-label) convention. + + The namespace of all stacks. Typically, there will be one namespace for the organization. + + Example: + + ```yaml + vars: + namespace: acme + ``` +
+ +
`metadata` (optional)
+
The `metadata` section extends functionality of the component.
+ +
`settings`
+
The `settings` block is a free-form map used to pass configuration information to [integrations](/integrations).
+ +
`base_path`
+
The root directory where the Helmfile components and configurations are located. This path serves as the starting point for resolving any relative paths within the Helmfile setup.
+ +
`use_eks` (default: `false`)
+
A flag indicating whether the component is configured to use Amazon EKS (Elastic Kubernetes Service). When set to `true`, the component will interact with EKS for provisioning and managing Kubernetes clusters. Also, it means `cluster_name_pattern` must be defined.
+ +
`kubeconfig_path`
+
The file path to the `kubeconfig` file, which contains the necessary authentication and configuration details to interact with the Kubernetes cluster. This path is essential for managing cluster resources using Helmfile.
+ +
`helm_aws_profile_pattern`
+
A pattern that defines which AWS CLI profiles should be used by Helm when interacting with AWS services, such as EKS. This allows for dynamic selection of AWS credentials based on the environment or cluster.
+ +
`cluster_name_pattern` (required when `use_eks=true`)
+
A naming pattern used to identify and select the Kubernetes cluster within the Helmfile configuration. This pattern helps automate the management of different clusters by matching their names based on the specified criteria.
+ +
+ + ### Types of Components +In Atmos, each component configuration defines its type through the `metadata.type` parameter. This defines how the component behaves—whether it can be used directly to provision resources or serves as a base configuration for other components. + The type of component is expressed in the `metadata.type` parameter of a given component configuration. There are two types of components:
`real`
-
is a ["concrete"](https://en.wikipedia.org/wiki/Concrete_class) component instance that can be provisioned
+
Think of a `real` component as one that can be deployed. It’s fully configured and ready to be provisioned, similar to a "concrete" class in programming. Once defined, you can use it to create resources or services directly in your infrastructure.
`abstract`
-
a component configuration, which cannot be instantiated directly. The concept is borrowed from ["abstract base classes"](https://en.wikipedia.org/wiki/Abstract_type) of Object-Oriented Programming
+
An `abstract` component is more like a blueprint. It can’t be deployed on its own. Instead, it’s a base configuration that needs to be extended or inherited by other components. This is similar to an ["abstract base classes"](https://en.wikipedia.org/wiki/Abstract_type) in programming—it defines reusable configurations, but it’s not complete enough to be deployed directly.
diff --git a/website/docs/core-concepts/stacks/stacks.mdx b/website/docs/core-concepts/stacks/stacks.mdx index 1efc276bc..580c46c8c 100644 --- a/website/docs/core-concepts/stacks/stacks.mdx +++ b/website/docs/core-concepts/stacks/stacks.mdx @@ -66,8 +66,6 @@ Components are different from Stacks. When a component is added to a stack, we call that a "Component Instance" - - ### Parent Stacks vs Child Stacks
@@ -150,27 +148,33 @@ components: ### Stack Attributes -#### components - -The `components` is the list of all the building blocks. - -Example: +
+
`components`
+
+ The `components` is the list of all the building blocks. -```yaml -components: - sometool: # "sometool" can be any tool - somecomponent: # "somecomponent" can be the name of any "sometool" component - vars: # etc... -``` + Example: -So for `terraform`, it might look something like this: + ```yaml + components: + sometool: # "sometool" can be any tool + somecomponent: # "somecomponent" can be the name of any "sometool" component + vars: # etc... + ``` +
-```yaml -components: - terraform: - vpc: - vars: # etc... -``` +
`components.terraform`
+
+ So for `terraform`, it might look something like this: + + ```yaml + components: + terraform: + vpc: + vars: # etc... + ``` +
+
## Stack Files diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 66d0d3e60..f8b910733 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -424,6 +424,33 @@ Pulling sources for the component 'vpc-flow-logs-bucket' from 'github.com/cloudp ``` +## Vendoring Multiple Versions of Components + +Atmos supports vendoring multiple versions of the same component. This is useful when you need to pin stacks to different versions of the same component. + +When vendoring multiple versions of the same component, use the `{{ .Version }}` template parameter in the `targets` attribute to ensure the components are vendored into different folders. Then update the stack configuration to point to the correct version of the component, and ensure the `backend.s3.workspace_key_prefix` is defined _without the version_ to ensure you can seamlessly upgrade between versions of a component wihtout losing the state. By default the `workspace_key_prefix` incorporates the `component` relative path, which will include the version if it's included in the path. + +``` +components: + terraform: + # `vpc` is the Atmos component name + vpc: + # Backend configuration for the component + backend: + s3: + # Ensure the path in the bucket is stable across versions + # IMPORTANT: If not explicilty set, the `workspace_key_prefix` will include the version + # This will cause the state to be lost when upgrading between versions. + workspace_key_prefix: vpc + metadata: + # Point to the Terraform component on the filesystem + component: vpc/1.2.3 +``` + +:::important +If not using the S3 backend, use the appropriate parameter for your backend to ensure the workspace is stable across versions of the component deployed. +::: + ## Vendoring from OCI Registries Atmos supports vendoring from [OCI registries](https://opencontainers.org). diff --git a/website/docs/core-concepts/vendor/vendor.mdx b/website/docs/core-concepts/vendor/vendor.mdx index 20d51d8d1..d5bdeb523 100644 --- a/website/docs/core-concepts/vendor/vendor.mdx +++ b/website/docs/core-concepts/vendor/vendor.mdx @@ -17,38 +17,6 @@ Vendoring standardizes dependency management, encourages enterprise component re ## Use-cases -Use vendoring to maintain a local copy of external dependencies critical for managing your infrastructure. Organize the dependencies in the manner that best suits your project's structure. Even create multiple vendor manifests, for example, tailored to specific layers, products, or teams. Then easily -update those dependencies by bumping the versions in the vendor manifest. - -- **Managing Third-Party Dependencies**: Use vendoring in Atmos to efficiently manage and version-control third-party Terraform components, modules or other infrastructure dependencies. This approach is crucial for teams relying on external libraries, root modules, and configurations from sources such as Git repositories. Vendoring these dependencies into your project repository ensures that every team member and CI/CD pipeline works with the same dependency versions, enhancing consistency and reliability across development, testing, and production environments. -- **Sharing Components Across an Enterprise**: Utilize Atmos vendoring to access a centralized component library, promoting code reuse and efficiency across teams while enabling customization and independent version control post-vendoring. This approach enhances collaboration without sacrificing the flexibility for teams to tailor components to their specific needs or update them at their preferred pace. -- **Maintaining Immutable Artifacts for Compliance**: Employ vendoring through Atmos to maintain local, immutable copies of remote dependencies, essential for meeting compliance and regulatory requirements. Keeping a local version-controlled artifact of dependencies ensures that your infrastructure complies with policies that mandate a record of all external components used within the system. This practice supports auditability and traceability, key aspects of maintaining a secure and compliant infrastructure. -- **Overcoming Tooling Limitations with Remote Dependencies**: Utilize Atmos vendoring as a practical solution when your tooling lacks native support for managing remote dependencies. By copying these dependencies into your project repository, you can work around these limitations, ensuring that your infrastructure can still leverage essential external modules and configurations. This approach allows for greater flexibility in infrastructure management, adapting to tooling constraints while still benefiting from the broad ecosystem of available infrastructure modules and configurations. -- **Optimize Boilerplate Code Reusability with Vendoring** Developers can utilize Atmos vendoring together with components to consolidate code by sourcing mixins (e.g. `providers.tf`, `context.tf`, etc) and boilerplate from centralized locations, streamlining development with DRY principles and immutable infrastructure. - -:::tip Pro Tip! Use GitOps -Vendoring plays nicely with GitOps practices, especially when leveraging [GitHub Actions](/integrations/github-actions/). -Use a workflow that automatically updates the vendor manifest and opens a pull request (PR) with all the changes. -This allows you to inspect and precisely assess the impact of any upgrades before merging by reviewing the job summary of the PR. -::: - -## Features - -With Atmos vendoring, you can copy components and other artifacts from the following sources: - -- Copy all files from an [OCI Registry](https://opencontainers.org) into a local folder -- Copy all files from Git, Mercurial, Amazon S3, Google GCP into a local folder -- Copy all files from an HTTP/HTTPS endpoint into a local folder -- Copy a single file from an HTTP/HTTPS endpoint to a local file -- Copy a local file into a local folder (keeping the same file name) -- Copy a local file to a local file with a different file name -- Copy a local folder (all files) into a local folder - -Our implementation is primarily inspired by the excellent tool by VMware Tanzu, called [`vendir`](https://github.com/vmware-tanzu/carvel-vendir). -While Atmos does not call `vendir`, it functions and supports a subset of the configuration that is very similar. - -## Use-cases - Atmos vendoring streamlines component sharing and version control across an enterprise, enhancing efficiency and collaboration while offering the flexibility to customize and manage multiple versions of dependencies, ensuring best practices in DevOps environments. - **Sharing Components Across an Enterprise**: Utilize Atmos vendoring to access a centralized component library, promoting code reuse and @@ -75,3 +43,26 @@ Refer to [`Atmos Vendoring`](/core-concepts/vendor) for more details The `component.yaml` vendoring manifest is used to vendor components from remote repositories. A `component.yaml` file placed into a component's directory is used to describe the vendoring config for one component only. + + +:::tip Pro Tip! Use GitOps +Vendoring plays nicely with GitOps practices, especially when leveraging [GitHub Actions](/integrations/github-actions/). +Use a workflow that automatically updates the vendor manifest and opens a pull request (PR) with all the changes. +This allows you to inspect and precisely assess the impact of any upgrades before merging by reviewing the job summary of the PR. +::: + +## Features + +With Atmos vendoring, you can copy components and other artifacts from the following sources: + +- Copy all files from an [OCI Registry](https://opencontainers.org) into a local folder +- Copy all files from Git, Mercurial, Amazon S3, Google GCP into a local folder +- Copy all files from an HTTP/HTTPS endpoint into a local folder +- Copy a single file from an HTTP/HTTPS endpoint to a local file +- Copy a local file into a local folder (keeping the same file name) +- Copy a local file to a local file with a different file name +- Copy a local folder (all files) into a local folder + +Our implementation is primarily inspired by the excellent tool by VMware Tanzu, called [`vendir`](https://github.com/vmware-tanzu/carvel-vendir). +While Atmos does not call `vendir`, it functions and supports a subset of the configuration that is very similar. + diff --git a/website/docs/quick-start/install-atmos.mdx b/website/docs/quick-start/install-atmos.mdx index 6ef0e0948..0b387dc24 100644 --- a/website/docs/quick-start/install-atmos.mdx +++ b/website/docs/quick-start/install-atmos.mdx @@ -11,6 +11,7 @@ import Link from '@docusaurus/Link'; import Intro from '@site/src/components/Intro' import LatestRelease from '@site/src/components/LatestRelease' import File from '@site/src/components/File' +import CodeBlock from '@theme/CodeBlock'; There are many ways to install Atmos. Choose the method that works best for you! @@ -97,7 +98,7 @@ Atmos has native packages for macOS and every major Linux distribution. We also nix-env -iA nixpkgs.atmos ``` - To get the latest version, you may need to update the Nix config to add "unstable" packages. + To get the latest version, you may need to update the Nix config to add "unstable" packages. For example, in `~/.config/nixpkgs/config.nix` add the following line to the file: @@ -125,7 +126,13 @@ Atmos has native packages for macOS and every major Linux distribution. We also __NOTE:__ Don't have `scoop`? Install it first from https://scoop.sh/ + + +## Other Ways to Install + +Atmos has a few other ways to install, including using Go, asdf, mise, aqua, building from source, or using the automatic installer. + #### Install with Go @@ -137,9 +144,7 @@ Atmos has native packages for macOS and every major Linux distribution. We also Grab a specific version: - ```shell - go install github.com/cloudposse/atmos@v1.64.0 - ``` + go install github.com/cloudposse/atmos@ Or specifically request the latest version. @@ -147,8 +152,7 @@ Atmos has native packages for macOS and every major Linux distribution. We also go install github.com/cloudposse/atmos@latest ``` - __NOTE:__ Since the version is passed in via `-ldflags` during the build, when running `go install` without using `-ldflags`, the CLI will return `0.0.1` - when running `atmos version`. + __NOTE:__ Since the version is passed in via `-ldflags` during the build, when running `go install` without using `-ldflags`, the CLI will return `0.0.1` when running `atmos version`. @@ -168,9 +172,8 @@ Atmos has native packages for macOS and every major Linux distribution. We also Install a specified version: - ```shell - asdf install atmos - ``` + asdf install atmos + Alternatively, create a `.tool-versions` file in your project to specify a consistent version for the users: 
@@ -189,10 +192,7 @@ Atmos has native packages for macOS and every major Linux distribution. We also
     #### Install with Mise
 
     Install a specified version:
-
-    ```shell
-    mise use atmos@
-    ```
+    mise use atmos@
 
     Alternatively, create a `.mise.toml` file in your repository to specify a consistent version for the users:
     

From 9aef37e7bd949ddb9773174a1129ffd41800bc30 Mon Sep 17 00:00:00 2001
From: Erik Osterman 
Date: Mon, 28 Oct 2024 12:24:26 -0500
Subject: [PATCH 2/6] add helmfile configuration page

---
 .../projects/configuration/helmfile.mdx       |  68 +++++++++++
 .../stacks/define-components.mdx              |  15 ---
 website/docs/core-concepts/stacks/imports.mdx | 107 ++++++++++--------
 3 files changed, 130 insertions(+), 60 deletions(-)
 create mode 100644 website/docs/core-concepts/projects/configuration/helmfile.mdx

diff --git a/website/docs/core-concepts/projects/configuration/helmfile.mdx b/website/docs/core-concepts/projects/configuration/helmfile.mdx
new file mode 100644
index 000000000..6f27f8c69
--- /dev/null
+++ b/website/docs/core-concepts/projects/configuration/helmfile.mdx
@@ -0,0 +1,68 @@
+---
+title: Configure Helmfile
+sidebar_position: 3
+sidebar_label: Configure Helmfile
+id: helmfile
+---
+import Intro from '@site/src/components/Intro'
+import File from '@site/src/components/File'
+
+
+Atmos natively supports opinionated workflows for Helmfile. It's compatible with every version of helmfile and designed to work with multiple different versions of Helmfile concurrently.
+
+
+Keep in mind that Atmos does not handle the downloading or installation of Helmfile (or Kustomize, a dependency of Helmfile); it assumes that any required commands are already installed on your system. To automate this, consider creating a [Custom Command](/core-concepts/custom-commands) to install Helmfile.
+
+Atmos provides many settings that are specific to Helmfile which are configured in `atmos.yaml`.
+
+## CLI Configuration
+
+All of the following settings are defined by default in the [Atmos CLI Configuration](/cli/configuration) found in `atmos.yaml`.
+
+:::important
+At this time, these settings cannot be overridden in the [Stack](/core-concepts/stacks/#schema) configuration.
+:::
+
+The defaults for everything are defined underneath the `components.helmfile` section.
+
+```yaml
+components:
+  helmfile:
+    # ...
+```
+
+The following settings are available for Helmfile:
+

+
+  
`base_path`

+  
The root directory where the Helmfile components and configurations are located. This path serves as the starting point for resolving any relative paths within the Helmfile setup.

+
+  
`use_eks` (default: `false`)

+  
A flag indicating whether the component is configured to use Amazon EKS (Elastic Kubernetes Service). When set to `true`, the component will interact with EKS for provisioning and managing Kubernetes clusters. Also, it means `cluster_name_pattern` must be defined.

+
+  
`kubeconfig_path`

+  
The file path to the `kubeconfig` file, which contains the necessary authentication and configuration details to interact with the Kubernetes cluster. This path is essential for managing cluster resources using Helmfile.

+
+  
`helm_aws_profile_pattern`

+  
A pattern that defines which AWS CLI profiles should be used by Helm when interacting with AWS services, such as EKS. This allows for dynamic selection of AWS credentials based on the environment or cluster.

+
+  
`cluster_name_pattern` (required when `use_eks=true`)

+  
A naming pattern used to identify and select the Kubernetes cluster within the Helmfile configuration. This pattern helps automate the management of different clusters by matching their names based on the specified criteria.

+
+

+
+## Example Configuration
+
+Here is an example configuration for Helmfile that we use at Cloud Posse in our [refarch for AWS](https://docs.cloudposse.com/).
+
+
+```yaml
+components:
+  helmfile:
+    base_path: components/helmfile
+    use_eks: true
+    kubeconfig_path: /dev/shm
+    helm_aws_profile_pattern: '{namespace}-{tenant}-gbl-{stage}-helm'
+    cluster_name_pattern: '{namespace}-{tenant}-{environment}-{stage}-eks-cluster'
+```
+
diff --git a/website/docs/core-concepts/stacks/define-components.mdx b/website/docs/core-concepts/stacks/define-components.mdx
index 3ed4f5567..01bb82401 100644
--- a/website/docs/core-concepts/stacks/define-components.mdx
+++ b/website/docs/core-concepts/stacks/define-components.mdx
@@ -195,21 +195,6 @@ components:
   
`settings`

   
The `settings` block is a free-form map used to pass configuration information to [integrations](/integrations).

 
-  
`base_path`

-  
The root directory where the Helmfile components and configurations are located. This path serves as the starting point for resolving any relative paths within the Helmfile setup.

-
-  
`use_eks` (default: `false`)

-  
A flag indicating whether the component is configured to use Amazon EKS (Elastic Kubernetes Service). When set to `true`, the component will interact with EKS for provisioning and managing Kubernetes clusters. Also, it means `cluster_name_pattern` must be defined.

-
-  
`kubeconfig_path`

-  
The file path to the `kubeconfig` file, which contains the necessary authentication and configuration details to interact with the Kubernetes cluster. This path is essential for managing cluster resources using Helmfile.

-
-  
`helm_aws_profile_pattern`

-  
A pattern that defines which AWS CLI profiles should be used by Helm when interacting with AWS services, such as EKS. This allows for dynamic selection of AWS credentials based on the environment or cluster.

-
-  
`cluster_name_pattern` (required when `use_eks=true`)

-  
A naming pattern used to identify and select the Kubernetes cluster within the Helmfile configuration. This pattern helps automate the management of different clusters by matching their names based on the specified criteria.

-
diff --git a/website/docs/core-concepts/stacks/imports.mdx b/website/docs/core-concepts/stacks/imports.mdx index 18d28e992..69e394a7a 100644 --- a/website/docs/core-concepts/stacks/imports.mdx +++ b/website/docs/core-concepts/stacks/imports.mdx @@ -14,28 +14,29 @@ As your stacks grow taller with more and more component configurations, it often Each import overlays on top of others, and gets deep merged. Then we support [inheritance](/core-concepts/stacks/inheritance) and overrides to manage configuration variations, all without relying on templating. When none of these other methods work for your use-case, we provide an "Escape Hatch" with [templating](/core-concepts/stacks/templates). ## Use cases + - **DRY Configuration:** Imports are how we reduce duplication of configurations. - **Configuration Blueprints:** Define reusable baselines or "defaults". Think of them almost as blueprints, that you can reuse anytime you want some particular combination of components in a stack. - **Service Catalogs:** Provide a "Service Catalog" for your team with reusable configurations that anyone can use to easily compose architectures with golden-path configurations. :::warning Pitfalls! -Imports can make your configurations harder to understand if overused. Review our [best practices](/best-practices/stacks) for practical guidance. +Overusing imports can make configurations harder to understand. We recommend limiting import levels to maintain clarity. Review our [best practices](/best-practices/stacks) for practical guidance. ::: Imports may be used in Stack configurations together with [inheritance](/core-concepts/stacks/inheritance) -and [mixins](/core-concepts/stacks/inheritance/mixins) to produce an exceptionally DRY configuration in a way that is logically organized and easier to maintain -for your team. +and [mixins](/core-concepts/stacks/inheritance/mixins) to produce an exceptionally DRY configuration in a way that is logically organized and easier to maintain by your team. ## Configuration To import any stack configuration from the `catalog/`, simply define an `import` section at the top of any [Stack](/core-concepts/stacks) configuration. Technically, it can be placed anywhere in the file, but by convention we recommend putting it at the top. +Here are some simple examples of how to import configurations: ```yaml import: - - catalog/file1 - - catalog/file2 - - catalog/file2 + - catalog/file1 # First import "file1" from the catalog + - catalog/file2 # Second import "file2" from the catalog, deep merging on top of the first import + - catalog/file3 # Third import "file3" from the catalog, deep merging on top of the preceeding imports ``` The base path for imports is specified in the [`atmos.yaml`](/cli/configuration) in the `stacks.base_path` section. @@ -46,9 +47,9 @@ It's also possible to specify file extensions, although we do not recommend it. ```yaml import: - - catalog/file1.yml - - catalog/file2.yaml - - catalog/file2.YAML + - catalog/file1.yml # Expicitly load a file with a .yml extension + - catalog/file2.yaml # Expicitly load a file with a .yaml extension + - catalog/file3.YAML # Expicitly load a file with a .YAML extension ``` ## Conventions @@ -59,9 +60,11 @@ Use [mixins](/core-concepts/stacks/inheritance/mixins) for reusable snippets of ## Imports Schema -The `import` section supports the following two formats: +The `import` section supports two different formats, depending on whether the imported files use templates or not. One is a list of strings respresenting paths to the imported files, and the other is a list of objects with several feature flags. + +### Imports without Templates -- a list of paths to the imported files, for example: +For a list of paths to the imported files, just provide a list of strings like this: ```yaml title="stacks/orgs/cp/tenant1/test1/us-east-2.yaml" import: @@ -73,40 +76,55 @@ The `import` section supports the following two formats: - catalog/helmfile/echo-server ``` -- a list of objects with the following schema: +### Imports with Templates + +Sometimes you may want to import files that use Go templates. In this case, you can provide a list of objects with `context` that can be used to dynamically generate a stack configuration. + + +:::important + The imported files must have the `.yaml.tmpl` extension to be processed as Go templates and to be able to use the `context` variables. + Additionally, the `skip_templates_processing` flag can be used to skip template processing for the imported file. + Templating must be enabled in [`atmos.yaml`](/core-concepts/stacks/templates) for Atmos to process the imported files as Go templates. +::: + +For example, here we import a file with a template and provide a `context` to passing two variables. ```yaml import: - - path: "" - context: {} + - path: "catalog/something.yaml.tmpl" # Path to the imported file with the required .tmpl extension for Go templates + context: + foo: bar + baz: qux skip_templates_processing: false ignore_missing_template_values: false skip_if_missing: false - - path: "" + - path: "catalog/something.yaml.tmpl" context: {} skip_templates_processing: false ignore_missing_template_values: true skip_if_missing: true ``` -where: +The `import` section supports the following fields: + +
+
`path` - (string) **required**
+
The path to the imported file
-- `path` - (string) The path to the imported file +
`context` - (map)
+
An optional freeform map of context variables that are applied as template variables to the imported file (if the imported file is + a [Go template](https://pkg.go.dev/text/template))
-- `context` - (map) An optional freeform map of context variables that are applied as template variables to the imported file (if the imported file is - a [Go template](https://pkg.go.dev/text/template)) +
`skip_templates_processing` - (boolean)
+
Skip template processing for the imported file. Can be used if the imported file uses `Go` templates that should not be interpretted by atmos. For example, sometimes configurations for components may pass Go template strings not intended for atmos.
-- `skip_templates_processing` - (boolean) Skip template processing for the imported file. Can be used if the imported file uses `Go` templates - to configure external systems, e.g. Datadog +
`ignore_missing_template_values` - (boolean)
+
Ignore the missing template values in the imported file. Can be used if the imported file uses `Go` templates to configure external systems, e.g. Datadog. In this case, Atmos will process all template values that are provided in the `context`, and will skip the missing values in the templates for the external systems without throwing an error. The `ignore_missing_template_values` setting is different from `skip_templates_processing` in that `skip_templates_processing` skips the template processing completely in the imported file, while `ignore_missing_template_values` processes the templates using the values provided in the `context` and skips all the missing values
-- `ignore_missing_template_values` - (boolean) Ignore the missing template values in the imported file. Can be used if the imported file uses `Go` - templates to configure external systems, e.g. Datadog. In this case, Atmos will process all template values that are provided in the `context`, - and will skip the missing values in the templates for the external systems without throwing an error. The `ignore_missing_template_values` setting - is different from `skip_templates_processing` in that `skip_templates_processing` skips the template processing completely in the imported file, - while `ignore_missing_template_values` processes the templates using the values provided in the `context` and skips all the missing values +
`skip_if_missing` - (boolean)
+
Set it to `true` to ignore the imported manifest if it does not exist, and don't throw an error. This is useful when generating Atmos manifests using other tools, but the imported files are not present yet at the generation time.
-- `skip_if_missing` - (boolean) Set it to `true` to ignore the imported manifest if it does not exist, and don't throw an error. - This is useful when generating Atmos manifests using other tools, but the imported files are not present yet at the generation time. +
A combination of the two formats is also supported: @@ -128,9 +146,9 @@ Atmos supports all the functionality of [Go templates](https://pkg.go.dev/text/t Stack configurations can be templatized and then reused with different settings provided via the import `context` section. -For example, we can define the following configuration for EKS Atmos components in the `catalog/terraform/eks_cluster_tmpl.yaml` template file: +For example, we can define the following configuration for EKS Atmos components in the `catalog/terraform/eks_cluster.yaml.tmpl` template file: -```yaml title="stacks/catalog/terraform/eks_cluster_tmpl.yaml" +```yaml title="stacks/catalog/terraform/eks_cluster.yaml.tmpl" # Imports can also be parameterized using `Go` templates import: [] @@ -150,8 +168,7 @@ components: :::note -Since `Go` processes templates as text files, we can parameterize the Atmos component name `eks-{{ .flavor }}/cluster` and any values in any -sections (`vars`, `settings`, `env`, `backend`, etc.), and even the `import` section in the imported file (if the file imports other configurations). +Since `Go` processes files ending in `.yaml.tmpl` text files with templates, we can parameterize the Atmos component name `eks-{{ .flavor }}/cluster` and any values in any sections (`vars`, `settings`, `env`, `backend`, etc.), and even the `import` section in the imported file (if the file imports other configurations). ::: @@ -164,7 +181,7 @@ import: # This import with the provided context will dynamically generate # a new Atmos component `eks-blue/cluster` in the current stack - - path: "catalog/terraform/eks_cluster_tmpl" + - path: "catalog/terraform/eks_cluster.yaml.tmpl" context: flavor: "blue" enabled: true @@ -173,7 +190,7 @@ import: # This import with the provided context will dynamically generate # a new Atmos component `eks-green/cluster` in the current stack - - path: "catalog/terraform/eks_cluster_tmpl" + - path: "catalog/terraform/eks_cluster.yaml.tmpl" context: flavor: "green" enabled: false @@ -228,14 +245,14 @@ vars: Atmos supports hierarchical imports with context. This will allow you to parameterize the entire chain of stack configurations and dynamically generate components in stacks. -For example, let's create the configuration `stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml` with the following content: +For example, let's create the configuration `stacks/catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` with the following content: -```yaml title="stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml" +```yaml title="stacks/catalog/terraform/eks_cluster_hierarchical.yaml.tmpl" import: - # Use `region_tmpl` `Go` template and provide `context` for it. + # Use `region.yaml.tmpl` `Go` template and provide `context` for it. # This can also be done by using `Go` templates in the import path itself. # - path: "mixins/region/{{ .region }}" - - path: "mixins/region/region_tmpl" + - path: "mixins/region/region.yaml.tmpl" # `Go` templates in `context` context: region: "{{ .region }}" @@ -261,14 +278,14 @@ components: ``` Then we can import the template into a top-level stack multiple times providing different context variables to each import and to the imports for -the entire inheritance chain (which `catalog/terraform/eks_cluster_tmpl_hierarchical` imports itself): +the entire inheritance chain (which `catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` imports itself): ```yaml title="stacks/orgs/cp/tenant1/test1/us-west-1.yaml" import: # This import with the provided hierarchical context will dynamically generate # a new Atmos component `eks-blue/cluster` in the `tenant1-uw1-test1` stack - - path: "catalog/terraform/eks_cluster_tmpl_hierarchical" + - path: "catalog/terraform/eks_cluster_hierarchical.yaml.tmpl" context: # Context variables for the EKS component flavor: "blue" @@ -276,7 +293,7 @@ import: service_1_name: "blue-service-1" service_2_name: "blue-service-2" # Context variables for the hierarchical imports - # `catalog/terraform/eks_cluster_tmpl_hierarchical` imports other parameterized configurations + # `catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` imports other parameterized configurations tenant: "tenant1" region: "us-west-1" environment: "uw1" @@ -284,7 +301,7 @@ import: # This import with the provided hierarchical context will dynamically generate # a new Atmos component `eks-green/cluster` in the `tenant1-uw1-test1` stack - - path: "catalog/terraform/eks_cluster_tmpl_hierarchical" + - path: "catalog/terraform/eks_cluster_hierarchical.yaml.tmpl" context: # Context variables for the EKS component flavor: "green" @@ -292,7 +309,7 @@ import: service_1_name: "green-service-1" service_2_name: "green-service-2" # Context variables for the hierarchical imports - # `catalog/terraform/eks_cluster_tmpl_hierarchical` imports other parameterized configurations + # `catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` imports other parameterized configurations tenant: "tenant1" region: "us-west-1" environment: "uw1" @@ -309,9 +326,9 @@ In the case of hierarchical imports, Atmos performs the following steps: effectively processing a graph of imports and deep-merging the contexts on all levels For example, in the `stacks/orgs/cp/tenant1/test1/us-west-1.yaml` configuration above, we first import -the `catalog/terraform/eks_cluster_tmpl_hierarchical` and provide it with the `context` which includes the context variables for the EKS component +the `catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` and provide it with the `context` which includes the context variables for the EKS component itself, as well as the context variables for all the hierarchical imports. Then, when processing -the `stacks/catalog/terraform/eks_cluster_tmpl_hierarchical` configuration, Atmos deep-merges the parent `context` (from +the `stacks/catalog/terraform/eks_cluster_hierarchical.yaml.tmpl` configuration, Atmos deep-merges the parent `context` (from `stacks/orgs/cp/tenant1/test1/us-west-1.yaml`) with the current `context` and processes the `Go` templates. We are now able to dynamically generate the components `eks-blue/cluster` and `eks-green/cluster` in the stack `tenant1-uw1-test1` and can From 760b41e1ae4bb6a8bb7d5d7aa46d440801c42c2b Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Mon, 28 Oct 2024 12:29:53 -0500 Subject: [PATCH 3/6] Update website/docs/core-concepts/stacks/define-components.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- website/docs/core-concepts/stacks/define-components.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/docs/core-concepts/stacks/define-components.mdx b/website/docs/core-concepts/stacks/define-components.mdx index 01bb82401..e7406c9eb 100644 --- a/website/docs/core-concepts/stacks/define-components.mdx +++ b/website/docs/core-concepts/stacks/define-components.mdx @@ -161,11 +161,11 @@ components: inherits: - example-defaults - # Define the terraform variables, which will get deep-merged and exported to a `.tfvars` file by atmos. + # Define the Helmfile variables, which will get deep-merged into the Helmfile configuration. vars: enabled: true - name: superduper - nodes: 10 + release_name: my-release + chart_version: "1.2.3" ``` #### Helmfile Attributes From 049d57c4f58866463e3354c9ebd695e9a7df81b8 Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Mon, 28 Oct 2024 12:30:56 -0500 Subject: [PATCH 4/6] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index f8b910733..55f6e319c 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -439,7 +439,7 @@ components: backend: s3: # Ensure the path in the bucket is stable across versions - # IMPORTANT: If not explicilty set, the `workspace_key_prefix` will include the version + # IMPORTANT: If not explicitly set, the `workspace_key_prefix` will include the version # This will cause the state to be lost when upgrading between versions. workspace_key_prefix: vpc metadata: From 08946a8363ed513488e86cbd434f558bfd0aab36 Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Mon, 28 Oct 2024 12:33:10 -0500 Subject: [PATCH 5/6] Update website/docs/core-concepts/projects/configuration/helmfile.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../docs/core-concepts/projects/configuration/helmfile.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/docs/core-concepts/projects/configuration/helmfile.mdx b/website/docs/core-concepts/projects/configuration/helmfile.mdx index 6f27f8c69..422ec94ef 100644 --- a/website/docs/core-concepts/projects/configuration/helmfile.mdx +++ b/website/docs/core-concepts/projects/configuration/helmfile.mdx @@ -11,7 +11,11 @@ import File from '@site/src/components/File' Atmos natively supports opinionated workflows for Helmfile. It's compatible with every version of helmfile and designed to work with multiple different versions of Helmfile concurrently. -Keep in mind that Atmos does not handle the downloading or installation of Helmfile (or Kustomize, a dependency of Helmfile); it assumes that any required commands are already installed on your system. To automate this, consider creating a [Custom Command](/core-concepts/custom-commands) to install Helmfile. +Keep in mind that Atmos does not handle the downloading or installation of Helmfile (or its dependency Kustomize); it assumes these commands are already installed on your system. For installation instructions, refer to: +- [Helmfile Installation Guide](https://helmfile.readthedocs.io/en/latest/#installation) +- [Kustomize Installation Guide](https://kubectl.docs.kubernetes.io/installation/kustomize/) + +To automate the installation process, consider creating a [Custom Command](/core-concepts/custom-commands). Atmos provides many settings that are specific to Helmfile which are configured in `atmos.yaml`. From 7c8a07f2cbeb07c1ff32fda0c3badcc74e7002f4 Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Mon, 28 Oct 2024 12:33:49 -0500 Subject: [PATCH 6/6] Update website/docs/core-concepts/projects/configuration/helmfile.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- website/docs/core-concepts/projects/configuration/helmfile.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/projects/configuration/helmfile.mdx b/website/docs/core-concepts/projects/configuration/helmfile.mdx index 422ec94ef..4e2255ee1 100644 --- a/website/docs/core-concepts/projects/configuration/helmfile.mdx +++ b/website/docs/core-concepts/projects/configuration/helmfile.mdx @@ -17,7 +17,7 @@ Keep in mind that Atmos does not handle the downloading or installation of Helmf To automate the installation process, consider creating a [Custom Command](/core-concepts/custom-commands). -Atmos provides many settings that are specific to Helmfile which are configured in `atmos.yaml`. +Atmos provides many settings that are specific to Helmfile, which are configured in `atmos.yaml`. ## CLI Configuration