diff --git a/.github/actions/terratest/Dockerfile b/.github/actions/terratest/Dockerfile index 5dcf9e8e..f8b9a576 100644 --- a/.github/actions/terratest/Dockerfile +++ b/.github/actions/terratest/Dockerfile @@ -1,10 +1,9 @@ -FROM golang:1.17.3-alpine3.13 +FROM golang:1.19-alpine3.16 WORKDIR / -ARG TERRAFORM_VERSION=1.2.3 -ARG KUBECTL_VERSION=1.22.11 - +ARG TERRAFORM_VERSION=1.2.7 +ARG KUBECTL_VERSION=1.23.7 RUN apk add --no-cache \ bash \ @@ -14,7 +13,8 @@ RUN apk add --no-cache \ git \ jq \ perl-utils \ - aws-cli && \ + py3-pip && \ + pip install awscli && \ git clone https://github.com/tfutils/tfenv.git ~/.tfenv && \ echo 'export PATH="$HOME/.tfenv/bin:$PATH"' >> ~/.bash_profile && ln -s ~/.tfenv/bin/* /usr/local/bin && \ tfenv install $TERRAFORM_VERSION && \ diff --git a/.terraform-version b/.terraform-version index 0495c4a8..c04c650a 100644 --- a/.terraform-version +++ b/.terraform-version @@ -1 +1 @@ -1.2.3 +1.2.7 diff --git a/examples/cluster/main.tf b/examples/cluster/main.tf index 3978b8d0..019423e1 100644 --- a/examples/cluster/main.tf +++ b/examples/cluster/main.tf @@ -24,8 +24,6 @@ module "cluster" { vpc_config = data.terraform_remote_state.environment.outputs.vpc_config iam_config = data.terraform_remote_state.environment.outputs.iam_config - aws_ebs_csi_driver = var.aws_ebs_csi_driver - critical_addons_node_group_key_name = "development" endpoint_public_access = true diff --git a/examples/cluster/variables.tf b/examples/cluster/variables.tf index ed7986e2..145811ab 100644 --- a/examples/cluster/variables.tf +++ b/examples/cluster/variables.tf @@ -2,8 +2,3 @@ variable "cluster_name" { type = string default = "test-cluster" } - -variable "aws_ebs_csi_driver" { - type = bool - default = true -} diff --git a/hack/generate_addons.sh b/hack/generate_addons.sh index 0cf8f7ab..14be7f43 100755 --- a/hack/generate_addons.sh +++ b/hack/generate_addons.sh @@ -11,4 +11,4 @@ helm_template() { helm template --no-hooks --namespace=kube-system --version $3 -f $ADDONS_DIR/helm/$2.yaml $2 $1/$2${4:-} | grep -v Helm > $ADDONS_DIR/$2.yaml } -helm_template autoscaler cluster-autoscaler 9.18.1 +helm_template autoscaler cluster-autoscaler 9.19.3 diff --git a/modules/asg_node_group/README.md b/modules/asg_node_group/README.md index b9ec6155..d0fef8b7 100644 --- a/modules/asg_node_group/README.md +++ b/modules/asg_node_group/README.md @@ -214,10 +214,6 @@ module "bottlerocket_nodes" { bottlerocket = true } ``` -⚠️ If you are using bottlerocket nodes and need EBS persistent volumes you must -enable the [AWS EBS CSI driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) by setting `aws_ebs_csi_driver = true` on the cluster module. -see: https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md#csi-plugin - ⚠️ Bottlerocket now [supports GPU nodes](https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md#aws-k8s--nvidia-variants), set `gpu = true` to enable them. Ensure that you set `instance_types` to a GPU instance type. 📝 If you want to get a shell session on your instances via Bottlerocket's SSM agent @@ -230,4 +226,4 @@ provision your node role, then this is done by default! By default, IMDSv2 will be enabled through the variable nodes_metadata_http_tokens. ⚠️ If you are using kube2iam change the default value to "optional". [terraform IMDSv2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template#metadata-options) -Once we don't have any cluster using kube2iam, this variable can be removed and forced to be required the token. \ No newline at end of file +Once we don't have any cluster using kube2iam, this variable can be removed and forced to be required the token. diff --git a/modules/asg_node_group/main.tf b/modules/asg_node_group/main.tf index a23f734f..6118384f 100644 --- a/modules/asg_node_group/main.tf +++ b/modules/asg_node_group/main.tf @@ -1,5 +1,5 @@ locals { - k8s_version = "1.22" + k8s_version = "1.23" preset_instance_families = { memory_optimized = ["r5", "r5d", "r5n", "r5dn", "r5a", "r5ad"] general_purpose = ["m5", "m5d", "m5n", "m5dn", "m5a", "m5ad"] diff --git a/modules/asg_node_group/variables.tf b/modules/asg_node_group/variables.tf index 3baa9c39..acbcb69e 100644 --- a/modules/asg_node_group/variables.tf +++ b/modules/asg_node_group/variables.tf @@ -8,7 +8,6 @@ variable "cluster_config" { node_security_group = string node_instance_profile = string tags = map(string) - aws_ebs_csi_driver = bool }) } diff --git a/modules/cluster/README.md b/modules/cluster/README.md index cbcdad6c..cff86889 100644 --- a/modules/cluster/README.md +++ b/modules/cluster/README.md @@ -85,7 +85,6 @@ specify the arn of an existing key by setting `kms_cmk_arn` | addon | variable | default | iam role variable | |-------|----------|---------|-------------------| | [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) | `cluster_autoscaler` | ✅ enabled | `cluster_autoscaler_iam_role_arn` | -| [Amazon Elastic Block Store (EBS) CSI driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/) | `aws_ebs_csi_driver` | ❌ disabled | `aws_ebs_csi_driver_iam_role_arn` | Note that setting these variables to false will not remove provisioned add-ons from an existing cluster. diff --git a/modules/cluster/addons.tf b/modules/cluster/addons.tf index 63f6e134..2b4a3fa7 100644 --- a/modules/cluster/addons.tf +++ b/modules/cluster/addons.tf @@ -20,6 +20,10 @@ module "critical_addons_node_group" { taints = { "CriticalAddonsOnly" = "true:NoSchedule" } + + depends_on = [ + module.aws_auth + ] } data "aws_region" "current" {} @@ -29,31 +33,37 @@ data "aws_region" "current" {} resource "aws_eks_addon" "vpc-cni" { cluster_name = local.config.name addon_name = "vpc-cni" - addon_version = "v1.11.0-eksbuild.1" + addon_version = "v1.11.2-eksbuild.1" resolve_conflicts = "OVERWRITE" } resource "aws_eks_addon" "kube-proxy" { cluster_name = local.config.name addon_name = "kube-proxy" - addon_version = "v1.22.6-eksbuild.1" + addon_version = "v1.23.7-eksbuild.1" resolve_conflicts = "OVERWRITE" } resource "aws_eks_addon" "coredns" { cluster_name = local.config.name addon_name = "coredns" - addon_version = "v1.8.7-eksbuild.1" + addon_version = "v1.8.7-eksbuild.2" resolve_conflicts = "OVERWRITE" + depends_on = [ + module.critical_addons_node_group + ] } resource "aws_eks_addon" "ebs-csi" { - count = var.aws_ebs_csi_driver ? 1 : 0 + count = 1 cluster_name = local.config.name addon_name = "aws-ebs-csi-driver" - addon_version = "v1.6.1-eksbuild.1" + addon_version = "v1.10.0-eksbuild.1" service_account_role_arn = local.aws_ebs_csi_driver_iam_role_arn resolve_conflicts = "OVERWRITE" + depends_on = [ + module.critical_addons_node_group + ] } module "cluster_autoscaler" { @@ -68,4 +78,7 @@ module "cluster_autoscaler" { aws_region = data.aws_region.current.name } ) + depends_on = [ + module.critical_addons_node_group + ] } diff --git a/modules/cluster/addons/cluster-autoscaler.yaml b/modules/cluster/addons/cluster-autoscaler.yaml index 2a0a3ef1..dbf148c3 100644 --- a/modules/cluster/addons/cluster-autoscaler.yaml +++ b/modules/cluster/addons/cluster-autoscaler.yaml @@ -7,7 +7,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" spec: # Prevents running in privileged mode privileged: false @@ -50,7 +50,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system spec: @@ -68,7 +68,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system annotations: @@ -83,7 +83,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler rules: - apiGroups: @@ -232,7 +232,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler roleRef: apiGroup: rbac.authorization.k8s.io @@ -250,7 +250,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system rules: @@ -278,7 +278,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system roleRef: @@ -297,7 +297,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system spec: @@ -320,7 +320,7 @@ metadata: labels: app.kubernetes.io/instance: "cluster-autoscaler" app.kubernetes.io/name: "aws-cluster-autoscaler" - helm.sh/chart: "cluster-autoscaler-9.18.1" + helm.sh/chart: "cluster-autoscaler-9.19.3" name: cluster-autoscaler namespace: kube-system spec: @@ -339,7 +339,7 @@ spec: dnsPolicy: "ClusterFirst" containers: - name: aws-cluster-autoscaler - image: "us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler:v1.22.2" + image: "k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.1" imagePullPolicy: "IfNotPresent" command: - ./cluster-autoscaler diff --git a/modules/cluster/addons/helm/cluster-autoscaler.yaml b/modules/cluster/addons/helm/cluster-autoscaler.yaml index 7b868977..4769a40d 100644 --- a/modules/cluster/addons/helm/cluster-autoscaler.yaml +++ b/modules/cluster/addons/helm/cluster-autoscaler.yaml @@ -21,8 +21,8 @@ extraArgs: expander: least-waste balance-similar-node-groups: true image: - repository: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler - tag: v1.22.2 + repository: k8s.gcr.io/autoscaling/cluster-autoscaler + tag: v1.23.1 fullnameOverride: cluster-autoscaler nameOverride: aws-cluster-autoscaler resources: diff --git a/modules/cluster/aws_ebs_csi_driver_iam.tf b/modules/cluster/aws_ebs_csi_driver_iam.tf index dad92c2e..b25325f8 100644 --- a/modules/cluster/aws_ebs_csi_driver_iam.tf +++ b/modules/cluster/aws_ebs_csi_driver_iam.tf @@ -1,5 +1,5 @@ locals { - aws_ebs_csi_driver_iam_role_count = length(var.aws_ebs_csi_driver_iam_role_arn) == 0 && var.aws_ebs_csi_driver ? 1 : 0 + aws_ebs_csi_driver_iam_role_count = length(var.aws_ebs_csi_driver_iam_role_arn) == 0 ? 1 : 0 aws_ebs_csi_driver_iam_role_arn = length(var.aws_ebs_csi_driver_iam_role_arn) > 0 ? var.aws_ebs_csi_driver_iam_role_arn : join("", aws_iam_role.aws_ebs_csi_driver.*.arn) } diff --git a/modules/cluster/kubectl/kubeconfig.yaml b/modules/cluster/kubectl/kubeconfig.yaml index 185ad7e0..7411640b 100644 --- a/modules/cluster/kubectl/kubeconfig.yaml +++ b/modules/cluster/kubectl/kubeconfig.yaml @@ -9,8 +9,9 @@ users: - name: ${cluster_name} user: exec: - apiVersion: client.authentication.k8s.io/v1alpha1 + apiVersion: client.authentication.k8s.io/v1beta1 command: aws + interactiveMode: IfAvailable args: - "eks" - "get-token" diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index 77000174..7ae6315f 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -6,7 +6,7 @@ data "aws_iam_role" "service_role" { name = var.iam_config.service_role } locals { - k8s_version = "1.22" + k8s_version = "1.23" } resource "aws_eks_cluster" "control_plane" { @@ -96,8 +96,8 @@ module "storage_classes" { manifest = templatefile( "${path.module}/storage_classes.yaml.tmpl", { - provisioner = var.aws_ebs_csi_driver ? "ebs.csi.aws.com" : "kubernetes.io/aws-ebs", - fstype = var.aws_ebs_csi_driver ? "csi.storage.k8s.io/fstype: ${var.pv_fstype}" : "fsType: ${var.pv_fstype}" + provisioner = "ebs.csi.aws.com", + fstype = "csi.storage.k8s.io/fstype: ${var.pv_fstype}", } ) } diff --git a/modules/cluster/outputs.tf b/modules/cluster/outputs.tf index 9d57fb78..c1e0ef1f 100644 --- a/modules/cluster/outputs.tf +++ b/modules/cluster/outputs.tf @@ -8,7 +8,6 @@ locals { node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id node_instance_profile = var.iam_config.node_role tags = var.tags - aws_ebs_csi_driver = var.aws_ebs_csi_driver } } diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf index 72d202b6..e6999552 100644 --- a/modules/cluster/variables.tf +++ b/modules/cluster/variables.tf @@ -68,12 +68,6 @@ variable "cluster_autoscaler_iam_role_arn" { description = "The IAM role for the cluster_autoscaler, if omitted then an IAM role will be created" } -variable "aws_ebs_csi_driver" { - type = bool - default = false - description = "Should the Amazon Elastic Block Store (EBS) CSI driver be deployed" -} - variable "aws_ebs_csi_driver_iam_role_arn" { type = string default = "" diff --git a/test/cluster_test.go b/test/cluster_test.go index 62f864f8..19b430dd 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -40,7 +40,6 @@ func TestTerraformAwsEksCluster(t *testing.T) { deployTerraform(t, environmentDir, map[string]interface{}{}) deployTerraform(t, workingDir, map[string]interface{}{ "cluster_name": clusterName, - "aws_ebs_csi_driver": false, }) }) @@ -82,10 +81,6 @@ func TestTerraformAwsEksCluster(t *testing.T) { validateClusterAutoscaler(t, kubeconfig) validateKubeBench(t, kubeconfig) validateStorage(t, kubeconfig) - overideAndApplyTerraform(t, workingDir, map[string]interface{}{ - "aws_ebs_csi_driver": true, - }) - validateStorage(t, kubeconfig) }) test_structure.RunTestStage(t, "validate_bottlerocket_node_group", func() { @@ -93,9 +88,6 @@ func TestTerraformAwsEksCluster(t *testing.T) { kubeconfig := writeKubeconfig(t, terraform.Output(t, terraformOptions, "cluster_name")) defer os.Remove(kubeconfig) nodeGroupDir := "../examples/cluster/bottlerocket_node_group" - overideAndApplyTerraform(t, workingDir, map[string]interface{}{ - "aws_ebs_csi_driver": true, - }) deployTerraform(t, nodeGroupDir, map[string]interface{}{}) defer cleanupTerraform(t, nodeGroupDir) validateClusterAutoscaler(t, kubeconfig)