Skip to content

Latest commit

 

History

History
418 lines (387 loc) · 28.3 KB

README.md

File metadata and controls

418 lines (387 loc) · 28.3 KB

AWS Elastic Kubernetes Service Terraform module

Terraform module which creates EKS Cluster and dependent resources on AWS.

Notice

This module install several Helm charts with limited inputs for their configuration, in order to keep it easy and simple. For a more accurate configuration we recommend to read their documentation and make your own installation of your desired Helm charts:

Usage

Terraform required version >= 0.14.8

Elastic Kubernetes Service

EKS Cluster with ELB:

locals {
  configmap_roles = [
    {
      "role_arn" = aws_iam_role.jobs_runner.arn 
      "k8s_user" = "jobs-runner"
      "k8s_groups" = [
        "system:masters"
        ]
      }
    ]

  configmap_users = [
    {
      "user_arn" = "arn:aws:iam::123456789123:user/demo"
      "k8s_user" = "demo"
      "k8s_groups" = [
        "system:masters",
        "system:developers"
        ]
      }
    ]

  prometheus_additional_scrape_configs = file("${path.root}/${var.prometheus_additional_scrape_configs_path}")
}


module "eks_main" {
  source                                      = "github.com/nimbux911/terraform-aws-eks.git"
  environment                                 = "dev"
  cluster_name                                = "dev-eks-main"
  cluster_version                             = "1.23"
  vpc_id                                      = "vpc-abcd1234"
  subnets_ids                                 = ["subnet-abc1234", "subnet-efgh5678"]
  eks_api_private                             = true
  enabled_cluster_log_types                   = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
  aws_auth_ignore_changes                     = false
  add_configmap_roles                         = local.configmap_roles
  target_group_arns                           = local.tg_arns
  eks_tags                                    = var.eks_tags
  health_check_type                           = "ELB"  

  # Example for ingress-nginx log format in logfmt using real source ip as client_ip
  ingress_custom_configuration                = "log-format-upstream: timestamp=$time_iso8601 client_ip=$http_x_forwarded_for method=$request_method uri=$request_uri status=$status http_user_agent=$http_user_agent request_length=$request_length request_time=$request_time proxy_upstream_name=$proxy_upstream_name upstream_addr=$upstream_addr upstream_response_length=$upstream_response_length upstream_response_time=$upstream_response_time upstream_status=$upstream_status req_id=$req_id" 

  managed_node_groups = [
    { 
      name    = "monitoring-${var.cluster_name}"
      values  = {
        ami_id            = var.eks_worker_ami_id,
        instance_type     = "m6a.large",
        asg_min           = 3,
        asg_max           = 4,
        subnets_ids       = ["subnet-abc1234", "subnet-efgh5678"],
        volume_type       = "gp3",
        volume_size       = 100,
        volume_iops       = 4000,
        k8s_labels        = {
          nodegroup       = "monitoring-${var.cluster_name}"
        }
      }
    }
  ]

  custom_node_groups = [
    {
      name    = "${var.environment}-${var.cluster_name}"
      values  = {
        ami_id            = var.eks_worker_ami_id,
        instance_type     = "t3.medium",
        asg_min           = 4,
        asg_max           = 8,
        subnets_ids       = ["subnet-abc1234", "subnet-efgh5678"],
        volume_type       = "gp2",
        volume_size       = 100,
        asg_tags          = var.asg_tags
        k8s_labels        = {
          nodegroup       = "${var.environment}-${var.cluster_name}"
        spot_nodes_enabled = true, // Just for custom node groups on Launch templates: https://docs.aws.amazon.com/eks/latest/APIReference/API_LaunchTemplateSpecification.html
        spot_options       = {
          max_price  = "0.0416" # t3.medium on-demand price
        }
      }
    }
  ]

  helm_ingress_nginx_enabled           = true 
  helm_cluster_autoscaler_enabled      = true
  cluster_autoscaler_extra_helm_values = file("${path.root}/resources/helm/values/cluster-autoscaler.yaml")
  helm_metrics_server_enabled          = true 
  helm_cert_manager_enabled            = true
  ingress_node_affinity                = {
    enabled     = true,
    label_key   = "nodegroup",
    label_value = "${var.env}-eks-spot"
  }

  create_ebs_csi_role = true

  eks_addons = {
    vpc-cni = {
      version              = "v1.12.6-eksbuild.1"
      configuration_values = {
        env = {
          ENABLE_PREFIX_DELEGATION = "true"
        }
      }
    },
    coredns = {
      version = "v1.8.7-eksbuild.4"
    },
    kube-proxy = {
      version = "v1.22.17-eksbuild.2"
    },
    aws-ebs-csi-driver = {
      version = "v1.17.0-eksbuild.1"
      service_account_role_arn = "arn:aws:iam::${var.current_account_id}:role/test-eks-main-eks-ebs-csi-controller"
    }
  }

# ================== loki-distributed ================= #
  helm_loki_enabled                     = true
  loki_storage_s3_bucket                = "my-bucket-loki-logs"
  loki_s3_bucket_region                 = "us-east-1"
  loki_ingester_replicas                = 3
  loki_ingester_node_selector           = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_distributor_min_replicas         = 2
  loki_distributor_node_selector        = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_distributor_max_replicas         = 4
  loki_querier_min_replicas             = 2
  loki_querier_max_replicas             = 4
  loki_querier_node_selector            = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_query_frontend_min_replicas      = 2
  loki_query_frontend_max_replicas      = 4
  loki_query_frontend_node_selector     = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_gateway_enabled                  = true
  loki_gateway_min_replicas             = 2
  loki_gateway_max_replicas             = 4
  loki_gateway_node_selector            = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_gateway_ingress_enabled          = true
  loki_gateway_ingress_host             = "loki.example.com"
  loki_compactor_enabled                = true
  loki_compactor_node_selector          = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  loki_index_gateway_enabled            = true
  loki_index_gateway_replicas           = 1
  loki_index_gateway_node_selector      = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }

# ================== fluent-bit ================== #
  helm_fluent_bit_enabled = true

# ================== prometheus ================== #
  helm_prometheus_enabled               = true
  prometheus_replicas                   = 2
  prometheus_ingress_enabled            = true
  prometheus_ingress_host               = "prometheus.example.com"
  prometheus_requests_cpu               = "200m"
  prometheus_requests_memory            = "1024Mi"
  prometheus_limits_cpu                 = "500m"
  prometheus_limits_memory              = "2048Mi"
  prometheus_node_selector              = { "eks\\.amazonaws\\.com/nodegroup" = "monitoring-${var.cluster_name}" }
  prometheus_additional_scrape_configs  = local.prometheus_additional_scrape_configs

# ================== tempo ================== #
  helm_tempo_enabled            = true
  tempo_storage_s3_bucket       = "my-bucket-tempo-traces"
  tempo_s3_bucket_region        = "us-east-1"
  tempo_gateway_enabled         = true
  tempo_gateway_ingress_enabled = true
  tempo_gateway_ingress_host    = "tempo.example.com"

# open-telemetry
  k8s_opentelemetry_enabled = true

# =================== grafana ================== #
  helm_grafana_enabled    = true
  grafana_ingress_enabled = true
  grafana_ingress_host    = "grafana.example.com"


}

Inputs

Name Description Type Default Required
environment Environment name of the resources. string "" yes
cluster_name Cluster name string "" yes
cluster_version Kubernetes version of the cluster. string "" yes
managed_node_groups AWS managed node groups configurations object(...) null no
custom_node_groups Custom node groups configurations object(...) null no
k8s_auth_api Kubernetes authentication API for Terraform providers. string client.authentication.k8s.io/v1beta1 no
vpc_id VPC ID where cluster will be deployed. string "" yes
subnets_ids Subnets ids from the VPC ID where the workers will be deployed. They must be, at least, from 2 differents AZs. list[string] [] yes
max_pods_per_node Max pods per Kubernetes worker node. string "100" no
target_group_arns ARNs of the target groups for using the worker nodes behind of ELB list[string] [] no
health_check_type Health check type for the worker nodes. string "EC2" no
on_demand_percentage_above_base_capacity Percentage split between on-demand and Spot instances above the base on-demand capacity. number 100 no
spot_allocation_strategy string "capacity-optimized" no
spot_instance_pools Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify. number 2 no
eks_tags Tags to add to all resources except the autoscaling group. map {} no
eks_api_private Defines it the Kubernetes API will be private or public. bool false no
eks_addons Adds EKS addons. map(map(string)) {} no
enable_irsa Determines whether to create an OpenID Connect Provider for EKS to enable IRSA. bool true no
openid_connect_audiences List of OpenID Connect audience client IDs to add to the IRSA provider. list[string] [] no
custom_oidc_thumbprints Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s). list[string] [] no
create_ebs_csi_role Indicates whether or not to create an IAM assumable role with oidc for EKS ebs-csi-controller. If true, the role will be created with the following name: <cluster_name>-ebs-csi-controller bool false no
add_configmap_roles List of maps with the information of the IAM roles to be added to aws-auth configmap. list[map] [] no
add_configmap_users List of maps with the information of the IAM users to be added to aws-auth configmap. list[map] [] no
aws_auth_ignore_changes Set if aws-auth configmap will be managed by Terraform or ignored. bool true no
eks_worker_max_pods_enabled Enable --max-pods flag in workers bootstrap bool false no
eks_worker_ssh_cidrs Add SSH ingress rule to eks workers list [] no
enabled_cluster_log_types Enable CloudWatch Logs for control plane components list[string] [] no
helm_ingress_nginx_enabled Set if ingress-nginx Helm chart will be installed on the cluster. bool false no
ingress_chart_version Set the version for the chart string 4.0.18 no
ingress_custom_configuration Add custom configuration options (see example above in module call inputs and https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml#L52) string null no
ingress_http_nodeport Set port for ingress http nodePort int 32080 no
ingress_https_nodeport Set port for ingress https nodePort int 32443 no
ingress_https_traffic_enabled Set https traffic for ingress bool false no
ingress_node_affinity Set nodeAffinity for ingress map { enabled = false, label_key = null, label_value = null} no
ingress_requests_cpu Set how much cpu will be assigned to the request string 100m no
ingress_requests_memory Set how much memory will be assigned to the request string 90Mi no
ingress_service_monitor_enabled Enable serviceMonitor for ingress-nginx helm chart bool false no
ingress_priority_class_name allows you to set a priority class string "" no
ingress_replicacount Minimum Replicas count of ingress number "1" no
helm_ingress_nginx_additional_enabled Set if additional ingress-nginx Helm chart will be installed on the cluster. bool false no
ingress_additional_chart_version Set the version for the chart string 4.0.18 no
ingress_additional_http_nodeport Set port for additional ingress http nodePort int 31080 no
ingress_additional_https_nodeport Set port for additional ingress https nodePort int 31443 no
ingress_additional_https_traffic_enabled Set https traffic for additional ingress bool false no
ingress_additional_requests_cpu Set how much cpu will be assigned to the request string 100m no
ingress_additional_requests_memory Set how much memory will be assigned to the request string 90Mi no
ingress_additional_priority_class_name allows you to set a priority class string "" no
ingress_additional_replicacount Minimum Replicas count of ingress additional number "1" no
helm_cluster_autoscaler_enabled Set if cluster-autoscaler Helm chart will be installed on the cluster. bool false no
cluster_autoscaler_chart_version Set the version for the chart string 9.16.1 no
cluster_autoscaler_priority_class_name allows you to set a priority class string "" no
cluster_autoscaler_extra_helm_values Raw YAML containing additional cluster-autoscaler Helm values string "" no
helm_metrics_server_enabled Set if metrics-server Helm chart will be installed on the cluster. bool false no
metrics_server_chart_version Set the version for the chart string 1.6.1 no
metrics_server_priority_class_name allows you to set a priority class string "" no
helm_cert_manager_enabled Set if cert-manager helm chart will be installed on the cluster bool false no
cert_manager_chart_version Set the version for the chart string 6.0.5 no
cert_manager_priority_class_name allows you to set a priority class string "" no
helm_loki_enabled Set if loki-stack Helm chart will be installed on the cluster. bool false no
loki_chart_version Set the version for the chart string 0.48.3 no
loki_storage_s3_bucket s3 bucket for loki logs string "" yes
loki_s3_bucket_region s3 bucket for loki logs region string "" yes
loki_logs_retention_enabled Enable logs retention. If s3 storage never stop growing bool false no
loki_logs_retention Set logs retention period string 744h no
loki_ingester_replicas Loki ingester replicas int 1 no
loki_ingester_node_selector Loki ingester nodeSelector map{} null no
loki_ingester_storage_class storageClass for ingesters pv string gp2 no
loki_ingester_storage_size size of ingesters pv string 10Gi no
loki_ingester_requests_cpu resources config for kubernetes pod string null no
loki_ingester_requests_memory resources config for kubernetes pod string null no
loki_ingester_limits_cpu resources config for kubernetes pod string null no
loki_ingester_limits_memory resources config for kubernetes pod string null no
loki_distributor_node_selector Loki distributor nodeSelector map{} null no
loki_distributor_min_replicas loki distributor hpa min replicas int 1 no
loki_distributor_requests_cpu resources config for kubernetes pod string null no
loki_distributor_requests_memory resources config for kubernetes pod string null no
loki_distributor_limits_cpu resources config for kubernetes pod string null no
loki_distributor_limits_memory resources config for kubernetes pod string null no
loki_distributor_max_replicas loki distributor hpa max replicas int 1 no
loki_querier_node_selector Loki querier nodeSelector map{} null no
loki_querier_min_replicas loki querier hpa min replicas int 1 no
loki_querier_max_replicas loki querier hpa max replicas int 1 no
loki_querier_requests_cpu resources config for kubernetes pod string null no
loki_querier_requests_memory resources config for kubernetes pod string null no
loki_querier_limits_cpu resources config for kubernetes pod string null no
loki_querier_limits_memory resources config for kubernetes pod string null no
loki_query_frontend_node_selector Loki query-frontend nodeSelector map{} null no
loki_query_frontend_min_replicas loki query-frontend hpa min replicas int 1 no
loki_query_frontend_max_replicas loki query-frontend hpa max replicas int 1 no
loki_query_frontend_requests_cpu resources config for kubernetes pod string null no
loki_query_frontend_requests_memory resources config for kubernetes pod string null no
loki_query_frontend_limits_cpu resources config for kubernetes pod string null no
loki_query_frontend_limits_memory resources config for kubernetes pod string null no
loki_max_query_length The limit to length of chunk store queries string 721h no
loki_gateway_enabled Enable loki gateway bool false no
loki_gateway_node_selector Loki gateway nodeSelector map{} null no
loki_gateway_min_replicas loki gateway hpa min replicas int 1 no
loki_gateway_max_replicas loki gateway hpa max replicas int 1 no
loki_gateway_ingress_enabled Enable ingress for loki gateway bool false no
loki_gateway_ingress_host Host for ingress rule string "" no
loki_gateway_ingress_path Path for ingress rule string / no
loki_gateway_ingress_path_type Path type for ingress rule string Prefix no
loki_gateway_ingress_class_name Set ingress class name string nginx no
loki_gateway_requests_cpu resources config for kubernetes pod string null no
loki_gateway_requests_memory resources config for kubernetes pod string null no
loki_gateway_limits_cpu resources config for kubernetes pod string null no
loki_gateway_limits_memory resources config for kubernetes pod string null no
loki_compactor_enabled Enable loki compactor bool false no
loki_compactor_node_selector Loki compactor nodeSelector map{} null no
loki_compactor_requests_cpu resources config for kubernetes pod string null no
loki_compactor_requests_memory resources config for kubernetes pod string null no
loki_compactor_limits_cpu resources config for kubernetes pod string null no
loki_compactor_limits_memory resources config for kubernetes pod string null no
loki_index_gateway_enabled Enable loki index gateway bool false no
loki_index_gateway_node_selector Loki _index gateway nodeSelector map{} null no
loki_index_gateway_replicas Set loki index gateway replicas int 1 no
loki_index_gateway_storage_class storageClass for index gateway pv string gp2 no
loki_index_gateway_storage_size storage size for index gateway pv string 10Gi no
loki_index_gateway_requests_cpu resources config for kubernetes pod string null no
loki_index_gateway_requests_memory resources config for kubernetes pod string null no
loki_index_gateway_limits_cpu resources config for kubernetes pod string null no
loki_index_gateway_limits_memory resources config for kubernetes pod string null no
loki_priority_class_name allows you to set a priority class string "" no
helm_fluent_bit_enabled install fluent-bit helm chart bool false no
fluent_bit_chart_version Set the version for the chart string 0.19.24 no
fluent_bit_priority_class_name allows you to set a priority class string "" no
k8s_opentelemetry_enabled install opentelemetry manifests bool false no
helm_prometheus_enabled install kube-prometheus-stack helm chart bool false no
prometheus_chart_version Set the version for the chart string 35.0.3 no
prometheus_node_selector Prometheus components nodeSelector map{} null no
prometheus_replicas prometheus server replicas int 1 no
prometheus_requests_cpu resources config for kubernetes pod string null no
prometheus_requests_memory resources config for kubernetes pod string null no
prometheus_limits_cpu resources config for kubernetes pod string null no
prometheus_limits_memory resources config for kubernetes pod string null no
prometheus_ingress_enabled Enable ingress for prometheus server bool false no
prometheus_ingress_host Host for ingress rule string "" no
prometheus_ingress_path Path for ingress rule string / no
prometheus_ingress_path_type Path type for ingress rule string Prefix no
prometheus_ingress_class_name Prometheus Ingress className string nginx no
prometheus_storage_class_name Prometheus storage className for pv string gp2 no
prometheus_storage_size Prometheus storage size string 20Gi no
prometheus_metrics_retention Prometheus metrics period retention string 14d no
prometheus_priority_class_name allows you to set a priority class string "" no
prometheus_additional_scrape_configs allows you to set a additional scrape config string "" no
helm_tempo_enabled Install tempo-distributed helm chart bool false no
tempo_chart_version Set the version for the chart string 0.17.1 no
tempo_compactor_requests_cpu resources config for kubernetes pod string null no
tempo_compactor_requests_memory resources config for kubernetes pod string null no
tempo_compactor_limits_cpu resources config for kubernetes pod string null no
tempo_compactor_limits_memory resources config for kubernetes pod string null no
tempo_distributor_requests_cpu resources config for kubernetes pod string null no
tempo_distributor_requests_memory resources config for kubernetes pod string null no
tempo_distributor_limits_cpu resources config for kubernetes pod string null no
tempo_distributor_limits_memory resources config for kubernetes pod string null no
tempo_storage_s3_bucket s3 bucket for tempo traces string "" no
tempo_s3_bucket_region s3 bucket regino for tempo traces string "" no
tempo_ingester_requests_cpu resources config for kubernetes pod string null no
tempo_ingester_requests_memory resources config for kubernetes pod string null no
tempo_ingester_limits_cpu resources config for kubernetes pod string null no
tempo_ingester_limits_memory resources config for kubernetes pod string null no
tempo_querier_requests_cpu resources config for kubernetes pod string null no
tempo_querier_requests_memory resources config for kubernetes pod string null no
tempo_querier_limits_cpu resources config for kubernetes pod string null no
tempo_querier_limits_memory resources config for kubernetes pod string null no
tempo_query_frontend_requests_cpu resources config for kubernetes pod string null no
tempo_query_frontend_requests_memory resources config for kubernetes pod string null no
tempo_query_frontend_limits_cpu resources config for kubernetes pod string null no
tempo_query_frontend_limits_memory resources config for kubernetes pod string null no
tempo_gateway_enabled enable tempo gateway bool false no
tempo_gateway_requests_cpu resources config for kubernetes pod string null no
tempo_gateway_requests_memory resources config for kubernetes pod string null no
tempo_gateway_limits_cpu resources config for kubernetes pod string null no
tempo_gateway_limits_memory resources config for kubernetes pod string null no
tempo_gateway_ingress_enabled Enable ingress for tempo gateway bool false no
tempo_gateway_ingress_host Host for ingress rule string "" no
tempo_gateway_ingress_path Path for ingress rule string / no
tempo_ingress_path_type Path type for ingress rule string Prefix no
tempo_ingress_class_name ingress className string nginx no
tempo_priority_class_name allows you to set a priority class string "" no
helm_grafana_enabled install grafana helm chart bool false no
grafana_chart_version Set the version for the chart string 6.45.0 no
grafana_ingress_enabled Enable ingress for grafana bool false no
grafana_ingress_host Host for ingress rule string "" no
grafana_ingress_path Path for ingress rule string / no
grafana_ingress_path_type Path type for ingress rule string Prefix no
grafana_ingress_class_name ingress className string nginx no
grafana_persistence_enabled Persistent volume bool false no
grafana_priority_class_name allows you to set a priority class string "" no
k8s_image_registry Kubernetes image registry. string registry.k8s.io no

Outputs

Name Description
security_group_worker_arn The ARN of the workers security group.
worker_role_arn The ARN of the workers IAM Role.
worker_role_id The ID of the workers IAM Role.
asg_name Name of the of the workers Autoscaling Group.
eks_certificate_authority Cluster's certificate authority.
eks_endpoint Cluster's endpoint.
eks_managed_node_groups_autoscaling_group_names List of the autoscaling group names created by EKS managed node groups.
oidc_provider The OpenID Connect identity provider (issuer URL without leading https://).
oidc_provider_arn The ARN of the OIDC Provider if enable_irsa = true.
cluster_tls_certificate_sha1_fingerprint The SHA1 fingerprint of the public key of the cluster's certificate.
ebs_csi_iam_role_arn The arn of the role created for ebs csi driver.