diff --git a/addons/core_dns_hpa/Chart.yaml b/addons/core_dns_hpa/Chart.yaml new file mode 100644 index 0000000..632a60c --- /dev/null +++ b/addons/core_dns_hpa/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: A Helm chart add hpa on coredns +name: corednshpa +version: 1.0.0 \ No newline at end of file diff --git a/addons/core_dns_hpa/templates/hpa.yaml b/addons/core_dns_hpa/templates/hpa.yaml new file mode 100644 index 0000000..b1b0bc0 --- /dev/null +++ b/addons/core_dns_hpa/templates/hpa.yaml @@ -0,0 +1,35 @@ +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: core-dns-hpa-cpu + namespace: kube-system +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.corednsdeploymentname}} + minReplicas: {{ .Values.minReplicas}} + maxReplicas: {{ .Values.maxReplicas}} + targetCPUUtilizationPercentage: {{ .Values.targetCPUUtilizationPercentage}} + +--- +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: core-dns-hpa-memory + namespace: kube-system +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: coredns + minReplicas: {{ .Values.minReplicas}} + maxReplicas: {{ .Values.maxReplicas}} + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageValue: {{ .Values.targetMemoryUtilizationPercentage}} \ No newline at end of file diff --git a/addons/core_dns_hpa/values.yaml b/addons/core_dns_hpa/values.yaml new file mode 100644 index 0000000..22667f1 --- /dev/null +++ b/addons/core_dns_hpa/values.yaml @@ -0,0 +1,5 @@ +corednsdeploymentname: ${corednsdeploymentname} +minReplicas: ${minReplicas} +maxReplicas: ${maxReplicas} +targetCPUUtilizationPercentage: ${targetCPUUtilizationPercentage} +targetMemoryUtilizationPercentage: ${targetMemoryUtilizationPercentage} \ No newline at end of file diff --git a/addons/metrics_server/metrics_server.yaml b/addons/metrics_server/metrics_server.yaml index 375084e..dc0a883 100644 --- a/addons/metrics_server/metrics_server.yaml +++ b/addons/metrics_server/metrics_server.yaml @@ -31,3 +31,5 @@ resources: podAnnotations: co.elastic.logs/enabled: "true" + +replicas: 2 \ No newline at end of file diff --git a/addons/metrics_server_vpa/Chart.yaml b/addons/metrics_server_vpa/Chart.yaml new file mode 100644 index 0000000..cc738de --- /dev/null +++ b/addons/metrics_server_vpa/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: A Helm chart add vpa on metrics-server +name: metricsservervpa +version: 1.0.0 \ No newline at end of file diff --git a/addons/metrics_server_vpa/templates/vpa.yaml b/addons/metrics_server_vpa/templates/vpa.yaml new file mode 100644 index 0000000..ee691dc --- /dev/null +++ b/addons/metrics_server_vpa/templates/vpa.yaml @@ -0,0 +1,22 @@ +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: metrics-server-vpa + namespace: kube-system +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{ .Values.metricsServerDeploymentName}} + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: '*' + minAllowed: + cpu: {{ .Values.minCPU}} + memory: {{ .Values.minMemory}} + maxAllowed: + cpu: {{ .Values.maxCPU}} + memory: {{ .Values.maxMemory}} + controlledResources: ["cpu", "memory"] diff --git a/addons/metrics_server_vpa/values.yaml b/addons/metrics_server_vpa/values.yaml new file mode 100644 index 0000000..0e2371a --- /dev/null +++ b/addons/metrics_server_vpa/values.yaml @@ -0,0 +1,5 @@ +metricsServerDeploymentName: ${metricsServerDeploymentName} +minCPU: ${minCPU} +minMemory: ${minMemory} +maxCPU: ${maxCPU} +maxMemory: ${maxMemory} \ No newline at end of file diff --git a/addons/vpa_crds/values.yaml b/addons/vpa_crds/values.yaml new file mode 100644 index 0000000..aa971da --- /dev/null +++ b/addons/vpa_crds/values.yaml @@ -0,0 +1,627 @@ +# Default values for vertical-pod-autoscaler. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + # Global Docker image registry + imageRegistry: "" + + # Global Docker registry secret names as an array + imagePullSecrets: [] + +# Override Kubernetes version +kubeVersion: "" + +nameOverride: "" +fullnameOverride: "" + +# Annotations to add to all deployed objects +commonAnnotations: {} + +# Labels to add to all deployed objects +commonLabels: {} + +# Array of extra objects to deploy with the release +extraDeploy: [] + +admissionController: + # Enable the component + enabled: true + + replicaCount: 1 + + image: + # Image registry + registry: registry.k8s.io + + # Image repository + repository: autoscaling/vpa-admission-controller + + # Image tag + tag: 0.14.0 + + # Image digest + digest: "" + + # Image pull policy + pullPolicy: IfNotPresent + + pdb: + # Specifies whether a pod disruption budget should be created + create: false + + # Minimum number/percentage of pods that should remain scheduled + minAvailable: 1 + + # Maximum number/percentage of pods that may be made unavailable + # maxUnavailable: 1 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # Additional pod annotations + podAnnotations: {} + + # Additional pod labels + podLabels: {} + + podSecurityContext: + # fsGroup: 2000 + runAsNonRoot: true + runAsUser: 65534 + + # Priority class name + # priorityClassName : high-priority + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + livenessProbe: + # Enable liveness probe + enabled: true + + # Delay before the liveness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the liveness probe + periodSeconds: 10 + + # When the liveness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the liveness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the liveness probe to be considered successful after having failed + successThreshold: 1 + + readinessProbe: + # Enable readiness probe + enabled: true + + # Delay before the readiness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the readiness probe + periodSeconds: 10 + + # When the readiness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the readiness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the readiness probe to be considered successful after having failed + successThreshold: 1 + + service: + # Service annotations + annotations: {} + + # Service type + type: ClusterIP + + # Static cluster IP address or None for headless service when service type is ClusterIP + # clusterIP: 10.43.0.100 + + # Service port + # port: 8000 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 512Mi + # requests: + # cpu: 50m + # memory: 256Mi + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + extraArgs: + # kube-api-burst: 10 + # kube-api-qps: 5 + v: 2 + # vpa-object-namespace: "" + # webhook-timeout-seconds: 30 + + # Additional container environment variables + extraEnvVars: [] + # - name: MY-NAME + # value: "MY-VALUE" + + # Name of existing ConfigMap containing additional container environment variables + extraEnvVarsCM: + + # Name of existing Secret containing additional container environment variables + extraEnvVarsSecret: + + metrics: + service: + # Metrics service annotations + annotations: {} + + # Metrics service type + type: ClusterIP + + # Metrics static cluster IP address or None for headless service when service type is ClusterIP + # clusterIP: 10.43.0.100 + + # Metrics service port + port: 8944 + + serviceMonitor: + # Specifies whether a service monitor should be created + enabled: false + # Extra annotations for the ServiceMonitor + annotations: {} + # Extra labels for the ServiceMonitor + labels: {} + # The name of the label on the target service to use as the job name in Prometheus + jobLabel: "" + # How frequently to scrape metrics + interval: "" + # Timeout after which the scrape is ended + scrapeTimeout: "" + # Specify additional relabeling of metrics + metricRelabelings: [] + # Specify general relabeling + relabelings: [] + + tls: + caCert: "" + cert: "" + key: "" + existingSecret: "" + +recommender: + replicaCount: 1 + + image: + # Image registry + registry: registry.k8s.io + + # Image repository + repository: autoscaling/vpa-recommender + + # Image tag + tag: 0.14.0 + + # Image digest + digest: "" + + # Image pull policy + pullPolicy: IfNotPresent + + pdb: + # Specifies whether a pod disruption budget should be created + create: false + + # Minimum number/percentage of pods that should remain scheduled + minAvailable: 1 + + # Maximum number/percentage of pods that may be made unavailable + # maxUnavailable: 1 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # Additional pod annotations + podAnnotations: {} + + # Additional pod labels + podLabels: {} + + podSecurityContext: + # fsGroup: 2000 + runAsNonRoot: true + runAsUser: 65534 + + # Priority class name + # priorityClassName : high-priority + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + livenessProbe: + # Enable liveness probe + enabled: true + + # Delay before the liveness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the liveness probe + periodSeconds: 10 + + # When the liveness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the liveness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the liveness probe to be considered successful after having failed + successThreshold: 1 + + readinessProbe: + # Enable readiness probe + enabled: true + + # Delay before the readiness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the readiness probe + periodSeconds: 10 + + # When the readiness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the readiness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the readiness probe to be considered successful after having failed + successThreshold: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 1024Mi + # requests: + # cpu: 50m + # memory: 512Mi + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + extraArgs: + # checkpoints-gc-interval: 10m0s + # checkpoints-timeout: 1m0s + # container-name-label: name + # container-namespace-label: namespace + # container-pod-name-label: pod_name + # cpu-histogram-decay-half-life: 24h0m0s + # cpu-integer-post-processor-enabled: false + # history-length: 8d + # history-resolution: 1h + # kube-api-burst: 10 + # kube-api-qps: 5 + # memory-aggregation-interval: 24h0m0s + # memory-aggregation-interval-count: 8 + # memory-histogram-decay-half-life: 24h0m0s + # memory-saver: false + # metric-for-pod-labels: up{job="kubernetes-pods"} + # min-checkpoints: 10 + # oom-bump-up-ratio: 1.2 + # oom-min-bump-up-bytes: 104857600 + # pod-label-prefix: pod_label_ + # pod-name-label: kubernetes_pod_name + # pod-namespace-label: kubernetes_namespace + # pod-recommendation-min-cpu-millicores: 25 + # pod-recommendation-min-memory-mb: 250 + # prometheus-address: "" + # prometheus-cadvisor-job-name: kubernetes-cadvisor + # prometheus-query-timeout: 5m + # recommendation-margin-fraction: 0.15 + # recommender-interval: 1m0s + # recommender-name: default + # storage: checkpoint + # target-cpu-percentile: 0.9 + v: 2 + # vpa-object-namespace: "" + + # Additional container environment variables + extraEnvVars: [] + # - name: MY-NAME + # value: "MY-VALUE" + + # Name of existing ConfigMap containing additional container environment variables + extraEnvVarsCM: + + # Name of existing Secret containing additional container environment variables + extraEnvVarsSecret: + + metrics: + service: + # Metrics service annotations + annotations: {} + + # Metrics service type + type: ClusterIP + + # Metrics static cluster IP address or None for headless service when service type is ClusterIP + # clusterIP: 10.43.0.100 + + # Metrics service port + port: 8942 + + serviceMonitor: + # Specifies whether a service monitor should be created + enabled: false + # Extra annotations for the ServiceMonitor + annotations: {} + # Extra labels for the ServiceMonitor + labels: {} + # The name of the label on the target service to use as the job name in Prometheus + jobLabel: "" + # How frequently to scrape metrics + interval: "" + # Timeout after which the scrape is ended + scrapeTimeout: "" + # Specify additional relabeling of metrics + metricRelabelings: [] + # Specify general relabeling + relabelings: [] + +updater: + # Enable the component + enabled: true + + replicaCount: 1 + + image: + # Image registry + registry: registry.k8s.io + + # Image repository + repository: autoscaling/vpa-updater + + # Image tag + tag: 0.14.0 + + # Image digest + digest: "" + + # Image pull policy + pullPolicy: IfNotPresent + + pdb: + # Specifies whether a pod disruption budget should be created + create: false + + # Minimum number/percentage of pods that should remain scheduled + minAvailable: 1 + + # Maximum number/percentage of pods that may be made unavailable + # maxUnavailable: 1 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # Additional pod annotations + podAnnotations: {} + + # Additional pod labels + podLabels: {} + + podSecurityContext: + # fsGroup: 2000 + runAsNonRoot: true + runAsUser: 65534 + + # Priority class name + # priorityClassName : high-priority + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + livenessProbe: + # Enable liveness probe + enabled: true + + # Delay before the liveness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the liveness probe + periodSeconds: 10 + + # When the liveness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the liveness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the liveness probe to be considered successful after having failed + successThreshold: 1 + + readinessProbe: + # Enable readiness probe + enabled: true + + # Delay before the readiness probe is initiated + initialDelaySeconds: 0 + + # How often to perform the readiness probe + periodSeconds: 10 + + # When the readiness probe times out + timeoutSeconds: 1 + + # Minimum consecutive failures for the readiness probe to be considered failed after having succeeded + failureThreshold: 3 + + # Minimum consecutive successes for the readiness probe to be considered successful after having failed + successThreshold: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 1024Mi + # requests: + # cpu: 50m + # memory: 512Mi + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + extraArgs: + # evict-after-oom-threshold: 10m0s + # eviction-rate-burst: 1 + # eviction-rate-limit: -1 + # eviction-tolerance: 0.5 + # in-recommendation-bounds-eviction-lifetime-threshold: 12h0m0s + # kube-api-burst: 10 + # kube-api-qps: 5 + # min-replicas: 2 + # pod-update-threshold: 0.1 + # updater-interval: 1m0s + # use-admission-controller-status: true + v: 2 + # vpa-object-namespace: "" + + # Additional container environment variables + extraEnvVars: [] + # - name: MY-NAME + # value: "MY-VALUE" + + # Name of existing ConfigMap containing additional container environment variables + extraEnvVarsCM: + + # Name of existing Secret containing additional container environment variables + extraEnvVarsSecret: + + metrics: + service: + # Metrics service annotations + annotations: {} + + # Metrics service type + type: ClusterIP + + # Metrics static cluster IP address or None for headless service when service type is ClusterIP + # clusterIP: 10.43.0.100 + + # Metrics service port + port: 8943 + serviceMonitor: + # Specifies whether a service monitor should be created + enabled: false + # Extra annotations for the ServiceMonitor + annotations: {} + # Extra labels for the ServiceMonitor + labels: {} + # The name of the label on the target service to use as the job name in Prometheus + jobLabel: "" + # How frequently to scrape metrics + interval: "" + # Timeout after which the scrape is ended + scrapeTimeout: "" + # Specify additional relabeling of metrics + metricRelabelings: [] + # Specify general relabeling + relabelings: [] + +crds: + image: + # Image registry + registry: docker.io + + # Image repository + repository: bitnami/kubectl + + # Image tag + tag: 1.26.3 + + # Image digest + digest: "" + + # Image pull policy + pullPolicy: IfNotPresent + + # Additional pod annotations + podAnnotations: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +tests: + image: + # Image registry + registry: ghcr.io + + # Image repository + repository: cowboysysop/pytest + + # Image tag + tag: 1.0.35 + + # Image digest + digest: "" + + # Image pull policy + pullPolicy: IfNotPresent diff --git a/main.tf b/main.tf index 07c5b8c..0e73513 100644 --- a/main.tf +++ b/main.tf @@ -353,3 +353,51 @@ resource "kubernetes_ingress_v1" "kubecost" { } } } + +#hpa-coredns +resource "helm_release" "coredns-hpa" { + name = "corednshpa" + namespace = "kube-system" + chart = "${path.module}/addons/core_dns_hpa/" + timeout = 600 + values = [ + templatefile("${path.module}/addons/core_dns_hpa/values.yaml", { + minReplicas = var.core_dns_hpa_config.minReplicas, + maxReplicas = var.core_dns_hpa_config.maxReplicas, + corednsdeploymentname = var.core_dns_hpa_config.corednsdeploymentname, + targetCPUUtilizationPercentage = var.core_dns_hpa_config.targetCPUUtilizationPercentage, + targetMemoryUtilizationPercentage = var.core_dns_hpa_config.targetMemoryUtilizationPercentage + }) + ] +} + +resource "helm_release" "vpa-crds" { + count = var.metrics_server_enabled ? 1 : 0 + name = "vertical-pod-autoscaler" + namespace = "kube-system" + repository = "https://cowboysysop.github.io/charts/" + chart = "vertical-pod-autoscaler" + version = "7.2.0" + timeout = 600 + values = [ + file("${path.module}/addons/vpa_crds/values.yaml") + ] +} + +resource "helm_release" "metrics-server-vpa" { + count = var.metrics_server_enabled ? 1 : 0 + depends_on = ["helm_release.vpa-crds"] + name = "metricsservervpa" + namespace = "kube-system" + chart = "${path.module}/addons/metrics_server_vpa/" + timeout = 600 + values = [ + templatefile("${path.module}/addons/metrics_server_vpa/values.yaml", { + minCPU = var.metrics_server_vpa_config.minCPU, + minMemory = var.metrics_server_vpa_config.minMemory, + maxCPU = var.metrics_server_vpa_config.maxCPU, + maxMemory = var.metrics_server_vpa_config.maxMemory, + metricsServerDeploymentName = var.metrics_server_vpa_config.metricsServerDeploymentName + }) + ] +} diff --git a/variables.tf b/variables.tf index fe19eb6..f8172ae 100644 --- a/variables.tf +++ b/variables.tf @@ -284,3 +284,30 @@ variable "cluster_issuer" { default = "letsencrypt-prod" type = string } + +#core-dns-hpa +variable "core_dns_hpa_config" { + description = "Configuration to provide settings of hpa over core dns" + default = { + minReplicas = 2 + maxReplicas = 10 + corednsdeploymentname = "coredns" + targetCPUUtilizationPercentage = 80 + targetMemoryUtilizationPercentage = "150Mi" + } + type = any +} + +#metrics-server-vpa +variable "metrics_server_vpa_config" { + description = "Configuration to provide settings of vpa over metrics server" + default = { + + minCPU = "25m" + maxCPU = "100m" + minMemory = "150Mi" + maxMemory = "500Mi" + metricsServerDeploymentName = "metrics-server" + } + type = any +} \ No newline at end of file