chore(prefect-worker): add CPU/memory-based HPA (#338)

* chore(prefect-worker): add CPU/memory-based HPA * default fals
PrefectHQ · May 31, 2024 · 683145d · 683145d
1 parent c1c83cb
commit 683145d
Show file tree

Hide file tree

Showing 5 changed files with 98 additions and 2 deletions.
diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md
@@ -179,8 +179,8 @@ No secrets are created when providing an existing secret.
 | server.readinessProbe.config.timeoutSeconds | int | `5` | The number of seconds to wait for a probe response before considering it as failed. |
 | server.readinessProbe.enabled | bool | `false` |  |
 | server.replicaCount | int | `1` | number of server replicas to deploy |
-| server.resources.limits | object | `{}` | the requested limits for the server container |
-| server.resources.requests | object | `{}` | the requested resources for the server container |
+| server.resources.limits | object | `{"cpu":"1","memory":"1Gi"}` | the requested limits for the server container |
+| server.resources.requests | object | `{"cpu":"500m","memory":"512Mi"}` | the requested resources for the server container |
 | server.tolerations | list | `[]` | tolerations for server pods assignment |
 | server.uiConfig.enabled | bool | `true` | set PREFECT_UI_ENABLED; enable the UI on the server |
 | server.uiConfig.prefectUiApiUrl | string | `""` | sets PREFECT_UI_API_URL |

diff --git a/charts/prefect-worker/README.md b/charts/prefect-worker/README.md
@@ -244,6 +244,11 @@ helm install prefect-worker prefect/prefect-worker -f values.yaml --set-file wor
 | serviceAccount.name | string | `""` | the name of the ServiceAccount to use. if not set and create is true, a name is generated using the common.names.fullname template |
 | worker.affinity | object | `{}` | affinity for worker pods assignment |
 | worker.apiConfig | string | `"cloud"` | one of 'cloud', 'selfHosted', or 'server' |
+| worker.autoscaling.enabled | bool | `false` | enable autoscaling for the worker |
+| worker.autoscaling.maxReplicas | int | `1` | maximum number of replicas to scale up to |
+| worker.autoscaling.minReplicas | int | `1` | minimum number of replicas to scale down to |
+| worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | target CPU utilization percentage for scaling the worker |
+| worker.autoscaling.targetMemoryUtilizationPercentage | int | `80` | target memory utilization percentage for scaling the worker |
 | worker.cloudApiConfig.accountId | string | `""` | prefect account ID |
 | worker.cloudApiConfig.apiKeySecret.key | string | `"key"` | prefect API secret key |
 | worker.cloudApiConfig.apiKeySecret.name | string | `"prefect-api-key"` | prefect API secret name |

diff --git a/charts/prefect-worker/templates/hpa.yaml b/charts/prefect-worker/templates/hpa.yaml
@@ -0,0 +1,40 @@
+{{- if and .Values.worker.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "common.names.fullname" . }}
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: worker
+    prefect-version: {{ .Chart.AppVersion }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  scaleTargetRef:
+    apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
+    kind: Deployment
+    name: {{ template "common.names.fullname" . }}
+  minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+{{- end }}
diff --git a/charts/prefect-worker/values.schema.json b/charts/prefect-worker/values.schema.json
@@ -40,6 +40,45 @@
       "form": true,
       "additionalProperties": false,
       "properties": {
+        "autoscaling": {
+          "type": "object",
+          "title": "Autoscaling",
+          "description": "autoscaling configuration for the worker",
+          "form": true,
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "title": "Enabled",
+              "description": "enable autoscaling for the worker",
+              "form": true
+            },
+            "minReplicas": {
+              "type": "integer",
+              "title": "Min Replicas",
+              "description": "minimum number of replicas to scale down to",
+              "form": true
+            },
+            "maxReplicas": {
+              "type": "integer",
+              "title": "Max Replicas",
+              "description": "maximum number of replicas to scale up to",
+              "form": true
+            },
+            "targetCPUUtilizationPercentage": {
+              "type": "integer",
+              "title": "Target CPU Utilization Percentage",
+              "description": "target CPU utilization percentage for scaling the worker",
+              "form": true
+            },
+            "targetMemoryUtilizationPercentage": {
+              "type": "integer",
+              "title": "Target Memory Utilization Percentage",
+              "description": "target memory utilization percentage for scaling the worker",
+              "form": true
+            }
+          }
+        },
         "clusterUid": {
           "type": "string",
           "title": "Cluster UID",

diff --git a/charts/prefect-worker/values.yaml b/charts/prefect-worker/values.yaml
@@ -12,6 +12,18 @@ commonAnnotations: {}
 
 ## Deployment Configuration
 worker:
+  autoscaling:
+    # -- enable autoscaling for the worker
+    enabled: false
+    # -- minimum number of replicas to scale down to
+    minReplicas: 1
+    # -- maximum number of replicas to scale up to
+    maxReplicas: 1
+    # -- target CPU utilization percentage for scaling the worker
+    targetCPUUtilizationPercentage: 80
+    # -- target memory utilization percentage for scaling the worker
+    targetMemoryUtilizationPercentage: 80
+
   # -- unique cluster identifier, if none is provided this value will be infered at time of helm install
   clusterUid: ""