diff --git a/charts/overprovisioner/.helmignore b/charts/overprovisioner/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/overprovisioner/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/overprovisioner/Chart.yaml b/charts/overprovisioner/Chart.yaml new file mode 100644 index 0000000..e694358 --- /dev/null +++ b/charts/overprovisioner/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: overprovisioner +description: Overprovisioner for cluster-autoscaler. +type: application + +version: 0.1.0 +appVersion: 0.0.1 diff --git a/charts/overprovisioner/templates/deployment.yaml b/charts/overprovisioner/templates/deployment.yaml new file mode 100644 index 0000000..af86740 --- /dev/null +++ b/charts/overprovisioner/templates/deployment.yaml @@ -0,0 +1,55 @@ +{{- range $g := .Values.nodeGroups }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: overprovisioner-{{ $g.nickname }} + {{- with $g.deploymentLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ $g.count | int | toYaml }} + selector: + matchLabels: + app.kubernetes.io/name: {{ $.Chart.Name | toString | toYaml }} + app.kubernetes.io/instance: {{ $.Release.Name | toString | toYaml }} + overprovisioner.v1.vessl.ai/instance: {{ $g.nickname }} + strategy: + type: Recreate + template: + metadata: + {{- with $g.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ $.Chart.Name | toString | toYaml }} + app.kubernetes.io/instance: {{ $.Release.Name | toString | toYaml }} + overprovisioner.v1.vessl.ai/instance: {{ $g.nickname }} + {{- with $g.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + priorityClassName: {{ $.Values.priorityClass.name }} + containers: + - name: {{ $.Chart.Name }} + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with $g.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $g.nodeSelectorLabels }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $g.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $g.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +{{- end }} diff --git a/charts/overprovisioner/templates/priority-class.yaml b/charts/overprovisioner/templates/priority-class.yaml new file mode 100644 index 0000000..c5e783f --- /dev/null +++ b/charts/overprovisioner/templates/priority-class.yaml @@ -0,0 +1,12 @@ +{{- if .Values.priorityClass.create }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.priorityClass.name }} +value: {{ .Values.priorityClass.value | int | toYaml }} +description: | + Used by overprovisioner. + Pods with this priority class can be evicted (preempted) to yield resources + whenever workload pods need to be scheduled. +{{- end }} diff --git a/charts/overprovisioner/values.yaml b/charts/overprovisioner/values.yaml new file mode 100644 index 0000000..cb17e48 --- /dev/null +++ b/charts/overprovisioner/values.yaml @@ -0,0 +1,45 @@ +nodeGroups: + - nickname: l4-spot + count: 1 + nodeSelector: {} + tolerations: + - key: nvidia.com/gpu + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: v1.k8s.vessl.ai/v1.l4-1.mem-42.spot + operator: In + values: + - 'true' + resources: + requests: { cpu: 6100m } + + deploymentLabels: {} + podAnnotations: {} + podLabels: {} + +priorityClass: + create: true + name: overprovisioner-placeholder + + # NOTE: value should be... + # + # - LESS than workload pod's priority. + # By default, we do not set PriorityClass to workload pods, and thus it + # uses cluster's default PriorityClass if exists. In most of our configuration, + # there is no default PriorityClass, and priority value falls back to zero. + # Thus, in most configurations, this value should be < 0. + # + # - GREATER OR EQUAL than cluster autoscaler's priority cutoff. + # If the placeholder pods are deemed too little to cluster-autoscaler, + # it won't allow the pods to hold onto resources and evict the nodes. + # Default cutoff is -10, so this value should be >= -10 in most configurations. + value: -5 + +image: + repository: k8s.gcr.io/pause + tag: 3.9 + pullPolicy: IfNotPresent