✨ Add topology spread constraints (#383)

* ☸️ Add topologySpreadConstraints Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com> * 📝 write about pods topology spread constraints Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com> * 📝 Revise pod scheduling section Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com> * 🤖 Update license headers and formatting go codes Signed-off-by: vdaas-ci <ci@vdaas.org> Co-authored-by: vdaas-ci <ci@vdaas.org>
vdaas · May 18, 2020 · 895ae0b · 895ae0b
1 parent a1e7809
commit 895ae0b
Show file tree

Hide file tree

Showing 19 changed files with 81 additions and 3 deletions.
diff --git a/charts/vald/templates/agent/daemonset.yaml b/charts/vald/templates/agent/daemonset.yaml
@@ -50,6 +50,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.agent.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.agent.name }}
           image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}"

diff --git a/charts/vald/templates/agent/deployment.yaml b/charts/vald/templates/agent/deployment.yaml
@@ -57,6 +57,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.agent.name }}
           image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}"

diff --git a/charts/vald/templates/agent/statefulset.yaml b/charts/vald/templates/agent/statefulset.yaml
@@ -57,6 +57,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.agent.name }}
           image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}"

diff --git a/charts/vald/templates/discoverer/daemonset.yaml b/charts/vald/templates/discoverer/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.discoverer.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.discoverer.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.discoverer.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.discoverer.name }}
           image: "{{ .Values.discoverer.image.repository }}:{{ default .Values.defaults.image.tag .Values.discoverer.image.tag }}"

diff --git a/charts/vald/templates/discoverer/deployment.yaml b/charts/vald/templates/discoverer/deployment.yaml
@@ -56,6 +56,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.discoverer.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.discoverer.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.discoverer.name }}
           image: "{{ .Values.discoverer.image.repository }}:{{ default .Values.defaults.image.tag .Values.discoverer.image.tag }}"

diff --git a/charts/vald/templates/gateway/vald/daemonset.yaml b/charts/vald/templates/gateway/vald/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.gateway.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.gateway.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.gateway.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.gateway.name }}
           image: "{{ .Values.gateway.image.repository }}:{{ default .Values.defaults.image.tag .Values.gateway.image.tag }}"

diff --git a/charts/vald/templates/gateway/vald/deployment.yaml b/charts/vald/templates/gateway/vald/deployment.yaml
@@ -56,6 +56,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.gateway.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.gateway.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.gateway.name }}
           image: "{{ .Values.gateway.image.repository }}:{{ default .Values.defaults.image.tag .Values.gateway.image.tag }}"

diff --git a/charts/vald/templates/manager/backup/daemonset.yaml b/charts/vald/templates/manager/backup/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.backupManager.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.backupManager.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.backupManager.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.backupManager.name }}
           image: "{{ .Values.backupManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.backupManager.image.tag }}"

diff --git a/charts/vald/templates/manager/backup/deployment.yaml b/charts/vald/templates/manager/backup/deployment.yaml
@@ -56,6 +56,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.backupManager.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.backupManager.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.backupManager.name }}
           image: "{{ .Values.backupManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.backupManager.image.tag }}"

diff --git a/charts/vald/templates/manager/compressor/daemonset.yaml b/charts/vald/templates/manager/compressor/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.compressor.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.compressor.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.compressor.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.compressor.name }}
           image: "{{ .Values.compressor.image.repository }}:{{ default .Values.defaults.image.tag .Values.compressor.image.tag }}"

diff --git a/charts/vald/templates/manager/compressor/deployment.yaml b/charts/vald/templates/manager/compressor/deployment.yaml
@@ -56,6 +56,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.compressor.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.compressor.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.compressor.name }}
           image: "{{ .Values.compressor.image.repository }}:{{ default .Values.defaults.image.tag .Values.compressor.image.tag }}"

diff --git a/charts/vald/templates/manager/index/daemonset.yaml b/charts/vald/templates/manager/index/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.indexManager.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.indexManager.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.indexManager.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.indexManager.name }}
           image: "{{ .Values.indexManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.indexManager.image.tag }}"

diff --git a/charts/vald/templates/manager/index/deployment.yaml b/charts/vald/templates/manager/index/deployment.yaml
@@ -54,6 +54,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.indexManager.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.indexManager.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.indexManager.name }}
           image: "{{ .Values.indexManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.indexManager.image.tag }}"

diff --git a/charts/vald/templates/meta/daemonset.yaml b/charts/vald/templates/meta/daemonset.yaml
@@ -49,6 +49,10 @@ spec:
         {{- $initContainers := dict "initContainers" .Values.meta.initContainers "Values" .Values "namespace" .Release.Namespace -}}
         {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
       {{- end }}
+      affinity:
+        {{- include "vald.affinity" .Values.meta.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.meta.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.meta.name }}
           image: "{{ .Values.meta.image.repository }}:{{ default .Values.defaults.image.tag .Values.meta.image.tag }}"

diff --git a/charts/vald/templates/meta/deployment.yaml b/charts/vald/templates/meta/deployment.yaml
@@ -56,6 +56,8 @@ spec:
       {{- end }}
       affinity:
         {{- include "vald.affinity" .Values.meta.affinity | nindent 8 }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.meta.topologySpreadConstraints | nindent 8 }}
       containers:
         - name: {{ .Values.meta.name }}
           image: "{{ .Values.meta.image.repository }}:{{ default .Values.defaults.image.tag .Values.meta.image.tag }}"

diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml
@@ -536,6 +536,8 @@ gateway:
                     - vald-gateway
       # gateway.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # gateway.topologySpreadConstraints -- topology spread constraints of gateway pods
+  topologySpreadConstraints: []
   # gateway.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -715,6 +717,8 @@ agent:
                     - vald-agent-ngt
       # agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # agent.topologySpreadConstraints -- topology spread constraints for agent pods
+  topologySpreadConstraints: []
   # agent.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -867,6 +871,8 @@ discoverer:
                     - vald-discoverer
       # discoverer.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # discoverer.topologySpreadConstraints -- topology spread constraints of discoverer pods
+  topologySpreadConstraints: []
   # discoverer.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -1011,6 +1017,8 @@ compressor:
       preferredDuringSchedulingIgnoredDuringExecution: []
       # compressor.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # compressor.topologySpreadConstraints -- topology spread constraints of compressor pods
+  topologySpreadConstraints: []
   # compressor.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -1207,6 +1215,8 @@ backupManager:
       preferredDuringSchedulingIgnoredDuringExecution: []
       # backupManager.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # backupManager.topologySpreadConstraints -- topology spread constraints of backup manager pods
+  topologySpreadConstraints: []
   # backupManager.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -1504,6 +1514,8 @@ indexManager:
       preferredDuringSchedulingIgnoredDuringExecution: []
       # indexManager.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # indexManager.topologySpreadConstraints -- topology spread constraints of index manager pods
+  topologySpreadConstraints: []
   # indexManager.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:
@@ -1697,6 +1709,8 @@ meta:
       preferredDuringSchedulingIgnoredDuringExecution: []
       # meta.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms
       requiredDuringSchedulingIgnoredDuringExecution: []
+  # meta.topologySpreadConstraints -- topology spread constraints of meta pods
+  topologySpreadConstraints: []
   # meta.server_config -- server config (overrides defaults.server_config)
   server_config:
     servers:

diff --git a/docs/guides/configurations.md b/docs/guides/configurations.md
@@ -197,10 +197,10 @@ Pod priorities are also useful for saving agent pods from eviction.
 By default, very high priority is set to agent pods in the Chart.
 
 
-#### Pod anti-affinity
+#### Pod scheduling
 
 It is recommended to schedule agent pods on different nodes as much as possible.
-To achieve this, the following podAntiAffinity is set by default.
+To achieve this, the following [podAntiAffinity][k8s-affinity-antiaffinity] is set by default.
 
 ```yaml
 agent:
@@ -218,6 +218,22 @@ agent:
                     - vald-agent-ngt
 ```
 
+It can be also achieved by using [pod topology spread constraints][k8s-topology-spread-constraints].
+
+```yaml
+agent:
+  topologySpreadConstraints:
+    - topologyKey: node
+      maxSkew: 1
+      whenUnsatisfiable: ScheduleAnyway
+      labelSelector:
+        matchLabels:
+          app: vald-agent-ngt
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution: [] # to disable default settings
+```
+
 ### Gateway
 
 #### Ingress
@@ -399,5 +415,7 @@ For further details, there are references of Helm values in GitHub Vald reposito
 [google-pprof]: https://github.com/google/pprof
 [prometheus-io]: https://prometheus.io/
 [k8s-liveness-readiness]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+[k8s-affinity-antiaffinity]: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
+[k8s-topology-spread-constraints]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
 [yj-ngt]: https://github.com/yahoojapan/NGT
 [yj-ngt-wiki]: https://github.com/yahoojapan/NGT/wiki
diff --git a/go.mod b/go.mod
@@ -55,7 +55,7 @@ require (
 	go.uber.org/goleak v1.0.0
 	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
-	golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d // indirect
+	golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 // indirect
 	gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
 	gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a // indirect
 	gonum.org/v1/plot v0.7.0

diff --git a/go.sum b/go.sum
@@ -643,6 +643,8 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d h1:n6zwymXmN9rCClNNmCWwV3qwMmBcRw/WeIGDK8Qnzk4=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 h1:vmsb6v0zUdmUlXfwKaYrHPPRCV0lHq/IwNIf0ASGjyQ=
+golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=