From 895ae0b46688ed57124774882e759fe5bcc3cf28 Mon Sep 17 00:00:00 2001 From: Rintaro Okamura Date: Mon, 18 May 2020 12:56:03 +0900 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20topology=20spread=20constrain?= =?UTF-8?q?ts=20(#383)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :wheel_of_dharma: Add topologySpreadConstraints Signed-off-by: Rintaro Okamura * :pencil: write about pods topology spread constraints Signed-off-by: Rintaro Okamura * :pencil: Revise pod scheduling section Signed-off-by: Rintaro Okamura * :robot: Update license headers and formatting go codes Signed-off-by: vdaas-ci Co-authored-by: vdaas-ci --- charts/vald/templates/agent/daemonset.yaml | 4 ++++ charts/vald/templates/agent/deployment.yaml | 2 ++ charts/vald/templates/agent/statefulset.yaml | 2 ++ .../vald/templates/discoverer/daemonset.yaml | 4 ++++ .../vald/templates/discoverer/deployment.yaml | 2 ++ .../templates/gateway/vald/daemonset.yaml | 4 ++++ .../templates/gateway/vald/deployment.yaml | 2 ++ .../templates/manager/backup/daemonset.yaml | 4 ++++ .../templates/manager/backup/deployment.yaml | 2 ++ .../manager/compressor/daemonset.yaml | 4 ++++ .../manager/compressor/deployment.yaml | 2 ++ .../templates/manager/index/daemonset.yaml | 4 ++++ .../templates/manager/index/deployment.yaml | 2 ++ charts/vald/templates/meta/daemonset.yaml | 4 ++++ charts/vald/templates/meta/deployment.yaml | 2 ++ charts/vald/values.yaml | 14 ++++++++++++ docs/guides/configurations.md | 22 +++++++++++++++++-- go.mod | 2 +- go.sum | 2 ++ 19 files changed, 81 insertions(+), 3 deletions(-) diff --git a/charts/vald/templates/agent/daemonset.yaml b/charts/vald/templates/agent/daemonset.yaml index 2d86ea8940..e4cdc20843 100644 --- a/charts/vald/templates/agent/daemonset.yaml +++ b/charts/vald/templates/agent/daemonset.yaml @@ -50,6 +50,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.agent.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.agent.name }} image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}" diff --git a/charts/vald/templates/agent/deployment.yaml b/charts/vald/templates/agent/deployment.yaml index a9e028f532..c2b8bc792d 100644 --- a/charts/vald/templates/agent/deployment.yaml +++ b/charts/vald/templates/agent/deployment.yaml @@ -57,6 +57,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.agent.name }} image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}" diff --git a/charts/vald/templates/agent/statefulset.yaml b/charts/vald/templates/agent/statefulset.yaml index 72ede9783b..27ef9df59f 100644 --- a/charts/vald/templates/agent/statefulset.yaml +++ b/charts/vald/templates/agent/statefulset.yaml @@ -57,6 +57,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.agent.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.agent.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.agent.name }} image: "{{ .Values.agent.image.repository }}:{{ default .Values.defaults.image.tag .Values.agent.image.tag }}" diff --git a/charts/vald/templates/discoverer/daemonset.yaml b/charts/vald/templates/discoverer/daemonset.yaml index c0d2e5d44d..cbad3f7fd4 100644 --- a/charts/vald/templates/discoverer/daemonset.yaml +++ b/charts/vald/templates/discoverer/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.discoverer.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.discoverer.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.discoverer.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.discoverer.name }} image: "{{ .Values.discoverer.image.repository }}:{{ default .Values.defaults.image.tag .Values.discoverer.image.tag }}" diff --git a/charts/vald/templates/discoverer/deployment.yaml b/charts/vald/templates/discoverer/deployment.yaml index 4d1444061d..97b566c9dc 100644 --- a/charts/vald/templates/discoverer/deployment.yaml +++ b/charts/vald/templates/discoverer/deployment.yaml @@ -56,6 +56,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.discoverer.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.discoverer.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.discoverer.name }} image: "{{ .Values.discoverer.image.repository }}:{{ default .Values.defaults.image.tag .Values.discoverer.image.tag }}" diff --git a/charts/vald/templates/gateway/vald/daemonset.yaml b/charts/vald/templates/gateway/vald/daemonset.yaml index 93039a1623..766d891c32 100644 --- a/charts/vald/templates/gateway/vald/daemonset.yaml +++ b/charts/vald/templates/gateway/vald/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.gateway.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.gateway.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.gateway.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.gateway.name }} image: "{{ .Values.gateway.image.repository }}:{{ default .Values.defaults.image.tag .Values.gateway.image.tag }}" diff --git a/charts/vald/templates/gateway/vald/deployment.yaml b/charts/vald/templates/gateway/vald/deployment.yaml index f355211d71..e549f2171d 100644 --- a/charts/vald/templates/gateway/vald/deployment.yaml +++ b/charts/vald/templates/gateway/vald/deployment.yaml @@ -56,6 +56,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.gateway.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.gateway.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.gateway.name }} image: "{{ .Values.gateway.image.repository }}:{{ default .Values.defaults.image.tag .Values.gateway.image.tag }}" diff --git a/charts/vald/templates/manager/backup/daemonset.yaml b/charts/vald/templates/manager/backup/daemonset.yaml index 62e87f6089..07e7fe0185 100644 --- a/charts/vald/templates/manager/backup/daemonset.yaml +++ b/charts/vald/templates/manager/backup/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.backupManager.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.backupManager.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.backupManager.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.backupManager.name }} image: "{{ .Values.backupManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.backupManager.image.tag }}" diff --git a/charts/vald/templates/manager/backup/deployment.yaml b/charts/vald/templates/manager/backup/deployment.yaml index 21492b546a..445425953a 100644 --- a/charts/vald/templates/manager/backup/deployment.yaml +++ b/charts/vald/templates/manager/backup/deployment.yaml @@ -56,6 +56,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.backupManager.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.backupManager.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.backupManager.name }} image: "{{ .Values.backupManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.backupManager.image.tag }}" diff --git a/charts/vald/templates/manager/compressor/daemonset.yaml b/charts/vald/templates/manager/compressor/daemonset.yaml index 6ca8ae31a2..c35d12b7a3 100644 --- a/charts/vald/templates/manager/compressor/daemonset.yaml +++ b/charts/vald/templates/manager/compressor/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.compressor.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.compressor.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.compressor.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.compressor.name }} image: "{{ .Values.compressor.image.repository }}:{{ default .Values.defaults.image.tag .Values.compressor.image.tag }}" diff --git a/charts/vald/templates/manager/compressor/deployment.yaml b/charts/vald/templates/manager/compressor/deployment.yaml index b08ab74048..f1a3b68e54 100644 --- a/charts/vald/templates/manager/compressor/deployment.yaml +++ b/charts/vald/templates/manager/compressor/deployment.yaml @@ -56,6 +56,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.compressor.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.compressor.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.compressor.name }} image: "{{ .Values.compressor.image.repository }}:{{ default .Values.defaults.image.tag .Values.compressor.image.tag }}" diff --git a/charts/vald/templates/manager/index/daemonset.yaml b/charts/vald/templates/manager/index/daemonset.yaml index 4ff8b8450f..678452ef18 100644 --- a/charts/vald/templates/manager/index/daemonset.yaml +++ b/charts/vald/templates/manager/index/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.indexManager.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.indexManager.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.indexManager.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.indexManager.name }} image: "{{ .Values.indexManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.indexManager.image.tag }}" diff --git a/charts/vald/templates/manager/index/deployment.yaml b/charts/vald/templates/manager/index/deployment.yaml index 2981baa004..ce0a448c8e 100644 --- a/charts/vald/templates/manager/index/deployment.yaml +++ b/charts/vald/templates/manager/index/deployment.yaml @@ -54,6 +54,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.indexManager.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.indexManager.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.indexManager.name }} image: "{{ .Values.indexManager.image.repository }}:{{ default .Values.defaults.image.tag .Values.indexManager.image.tag }}" diff --git a/charts/vald/templates/meta/daemonset.yaml b/charts/vald/templates/meta/daemonset.yaml index f12b097d15..5876455395 100644 --- a/charts/vald/templates/meta/daemonset.yaml +++ b/charts/vald/templates/meta/daemonset.yaml @@ -49,6 +49,10 @@ spec: {{- $initContainers := dict "initContainers" .Values.meta.initContainers "Values" .Values "namespace" .Release.Namespace -}} {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} {{- end }} + affinity: + {{- include "vald.affinity" .Values.meta.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.meta.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.meta.name }} image: "{{ .Values.meta.image.repository }}:{{ default .Values.defaults.image.tag .Values.meta.image.tag }}" diff --git a/charts/vald/templates/meta/deployment.yaml b/charts/vald/templates/meta/deployment.yaml index 3890a1ba63..dfad0d3097 100644 --- a/charts/vald/templates/meta/deployment.yaml +++ b/charts/vald/templates/meta/deployment.yaml @@ -56,6 +56,8 @@ spec: {{- end }} affinity: {{- include "vald.affinity" .Values.meta.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.meta.topologySpreadConstraints | nindent 8 }} containers: - name: {{ .Values.meta.name }} image: "{{ .Values.meta.image.repository }}:{{ default .Values.defaults.image.tag .Values.meta.image.tag }}" diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 83530f2ff2..880fcae7e3 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -536,6 +536,8 @@ gateway: - vald-gateway # gateway.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # gateway.topologySpreadConstraints -- topology spread constraints of gateway pods + topologySpreadConstraints: [] # gateway.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -715,6 +717,8 @@ agent: - vald-agent-ngt # agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # agent.topologySpreadConstraints -- topology spread constraints for agent pods + topologySpreadConstraints: [] # agent.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -867,6 +871,8 @@ discoverer: - vald-discoverer # discoverer.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # discoverer.topologySpreadConstraints -- topology spread constraints of discoverer pods + topologySpreadConstraints: [] # discoverer.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -1011,6 +1017,8 @@ compressor: preferredDuringSchedulingIgnoredDuringExecution: [] # compressor.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # compressor.topologySpreadConstraints -- topology spread constraints of compressor pods + topologySpreadConstraints: [] # compressor.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -1207,6 +1215,8 @@ backupManager: preferredDuringSchedulingIgnoredDuringExecution: [] # backupManager.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # backupManager.topologySpreadConstraints -- topology spread constraints of backup manager pods + topologySpreadConstraints: [] # backupManager.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -1504,6 +1514,8 @@ indexManager: preferredDuringSchedulingIgnoredDuringExecution: [] # indexManager.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # indexManager.topologySpreadConstraints -- topology spread constraints of index manager pods + topologySpreadConstraints: [] # indexManager.server_config -- server config (overrides defaults.server_config) server_config: servers: @@ -1697,6 +1709,8 @@ meta: preferredDuringSchedulingIgnoredDuringExecution: [] # meta.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] + # meta.topologySpreadConstraints -- topology spread constraints of meta pods + topologySpreadConstraints: [] # meta.server_config -- server config (overrides defaults.server_config) server_config: servers: diff --git a/docs/guides/configurations.md b/docs/guides/configurations.md index 359e88ace3..e3a7e8d827 100644 --- a/docs/guides/configurations.md +++ b/docs/guides/configurations.md @@ -197,10 +197,10 @@ Pod priorities are also useful for saving agent pods from eviction. By default, very high priority is set to agent pods in the Chart. -#### Pod anti-affinity +#### Pod scheduling It is recommended to schedule agent pods on different nodes as much as possible. -To achieve this, the following podAntiAffinity is set by default. +To achieve this, the following [podAntiAffinity][k8s-affinity-antiaffinity] is set by default. ```yaml agent: @@ -218,6 +218,22 @@ agent: - vald-agent-ngt ``` +It can be also achieved by using [pod topology spread constraints][k8s-topology-spread-constraints]. + +```yaml +agent: + topologySpreadConstraints: + - topologyKey: node + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: vald-agent-ngt + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: [] # to disable default settings +``` + ### Gateway #### Ingress @@ -399,5 +415,7 @@ For further details, there are references of Helm values in GitHub Vald reposito [google-pprof]: https://github.com/google/pprof [prometheus-io]: https://prometheus.io/ [k8s-liveness-readiness]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +[k8s-affinity-antiaffinity]: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +[k8s-topology-spread-constraints]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ [yj-ngt]: https://github.com/yahoojapan/NGT [yj-ngt-wiki]: https://github.com/yahoojapan/NGT/wiki diff --git a/go.mod b/go.mod index 9b0327ad03..69f161bd30 100755 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( go.uber.org/goleak v1.0.0 golang.org/x/mod v0.3.0 // indirect golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 - golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d // indirect + golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 // indirect gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614 gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a // indirect gonum.org/v1/plot v0.7.0 diff --git a/go.sum b/go.sum index d8ef847b2b..647c1c329a 100644 --- a/go.sum +++ b/go.sum @@ -643,6 +643,8 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d h1:n6zwymXmN9rCClNNmCWwV3qwMmBcRw/WeIGDK8Qnzk4= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 h1:vmsb6v0zUdmUlXfwKaYrHPPRCV0lHq/IwNIf0ASGjyQ= +golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=