diff --git a/embed/templates/scripts/run_tidb.sh.tpl b/embed/templates/scripts/run_tidb.sh.tpl index 6c30088e9f..5a2925471f 100644 --- a/embed/templates/scripts/run_tidb.sh.tpl +++ b/embed/templates/scripts/run_tidb.sh.tpl @@ -17,7 +17,9 @@ cd "${DEPLOY_DIR}" || exit 1 {{- end}} {{- end}} -{{- if .NumaNode}} +{{- if and .NumaNode .NumaCores}} +exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} -C {{.NumaCores}} env GODEBUG=madvdontneed=1 bin/tidb-server \ +{{- else if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/tidb-server \ {{- else}} exec env GODEBUG=madvdontneed=1 bin/tidb-server \ diff --git a/embed/templates/scripts/run_tiflash.sh.tpl b/embed/templates/scripts/run_tiflash.sh.tpl index 10e87cb579..fec3523095 100644 --- a/embed/templates/scripts/run_tiflash.sh.tpl +++ b/embed/templates/scripts/run_tiflash.sh.tpl @@ -15,7 +15,9 @@ stat=$(time sync) echo ok echo $stat -{{- if .NumaNode}} +{{- if and .NumaNode .NumaCores}} +exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} -C {{.NumaCores}} bin/tiflash/tiflash server \ +{{- else if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tiflash/tiflash server \ {{- else}} exec bin/tiflash/tiflash server \ diff --git a/embed/templates/scripts/run_tikv.sh.tpl b/embed/templates/scripts/run_tikv.sh.tpl index fad4935a04..789f21bdac 100644 --- a/embed/templates/scripts/run_tikv.sh.tpl +++ b/embed/templates/scripts/run_tikv.sh.tpl @@ -22,7 +22,9 @@ echo $stat export MALLOC_CONF="prof:true,prof_active:false" -{{- if .NumaNode}} +{{- if and .NumaNode .NumaCores}} +exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} -C {{.NumaCores}} bin/tikv-server \ +{{- else if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tikv-server \ {{- else}} exec bin/tikv-server \ diff --git a/go.mod b/go.mod index 31de7df142..f2897494ff 100644 --- a/go.mod +++ b/go.mod @@ -57,8 +57,8 @@ require ( go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 - golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 golang.org/x/text v0.3.7 google.golang.org/genproto v0.0.0-20220525015930-6ca3db687a9d @@ -88,7 +88,6 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/net v0.0.0-20220524220425-1d687d428aca // indirect - golang.org/x/tools v0.1.11 // indirect + golang.org/x/tools v0.1.12 // indirect google.golang.org/appengine v1.6.7 // indirect ) diff --git a/go.sum b/go.sum index 3ad5e9e12a..7b27f8cfb7 100644 --- a/go.sum +++ b/go.sum @@ -537,7 +537,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -668,12 +668,11 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8= -golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -692,8 +691,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -758,14 +757,14 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -835,8 +834,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= -golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/cluster/spec/tidb.go b/pkg/cluster/spec/tidb.go index 469dc22104..0d41cc62ee 100644 --- a/pkg/cluster/spec/tidb.go +++ b/pkg/cluster/spec/tidb.go @@ -41,6 +41,7 @@ type TiDBSpec struct { DeployDir string `yaml:"deploy_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]interface{} `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` Arch string `yaml:"arch,omitempty"` @@ -142,6 +143,7 @@ func (i *TiDBInstance) InitConfig( NewTiDBScript(i.GetHost(), paths.Deploy, paths.Log). WithPort(spec.Port). WithNumaNode(spec.NumaNode). + WithNumaCores(spec.NumaCores). WithStatusPort(spec.StatusPort). AppendEndpoints(topo.Endpoints(deployUser)...). WithListenHost(i.GetListenHost()). diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index a4d24e5f88..0dad9aa6b8 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -58,6 +58,7 @@ type TiFlashSpec struct { TmpDir string `yaml:"tmp_path,omitempty"` Offline bool `yaml:"offline,omitempty"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]interface{} `yaml:"config,omitempty" validate:"config:ignore"` LearnerConfig map[string]interface{} `yaml:"learner_config,omitempty" validate:"learner_config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -652,6 +653,7 @@ func (i *TiFlashInstance) InitConfig( WithStatusPort(spec.StatusPort). WithTmpDir(spec.TmpDir). WithNumaNode(spec.NumaNode). + WithNumaCores(spec.NumaCores). AppendEndpoints(topo.Endpoints(deployUser)...) fp := filepath.Join(paths.Cache, fmt.Sprintf("run_tiflash_%s_%d.sh", i.GetHost(), i.GetPort())) diff --git a/pkg/cluster/spec/tikv.go b/pkg/cluster/spec/tikv.go index 7cc1478625..1b551bcd4f 100644 --- a/pkg/cluster/spec/tikv.go +++ b/pkg/cluster/spec/tikv.go @@ -60,6 +60,7 @@ type TiKVSpec struct { LogDir string `yaml:"log_dir,omitempty"` Offline bool `yaml:"offline,omitempty"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]interface{} `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` Arch string `yaml:"arch,omitempty"` @@ -216,6 +217,7 @@ func (i *TiKVInstance) InitConfig( cfg := scripts. NewTiKVScript(clusterVersion, i.GetHost(), spec.Port, spec.StatusPort, paths.Deploy, paths.Data[0], paths.Log). WithNumaNode(spec.NumaNode). + WithNumaCores(spec.NumaCores). AppendEndpoints(topo.Endpoints(deployUser)...). WithListenHost(i.GetListenHost()). WithAdvertiseAddr(spec.AdvertiseAddr). diff --git a/pkg/cluster/template/scripts/tidb.go b/pkg/cluster/template/scripts/tidb.go index 04f8c912a3..196b354889 100644 --- a/pkg/cluster/template/scripts/tidb.go +++ b/pkg/cluster/template/scripts/tidb.go @@ -32,6 +32,7 @@ type TiDBScript struct { DeployDir string LogDir string NumaNode string + NumaCores string SupportSecboot bool Endpoints []*PDScript } @@ -80,6 +81,12 @@ func (c *TiDBScript) WithNumaNode(numa string) *TiDBScript { return c } +// WithNumaCores set NumaCores field of TiDBScript +func (c *TiDBScript) WithNumaCores(numaCores string) *TiDBScript { + c.NumaCores = numaCores + return c +} + // SupportSecureBootstrap set SupportSecboot field of TiDBScript func (c *TiDBScript) SupportSecureBootstrap(s bool) *TiDBScript { c.SupportSecboot = s diff --git a/pkg/cluster/template/scripts/tiflash.go b/pkg/cluster/template/scripts/tiflash.go index db40634df1..39296e17cc 100644 --- a/pkg/cluster/template/scripts/tiflash.go +++ b/pkg/cluster/template/scripts/tiflash.go @@ -38,6 +38,7 @@ type TiFlashScript struct { LogDir string TmpDir string NumaNode string + NumaCores string Endpoints []*PDScript TiDBStatusAddrs string PDAddrs string @@ -112,6 +113,12 @@ func (c *TiFlashScript) WithNumaNode(numa string) *TiFlashScript { return c } +// WithNumaCores set NumaCores field of TiFlashScript +func (c *TiFlashScript) WithNumaCores(numaCores string) *TiFlashScript { + c.NumaCores = numaCores + return c +} + // AppendEndpoints add new PDScript to Endpoints field func (c *TiFlashScript) AppendEndpoints(ends ...*PDScript) *TiFlashScript { c.Endpoints = append(c.Endpoints, ends...) diff --git a/pkg/cluster/template/scripts/tikv.go b/pkg/cluster/template/scripts/tikv.go index d5ba933f83..53d7608544 100644 --- a/pkg/cluster/template/scripts/tikv.go +++ b/pkg/cluster/template/scripts/tikv.go @@ -37,6 +37,7 @@ type TiKVScript struct { LogDir string SupportAdvertiseStatusAddr bool NumaNode string + NumaCores string Endpoints []*PDScript } @@ -83,6 +84,12 @@ func (c *TiKVScript) WithNumaNode(numa string) *TiKVScript { return c } +// WithNumaCores set NumaCores field of TiKVScript +func (c *TiKVScript) WithNumaCores(numaCores string) *TiKVScript { + c.NumaCores = numaCores + return c +} + // AppendEndpoints add new PDScript to Endpoints field func (c *TiKVScript) AppendEndpoints(ends ...*PDScript) *TiKVScript { c.Endpoints = append(c.Endpoints, ends...)