From 2340f985664f404dd8bf7bf13d3589bc2b93aea1 Mon Sep 17 00:00:00 2001 From: lucklove Date: Fri, 24 Jul 2020 20:26:21 +0800 Subject: [PATCH 1/3] Refresh monitor configs on reload Fix https://github.com/pingcap/tiup/issues/628 Signed-off-by: lucklove --- components/cluster/command/reload.go | 60 +++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/components/cluster/command/reload.go b/components/cluster/command/reload.go index f231311576..6a5d385301 100644 --- a/components/cluster/command/reload.go +++ b/components/cluster/command/reload.go @@ -15,6 +15,9 @@ package command import ( "errors" + "fmt" + "path/filepath" + "strings" "github.com/joomcode/errorx" perrs "github.com/pingcap/errors" @@ -100,8 +103,17 @@ func buildReloadTask( topo := metadata.Topology hasImported := false + uniqueHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch topo.IterInstance(func(inst spec.Instance) { + if _, found := uniqueHosts[inst.GetHost()]; !found { + uniqueHosts[inst.GetHost()] = hostInfo{ + ssh: inst.GetSSHPort(), + os: inst.OS(), + arch: inst.Arch(), + } + } + deployDir := clusterutil.Abs(metadata.User, inst.DeployDir()) // data dir would be empty for components which don't need it dataDirs := clusterutil.MultiDirAbs(metadata.User, inst.DataDir()) @@ -142,6 +154,8 @@ func buildReloadTask( refreshConfigTasks = append(refreshConfigTasks, t) }) + monitorConfigTasks := refreshMonitoredConfigTask(clusterName, uniqueHosts, topo.GlobalOptions, topo.MonitoredOptions) + // handle dir scheme changes if hasImported { if err := spec.HandleImportPathMigration(clusterName); err != nil { @@ -154,13 +168,57 @@ func buildReloadTask( spec.ClusterPath(clusterName, "ssh", "id_rsa"), spec.ClusterPath(clusterName, "ssh", "id_rsa.pub")). ClusterSSH(metadata.Topology, metadata.User, gOpt.SSHTimeout). - Parallel(refreshConfigTasks...) + Parallel(refreshConfigTasks...). + ParallelStep("+ Refresh monitor configs", monitorConfigTasks...) if !skipRestart { tb = tb.ClusterOperate(metadata.Topology, operator.UpgradeOperation, options) } return tb.Build(), nil } +func refreshMonitoredConfigTask( + clusterName string, + uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch + globalOptions spec.GlobalOptions, + monitoredOptions spec.MonitoredOptions, +) []*task.StepDisplay { + tasks := []*task.StepDisplay{} + // monitoring agents + for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { + for host, info := range uniqueHosts { + deployDir := clusterutil.Abs(globalOptions.User, monitoredOptions.DeployDir) + // data dir would be empty for components which don't need it + dataDir := monitoredOptions.DataDir + // the default data_dir is relative to deploy_dir + if dataDir != "" && !strings.HasPrefix(dataDir, "/") { + dataDir = filepath.Join(deployDir, dataDir) + } + // log dir will always be with values, but might not used by the component + logDir := clusterutil.Abs(globalOptions.User, monitoredOptions.LogDir) + // Deploy component + t := task.NewBuilder(). + UserSSH(host, info.ssh, globalOptions.User, gOpt.SSHTimeout). + MonitoredConfig( + clusterName, + comp, + host, + globalOptions.ResourceControl, + monitoredOptions, + globalOptions.User, + meta.DirPaths{ + Deploy: deployDir, + Data: []string{dataDir}, + Log: logDir, + Cache: spec.ClusterPath(clusterName, spec.TempConfigPath), + }, + ). + BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", comp, host)) + tasks = append(tasks, t) + } + } + return tasks +} + func validRoles(roles []string) error { for _, r := range roles { match := false From 725b7ffcc97893c774cb3a4c3ac16dcf9a186ee0 Mon Sep 17 00:00:00 2001 From: lucklove Date: Mon, 27 Jul 2020 11:34:04 +0800 Subject: [PATCH 2/3] Address comment Signed-off-by: lucklove --- components/cluster/command/reload.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/cluster/command/reload.go b/components/cluster/command/reload.go index 6a5d385301..274bfde1d2 100644 --- a/components/cluster/command/reload.go +++ b/components/cluster/command/reload.go @@ -99,7 +99,7 @@ func buildReloadTask( skipRestart bool, ) (task.Task, error) { - var refreshConfigTasks []task.Task + var refreshConfigTasks []*task.StepDisplay topo := metadata.Topology hasImported := false @@ -150,7 +150,8 @@ func buildReloadTask( Data: dataDirs, Log: logDir, Cache: spec.ClusterPath(clusterName, spec.TempConfigPath), - }).Build() + }). + BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", inst.ComponentName(), inst.GetHost())) refreshConfigTasks = append(refreshConfigTasks, t) }) @@ -168,7 +169,7 @@ func buildReloadTask( spec.ClusterPath(clusterName, "ssh", "id_rsa"), spec.ClusterPath(clusterName, "ssh", "id_rsa.pub")). ClusterSSH(metadata.Topology, metadata.User, gOpt.SSHTimeout). - Parallel(refreshConfigTasks...). + ParallelStep("+ Refresh instance configs", refreshConfigTasks...). ParallelStep("+ Refresh monitor configs", monitorConfigTasks...) if !skipRestart { tb = tb.ClusterOperate(metadata.Topology, operator.UpgradeOperation, options) @@ -195,7 +196,7 @@ func refreshMonitoredConfigTask( } // log dir will always be with values, but might not used by the component logDir := clusterutil.Abs(globalOptions.User, monitoredOptions.LogDir) - // Deploy component + // Generate configs t := task.NewBuilder(). UserSSH(host, info.ssh, globalOptions.User, gOpt.SSHTimeout). MonitoredConfig( From 11bd64675b9ef506dd3958190504fb4be782879a Mon Sep 17 00:00:00 2001 From: SIGSEGV Date: Mon, 27 Jul 2020 11:50:15 +0800 Subject: [PATCH 3/3] Update components/cluster/command/reload.go Co-authored-by: Lonng --- components/cluster/command/reload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/cluster/command/reload.go b/components/cluster/command/reload.go index 274bfde1d2..88b04dfd0a 100644 --- a/components/cluster/command/reload.go +++ b/components/cluster/command/reload.go @@ -151,7 +151,7 @@ func buildReloadTask( Log: logDir, Cache: spec.ClusterPath(clusterName, spec.TempConfigPath), }). - BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", inst.ComponentName(), inst.GetHost())) + BuildAsStep(fmt.Sprintf(" - Refresh config %s -> %s", inst.ComponentName(), inst.ID())) refreshConfigTasks = append(refreshConfigTasks, t) })