From f6eef14451b02e950c501f43ff2696fe308342fc Mon Sep 17 00:00:00 2001 From: emma Date: Thu, 23 Feb 2023 12:21:59 +0100 Subject: [PATCH 1/4] fix bug #469, don't send replication metric when topology galera or grouprep --- cluster/srv_snd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/srv_snd.go b/cluster/srv_snd.go index 235f0b058..efa728c93 100644 --- a/cluster/srv_snd.go +++ b/cluster/srv_snd.go @@ -24,7 +24,7 @@ func (server *ServerMonitor) GetDatabaseMetrics() []graphite.Metric { replacer := strings.NewReplacer("`", "", "?", "", " ", "_", ".", "-", "(", "-", ")", "-", "/", "_", "<", "-", "'", "-", "\"", "-") hostname := replacer.Replace(server.Variables["HOSTNAME"]) var metrics []graphite.Metric - if server.IsSlave { + if server.IsSlave && server.GetCluster().GetTopology() != topoMultiMasterWsrep && server.GetCluster().GetTopology() != topoMultiMasterGrouprep { m := graphite.NewMetric(fmt.Sprintf("mysql.%s.mysql_slave_status_seconds_behind_master", hostname), fmt.Sprintf("%d", server.SlaveStatus.SecondsBehindMaster.Int64), time.Now().Unix()) metrics = append(metrics, m) metrics = append(metrics, graphite.NewMetric(fmt.Sprintf("mysql.%s.mysql_slave_status_exec_master_log_pos", hostname), fmt.Sprintf("%s", server.SlaveStatus.ExecMasterLogPos.String), time.Now().Unix())) From 5854eab1ea44c7cc3514b41c76fe818b1f930172 Mon Sep 17 00:00:00 2001 From: emma Date: Thu, 23 Feb 2023 12:38:06 +0100 Subject: [PATCH 2/4] fix bug #471, database directory can be wrongly created --- cluster/srv_get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/srv_get.go b/cluster/srv_get.go index 5a1948648..1bbaa98ba 100644 --- a/cluster/srv_get.go +++ b/cluster/srv_get.go @@ -618,7 +618,7 @@ func (server *ServerMonitor) GetDatabaseBasedir() string { } else if server.ClusterGroup.Conf.ProvOrchestrator == config.ConstOrchestratorSlapOS { return server.SlapOSDatadir } - return "" + return server.Datadir } func (server *ServerMonitor) GetTablePK(schema string, table string) (string, error) { From f6327a92bce3b712f23844c581f378a61f72fbe5 Mon Sep 17 00:00:00 2001 From: emma Date: Fri, 24 Feb 2023 14:44:07 +0100 Subject: [PATCH 3/4] #470 fix Replication manager pro don't start if account.yaml --- cluster/cluster.go | 8 ++++++++ config/config.go | 3 ++- server/server.go | 4 ++-- server/server_monitor.go | 26 +++++++++++--------------- share/opensvc/account.yaml | 2 +- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index b0675cf5d..67bb597fa 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -171,6 +171,7 @@ type Cluster struct { WaitingFailover int `json:"waitingFailover"` Configurator configurator.Configurator `json:"configurator"` DiffVariables []VariableDiff `json:"diffVariables"` + insideInitNodes bool `json:"-"` sync.Mutex crcTable *crc64.Table } @@ -359,8 +360,15 @@ func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.T return nil } + func (cluster *Cluster) initOrchetratorNodes() { + if cluster.insideInitNodes { + return + } + cluster.insideInitNodes = true + defer func() { cluster.insideInitNodes = false }() + //defer cluster.insideInitNodes = false //cluster.LogPrintf(LvlInfo, "Loading nodes from orchestrator %s", cluster.Conf.ProvOrchestrator) switch cluster.GetOrchestrator() { case config.ConstOrchestratorOpenSVC: diff --git a/config/config.go b/config/config.go index ee04af08f..a505b9c66 100644 --- a/config/config.go +++ b/config/config.go @@ -345,6 +345,7 @@ type Config struct { ProvOpensvcP12Certificate string `mapstructure:"opensvc-p12-certificate" toml:"opensvc-p12-certificat" json:"opensvcP12Certificate"` ProvOpensvcP12Secret string `mapstructure:"opensvc-p12-secret" toml:"opensvc-p12-secret" json:"opensvcP12Secret"` ProvOpensvcUseCollectorAPI bool `mapstructure:"opensvc-use-collector-api" toml:"opensvc-use-collector-api" json:"opensvcUseCollectorApi"` + ProvOpensvcCollectorAccount string `mapstructure:"opensvc-collector-account" toml:"opensvc-collector-account" json:"opensvcCollectorAccount"` ProvRegister bool `mapstructure:"opensvc-register" toml:"opensvc-register" json:"opensvcRegister"` ProvAdminUser string `mapstructure:"opensvc-admin-user" toml:"opensvc-admin-user" json:"opensvcAdminUser"` ProvUser string `mapstructure:"opensvc-user" toml:"opensvc-user" json:"opensvcUser"` @@ -529,7 +530,7 @@ type ConfigVariableType struct { Label string `json:"label"` } -//Compliance created in OpenSVC collector and exported as JSON +// Compliance created in OpenSVC collector and exported as JSON type Compliance struct { Filtersets []struct { ID uint `json:"id"` diff --git a/server/server.go b/server/server.go index e702ca534..9121dc806 100644 --- a/server/server.go +++ b/server/server.go @@ -99,7 +99,7 @@ const ( ConstMonitorStandby string = "S" ) -//Unused in server still used in client cmd line +// Unused in server still used in client cmd line type Settings struct { Enterprise string `json:"enterprise"` Interactive string `json:"interactive"` @@ -573,7 +573,7 @@ func (repman *ReplicationManager) Run() error { repman.OpenSVC.CertsDERSecret = repman.Conf.ProvOpensvcP12Secret err := repman.OpenSVC.LoadCert(repman.Conf.ProvOpensvcP12Certificate) if err != nil { - log.Printf("Cannot load OpenSVC cluster certificate %s ", err) + log.Fatalf("Cannot load OpenSVC cluster certificate %s ", err) } } //don't Bootstrap opensvc to speedup test diff --git a/server/server_monitor.go b/server/server_monitor.go index cf5063a46..316006205 100644 --- a/server/server_monitor.go +++ b/server/server_monitor.go @@ -519,26 +519,22 @@ func init() { monitorCmd.Flags().StringVar(&conf.ProvOpensvcP12Certificate, "opensvc-p12-certificate", "/etc/replication-manager/s18.p12", "Certicate used for socket vs collector API opensvc-host refer to a cluster VIP") monitorCmd.Flags().BoolVar(&conf.ProvOpensvcUseCollectorAPI, "opensvc-use-collector-api", false, "Use the collector API instead of cluster VIP") monitorCmd.Flags().StringVar(&conf.KubeConfig, "kube-config", "", "path to ks8 config file") + monitorCmd.Flags().StringVar(&conf.ProvOpensvcCollectorAccount, "opensvc-collector-account", "/etc/replication-manager/account.yaml", "Openscv collector account") dbConfig := viper.New() - dbConfig.SetConfigType("yaml") - file, err := ioutil.ReadFile(conf.ConfDir + "/account.yaml") - if err != nil { - file, err = ioutil.ReadFile(conf.ShareDir + "/opensvc/account.yaml") + if conf.ProvOpensvcUseCollectorAPI { + dbConfig.SetConfigType("yaml") + file, err := ioutil.ReadFile(conf.ProvOpensvcCollectorAccount) if err != nil { - log.Errorf("%s", err) + log.Errorf("Provide OpenSVC account file : %s", err) + } + + dbConfig.ReadConfig(bytes.NewBuffer(file)) + conf.ProvUser = dbConfig.Get("email").(string) + ":" + dbConfig.Get("hashed_password").(string) + crcTable := crc64.MakeTable(crc64.ECMA) + conf.ProvCodeApp = "ns" + strconv.FormatUint(crc64.Checksum([]byte(dbConfig.Get("email").(string)), crcTable), 10) } - dbConfig.ReadConfig(bytes.NewBuffer(file)) - // log.Printf("OpenSVC user account: %s", dbConfig.Get("email").(string)) - conf.ProvUser = dbConfig.Get("email").(string) + ":" + dbConfig.Get("hashed_password").(string) - crcTable := crc64.MakeTable(crc64.ECMA) - conf.ProvCodeApp = "ns" + strconv.FormatUint(crc64.Checksum([]byte(dbConfig.Get("email").(string)), crcTable), 10) - // log.Printf("OpenSVC code application: %s", conf.ProvCodeApp) - // } else { - // monitorCmd.Flags().StringVar(&conf.ProvUser, "opensvc-user", "replication-manager@localhost.localdomain:mariadb", "OpenSVC collector provisioning user") - // monitorCmd.Flags().StringVar(&conf.ProvCodeApp, "opensvc-codeapp", "MariaDB", "OpenSVC collector applicative code") - // } } } diff --git a/share/opensvc/account.yaml b/share/opensvc/account.yaml index 002c35aa4..2425ffe5f 100644 --- a/share/opensvc/account.yaml +++ b/share/opensvc/account.yaml @@ -3,4 +3,4 @@ state: enabled access: site: login: 'true' -hashed_password: repman +hashed_password: xxxx From f58e5bfd774f3c80ce50aec8d8b29522bd9d3b94 Mon Sep 17 00:00:00 2001 From: emma Date: Mon, 27 Feb 2023 12:14:34 +0100 Subject: [PATCH 4/4] Concurrent get orchestrator nodes #475 --- cluster/cluster.go | 37 +++++++++++++++++++++++++------------ cluster/prov_opensvc.go | 4 +++- cluster/srv_set.go | 3 +-- server/server_monitor.go | 2 +- utils/state/state.go | 9 +++++++++ 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index 67bb597fa..41d4b611b 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -171,7 +171,9 @@ type Cluster struct { WaitingFailover int `json:"waitingFailover"` Configurator configurator.Configurator `json:"configurator"` DiffVariables []VariableDiff `json:"diffVariables"` - insideInitNodes bool `json:"-"` + inInitNodes bool `json:"-"` + CanInitNodes bool `json:"canInitNodes"` + errorInitNodes error `json:"-"` sync.Mutex crcTable *crc64.Table } @@ -264,6 +266,7 @@ func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.T cluster.addtableCond = nbc.New() cluster.altertableCond = nbc.New() cluster.canFlashBack = true + cluster.CanInitNodes = true cluster.runOnceAfterTopology = true cluster.testStopCluster = true cluster.testStartCluster = true @@ -362,23 +365,23 @@ func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.T } func (cluster *Cluster) initOrchetratorNodes() { - if cluster.insideInitNodes { + if cluster.inInitNodes { return } - cluster.insideInitNodes = true - defer func() { cluster.insideInitNodes = false }() + cluster.inInitNodes = true + defer func() { cluster.inInitNodes = false }() //defer cluster.insideInitNodes = false //cluster.LogPrintf(LvlInfo, "Loading nodes from orchestrator %s", cluster.Conf.ProvOrchestrator) switch cluster.GetOrchestrator() { case config.ConstOrchestratorOpenSVC: - cluster.Agents, _ = cluster.OpenSVCGetNodes() + cluster.Agents, cluster.errorInitNodes = cluster.OpenSVCGetNodes() case config.ConstOrchestratorKubernetes: - cluster.Agents, _ = cluster.K8SGetNodes() + cluster.Agents, cluster.errorInitNodes = cluster.K8SGetNodes() case config.ConstOrchestratorSlapOS: - cluster.Agents, _ = cluster.SlapOSGetNodes() + cluster.Agents, cluster.errorInitNodes = cluster.SlapOSGetNodes() case config.ConstOrchestratorLocalhost: - cluster.Agents, _ = cluster.LocalhostGetNodes() + cluster.Agents, cluster.errorInitNodes = cluster.LocalhostGetNodes() case config.ConstOrchestratorOnPremise: default: log.Fatalln("prov-orchestrator not supported", cluster.Conf.ProvOrchestrator) @@ -452,7 +455,7 @@ func (cluster *Cluster) Run() { if !cluster.IsInFailover() { cluster.initProxies() } - cluster.initOrchetratorNodes() + go cluster.initOrchetratorNodes() cluster.ResticFetchRepo() cluster.runOnceAfterTopology = false } else { @@ -469,7 +472,7 @@ func (cluster *Cluster) Run() { cluster.InjectProxiesTraffic() } if cluster.sme.GetHeartbeats()%30 == 0 { - cluster.initOrchetratorNodes() + go cluster.initOrchetratorNodes() cluster.MonitorQueryRules() cluster.MonitorVariablesDiff() cluster.ResticFetchRepo() @@ -479,9 +482,12 @@ func (cluster *Cluster) Run() { cluster.sme.PreserveState("WARN0093") cluster.sme.PreserveState("WARN0084") cluster.sme.PreserveState("WARN0095") - cluster.sme.PreserveState("ERR00082") cluster.sme.PreserveState("WARN0101") } + if !cluster.CanInitNodes { + cluster.SetState("ERR00082", state.State{ErrType: "WARNING", ErrDesc: fmt.Sprintf(clusterError["ERR00082"], cluster.errorInitNodes), ErrFrom: "OPENSVC"}) + } + if cluster.sme.GetHeartbeats()%36000 == 0 { cluster.ResticPurgeRepo() } else { @@ -569,8 +575,13 @@ func (cluster *Cluster) StateProcessing() { } // cluster.statecloseChan <- s } + var states []string + if cluster.runOnceAfterTopology { + states = cluster.sme.GetFirstStates() - states := cluster.sme.GetStates() + } else { + states = cluster.sme.GetStates() + } for i := range states { cluster.LogPrintf("STATE", states[i]) } @@ -581,7 +592,9 @@ func (cluster *Cluster) StateProcessing() { } for _, s := range cluster.sme.GetLastOpenedStates() { + cluster.CheckAlert(s) + } cluster.sme.ClearState() diff --git a/cluster/prov_opensvc.go b/cluster/prov_opensvc.go index 59c46a588..69ea60456 100644 --- a/cluster/prov_opensvc.go +++ b/cluster/prov_opensvc.go @@ -81,8 +81,10 @@ func (cluster *Cluster) OpenSVCGetNodes() ([]Agent, error) { svc := cluster.OpenSVCConnect() hosts, err := svc.GetNodes() if err != nil { - cluster.SetState("ERR00082", state.State{ErrType: "WARNING", ErrDesc: fmt.Sprintf(clusterError["ERR00082"], err), ErrFrom: "OPENSVC"}) + cluster.CanInitNodes = false return nil, err + } else { + cluster.CanInitNodes = true } if hosts == nil { return nil, errors.New("Empty Opensvc Agent list") diff --git a/cluster/srv_set.go b/cluster/srv_set.go index 9c316bd13..68631c3e8 100644 --- a/cluster/srv_set.go +++ b/cluster/srv_set.go @@ -21,7 +21,6 @@ import ( "github.com/signal18/replication-manager/utils/dbhelper" "github.com/signal18/replication-manager/utils/misc" - "github.com/signal18/replication-manager/utils/state" ) func (server *ServerMonitor) SetPlacement(k int, ProvAgents string, SlapOSDBPartitions string, SchedulerReceiverPorts string) { @@ -230,7 +229,7 @@ func (server *ServerMonitor) SetCredential(url string, user string, pass string) server.Host, server.Port, server.PostgressDB = misc.SplitHostPortDB(url) server.IP, err = dbhelper.CheckHostAddr(server.Host) if err != nil { - server.ClusterGroup.SetState("ERR00062", state.State{ErrType: LvlWarn, ErrDesc: fmt.Sprintf(clusterError["ERR00062"], server.Host, err.Error()), ErrFrom: "TOPO"}) + server.GetCluster().LogPrintf(LvlErr, "Cannot resolved DNS for host %s, error: %s", server.Host, err.Error()) } if server.PostgressDB == "" { server.PostgressDB = "test" diff --git a/server/server_monitor.go b/server/server_monitor.go index 316006205..81f0dd135 100644 --- a/server/server_monitor.go +++ b/server/server_monitor.go @@ -521,8 +521,8 @@ func init() { monitorCmd.Flags().StringVar(&conf.KubeConfig, "kube-config", "", "path to ks8 config file") monitorCmd.Flags().StringVar(&conf.ProvOpensvcCollectorAccount, "opensvc-collector-account", "/etc/replication-manager/account.yaml", "Openscv collector account") - dbConfig := viper.New() if conf.ProvOpensvcUseCollectorAPI { + dbConfig := viper.New() dbConfig.SetConfigType("yaml") file, err := ioutil.ReadFile(conf.ProvOpensvcCollectorAccount) if err != nil { diff --git a/utils/state/state.go b/utils/state/state.go index dd459666a..e75547c86 100644 --- a/utils/state/state.go +++ b/utils/state/state.go @@ -298,6 +298,15 @@ func (SM *StateMachine) GetStates() []string { return log } +func (SM *StateMachine) GetFirstStates() []string { + var log []string + for key, value := range SM.GetLastOpenedStates() { + log = append(log, fmt.Sprintf("OPENED %s : %s", key, value.ErrDesc)) + } + + return log +} + func (SM *StateMachine) GetLastResolvedStates() map[string]State { resolved := make(map[string]State) SM.Lock()