From 399707f174abc1980a9101785cb27ca0e109a681 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 10 Feb 2019 13:07:07 +0200 Subject: [PATCH 1/3] detach-replica and reattach-replica use detach,reattach-replica--master_host --- go/app/cli.go | 24 ++-------- go/http/api.go | 48 +------------------- go/inst/instance_topology.go | 76 +------------------------------- go/inst/instance_topology_dao.go | 67 ---------------------------- 4 files changed, 8 insertions(+), 207 deletions(-) diff --git a/go/app/cli.go b/go/app/cli.go index 2f68f5ad6..c993d0beb 100644 --- a/go/app/cli.go +++ b/go/app/cli.go @@ -67,9 +67,11 @@ var commandSynonyms = map[string]string{ "which-cluster-osc-slaves": "which-cluster-osc-replicas", "which-cluster-gh-ost-slaves": "which-cluster-gh-ost-replicas", "which-slaves": "which-replicas", - "detach-slave": "detach-replica", - "reattach-slave": "reattach-replica", + "detach-slave": "detach-replica-master-host", + "detach-replica": "detach-replica-master-host", "detach-slave-master-host": "detach-replica-master-host", + "reattach-slave": "reattach-replica-master-host", + "reattach-replica": "reattach-replica-master-host", "reattach-slave-master-host": "reattach-replica-master-host", } @@ -622,24 +624,6 @@ func Cli(command string, strict bool, instance string, destination string, owner } fmt.Println(instanceKey.DisplayString()) } - case registerCliCommand("detach-replica", "Replication, general", `Stops replication and modifies binlog position into an impossible, yet reversible, value.`): - { - instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) - _, err := inst.DetachReplicaOperation(instanceKey) - if err != nil { - log.Fatale(err) - } - fmt.Println(instanceKey.DisplayString()) - } - case registerCliCommand("reattach-replica", "Replication, general", `Undo a detach-replica operation`): - { - instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) - _, err := inst.ReattachReplicaOperation(instanceKey) - if err != nil { - log.Fatale(err) - } - fmt.Println(instanceKey.DisplayString()) - } case registerCliCommand("detach-replica-master-host", "Replication, general", `Stops replication and modifies Master_Host into an impossible, yet reversible, value.`): { instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) diff --git a/go/http/api.go b/go/http/api.go index 7b201c5d0..cd28b63ca 100644 --- a/go/http/api.go +++ b/go/http/api.go @@ -599,50 +599,6 @@ func (this *HttpAPI) ResetSlave(params martini.Params, r render.Render, req *htt Respond(r, &APIResponse{Code: OK, Message: fmt.Sprintf("Replica reset on %+v", instance.Key), Details: instance}) } -// DetachReplica corrupts a replica's binlog corrdinates (though encodes it in such way -// that is reversible), effectively breaking replication -func (this *HttpAPI) DetachReplica(params martini.Params, r render.Render, req *http.Request, user auth.User) { - if !isAuthorizedForAction(req, user) { - Respond(r, &APIResponse{Code: ERROR, Message: "Unauthorized"}) - return - } - instanceKey, err := this.getInstanceKey(params["host"], params["port"]) - - if err != nil { - Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) - return - } - instance, err := inst.DetachReplicaOperation(&instanceKey) - if err != nil { - Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) - return - } - - Respond(r, &APIResponse{Code: OK, Message: fmt.Sprintf("Replica detached: %+v", instance.Key), Details: instance}) -} - -// ReattachReplica reverts a DetachReplica commands by reassigning the correct -// binlog coordinates to an instance -func (this *HttpAPI) ReattachReplica(params martini.Params, r render.Render, req *http.Request, user auth.User) { - if !isAuthorizedForAction(req, user) { - Respond(r, &APIResponse{Code: ERROR, Message: "Unauthorized"}) - return - } - instanceKey, err := this.getInstanceKey(params["host"], params["port"]) - - if err != nil { - Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) - return - } - instance, err := inst.ReattachReplicaOperation(&instanceKey) - if err != nil { - Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) - return - } - - Respond(r, &APIResponse{Code: OK, Message: fmt.Sprintf("Replica reattached: %+v", instance.Key), Details: instance}) -} - // DetachReplicaMasterHost detaches a replica from its master by setting an invalid // (yet revertible) host name func (this *HttpAPI) DetachReplicaMasterHost(params martini.Params, r render.Render, req *http.Request, user auth.User) { @@ -3547,8 +3503,8 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) { this.registerAPIRequest(m, "stop-slave/:host/:port", this.StopSlave) this.registerAPIRequest(m, "stop-slave-nice/:host/:port", this.StopSlaveNicely) this.registerAPIRequest(m, "reset-slave/:host/:port", this.ResetSlave) - this.registerAPIRequest(m, "detach-slave/:host/:port", this.DetachReplica) - this.registerAPIRequest(m, "reattach-slave/:host/:port", this.ReattachReplica) + this.registerAPIRequest(m, "detach-slave/:host/:port", this.DetachReplicaMasterHost) + this.registerAPIRequest(m, "reattach-slave/:host/:port", this.ReattachReplicaMasterHost) this.registerAPIRequest(m, "detach-slave-master-host/:host/:port", this.DetachReplicaMasterHost) this.registerAPIRequest(m, "reattach-slave-master-host/:host/:port", this.ReattachReplicaMasterHost) this.registerAPIRequest(m, "flush-binary-logs/:host/:port", this.FlushBinaryLogs) diff --git a/go/inst/instance_topology.go b/go/inst/instance_topology.go index 9790c31f4..3bff040db 100644 --- a/go/inst/instance_topology.go +++ b/go/inst/instance_topology.go @@ -1053,84 +1053,12 @@ Cleanup: // DetachReplicaOperation will detach a replica from its master by forcibly corrupting its replication coordinates func DetachReplicaOperation(instanceKey *InstanceKey) (*Instance, error) { - instance, err := ReadTopologyInstance(instanceKey) - if err != nil { - return instance, err - } - - log.Infof("Will detach %+v", instanceKey) - - if maintenanceToken, merr := BeginMaintenance(instanceKey, GetMaintenanceOwner(), "detach replica"); merr != nil { - err = fmt.Errorf("Cannot begin maintenance on %+v", *instanceKey) - goto Cleanup - } else { - defer EndMaintenance(maintenanceToken) - } - - if instance.IsReplica() { - instance, err = StopSlave(instanceKey) - if err != nil { - goto Cleanup - } - } - - instance, err = DetachReplica(instanceKey) - if err != nil { - goto Cleanup - } - -Cleanup: - instance, _ = StartSlave(instanceKey) - - if err != nil { - return instance, log.Errore(err) - } - - // and we're done (pending deferred functions) - AuditOperation("detach-replica", instanceKey, fmt.Sprintf("%+v replication detached", *instanceKey)) - - return instance, err + return DetachReplicaMasterHost(instanceKey) } // ReattachReplicaOperation will detach a replica from its master by forcibly corrupting its replication coordinates func ReattachReplicaOperation(instanceKey *InstanceKey) (*Instance, error) { - instance, err := ReadTopologyInstance(instanceKey) - if err != nil { - return instance, err - } - - log.Infof("Will reattach %+v", instanceKey) - - if maintenanceToken, merr := BeginMaintenance(instanceKey, GetMaintenanceOwner(), "detach replica"); merr != nil { - err = fmt.Errorf("Cannot begin maintenance on %+v", *instanceKey) - goto Cleanup - } else { - defer EndMaintenance(maintenanceToken) - } - - if instance.IsReplica() { - instance, err = StopSlave(instanceKey) - if err != nil { - goto Cleanup - } - } - - instance, err = ReattachReplica(instanceKey) - if err != nil { - goto Cleanup - } - -Cleanup: - instance, _ = StartSlave(instanceKey) - - if err != nil { - return instance, log.Errore(err) - } - - // and we're done (pending deferred functions) - AuditOperation("reattach-replica", instanceKey, fmt.Sprintf("%+v replication reattached", *instanceKey)) - - return instance, err + return ReattachReplicaMasterHost(instanceKey) } // DetachReplicaMasterHost detaches a replica from its master by corrupting the Master_Host (in such way that is reversible) diff --git a/go/inst/instance_topology_dao.go b/go/inst/instance_topology_dao.go index 4d30eac94..5604cfe44 100644 --- a/go/inst/instance_topology_dao.go +++ b/go/inst/instance_topology_dao.go @@ -862,73 +862,6 @@ func SkipQuery(instanceKey *InstanceKey) (*Instance, error) { return StartSlave(instanceKey) } -// DetachReplica detaches a replica from replication; forcibly corrupting the binlog coordinates (though in such way -// that is reversible) -func DetachReplica(instanceKey *InstanceKey) (*Instance, error) { - instance, err := ReadTopologyInstance(instanceKey) - if err != nil { - return instance, log.Errore(err) - } - - if !instance.ReplicationThreadsStopped() { - return instance, fmt.Errorf("Cannot detach slave on: %+v because replication threads are not stopped", instanceKey) - } - - isDetached, _ := instance.ExecBinlogCoordinates.ExtractDetachedCoordinates() - - if isDetached { - return instance, fmt.Errorf("Cannot (need not) detach slave on: %+v because slave is already detached", instanceKey) - } - - if *config.RuntimeCLIFlags.Noop { - return instance, fmt.Errorf("noop: aborting detach-slave operation on %+v; signalling error but nothing went wrong.", *instanceKey) - } - - detachedCoordinates := instance.ExecBinlogCoordinates.Detach() - // Encode the current coordinates within the log file name, in such way that replication is broken, but info can still be resurrected - _, err = ExecInstance(instanceKey, `change master to master_log_file=?, master_log_pos=?`, detachedCoordinates.LogFile, detachedCoordinates.LogPos) - if err != nil { - return instance, log.Errore(err) - } - - log.Infof("Detach slave %+v", instanceKey) - - instance, err = ReadTopologyInstance(instanceKey) - return instance, err -} - -// ReattachReplica restores a detached replica back into replication -func ReattachReplica(instanceKey *InstanceKey) (*Instance, error) { - instance, err := ReadTopologyInstance(instanceKey) - if err != nil { - return instance, log.Errore(err) - } - - if !instance.ReplicationThreadsStopped() { - return instance, fmt.Errorf("Cannot (need not) reattach slave on: %+v because replication threads are not stopped", instanceKey) - } - - isDetached, detachedCoordinates := instance.ExecBinlogCoordinates.ExtractDetachedCoordinates() - - if !isDetached { - return instance, fmt.Errorf("Cannot reattach slave on: %+v because slave is not detached", instanceKey) - } - - if *config.RuntimeCLIFlags.Noop { - return instance, fmt.Errorf("noop: aborting reattach-slave operation on %+v; signalling error but nothing went wrong.", *instanceKey) - } - - _, err = ExecInstance(instanceKey, `change master to master_log_file=?, master_log_pos=?`, detachedCoordinates.LogFile, detachedCoordinates.LogPos) - if err != nil { - return instance, log.Errore(err) - } - - log.Infof("Reattach slave %+v", instanceKey) - - instance, err = ReadTopologyInstance(instanceKey) - return instance, err -} - // MasterPosWait issues a MASTER_POS_WAIT() an given instance according to given coordinates. func MasterPosWait(instanceKey *InstanceKey, binlogCoordinates *BinlogCoordinates) (*Instance, error) { instance, err := ReadTopologyInstance(instanceKey) From 90d4b95903830491230548ed10aa0360679fed97 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 10 Feb 2019 13:08:37 +0200 Subject: [PATCH 2/3] cleanup --- go/inst/instance_topology.go | 10 ---------- go/logic/topology_recovery.go | 4 ++-- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/go/inst/instance_topology.go b/go/inst/instance_topology.go index 3bff040db..9a39b99e6 100644 --- a/go/inst/instance_topology.go +++ b/go/inst/instance_topology.go @@ -1051,16 +1051,6 @@ Cleanup: return instance, err } -// DetachReplicaOperation will detach a replica from its master by forcibly corrupting its replication coordinates -func DetachReplicaOperation(instanceKey *InstanceKey) (*Instance, error) { - return DetachReplicaMasterHost(instanceKey) -} - -// ReattachReplicaOperation will detach a replica from its master by forcibly corrupting its replication coordinates -func ReattachReplicaOperation(instanceKey *InstanceKey) (*Instance, error) { - return ReattachReplicaMasterHost(instanceKey) -} - // DetachReplicaMasterHost detaches a replica from its master by corrupting the Master_Host (in such way that is reversible) func DetachReplicaMasterHost(instanceKey *InstanceKey) (*Instance, error) { instance, err := ReadTopologyInstance(instanceKey) diff --git a/go/logic/topology_recovery.go b/go/logic/topology_recovery.go index 583518685..50b186644 100644 --- a/go/logic/topology_recovery.go +++ b/go/logic/topology_recovery.go @@ -537,7 +537,7 @@ func recoverDeadMaster(topologyRecovery *TopologyRecovery, candidateInstanceKey AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) for _, replica := range lostReplicas { replica := replica - inst.DetachReplicaOperation(&replica.Key) + inst.DetachReplicaMasterHost(&replica.Key) } return nil } @@ -1260,7 +1260,7 @@ func RecoverDeadCoMaster(topologyRecovery *TopologyRecovery, skipProcesses bool) AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadCoMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) for _, replica := range lostReplicas { replica := replica - inst.DetachReplicaOperation(&replica.Key) + inst.DetachReplicaMasterHost(&replica.Key) } return nil } From a7b8ebf220d85276a38cbe889435aa6b68917424 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 14 Feb 2019 13:01:02 +0200 Subject: [PATCH 3/3] doc update --- docs/first-steps.md | 2 +- resources/bin/orchestrator-client | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/first-steps.md b/docs/first-steps.md index a8736e7d4..63c45d669 100644 --- a/docs/first-steps.md +++ b/docs/first-steps.md @@ -122,7 +122,7 @@ You are easily able to see what the following do: $ orchestrator -c set-read-only -i a.replica.8.instance.com $ orchestrator -c set-writeable -i a.replica.8.instance.com -Break replication by messing with a replica's binlog coordinates: +Break replication by messing with a replica's master host: $ orchestrator -c detach-replica -i a.replica.8.instance.com diff --git a/resources/bin/orchestrator-client b/resources/bin/orchestrator-client index 161949e49..c88620c46 100755 --- a/resources/bin/orchestrator-client +++ b/resources/bin/orchestrator-client @@ -867,7 +867,7 @@ function run_command { "start-replica") general_instance_command ;; # Issue a START SLAVE on an instance "restart-replica") general_instance_command ;; # Issue STOP and START SLAVE on an instance "reset-replica") general_instance_command ;; # Issues a RESET SLAVE command; use with care - "detach-replica") general_instance_command ;; # Stops replication and modifies binlog position into an impossible yet reversible value. + "detach-replica") general_instance_command ;; # Stops replication and modifies Master_Host into an impossible yet reversible value. "reattach-replica") general_instance_command ;; # Undo a detach-replica operation "detach-replica-master-host") general_instance_command ;; # Stops replication and modifies Master_Host into an impossible yet reversible value. "reattach-replica-master-host") general_instance_command ;; # Undo a detach-replica-master-host operation