Merge pull request #445 from Slach/reanimate_agent

actualize Vagrant and fix problems on orchestrator-agent api with sqlite backend

.gitignore

@@ -2,6 +2,7 @@
 conf/orchestrator.conf.json
 .DS_Store
 .vagrant/
+vagrant/.sqlite
 vagrant/admin-post-install.sh
 vagrant/db-post-install.sh
 vagrant/db1-post-install.sh
@@ -13,3 +14,7 @@ vagrant/vagrant-ssh-key.pub
 Godeps/_workspace
 .gopath/
 main
+.idea/
+*.deb
+*.pcap
+*.log

Vagrantfile

@@ -16,8 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = BOX
   config.vm.box_download_insecure = true
   config.vm.box_check_update = false
-  config.vm.synced_folder '.', '/orchestrator', type: 'rsync',
-    rsync__auto: true
+  config.vm.synced_folder '.', '/orchestrator', type: 'rsync', rsync__auto: true
 
   (0..4).each do |n|
     name = (n > 0 ? ("db" + n.to_s) : "admin")

build.sh

@@ -87,7 +87,7 @@ function oinstall() {
   cd  $mydir
   gofmt -s -w  go/
   rsync -qa ./resources $builddir/orchestrator${prefix}/orchestrator/
-  rsync -qa ./conf/orchestrator-sample.* $builddir/orchestrator${prefix}/orchestrator/
+  rsync -qa ./conf/orchestrator-sample*.conf.json $builddir/orchestrator${prefix}/orchestrator/
   cp etc/init.d/orchestrator.bash $builddir/orchestrator/etc/init.d/orchestrator
   chmod +x $builddir/orchestrator/etc/init.d/orchestrator
 }
@@ -134,12 +134,14 @@ function package() {
   esac
 
   echo "---"
-  if cat /etc/centos-release | grep 'CentOS release 6' ; then
-    rm ${TOPDIR:-?}/orchestrator*.deb
-    rm ${TOPDIR:-?}/orchestrator*.tar.gz
-    # n CentOD 6 box: we only want the rpms for CentOS6
-    # Add "-centos6" to the file name.
-    ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done
+  if [[ -f /etc/centos-release ]]; then
+      if cat /etc/centos-release | grep 'CentOS release 6' ; then
+        rm ${TOPDIR:-?}/orchestrator*.deb
+        rm ${TOPDIR:-?}/orchestrator*.tar.gz
+        # n CentOD 6 box: we only want the rpms for CentOS6
+        # Add "-centos6" to the file name.
+        ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done
+      fi
   fi
   echo "Done. Find releases in $TOPDIR"
 }

conf/orchestrator-sample-sqlite.conf.json

@@ -0,0 +1,120 @@
+{
+  "Debug": true,
+  "EnableSyslog": false,
+  "ListenAddress": ":3000",
+  "MySQLTopologyUser": "orc_client_user",
+  "MySQLTopologyPassword": "orc_client_password",
+  "MySQLTopologyCredentialsConfigFile": "",
+  "MySQLTopologySSLPrivateKeyFile": "",
+  "MySQLTopologySSLCertFile": "",
+  "MySQLTopologySSLCAFile": "",
+  "MySQLTopologySSLSkipVerify": true,
+  "MySQLTopologyUseMutualTLS": false,
+  "BackendDB": "sqlite",
+  "SQLite3DataFile": "/usr/local/orchestrator/orchestrator.sqlite3",
+  "DefaultInstancePort": 3306,
+  "DiscoverByShowSlaveHosts": true,
+  "InstancePollSeconds": 5,
+  "UnseenInstanceForgetHours": 240,
+  "SnapshotTopologiesIntervalHours": 0,
+  "InstanceBulkOperationsWaitTimeoutSeconds": 10,
+  "HostnameResolveMethod": "default",
+  "MySQLHostnameResolveMethod": "@@hostname",
+  "SkipBinlogServerUnresolveCheck": true,
+  "ExpiryHostnameResolvesMinutes": 60,
+  "RejectHostnameResolvePattern": "",
+  "ReasonableReplicationLagSeconds": 10,
+  "ProblemIgnoreHostnameFilters": [],
+  "VerifyReplicationFilters": false,
+  "ReasonableMaintenanceReplicationLagSeconds": 20,
+  "CandidateInstanceExpireMinutes": 60,
+  "AuditLogFile": "",
+  "AuditToSyslog": false,
+  "RemoveTextFromHostnameDisplay": ".mydomain.com:3306",
+  "ReadOnly": false,
+  "AuthenticationMethod": "",
+  "HTTPAuthUser": "",
+  "HTTPAuthPassword": "",
+  "AuthUserHeader": "",
+  "PowerAuthUsers": [
+    "*"
+  ],
+  "ClusterNameToAlias": {
+    "127.0.0.1": "test suite"
+  },
+  "SlaveLagQuery": "",
+  "DetectClusterAliasQuery": "SELECT SUBSTRING_INDEX(@@hostname, '.', 1)",
+  "DetectClusterDomainQuery": "",
+  "DetectInstanceAliasQuery": "",
+  "DetectPromotionRuleQuery": "",
+  "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com",
+  "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com",
+  "PromotionIgnoreHostnameFilters": [],
+  "DetectSemiSyncEnforcedQuery": "",
+  "ServeAgentsHttp": false,
+  "AgentsServerPort": ":3001",
+  "AgentsUseSSL": false,
+  "AgentsUseMutualTLS": false,
+  "AgentSSLSkipVerify": false,
+  "AgentSSLPrivateKeyFile": "",
+  "AgentSSLCertFile": "",
+  "AgentSSLCAFile": "",
+  "AgentSSLValidOUs": [],
+  "UseSSL": false,
+  "UseMutualTLS": false,
+  "SSLSkipVerify": false,
+  "SSLPrivateKeyFile": "",
+  "SSLCertFile": "",
+  "SSLCAFile": "",
+  "SSLValidOUs": [],
+  "URLPrefix": "",
+  "StatusEndpoint": "/api/status",
+  "StatusSimpleHealth": true,
+  "StatusOUVerify": false,
+  "AgentPollMinutes": 60,
+  "UnseenAgentForgetHours": 6,
+  "StaleSeedFailMinutes": 60,
+  "SeedAcceptableBytesDiff": 8192,
+  "PseudoGTIDPattern": "",
+  "PseudoGTIDPatternIsFixedSubstring": false,
+  "PseudoGTIDMonotonicHint": "asc:",
+  "DetectPseudoGTIDQuery": "",
+  "BinlogEventsChunkSize": 10000,
+  "SkipBinlogEventsContaining": [],
+  "ReduceReplicationAnalysisCount": true,
+  "FailureDetectionPeriodBlockMinutes": 60,
+  "RecoveryPeriodBlockSeconds": 3600,
+  "RecoveryIgnoreHostnameFilters": [],
+  "RecoverMasterClusterFilters": [
+    "_master_pattern_"
+  ],
+  "RecoverIntermediateMasterClusterFilters": [
+    "_intermediate_master_pattern_"
+  ],
+  "OnFailureDetectionProcesses": [
+    "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log"
+  ],
+  "PreFailoverProcesses": [
+    "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log"
+  ],
+  "PostFailoverProcesses": [
+    "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log"
+  ],
+  "PostUnsuccessfulFailoverProcesses": [],
+  "PostMasterFailoverProcesses": [
+    "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log"
+  ],
+  "PostIntermediateMasterFailoverProcesses": [
+    "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log"
+  ],
+  "CoMasterRecoveryMustPromoteOtherCoMaster": true,
+  "DetachLostSlavesAfterMasterFailover": true,
+  "ApplyMySQLPromotionAfterMasterFailover": false,
+  "MasterFailoverDetachSlaveMasterHost": false,
+  "MasterFailoverLostInstancesDowntimeMinutes": 0,
+  "PostponeSlaveRecoveryOnLagMinutes": 0,
+  "OSCIgnoreHostnameFilters": [],
+  "GraphiteAddr": "",
+  "GraphitePath": "",
+  "GraphiteConvertHostnameDotsToUnderscores": true
+}

docs/configuration-sample.md

@@ -93,6 +93,7 @@ The following is a production configuration file, with some details redacted.
   "HttpTimeoutSeconds": 60,
   "StaleSeedFailMinutes": 60,
   "SeedAcceptableBytesDiff": 8192,
+  "SeedWaitSecondsBeforeSend": 2,
   "PseudoGTIDPattern": "drop view if exists `meta`.`_pseudo_gtid_hint__asc:",
   "PseudoGTIDPatternIsFixedSubstring": true,
   "PseudoGTIDMonotonicHint": "asc:",

etc/init.d/orchestrator.bash

@@ -54,9 +54,11 @@ start_daemon () {
 	post_start_daemon_hook 1>&2
 }
 
-# The file /etc/orchestrator_profile can be used to inject pre-service execution
+# This files can be used to inject pre-service execution
 # scripts, such as exporting variables or whatever. It's yours!
+[ -f /etc/default/orchestrator ] && . /etc/default/orchestrator
 [ -f /etc/orchestrator_profile ] && . /etc/orchestrator_profile
+[ -f /etc/profile.d/orchestrator ] && . /etc/profile.d/orchestrator
 
 case "$1" in
   start)

go/agent/agent_dao.go

@@ -106,9 +106,9 @@ func SubmitAgent(hostname string, port int, token string) (string, error) {
 	_, err := db.ExecOrchestrator(`
 			replace
 				into host_agent (
-					hostname, port, token, last_submitted
+					hostname, port, token, last_submitted, count_mysql_snapshots
 				) VALUES (
-					?, ?, ?, NOW()
+					?, ?, ?, NOW(), 0
 				)
 			`,
 		hostname,
@@ -164,7 +164,7 @@ func ReadOutdatedAgentsHosts() ([]string, error) {
 		from
 			host_agent
 		where
-			IFNULL(last_checked < now() - interval ? minute, true)
+			IFNULL(last_checked < now() - interval ? minute, 1)
 			`
 	err := db.QueryOrchestrator(query, sqlutils.Args(config.Config.AgentPollMinutes), func(m sqlutils.RowMap) error {
 		hostname := m.GetString("hostname")
@@ -461,7 +461,7 @@ func MountLV(hostname string, lv string) (Agent, error) {
 	return executeAgentCommand(hostname, fmt.Sprintf("mountlv?lv=%s", lv), nil)
 }
 
-// RemoveLV requests an agent to remvoe a snapshot
+// RemoveLV requests an agent to remove a snapshot
 func RemoveLV(hostname string, lv string) (Agent, error) {
 	return executeAgentCommand(hostname, fmt.Sprintf("removelv?lv=%s", lv), nil)
 }
@@ -547,8 +547,8 @@ func AbortSeed(seedId int64) error {
 }
 
 // PostCopy will request an agent to invoke post-copy commands
-func PostCopy(hostname string) (Agent, error) {
-	return executeAgentCommand(hostname, "post-copy", nil)
+func PostCopy(hostname, sourceHostname string) (Agent, error) {
+	return executeAgentCommand(hostname, fmt.Sprintf("post-copy/?sourceHost=%s", sourceHostname), nil)
 }
 
 // SubmitSeedEntry submits a new seed operation entry, returning its unique ID
@@ -693,7 +693,7 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err
 	}
 
 	seedFromLogicalVolume := sourceAgent.LogicalVolumes[0]
-	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Mounting logical volume: %s", seedFromLogicalVolume.Path), "")
+	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Mounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "")
 	_, err = MountLV(sourceHostname, seedFromLogicalVolume.Path)
 	if err != nil {
 		return updateSeedStateEntry(seedStateId, err)
@@ -722,8 +722,8 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err
 	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now receive data in background", targetHostname), "")
 	ReceiveMySQLSeedData(targetHostname, seedId)
 
-	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting some time for %s to start listening for incoming data", targetHostname), "")
-	time.Sleep(2 * time.Second)
+	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting %d seconds for %s to start listening for incoming data", config.Config.SeedWaitSecondsBeforeSend, targetHostname), "")
+	time.Sleep(time.Duration(config.Config.SeedWaitSecondsBeforeSend) * time.Second)
 
 	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now send data to %s in background", sourceHostname, targetHostname), "")
 	SendMySQLSeedData(sourceHostname, targetHostname, seedId)
@@ -774,12 +774,12 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err
 
 	// Cleanup:
 	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Executing post-copy command on %s", targetHostname), "")
-	_, err = PostCopy(targetHostname)
+	_, err = PostCopy(targetHostname, sourceHostname)
 	if err != nil {
 		return updateSeedStateEntry(seedStateId, err)
 	}
 
-	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Unmounting logical volume: %s", seedFromLogicalVolume.Path), "")
+	seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Unmounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "")
 	_, err = Unmount(sourceHostname)
 	if err != nil {
 		return updateSeedStateEntry(seedStateId, err)

go/config/config.go

@@ -204,6 +204,7 @@ type Configuration struct {
 	UnseenAgentForgetHours                     uint              // Number of hours after which an unseen agent is forgotten
 	StaleSeedFailMinutes                       uint              // Number of minutes after which a stale (no progress) seed is considered failed.
 	SeedAcceptableBytesDiff                    int64             // Difference in bytes between seed source & target data size that is still considered as successful copy
+	SeedWaitSecondsBeforeSend                  int64             // Number of seconds for waiting before start send data command on agent
 	AutoPseudoGTID                             bool              // Should orchestrator automatically inject Pseudo-GTID entries to the masters
 	PseudoGTIDPattern                          string            // Pattern to look for in binary logs that makes for a unique entry (pseudo GTID). When empty, Pseudo-GTID based refactoring is disabled.
 	PseudoGTIDPatternIsFixedSubstring          bool              // If true, then PseudoGTIDPattern is not treated as regular expression but as fixed substring, and can boost search time
@@ -365,6 +366,7 @@ func newConfiguration() *Configuration {
 		UnseenAgentForgetHours:                     6,
 		StaleSeedFailMinutes:                       60,
 		SeedAcceptableBytesDiff:                    8192,
+		SeedWaitSecondsBeforeSend:                  2,
 		AutoPseudoGTID:                             false,
 		PseudoGTIDPattern:                          "",
 		PseudoGTIDPatternIsFixedSubstring:          false,

vagrant/admin-build.sh

@@ -1,23 +1,33 @@
 #!/bin/bash
-
+set -xeuo pipefail
 # Install orchestrator
-rpm -i /tmp/orchestrator-release/orchestrator*.rpm
+if [[ -e /etc/redhat-release ]]; then
+  rpm -i /tmp/orchestrator-release/orchestrator*.rpm
+fi
+
+if [[ -e /etc/debian_version ]]; then
+  dpkg -i /tmp/orchestrator-release/orchestrator*.deb
+fi
+
+if [[ -e /orchestrator/vagrant/.sqlite ]]; then
+  cp -fv /usr/local/orchestrator/orchestrator-sample-sqlite.conf.json /etc/orchestrator.conf.json
+else
+  cp -fv /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json
+fi
 
 if [[ -e /etc/redhat-release ]]; then
 
   /sbin/chkconfig orchestrator on
-  cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json
   /sbin/service orchestrator start
 
 elif [[ -e /etc/debian_version ]]; then
 
   update-rc.d orchestrator defaults
-  cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json
   /usr/sbin/service orchestrator start
 
 fi
 
 echo '* * * * * root /usr/bin/orchestrator -c discover -i db1' > /etc/cron.d/orchestrator-discovery
 
 # Discover instances
-/usr/bin/orchestrator -c discover -i localhost
+/usr/bin/orchestrator --verbose --debug --stack -c discover -i localhost