Preflight checks added to determine upfront information for later sync

cmd/sync.go

@@ -19,6 +19,7 @@ var targetEnvironmentName string
 var SyncerType string
 var ServiceName string
 var configurationFile string
+var ExcludeTables string
 var noCliInteraction bool
 var dryRun bool
 var verboseSSH bool
@@ -81,6 +82,10 @@ var syncCmd = &cobra.Command{
 			utils.LogFatalError(err.Error(), nil)
 		}
 
+		commandOptions := synchers.SyncCommandOptions{
+			ExcludeTables: ExcludeTables,
+		}
+
 		if ProjectName == "" {
 			utils.LogFatalError("No Project name given", nil)
 		}
@@ -95,7 +100,7 @@ var syncCmd = &cobra.Command{
 			}
 		}
 
-		err = synchers.RunSyncProcess(sourceEnvironment, targetEnvironment, lagoonSyncer, SyncerType, dryRun, verboseSSH)
+		err = synchers.RunSyncProcess(sourceEnvironment, targetEnvironment, lagoonSyncer, SyncerType, commandOptions, dryRun, verboseSSH)
 		if err != nil {
 			utils.LogFatalError("There was an error running the sync process", err)
 		}
@@ -133,6 +138,7 @@ func init() {
 	syncCmd.PersistentFlags().StringVarP(&targetEnvironmentName, "target-environment-name", "t", "", "The target environment name (defaults to local)")
 	syncCmd.PersistentFlags().StringVarP(&ServiceName, "service-name", "s", "", "The service name (default is 'cli'")
 	syncCmd.PersistentFlags().StringVarP(&configurationFile, "configuration-file", "c", "", "File containing sync configuration.")
+	syncCmd.PersistentFlags().StringVarP(&ExcludeTables, "skip-tables-list", "x", "", "A comma-separated list of tables to exclude (e.g. 'cache_*'")
 	syncCmd.MarkPersistentFlagRequired("remote-environment-name")
 	syncCmd.PersistentFlags().BoolVar(&noCliInteraction, "no-interaction", false, "Disallow interaction")
 	syncCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Don't run the commands, just preview what will be run")

go.mod

@@ -8,8 +8,6 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
-	golang.org/dl v0.0.0-20210318190803-5564b1ea1f85 // indirect
 	golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
-	golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc
 	gopkg.in/yaml.v2 v2.2.8
 )

preflight/main.go

@@ -0,0 +1,5 @@
+package preflight
+
+func init() {
+}
+

preflight/preflightUtils.go

@@ -0,0 +1,75 @@
+package preflight
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func MysqlConnectionCommand(database string, hostname string, port string, username string, password string) string {
+	//@TODO: db password should be hidden from log output
+	// A solution here could be to write the credentials to file and read from 'mysql --defaults-file' - passing in username and password
+	// see CreateDBCredentialsTempFile()
+
+	return fmt.Sprintf(
+		"mysql -u%s -p%s --database=%s --host=%s --port=%s -A",
+		username,
+		password,
+		database,
+		hostname,
+		port,
+	)
+}
+
+func CreateDBCredentialsTempFile(username string, password string, dir string, debug bool) (string, error) {
+	tmpFile, err := ioutil.TempFile(dir, fmt.Sprintf("%s-", filepath.Base(os.Args[0])))
+	if err != nil {
+		log.Fatal("Could not create temporary file", err)
+	}
+	defer tmpFile.Close()
+
+	if (debug) {
+		fmt.Println("Created temp file: ", tmpFile.Name())
+	}
+	if _, err = tmpFile.WriteString(fmt.Sprintf("#This file was written by lagoon-sync.\n[client]\n%s\n%s\n", os.Getenv(username), os.Getenv(password))); err != nil {
+		if debug {
+			log.Println("Unable to write to temporary file", err)
+		}
+	} else {
+		if debug {
+			fmt.Println("Credentials have been written to file")
+		}
+	}
+
+	return tmpFile.Name(), nil
+}
+
+func StringIsWildcard (string string) bool {
+	// regexp.MatchString(".*", table)
+	return strings.Contains(string, "*")
+}
+
+func FindMatchingTablesFromWildcardPattern(option string, tablesList []string) []string {
+	var tables []string
+	var splitOption = strings.Split(option, "*")
+	ignoreTableOption := splitOption[0]
+
+	for i, table := range tablesList {
+		// remove first value as this will be the table header
+		if i == 0 {
+			continue
+		}
+
+		//fmt.Println(table, splitOption[0], strings.Contains(table, ignoreTableOption))
+
+		optionMatchesTable := strings.Contains(table, ignoreTableOption)
+		if  optionMatchesTable {
+			//fmt.Println("match; ", ignoreTableOption, table)
+			tables = append(tables, table)
+		}
+	}
+	return tables
+}

preflight/preflightdefs.go

@@ -0,0 +1 @@
+package preflight

synchers/drupalconfig.go

@@ -70,7 +70,15 @@ func (root DrupalconfigSyncRoot) GetPrerequisiteCommand(environment Environment,
 	return SyncCommand{}
 }
 
-func (root DrupalconfigSyncRoot) GetRemoteCommand(environment Environment) SyncCommand {
+func (root DrupalconfigSyncRoot) ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error) {
+	return root, nil
+}
+
+func (root DrupalconfigSyncRoot) GetPreflightCommand(environment Environment, debug bool) SyncCommand {
+	return SyncCommand{}
+}
+
+func (root DrupalconfigSyncRoot) GetRemoteCommand(environment Environment, options SyncCommandOptions) SyncCommand {
 	transferResource := root.GetTransferResource(environment)
 	return SyncCommand{
 		command: fmt.Sprintf("drush config-export --destination=%s || true", transferResource.Name),

synchers/files.go

@@ -90,7 +90,15 @@ func (root FilesSyncRoot) GetPrerequisiteCommand(environment Environment, comman
 	return SyncCommand{}
 }
 
-func (root FilesSyncRoot) GetRemoteCommand(environment Environment) SyncCommand {
+func (root FilesSyncRoot) GetPreflightCommand(environment Environment, debug bool) SyncCommand {
+	return SyncCommand{}
+}
+
+func (root FilesSyncRoot) ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error) {
+	return root, nil
+}
+
+func (root FilesSyncRoot) GetRemoteCommand(environment Environment, options SyncCommandOptions) SyncCommand {
 	return generateNoOpSyncCommand()
 }
 

synchers/mariadb.go

@@ -3,11 +3,13 @@ package synchers
 import (
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/amazeeio/lagoon-sync/preflight"
 	"github.com/amazeeio/lagoon-sync/utils"
 	"github.com/spf13/viper"
 )
@@ -123,7 +125,78 @@ func (root MariadbSyncRoot) GetPrerequisiteCommand(environment Environment, comm
 	}
 }
 
-func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) SyncCommand {
+func (root MariadbSyncRoot) ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error) {
+
+	tablesToIgnore := root.Config.IgnoreTable
+	if len(tablesToIgnore) != 0 {
+		fmt.Println(" has tables to ignore", tablesToIgnore)
+	}
+
+	if preflightResponse != "" {
+		tablesList := strings.Fields(preflightResponse)
+
+		if commandOptions.ExcludeTables == "" {
+			return root, nil
+		}
+
+		var excludeTables = strings.Split(commandOptions.ExcludeTables, ",")
+
+		for _, option := range excludeTables {
+			// check if wildcard option
+			isWildcardString := preflight.StringIsWildcard(option)
+			// if option is a wildcard, find a match
+			if isWildcardString {
+				matchedTables := preflight.FindMatchingTablesFromWildcardPattern(option, tablesList)
+
+				tablesToIgnore = append(tablesToIgnore, matchedTables...)
+				root.Config.IgnoreTable = tablesToIgnore
+			}
+		}
+	}
+
+	return root, nil
+}
+
+
+
+func (root MariadbSyncRoot) GetPreflightCommand(environment Environment, verboseSSH bool) SyncCommand {
+	var config = root.Config
+
+	var debug = viper.Get("show-debug")
+	fmt.Print("debug", debug)
+
+	// check if mysql is available first
+	// command -v mysql
+	// mysqlBin, err := utils.FindMySQLBin()
+
+	// Get db connection credentials - save to temp file to hide password being printed
+	//tmpCredFilePath, err := preflight.CreateDBCredentialsTempFile(
+	//	config.DbUsername,
+	//	config.DbPassword,
+	//	"/tmp",
+	//	true)
+	//if err != nil {
+	//	log.Print("Unable to create temp db credentials")
+	//}
+	//
+	//if tmpCredFilePath != "" {
+	//	fmt.Print(tmpCredFilePath)
+	//}
+
+	// Establish mysql db connection
+	sqlConnectionCommand := preflight.MysqlConnectionCommand(config.DbDatabase, config.DbHostname, config.DbPort, config.DbUsername, config.DbPassword)
+	log.Println(sqlConnectionCommand)
+
+	return SyncCommand{
+		command: fmt.Sprintf("{{ .command }} | {{ .sql }}"),
+		substitutions: map[string]interface{}{
+			"sql":     sqlConnectionCommand,
+			"command": "echo \"SHOW TABLES;\"",
+		},
+	}
+}
+
+func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment, commandOptions SyncCommandOptions) SyncCommand {
 	m := root.Config
 
 	if sourceEnvironment.EnvironmentName == LOCAL_ENVIRONMENT_NAME {
@@ -136,20 +209,27 @@ func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) Sync
 	for _, s := range m.IgnoreTable {
 		tablesToIgnore += fmt.Sprintf("--ignore-table=%s.%s ", m.DbDatabase, s)
 	}
+	if commandOptions.ExcludeTables != "" {
+		var excludeTables = strings.Split(commandOptions.ExcludeTables, ",")
+		for i := range excludeTables {
+			tablesToIgnore += fmt.Sprintf("--ignore-table=%s.%s ", m.DbDatabase, strings.TrimSpace(excludeTables[i]))
+		}
+	}
 
 	var tablesWhoseDataToIgnore string
 	for _, s := range m.IgnoreTableData {
 		tablesWhoseDataToIgnore += fmt.Sprintf("--ignore-table-data=%s.%s ", m.DbDatabase, s)
 	}
 
 	return SyncCommand{
-		command: fmt.Sprintf("mysqldump -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .tablesToIgnore }} {{ .database }} > {{ .transferResource }}"),
+		command: fmt.Sprintf("mysqldump -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .tablesToIgnore }} {{ .tableDataToIgnore }} {{ .database }} > {{ .transferResource }}"),
 		substitutions: map[string]interface{}{
 			"hostname":         m.DbHostname,
 			"username":         m.DbUsername,
 			"password":         m.DbPassword,
 			"port":             m.DbPort,
-			"tablesToIgnore":   tablesWhoseDataToIgnore,
+			"tablesToIgnore":   tablesToIgnore,
+			"tableDataToIgnore":   tablesWhoseDataToIgnore,
 			"database":         m.DbDatabase,
 			"transferResource": transferResource.Name,
 		},

synchers/mongodb.go

@@ -116,7 +116,15 @@ func (root MongoDbSyncRoot) GetPrerequisiteCommand(environment Environment, comm
 	}
 }
 
-func (root MongoDbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) SyncCommand {
+func (root MongoDbSyncRoot) GetPreflightCommand(environment Environment, debug bool) SyncCommand {
+	return SyncCommand{}
+}
+
+func (root MongoDbSyncRoot) ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error) {
+	return root, nil
+}
+
+func (root MongoDbSyncRoot) GetRemoteCommand(sourceEnvironment Environment, options SyncCommandOptions) SyncCommand {
 	m := root.Config
 
 	if sourceEnvironment.EnvironmentName == LOCAL_ENVIRONMENT_NAME {

synchers/postgres.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/amazeeio/lagoon-sync/utils"
@@ -117,14 +118,28 @@ func (root PostgresSyncRoot) GetPrerequisiteCommand(environment Environment, com
 	}
 }
 
-func (root PostgresSyncRoot) GetRemoteCommand(environment Environment) SyncCommand {
+func (root PostgresSyncRoot) GetPreflightCommand(environment Environment, debug bool) SyncCommand {
+	return SyncCommand{}
+}
+
+func (root PostgresSyncRoot) ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error) {
+	return root, nil
+}
+
+func (root PostgresSyncRoot) GetRemoteCommand(environment Environment, commandOptions SyncCommandOptions) SyncCommand {
 	m := root.Config
 	transferResource := root.GetTransferResource(environment)
 
 	var tablesToExclude string
 	for _, s := range m.ExcludeTable {
 		tablesToExclude += fmt.Sprintf("--exclude-table=%s.%s ", m.DbDatabase, s)
 	}
+	if commandOptions.ExcludeTables != "" {
+		var excludeTables = strings.Split(commandOptions.ExcludeTables, ",")
+		for i := range excludeTables {
+			tablesToExclude += fmt.Sprintf("--ignore-table=%s.%s ", m.DbDatabase, strings.TrimSpace(excludeTables[i]))
+		}
+	}
 
 	var tablesWhoseDataToExclude string
 	for _, s := range m.ExcludeTableData {

synchers/syncdefs.go

@@ -12,9 +12,13 @@ const LOCAL_ENVIRONMENT_NAME = "local"
 
 type Syncer interface {
 	// GetPrequisiteCommand will return the command to run on source or target environment to extract information.
-	GetPrerequisiteCommand(environmnt Environment, command string) SyncCommand
+	GetPrerequisiteCommand(environment Environment, command string) SyncCommand
+	// GetPreflightCommand will return the command used to initially run on source environment prior to remove command execution.
+	GetPreflightCommand(environment Environment, verboseSSH bool) SyncCommand
+	// Apply preflight changes if there are any
+	ApplyPreflightResponseChecks(preflightResponse string, commandOptions SyncCommandOptions) (Syncer, error)
 	// GetRemoteCommand will return the command to be run on the source system
-	GetRemoteCommand(environment Environment) SyncCommand
+	GetRemoteCommand(environment Environment, commandOptions SyncCommandOptions) SyncCommand
 	// GetLocalCommand will return the command to be run on the target system
 	GetLocalCommand(environment Environment) SyncCommand
 	// GetTransferResource will return the command that executes the transfer
@@ -27,6 +31,7 @@ type Syncer interface {
 
 type SyncCommand struct {
 	command       string
+	commandOptions SyncCommandOptions
 	substitutions map[string]interface{}
 	NoOp          bool // NoOp can be set to true if this command performs no operation (in situations like file transfers)
 }
@@ -48,6 +53,10 @@ type Environment struct {
 	RsyncLocalPath  string
 }
 
+type SyncCommandOptions struct {
+	ExcludeTables string
+}
+
 func (r Environment) GetOpenshiftProjectName() string {
 	return fmt.Sprintf("%s-%s", strings.ToLower(r.ProjectName), strings.ToLower(r.EnvironmentName))
 }