From ccc11e655f728ded7994705ba7ad540228becdda Mon Sep 17 00:00:00 2001 From: Joel Rieke Date: Tue, 3 Oct 2023 13:20:39 -0700 Subject: [PATCH 01/55] Fixing broken Darwin. --- buildopts/memprotectopts/options_darwin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildopts/memprotectopts/options_darwin.go b/buildopts/memprotectopts/options_darwin.go index e2d4fec67..a6ffb3888 100644 --- a/buildopts/memprotectopts/options_darwin.go +++ b/buildopts/memprotectopts/options_darwin.go @@ -19,5 +19,5 @@ func MemUnprotectAll(logger *log.Logger) error { } func MemProtect(logger *log.Logger, sensitive *string) error { - return mlock.Mlock(logger*log.Logger, sensitive*string) + return mlock.Mlock(logger, sensitive) } From b92d12ccd5b046b3bc7387a8c3e1a33eef0cb98c Mon Sep 17 00:00:00 2001 From: Joel Rieke Date: Tue, 3 Oct 2023 13:26:27 -0700 Subject: [PATCH 02/55] Get darwin to compile. --- buildopts/memprotectopts/options_darwin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildopts/memprotectopts/options_darwin.go b/buildopts/memprotectopts/options_darwin.go index a6ffb3888..5ca3d3f97 100644 --- a/buildopts/memprotectopts/options_darwin.go +++ b/buildopts/memprotectopts/options_darwin.go @@ -19,5 +19,6 @@ func MemUnprotectAll(logger *log.Logger) error { } func MemProtect(logger *log.Logger, sensitive *string) error { - return mlock.Mlock(logger, sensitive) + // TODO: is this correct? + return mlock.Mlock(logger) } From 24cd4151ae5a6b1a39074362342254835226e0d5 Mon Sep 17 00:00:00 2001 From: Joel Rieke Date: Tue, 3 Oct 2023 13:35:04 -0700 Subject: [PATCH 03/55] Still fixing darwin. --- buildopts/memprotectopts/options_darwin.go | 1 + 1 file changed, 1 insertion(+) diff --git a/buildopts/memprotectopts/options_darwin.go b/buildopts/memprotectopts/options_darwin.go index 5ca3d3f97..e6123ea4f 100644 --- a/buildopts/memprotectopts/options_darwin.go +++ b/buildopts/memprotectopts/options_darwin.go @@ -12,6 +12,7 @@ import ( func MemProtectInit(logger *log.Logger) error { mlock.Mlock(logger) + return nil } func MemUnprotectAll(logger *log.Logger) error { From 62cedc475f53b9b922fa86e2748bb8db27285773 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:58:36 -0800 Subject: [PATCH 04/55] Prepare for release (#774) * Making more util trcsub for common. * Apply state changes. * Checking in progress. * Reverting some hardcodings. * Adding some robustness handling and changes to initial states. * More state updates for windows side. * Allow for definition of service. * Adding Refresh syncMode (#775) * Adding refresh * Adding refresh logs --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --- Makefile.tc | 3 + trcflow/deploy/process.go | 2 +- trcflow/flows/process_dataflowstatistics.go | 64 +++++++++ trcinit/initlib/vsub.go | 71 +++++++++ trcsh/trcsh.go | 152 ++++++++++++++++---- trcsubbase/trcsub.go | 9 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 16 ++- trcvault/util/util.go | 5 +- 8 files changed, 281 insertions(+), 41 deletions(-) diff --git a/Makefile.tc b/Makefile.tc index 5a2c0affd..8d0649d04 100755 --- a/Makefile.tc +++ b/Makefile.tc @@ -70,6 +70,9 @@ pub: sub: @GOPATH=$(GOPATH) CGO_ENABLED=0 GOBIN=$(GOBIN) go install -tags "memonly" github.com/trimble-oss/tierceron/trcsub sudo setcap cap_ipc_lock=+ep $(GOBIN)/trcsub +# Warning! windows not as secure.... +subwin: + @GOPATH=$(GOPATH) GOBIN=$(GOBIN) GOOS=windows GOARCH=amd64 go build -tags "tc windows memonly" -o $(GOBIN)/trcsub.exe trcsub/trcsub.go certify: @GOPATH=$(GOPATH) CGO_ENABLED=0 GOBIN=$(GOBIN) go build -o $(GOBIN)/trcplgtool -tags "tc azrcr memonly" github.com/trimble-oss/tierceron/trcvault/trcplgtool sudo setcap cap_ipc_lock=+ep $(GOBIN)/trcplgtool diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index 206b59063..b23d0cf27 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -132,7 +132,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e return err } - vaultPluginSignature, ptcErr := trcvutils.GetPluginToolConfig(cConfig, cGoMod, pluginConfig) + vaultPluginSignature, ptcErr := trcvutils.GetPluginToolConfig(cConfig, cGoMod, pluginConfig, false) defer func(vaddrPtr *string, tPtr *string) { pluginConfig["vaddress"] = *vaddrPtr diff --git a/trcflow/flows/process_dataflowstatistics.go b/trcflow/flows/process_dataflowstatistics.go index 384733b07..d7570c628 100644 --- a/trcflow/flows/process_dataflowstatistics.go +++ b/trcflow/flows/process_dataflowstatistics.go @@ -23,6 +23,9 @@ import ( const flowGroupName = "Ninja" +var refresh = false +var endRefreshChan = make(chan bool, 1) + func GetDataflowStatIndexedPathExt(engine interface{}, rowDataMap map[string]interface{}, indexColumnNames interface{}, databaseName string, tableName string, dbCallBack func(interface{}, map[string]interface{}) (string, []string, [][]interface{}, error)) (string, error) { tenantIndexPath, _ := utilcore.GetDFSPathName() if _, ok := rowDataMap[dfssql.DataflowTestIdColumn].(string); ok { @@ -204,6 +207,9 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex stateUpdate.SyncFilter = "N/A" if previousState.State == stateUpdate.State && previousState.SyncMode == stateUpdate.SyncMode && previousState.SyncFilter == stateUpdate.SyncFilter && previousState.FlowAlias == stateUpdate.FlowAlias { continue + } else if previousState.SyncMode == "refreshingDaily" && stateUpdate.SyncMode != "refreshEnd" && stateUpdate.State == 2 && int(previousState.State) != utilcore.PreviousStateCheck(int(stateUpdate.State)) { + sPC <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: strconv.Itoa(int(stateUpdate.State)), SyncFilter: stateUpdate.SyncFilter, SyncMode: previousState.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} + break } else if int(previousState.State) != utilcore.PreviousStateCheck(int(stateUpdate.State)) && stateUpdate.State != previousState.State { sPC <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: strconv.Itoa(int(previousState.State)), SyncFilter: stateUpdate.SyncFilter, SyncMode: stateUpdate.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} continue @@ -243,6 +249,10 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex } else if tfContext.FlowState.State == 0 { tfContext.FlowLock.Unlock() tfmContext.Log("DataFlowStatistics flow is currently offline...", nil) + if tfContext.FlowState.SyncMode == "refreshingDaily" { + refresh = true + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "1", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: tfContext.FlowState.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} + } continue } else if tfContext.FlowState.State == 1 { tfContext.FlowLock.Unlock() @@ -261,6 +271,22 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex tfmContext.Log("Ignoring invalid flow.", nil) continue } + + tfContext.FlowLock.Lock() + if strings.HasPrefix(tfContext.FlowState.SyncMode, "refresh") { //This is to refresh from vault - different from pulling/pushing. + refreshSuffix, _ := strings.CutPrefix(tfContext.FlowState.SyncMode, "refresh") + if tfContext.FlowState.SyncMode == "refreshingDaily" { + if !refresh { //This is for if trcdb loads up in "refreshingDaily" -> need to kick off refresh again. + KickOffTimedRefresh(tfContext, stateUpdateChannel, "Daily") + } + } else if !KickOffTimedRefresh(tfContext, stateUpdateChannel, refreshSuffix) { + tfmContext.Log("DataFlowStatistics has an invalid refresh timing"+flowcorehelper.SyncCheck(tfContext.FlowState.SyncMode)+".", nil) + tfContext.FlowState.SyncMode = "InvalidRefreshMode" + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "InvalidRefreshMode", FlowAlias: tfContext.FlowState.FlowAlias} + } + } + tfContext.FlowLock.Unlock() + tfContext.FlowLock.Lock() if tfContext.FlowState.SyncMode == "pullonce" { tfContext.FlowState.SyncMode = "pullsynccomplete" @@ -277,3 +303,41 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex tfContext.CancelContext() return nil } + +func KickOffTimedRefresh(tfContext *flowcore.TrcFlowContext, stateUpdateChannel chan flowcorehelper.FlowStateUpdate, timing string) bool { + switch { //Always at midnight + case timing == "Daily": + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "refreshingDaily", FlowAlias: tfContext.FlowState.FlowAlias} + loc, _ := time.LoadLocation("America/Los_Angeles") + now := time.Now().In(loc) + midnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, loc) + timeTilMidnight := midnight.Sub(now) + go func(tfc *flowcore.TrcFlowContext, tilMidnight time.Duration) { + refresh = true + time.Sleep(tilMidnight) + refreshTime := time.Duration(time.Second * 0) + for { + select { + case <-endRefreshChan: + tfContext.Log.Println("Daily Refresh Ended - no longer refreshing DFS") + return + case <-time.After(refreshTime): + tfContext.Log.Println("Daily Refresh Triggered - refreshing DFS") + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfc.Flow.TableName(), StateUpdate: "3", SyncFilter: tfc.FlowState.SyncFilter, SyncMode: "refreshingDaily", FlowAlias: tfc.FlowState.FlowAlias} + refreshTime = time.Duration(time.Hour * 24) + } + + } + }(tfContext, timeTilMidnight) + case timing == "End": + endRefreshChan <- true + refresh = false + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "refreshEnded", FlowAlias: tfContext.FlowState.FlowAlias} + case timing == "Ended": + return true + default: + return false + } + + return true +} diff --git a/trcinit/initlib/vsub.go b/trcinit/initlib/vsub.go index 834de4a14..b4adf458d 100644 --- a/trcinit/initlib/vsub.go +++ b/trcinit/initlib/vsub.go @@ -12,6 +12,77 @@ import ( helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" ) +func DownloadTemplates(config *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string, logger *log.Logger, templatePaths *string) { + var filterTemplatePathSlice []string + if len(*templatePaths) > 0 { + filterTemplatePathSlice = strings.Split(*templatePaths, ",") + } + + for _, filterTemplatePath := range filterTemplatePathSlice { + path := filterTemplatePath + ext := "" + if !strings.HasSuffix(filterTemplatePath, "/") { + path = filterTemplatePath + "/" + } + tfMap, err := mod.ReadData(fmt.Sprintf("templates/%stemplate-file", path)) //Grab extention of file + if err != nil { + eUtils.LogErrorMessage(config, "Skipping template: "+path+" Error: "+err.Error(), false) + continue + } + if _, extOk := tfMap["ext"]; extOk { + ext = tfMap["ext"].(string) + } + + var data string + if _, dataOk := tfMap["data"]; dataOk { + data = tfMap["data"].(string) + } else { + // TODO: In recent run in prod, sub was printing an annoying warning here + // and yet correct templates seem to have gotten created... + fmt.Println("No data found for: " + filterTemplatePath + "template-file") + continue + } + templateBytes, decodeErr := base64.StdEncoding.DecodeString(data) + if decodeErr != nil { + eUtils.LogErrorMessage(config, "Couldn't decode data for: "+path+"template-file", false) + continue + } + //Ensure directory has been created + filePath := strings.Trim(path, "/") + templateAndFilePath := fmt.Sprintf("%s/%s", dirName, filePath) + dirPath := filepath.Dir(templateAndFilePath) + file := filepath.Base(templateAndFilePath) + fmt.Printf("templateDir: %s\n", templateAndFilePath) + fmt.Printf("Dir: %s\n", dirPath) + fmt.Printf("file: %s\n", file) + err = os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + eUtils.LogErrorMessage(config, "Couldn't make directory: "+dirName+filePath, false) + continue + } + //create new file + templateFile := fmt.Sprintf("%s/%s%s.tmpl", dirPath, file, ext) + newFile, err := os.Create(templateFile) + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't create file: %s", templateFile), false) + continue + } + //write to file + _, err = newFile.Write(templateBytes) + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't write file: %s", templateFile), false) + continue + } + err = newFile.Sync() + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't sync file: %s", templateFile), false) + continue + } + newFile.Close() + fmt.Printf("File has been writen to %s\n", templateFile) + } +} + func DownloadTemplateDirectory(config *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string, logger *log.Logger, templateFilter *string) ([]string, error) { var filterTemplateSlice []string diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 21ffae8d0..47a64bfbb 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -8,9 +8,11 @@ import ( "io" "log" "os" + "os/signal" "runtime" "strings" "sync" + "syscall" "time" "github.com/go-git/go-billy/v5" @@ -25,7 +27,6 @@ import ( "github.com/trimble-oss/tierceron/trcsh/trcshauth" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" - "github.com/trimble-oss/tierceron/trcvault/util" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -41,7 +42,7 @@ func main() { memprotectopts.MemProtectInit(nil) } eUtils.InitHeadless(true) - fmt.Println("trcsh Version: " + "1.12") + fmt.Println("trcsh Version: " + "1.20") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string if runtime.GOOS != "windows" { @@ -55,6 +56,13 @@ func main() { if strings.Contains(os.Args[1], "trc") && !strings.Contains(os.Args[1], "-c") { // Running as shell. os.Args[1] = "-c=" + os.Args[1] + // Initiate signal handling. + var ic chan os.Signal = make(chan os.Signal) + signal.Notify(ic, os.Interrupt, syscall.SIGTERM) + go func() { + x := <-ic + interruptChan <- x + }() } } envPtr = flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. @@ -117,14 +125,14 @@ func main() { // Preload agent synchronization configs gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) for { - if agentFlapMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + if deployFlapMode, deployEmitErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env); featherErr == nil && agentFlapMode == cap.MODE_FLAP { + cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); deployEmitErr == nil && strings.HasPrefix(deployFlapMode, cap.MODE_GAZE) { go func() { - continueMsg := true - for continueMsg { + for { + perching: select { case <-time.After(120 * time.Second): ctlMsg := "Deployment timed out after 120 seconds" @@ -132,29 +140,56 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) + cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - continueMsg = false case ctlMsg := <-gAgentConfig.CtlMessage: if ctlMsg != capauth.TrcCtlComplete { - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) + deployLineFlapMode := cap.MODE_FLAP + "_" + ctlMsg + ctlFlapMode := deployLineFlapMode + var err error + for { + if err == nil && ctlFlapMode == cap.MODE_PERCH { + // Acknowledge perching... + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) + ctlFlapMode = cap.MODE_PERCH + goto perching + } + + // Notify deployer of command run and wait for confirmation of msg received. + if err == nil && deployLineFlapMode != ctlFlapMode { + // Flap, Gaze, etc... + break + } else { + callFlap := deployLineFlapMode + if err == nil { + time.Sleep(200 * time.Millisecond) + } else { + if err.Error() != "init" { + time.Sleep(1 * time.Second) + } + } + deployLineFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, deployments+"."+*gAgentConfig.Env) + } + } } else { cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) - continueMsg = false + cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) } } } }() - - ProcessDeploy(*envPtr, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) + ProcessDeploy(*gAgentConfig.Env, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) } else { time.Sleep(500 * time.Millisecond) @@ -165,6 +200,23 @@ func main() { } +var interruptChan chan os.Signal = make(chan os.Signal) +var twoHundredMilliInterruptTicker *time.Ticker = time.NewTicker(200 * time.Millisecond) +var multiSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second) + +func interruptFun(tickerInterrupt *time.Ticker) { + select { + case <-interruptChan: + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env) + os.Exit(1) + case <-tickerInterrupt.C: + } +} + func featherCtlCb(agentName string) error { if gAgentConfig == nil { @@ -172,25 +224,31 @@ func featherCtlCb(agentName string) error { } else { gAgentConfig.Deployments = &agentName } - callFlap := cap.MODE_FLAP + callFlap := cap.MODE_GAZE for { // Azure deployment agent kicks off a deploy with a flap command... if ctlFlapMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env); featherErr == nil && ctlFlapMode == cap.MODE_PERCH { - fmt.Printf("\nDeployment complete.\n") + callFlap, agentName+"."+*gAgentConfig.Env); featherErr != nil || ctlFlapMode == cap.MODE_PERCH || ctlFlapMode == cap.MODE_GLIDE { + if featherErr != nil { + fmt.Printf("\nDeployment error.\n") + } else { + fmt.Printf("\nDeployment complete.\n") + } os.Exit(0) } else { - if strings.HasPrefix(ctlFlapMode, cap.MODE_GLIDE) { + if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { if strings.Contains(ctlFlapMode, "_") { - ctl := strings.Split(ctlFlapMode, "_") - fmt.Println(ctl) + ctlMessage := strings.Split(ctlFlapMode, "_") + if len(ctlMessage) > 1 { + fmt.Printf("%s\n", ctlMessage[1]) + } } - callFlap = cap.MODE_GLIDE } - time.Sleep(time.Second * 3) + callFlap = cap.MODE_GAZE + interruptFun(multiSecondInterruptTicker) } } @@ -341,7 +399,8 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, config.IsShellSubProcess = true trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + ResetModifier(config) + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. if !agentToken { token = "" @@ -349,6 +408,13 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, } case "trcconfig": configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } } } @@ -452,9 +518,33 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr // Chewbacca: Begin dbg comment var auth string + mergedVaultAddress := config.VaultAddress + mergedEnvRaw := config.EnvRaw + + if (approleId != nil && len(*approleId) == 0) || (secretId != nil && len(*secretId) == 0) { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil && gTrcshConfig.CToken != nil { + auth = *gTrcshConfig.CToken + } + } + + if len(mergedVaultAddress) == 0 { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil && gTrcshConfig.VaultAddress != nil { + mergedVaultAddress = *gTrcshConfig.VaultAddress + } + } + + if len(mergedEnvRaw) == 0 { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil { + mergedEnvRaw = gTrcshConfig.EnvContext + } + } + authTokenName := "vault_token_azuredeploy" authTokenEnv := "azuredeploy" - autoErr := eUtils.AutoAuth(config, secretId, approleId, &auth, &authTokenName, &authTokenEnv, &config.VaultAddress, &config.EnvRaw, "deployauth", false) + autoErr := eUtils.AutoAuth(config, secretId, approleId, &auth, &authTokenName, &authTokenEnv, &mergedVaultAddress, &mergedEnvRaw, "deployauth", false) if autoErr != nil || auth == "" { fmt.Println("Unable to auth.") fmt.Println(autoErr) @@ -472,7 +562,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr config.EnvRaw = env config.EndDir = "deploy" config.OutputMemCache = true - trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &token, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. var memFile billy.File @@ -609,9 +699,9 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr logger) } } - if runtime.GOOS == "windows" { - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - } + } + if runtime.GOOS == "windows" { + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete } //Make the arguments in the script -> os.args. diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index aa2d34c72..3c6b6d162 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -36,10 +36,11 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"sub.log", "Output path for log files") projectInfoPtr := flag.Bool("projectInfo", false, "Lists all project info") filterTemplatePtr := flag.String("templateFilter", "", "Specifies which templates to filter") + templatePathsPtr := flag.String("templatePaths", "", "Specifies which specific templates to download.") flag.Parse() - if len(*filterTemplatePtr) == 0 && !*projectInfoPtr { + if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") os.Exit(1) } @@ -80,7 +81,11 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { eUtils.CheckError(config, err, true) mod.Env = *envPtr - if *projectInfoPtr { + if *templatePathsPtr != "" { + fmt.Printf("Downloading templates from vault to %s\n", *dirPtr) + // The actual download templates goes here. + il.DownloadTemplates(config, mod, *dirPtr, logger, templatePathsPtr) + } else if *projectInfoPtr { templateList, err := mod.List("templates/", logger) if err != nil { eUtils.CheckError(config, err, true) diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index daef3fe5f..860d3c023 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -197,7 +197,7 @@ func CommonMain(envPtr *string, } // Get existing configs if they exist... - pluginToolConfig, plcErr := trcvutils.GetPluginToolConfig(configBase, mod, coreopts.ProcessDeployPluginEnvConfig(map[string]interface{}{})) + pluginToolConfig, plcErr := trcvutils.GetPluginToolConfig(configBase, mod, coreopts.ProcessDeployPluginEnvConfig(map[string]interface{}{}), *defineServicePtr) if plcErr != nil { fmt.Println(plcErr.Error()) os.Exit(1) @@ -307,13 +307,17 @@ func CommonMain(envPtr *string, } fmt.Printf("Service started: %s\n", pluginToolConfig["trcservicename"].(string)) } else if *codebundledeployPtr { - err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + if pluginToolConfig["trcsha256"] != nil && len(pluginToolConfig["trcsha256"].(string)) > 0 { + err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } } - if pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { + if pluginToolConfig["trcsha256"] != nil && + pluginToolConfig["imagesha256"] != nil && + pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { // Write the image to the destination... deployPath := fmt.Sprintf("%s\\%s", pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) fmt.Printf("Deploying image to: %s\n", deployPath) diff --git a/trcvault/util/util.go b/trcvault/util/util.go index a3f40a064..1e4238f2b 100644 --- a/trcvault/util/util.go +++ b/trcvault/util/util.go @@ -249,7 +249,7 @@ func SeedVaultById(config *eUtils.DriverConfig, goMod *helperkv.Modifier, servic return nil } -func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pluginConfig map[string]interface{}) (map[string]interface{}, error) { +func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pluginConfig map[string]interface{}, defineService bool) (map[string]interface{}, error) { config.Log.Println("GetPluginToolConfig begin processing plugins.") //templatePaths indexFound := false @@ -334,6 +334,9 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl } return pluginEnvConfigClone, err } else if !indexFound { + if defineService { + pluginEnvConfigClone["pluginpath"] = pluginToolConfig["pluginpath"] + } return pluginEnvConfigClone, nil } else { if _, ok := pluginEnvConfigClone["trcplugin"]; ok { From 472f3c16bda5a7195c35ab33fb13974ceef440de Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:09:12 -0800 Subject: [PATCH 05/55] Merging Develop into Comprehensive (#777) --- Makefile.tc | 8 + azure-pipelines-legacy.yml | 2 +- azure-pipelines-linux.yml | 2 +- azure-pipelines-plugin-develop.yml | 2 +- azure-pipelines-plugin.yml | 12 +- azure-pipelines.yml | 2 +- buildopts/coreopts/options_common.go | 4 + buildopts/coreopts/options_tc.go | 7 +- capauth/agentconfig.go | 234 +++++++ capauth/creds.go | 103 ++++ capauth/trcshconfig.go | 11 + go.mod | 28 +- go.sum | 76 +-- trcchatproxy/.gitignore | 48 ++ trcchatproxy/cmd/googlechatcmd.go | 29 + trcchatproxy/function.json | 18 + trcchatproxy/go.mod | 30 + trcchatproxy/go.sum | 143 +++++ trcchatproxy/googlechat.go | 43 -- trcchatproxy/googlechatapi.go | 72 +++ trcchatproxy/googlechatapi/function.json | 19 + trcchatproxy/googlechatapi/main.go | 41 -- trcchatproxy/host.json | 23 + trcchatproxy/pubsub/pubsub.go | 38 ++ trcchatproxy/{ => trcchat}/chatworld.go | 2 +- trcchatproxy/{main.go => trcchat/common.go} | 55 +- trcchatproxy/{ => trcchat}/dialogflow.go | 8 +- trcchatproxy/trcchat/googlechat.go | 74 +++ trcchatproxy/{ => trcchat}/tls/mashup.crt | 0 trcchatproxy/{ => trcchat}/tls/mashup.key | 0 trcflow/core/askflume/process_askflume.go | 19 +- trcflow/deploy/process.go | 363 +++++------ trcflow/flows/process_dataflowstatistics.go | 64 ++ trcflow/flumen/process.go | 6 +- trcinit/initlib/vsub.go | 71 +++ trcsh/kube/native/trckube.go | 6 +- trcsh/trcsh.go | 580 ++++++++++++++---- trcsh/trcshauth/auth.go | 148 ++--- trcsh/trcshauth/tls/mashup.crt | 24 - trcsh/trcshauth/tls/mashup.key | 28 - trcsubbase/trcsub.go | 9 +- trcvault/carrierfactory/backend.go | 83 ++- trcvault/carrierfactory/capauth/capauth.go | 115 ---- .../carrierfactory/servercapauth/capauth.go | 166 +++++ trcvault/deploy/deploy.sh | 63 +- trcvault/deploy/deployupdate.sh | 4 +- trcvault/deploy/refreshcarriertoken.sh | 23 +- trcvault/deploy/refreshtoken.sh | 35 +- trcvault/factory/backend.go | 246 ++++---- trcvault/plugins/carrier/main.go | 9 +- .../TrcVault/Certify/config.yml.tmpl | 4 +- .../TrcVault/TrcshAgent/config.yml.tmpl | 7 + trcvault/trcplgtool/trcplgtool.go | 11 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 413 +++++++++++-- trcvault/util/authutil.go | 34 + trcvault/util/properties.go | 101 +++ trcvault/util/repository/azrcr.go | 96 ++- trcvault/util/util.go | 86 ++- utils/driverconfig.go | 1 + utils/vaultUtil.go | 30 +- vaulthelper/kv/Modifier.go | 4 +- 61 files changed, 2886 insertions(+), 1097 deletions(-) create mode 100644 capauth/agentconfig.go create mode 100644 capauth/creds.go create mode 100644 capauth/trcshconfig.go create mode 100644 trcchatproxy/.gitignore create mode 100644 trcchatproxy/cmd/googlechatcmd.go create mode 100644 trcchatproxy/function.json create mode 100644 trcchatproxy/go.mod create mode 100644 trcchatproxy/go.sum delete mode 100644 trcchatproxy/googlechat.go create mode 100644 trcchatproxy/googlechatapi.go create mode 100644 trcchatproxy/googlechatapi/function.json delete mode 100644 trcchatproxy/googlechatapi/main.go create mode 100644 trcchatproxy/host.json create mode 100644 trcchatproxy/pubsub/pubsub.go rename trcchatproxy/{ => trcchat}/chatworld.go (99%) rename trcchatproxy/{main.go => trcchat/common.go} (70%) rename trcchatproxy/{ => trcchat}/dialogflow.go (86%) create mode 100644 trcchatproxy/trcchat/googlechat.go rename trcchatproxy/{ => trcchat}/tls/mashup.crt (100%) rename trcchatproxy/{ => trcchat}/tls/mashup.key (100%) delete mode 100644 trcsh/trcshauth/tls/mashup.crt delete mode 100644 trcsh/trcshauth/tls/mashup.key delete mode 100644 trcvault/carrierfactory/capauth/capauth.go create mode 100644 trcvault/carrierfactory/servercapauth/capauth.go create mode 100644 trcvault/trc_templates/TrcVault/TrcshAgent/config.yml.tmpl create mode 100644 trcvault/util/authutil.go diff --git a/Makefile.tc b/Makefile.tc index f74a67ac0..8d0649d04 100755 --- a/Makefile.tc +++ b/Makefile.tc @@ -70,9 +70,14 @@ pub: sub: @GOPATH=$(GOPATH) CGO_ENABLED=0 GOBIN=$(GOBIN) go install -tags "memonly" github.com/trimble-oss/tierceron/trcsub sudo setcap cap_ipc_lock=+ep $(GOBIN)/trcsub +# Warning! windows not as secure.... +subwin: + @GOPATH=$(GOPATH) GOBIN=$(GOBIN) GOOS=windows GOARCH=amd64 go build -tags "tc windows memonly" -o $(GOBIN)/trcsub.exe trcsub/trcsub.go certify: @GOPATH=$(GOPATH) CGO_ENABLED=0 GOBIN=$(GOBIN) go build -o $(GOBIN)/trcplgtool -tags "tc azrcr memonly" github.com/trimble-oss/tierceron/trcvault/trcplgtool sudo setcap cap_ipc_lock=+ep $(GOBIN)/trcplgtool +certifywin: + @GOPATH=$(GOPATH) GOBIN=$(GOBIN) GOOS=windows GOARCH=amd64 go build -tags "tc windows azrcr memonly" -o $(GOBIN)/trcplgtool.exe github.com/trimble-oss/tierceron/trcvault/trcplgtool gen: protoc --proto_path=. --twirp_out=. --go_out=. rpc/apinator/service.proto @@ -82,6 +87,9 @@ trcshellsha: sha256sum trcvault/deploy/target/trcsh | cut -d' ' -f1 > trcvault/deploy/target/trcsh.sha256 trcshell: trcshellbuild trcshellsha +trcshellwin: + @GOPATH=$(GOPATH) GOBIN=$(GOBIN) GOOS=windows GOARCH=amd64 go build -tags "tc windows azrcr memonly" -o $(GOBIN)/trcsh.exe github.com/trimble-oss/tierceron/trcsh + trckubectlbuild: @GOPATH=$(GOPATH) GOBIN=$(GOBIN) go build -o trcvault/deploy/target/kubectl -tags "memonly" github.com/trimble-oss/tierceron/trcsh trckubectlsha: diff --git a/azure-pipelines-legacy.yml b/azure-pipelines-legacy.yml index da4672490..06757e0da 100644 --- a/azure-pipelines-legacy.yml +++ b/azure-pipelines-legacy.yml @@ -75,7 +75,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.1' + version: '1.21.3' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-linux.yml b/azure-pipelines-linux.yml index b6f552982..c39a51a56 100644 --- a/azure-pipelines-linux.yml +++ b/azure-pipelines-linux.yml @@ -71,7 +71,7 @@ steps: - task: GoTool@0 inputs: - version: '1.21.1' + version: '1.21.3' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin-develop.yml b/azure-pipelines-plugin-develop.yml index 7760654e4..e4b420f62 100644 --- a/azure-pipelines-plugin-develop.yml +++ b/azure-pipelines-plugin-develop.yml @@ -73,7 +73,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.1' + version: '1.21.3' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index dd2decdb4..c6cf7f7ea 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -78,7 +78,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.1' + version: '1.21.3' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' @@ -121,12 +121,19 @@ steps: echo "Compile start" echo "Compile Linux" - GOBIN= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOCACHE=$GOCACHE go install -ldflags '-w' -tags "azure memonly tc" github.com/trimble-oss/tierceron/trcsh + GOBIN= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOCACHE=$GOCACHE go install -ldflags '-w' -tags "tc memonly azrcr azure" github.com/trimble-oss/tierceron/trcsh if [[ ! -f $GOBIN/trcsh ]] ; then echo File trcsh was not built, aborting. exit 1 fi + echo "Compile Windows" + GOBIN= CGO_ENABLED=0 GOOS=windows GOARCH=amd64 GOCACHE=$GOCACHE go install -tags "tc memonly azrcr windows" -ldflags '-w' -tags "tc" github.com/trimble-oss/tierceron/trcsh + if [[ ! -f $GOBIN/windows_amd64/trcsh.exe ]] ; then + echo File trcsh.exe was not built, aborting. + exit 1 + fi + echo "Compile Agent plugin trc-vault-carrier-plugin" GOBIN= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOCACHE=$GOCACHE go install -ldflags '-w' -tags "tc memonly azrcr insecure" github.com/trimble-oss/tierceron/trcvault/plugins/carrier mv $GOBIN/carrier $GOBIN/trc-vault-carrier-plugin @@ -152,6 +159,7 @@ steps: echo "Compile Complete" cp $GOBIN/trcsh $(Build.ArtifactStagingDirectory) cp $GOBIN/trcsh.sha256 $(Build.ArtifactStagingDirectory) + cp $GOBIN/windows_amd64/trc* $(Build.ArtifactStagingDirectory) cp $GOBIN/trc-vault-carrier-plugin $(Build.ArtifactStagingDirectory) cp $GOBIN/trc-vault-carrier-plugin.sha256 $(Build.ArtifactStagingDirectory) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ce5c974d0..238a79e59 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.1' + version: '1.21.3' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/buildopts/coreopts/options_common.go b/buildopts/coreopts/options_common.go index a0e6674a8..71b7a52ea 100644 --- a/buildopts/coreopts/options_common.go +++ b/buildopts/coreopts/options_common.go @@ -39,6 +39,10 @@ func GetLocalHost() string { return "" } +func GetRegion(hostName string) string { + return "" +} + func GetVaultHost() string { return "" } diff --git a/buildopts/coreopts/options_tc.go b/buildopts/coreopts/options_tc.go index 70b566f18..76406af2d 100644 --- a/buildopts/coreopts/options_tc.go +++ b/buildopts/coreopts/options_tc.go @@ -4,11 +4,12 @@ package coreopts import ( + "strings" + bcore "VaultConfig.Bootstrap/configcore" tcbuildopts "VaultConfig.TenantConfig/util/buildopts" trcprefix "VaultConfig.TenantConfig/util/buildopts/trcprefix" tccore "VaultConfig.TenantConfig/util/core" - "strings" "database/sql" ) @@ -39,6 +40,10 @@ func GetSupportedEndpoints() []string { return bcore.GetSupportedEndpoints() } +func GetRegion(hostName string) string { + return tccore.GetRegion(hostName) +} + func GetLocalHost() string { return bcore.LocalHost } diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go new file mode 100644 index 000000000..c12f57614 --- /dev/null +++ b/capauth/agentconfig.go @@ -0,0 +1,234 @@ +package capauth + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log" + "math/rand" + "net" + "os" + "strings" + "time" + + "github.com/trimble-oss/tierceron-hat/cap" + "github.com/trimble-oss/tierceron-hat/cap/tap" + "github.com/trimble-oss/tierceron/buildopts/coreopts" + "github.com/trimble-oss/tierceron/buildopts/memprotectopts" + eUtils "github.com/trimble-oss/tierceron/utils" + helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" + "google.golang.org/grpc" +) + +var TrcCtlComplete string = "trcctlcomplete" + +type AgentConfigs struct { + HandshakeHostPort *string + FeatherHostPort *string + HandshakeCode *string + DeployRoleID *string + EncryptPass *string + EncryptSalt *string + Deployments *string + Env *string + CtlMessage chan string +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + +var gTrcHatSecretsPort string = "" + +func randomString(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func ValidateVhost(host string) error { + protocolHost := "https://" + host + for _, endpoint := range coreopts.GetSupportedEndpoints() { + if strings.HasPrefix(endpoint, protocolHost) { + return nil + } + } + return errors.New("Bad host: " + host) +} + +func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error) { + penseCode := randomString(7 + rand.Intn(7)) + penseArray := sha256.Sum256([]byte(penseCode)) + penseSum := hex.EncodeToString(penseArray[:]) + + _, featherErr := cap.FeatherWriter(*agentconfig.EncryptPass, + *agentconfig.EncryptSalt, + *agentconfig.HandshakeHostPort, + *agentconfig.HandshakeCode, + penseSum) + if featherErr != nil { + return nil, featherErr + } + + creds, credErr := GetTransportCredentials() + + if credErr != nil { + return nil, credErr + } + + conn, err := grpc.Dial(*agentconfig.FeatherHostPort, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + c := cap.NewCapClient(conn) + + // Contact the server and print out its response. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + r, err := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + var penseProtect *string + rPense := r.GetPense() + penseProtect = &rPense + memprotectopts.MemProtect(nil, penseProtect) + + return penseProtect, nil +} + +func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, deployments string, env string) (*TrcShConfig, error) { + var trcshConfig *TrcShConfig + + mod, modErr := helperkv.NewModifier(false, agentToken, address, env, nil, true, nil) + if modErr != nil { + fmt.Println("trcsh Failed to bootstrap") + os.Exit(-1) + } + mod.Direct = true + mod.Env = env + + data, readErr := mod.ReadData("super-secrets/Restricted/TrcshAgent/config") + if readErr != nil { + return nil, readErr + } else { + trcHatEncryptPass := data["trcHatEncryptPass"].(string) + memprotectopts.MemProtect(nil, &trcHatEncryptPass) + trcHatEncryptSalt := data["trcHatEncryptSalt"].(string) + memprotectopts.MemProtect(nil, &trcHatEncryptSalt) + trcHatEnv := data["trcHatEnv"].(string) + memprotectopts.MemProtect(nil, &trcHatEnv) + trcHatHandshakeCode := data["trcHatHandshakeCode"].(string) + memprotectopts.MemProtect(nil, &trcHatHandshakeCode) + trcHatHandshakePort := data["trcHatHandshakePort"].(string) + memprotectopts.MemProtect(nil, &trcHatHandshakePort) + trcHatHost := data["trcHatHost"].(string) + memprotectopts.MemProtect(nil, &trcHatHost) + trcHatSecretsPort := data["trcHatSecretsPort"].(string) + memprotectopts.MemProtect(nil, &trcHatSecretsPort) + trcHandshakeHostPort := trcHatHost + ":" + trcHatHandshakePort + memprotectopts.MemProtect(nil, &trcHandshakeHostPort) + trcFeatherHostPort := trcHatHost + ":" + trcHatSecretsPort + memprotectopts.MemProtect(nil, &trcFeatherHostPort) + + agentconfig.HandshakeHostPort = &trcHandshakeHostPort + agentconfig.FeatherHostPort = &trcFeatherHostPort + agentconfig.HandshakeCode = &trcHatHandshakeCode + agentconfig.EncryptPass = &trcHatEncryptPass + agentconfig.EncryptSalt = &trcHatEncryptSalt + agentconfig.Deployments = &deployments + agentconfig.Env = &trcHatEnv + + trcshConfig = &TrcShConfig{Env: trcHatEnv, + EnvContext: trcHatEnv, + } + trcShConfigRole, penseError := agentconfig.PenseFeatherQuery("configrole") + if penseError != nil { + return nil, penseError + } + memprotectopts.MemProtect(nil, trcShConfigRole) + trcshConfig.ConfigRole = trcShConfigRole + + trcShCToken, penseError := agentconfig.PenseFeatherQuery("ctoken") + if penseError != nil { + return nil, penseError + } + memprotectopts.MemProtect(nil, trcShCToken) + trcshConfig.CToken = trcShCToken + } + + return trcshConfig, nil +} + +func PenseQuery(config *eUtils.DriverConfig, pense string) (*string, error) { + penseCode := randomString(7 + rand.Intn(7)) + penseArray := sha256.Sum256([]byte(penseCode)) + penseSum := hex.EncodeToString(penseArray[:]) + + penseEye, capWriteErr := tap.TapWriter(penseSum) + + if trcHtSp, trcHSPOk := penseEye["trcHatSecretsPort"]; trcHSPOk { + if gTrcHatSecretsPort != trcHtSp { + gTrcHatSecretsPort = trcHtSp + } + } + + if capWriteErr != nil || gTrcHatSecretsPort == "" { + fmt.Println("Code 54 failure...") + // 2023-06-30T01:29:21.7020686Z read unix @->/tmp/trccarrier/trcsnap.sock: read: connection reset by peer + // os.Exit(-1) // restarting carrier will rebuild necessary resources... + return new(string), errors.Join(errors.New("Tap writer error"), capWriteErr) + } + + localIP, err := LocalIp(config.EnvRaw) + if err != nil { + return nil, err + } + addrs, hostErr := net.LookupAddr(localIP) + if hostErr != nil { + return nil, hostErr + } + localHost := "" + if len(addrs) > 0 { + localHost = strings.TrimRight(addrs[0], ".") + if validErr := ValidateVhost(localHost); validErr != nil { + return nil, validErr + } + } else { + return nil, errors.New("Invalid host") + } + + // TODO: add domain if it's missing because that might actually happen... Pull the domain from + // vaddress in config.. that should always be the same... + + creds, err := GetTransportCredentials() + if err != nil { + return nil, err + } + + conn, err := grpc.Dial(fmt.Sprintf("%s:%s", localHost, gTrcHatSecretsPort), grpc.WithTransportCredentials(creds)) + if err != nil { + return new(string), err + } + defer conn.Close() + c := cap.NewCapClient(conn) + + // Contact the server and print out its response. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + r, penseErr := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) + if penseErr != nil { + return new(string), errors.Join(errors.New("pense error"), penseErr) + } + var penseProtect *string + rPense := r.GetPense() + penseProtect = &rPense + memprotectopts.MemProtect(nil, penseProtect) + + return penseProtect, nil +} diff --git a/capauth/creds.go b/capauth/creds.go new file mode 100644 index 000000000..3234865f3 --- /dev/null +++ b/capauth/creds.go @@ -0,0 +1,103 @@ +package capauth + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "log" + "math/rand" + "net" + "os" + "strings" + "time" + + "google.golang.org/grpc/credentials" +) + +const ( + ServCert = "/etc/opt/vault/certs/serv_cert.pem" + ServCertLocal = "./serv_cert.pem" + ServKey = "/etc/opt/vault/certs/serv_key.pem" +) + +var MashupCertPool *x509.CertPool + +func init() { + rand.Seed(time.Now().UnixNano()) + mashupCertBytes, err := os.ReadFile(ServCert) + if err != nil { + mashupCertBytes, err = os.ReadFile(ServCertLocal) + if err != nil { + fmt.Println("Cert read failure.") + return + } + } + + mashupBlock, _ := pem.Decode([]byte(mashupCertBytes)) + + mashupClientCert, parseErr := x509.ParseCertificate(mashupBlock.Bytes) + if parseErr != nil { + fmt.Println("Cert parse read failure.") + return + } + MashupCertPool = x509.NewCertPool() + MashupCertPool.AddCert(mashupClientCert) +} + +func LocalIp(env string) (string, error) { + if strings.Contains(env, "staging") || strings.Contains(env, "prod") { + return "127.0.0.1", nil + } + + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", err + } + for _, address := range addrs { + if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String(), nil + } + } + } + return "", err +} + +func GetTransportCredentials() (credentials.TransportCredentials, error) { + + mashupKeyBytes, err := os.ReadFile(ServCert) + if err != nil { + return nil, err + } + + return credentials.NewTLS(&tls.Config{ + ServerName: "", + Certificates: []tls.Certificate{ + { + Certificate: [][]byte{mashupKeyBytes}, + }, + }, + InsecureSkipVerify: false}), nil +} + +func GetServerCredentials(logger *log.Logger) (credentials.TransportCredentials, error) { + mashupCertBytes, err := os.ReadFile(ServCert) + if err != nil { + logger.Printf("Couldn't load cert: %v\n", err) + return nil, err + } + + mashupKeyBytes, err := os.ReadFile(ServKey) + if err != nil { + logger.Printf("Couldn't load key: %v\n", err) + return nil, err + } + + cert, err := tls.X509KeyPair(mashupCertBytes, mashupKeyBytes) + if err != nil { + logger.Printf("Couldn't load cert: %v\n", err) + return nil, err + } + return credentials.NewServerTLSFromCert(&cert), nil +} diff --git a/capauth/trcshconfig.go b/capauth/trcshconfig.go new file mode 100644 index 000000000..bec828ccc --- /dev/null +++ b/capauth/trcshconfig.go @@ -0,0 +1,11 @@ +package capauth + +type TrcShConfig struct { + Env string + EnvContext string // Current env context... + VaultAddress *string + CToken *string + ConfigRole *string + PubRole *string + KubeConfig *string +} diff --git a/go.mod b/go.mod index df8584942..af90651f5 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/trimble-oss/tierceron go 1.21 -toolchain go1.21.1 +toolchain go1.21.3 require ( VaultConfig.Bootstrap v0.0.0-00010101000000-000000000000 @@ -20,8 +20,8 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/twitchtv/twirp v5.12.1+incompatible github.com/xo/dburl v0.9.0 - golang.org/x/crypto v0.13.0 - golang.org/x/sys v0.12.0 + golang.org/x/crypto v0.14.0 + golang.org/x/sys v0.13.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -46,7 +46,6 @@ require ( github.com/go-git/go-billy/v5 v5.4.1 github.com/graphql-go/graphql v0.8.1-0.20220614210743-09272f350067 github.com/trimble-oss/tierceron-hat v0.0.0-00010101000000-000000000000 - google.golang.org/api v0.130.0 k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 k8s.io/cli-runtime v0.26.1 @@ -57,14 +56,10 @@ require ( ) require ( - cloud.google.com/go/compute v1.19.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/awnumar/memcall v0.1.2 // indirect - github.com/awnumar/memguard v0.22.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -84,14 +79,10 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -130,11 +121,10 @@ require ( github.com/tjfoc/gmsm v1.4.1 // indirect github.com/xlab/treeprint v1.1.0 // indirect github.com/xtaci/kcp-go/v5 v5.6.3 // indirect - go.opencensus.io v0.24.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/oauth2 v0.9.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/component-helpers v0.0.0-20230215120307-78f7b9c080f0 // indirect k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a // indirect @@ -203,7 +193,7 @@ require ( github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-hclog v1.0.0 @@ -236,13 +226,13 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.12.0 + golang.org/x/term v0.13.0 golang.org/x/text v0.13.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/grpc v1.56.1 + google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6f1b18f60..17f9b58b5 100644 --- a/go.sum +++ b/go.sum @@ -24,10 +24,6 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -101,10 +97,6 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/awnumar/memcall v0.1.2 h1:7gOfDTL+BJ6nnbtAp9+HQzUFjtP1hEseRQq8eP055QY= -github.com/awnumar/memcall v0.1.2/go.mod h1:S911igBPR9CThzd/hYQQmTc9SWNu3ZHIlCGaWsWsoJo= -github.com/awnumar/memguard v0.22.3 h1:b4sgUXtbUjhrGELPbuC62wU+BsPQy+8lkWed9Z+pj0Y= -github.com/awnumar/memguard v0.22.3/go.mod h1:mmGunnffnLHlxE5rRgQc3j+uwPZ27eYb61ccr8Clz2Y= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -145,10 +137,6 @@ github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21/go.mod github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -196,7 +184,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -303,8 +290,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -381,20 +366,14 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20211219123610-ec9572f70e60/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -880,13 +859,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -915,9 +891,8 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1018,12 +993,11 @@ golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1038,8 +1012,8 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= -golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1133,12 +1107,12 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1149,7 +1123,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1249,8 +1222,6 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ= -google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1305,12 +1276,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1339,9 +1306,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1407,26 +1373,16 @@ k8s.io/api v0.0.0-20230228090259-b5b22ca1babf h1:nOM4wjFnU0nlgQ5xSvS24ayMbBOFw1N k8s.io/api v0.0.0-20230228090259-b5b22ca1babf/go.mod h1:XAyCRdxxjlGJLzvSsIhA8Ccnsryd6Xh0CgmB3ah3AmQ= k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/cli-runtime v0.26.1 h1:f9+bRQ1V3elQsx37KmZy5fRAh56mVLbE9A7EMdlqVdI= -k8s.io/cli-runtime v0.26.1/go.mod h1:+e5Ym/ARySKscUhZ8K3hZ+ZBo/wYPIcg+7b5sFYi6Gg= -k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= -k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= k8s.io/component-helpers v0.0.0-20230215120307-78f7b9c080f0 h1:2lV0FlxzZ3gJgDcpiJz54cZgDxHjRn8u9xwoSxkYtD4= k8s.io/component-helpers v0.0.0-20230215120307-78f7b9c080f0/go.mod h1:iADql9qALcuDPa5sGm2MRHnCQA5DGvV+LIPanigEbyo= -k8s.io/component-helpers v0.26.1 h1:Y5h1OYUJTGyHZlSAsc7mcfNsWF08S/MlrQyF/vn93mU= -k8s.io/component-helpers v0.26.1/go.mod h1:jxNTnHb1axLe93MyVuvKj9T/+f4nxBVrj/xf01/UNFk= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a h1:s6zvHjyDQX1NtVT88pvw2tddqhqY0Bz0Gbnn+yctsFU= k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a/go.mod h1:/BYxry62FuDzmI+i9B+X2pqfySRmSOW2ARmj5Zbqhj0= -k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s= -k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo= k8s.io/metrics v0.0.0-20230215135002-90fa97165491 h1:f8AcsXhdn+GEhEYVFBhYM8hPg1nMYypr0F3R5A/LWl4= k8s.io/metrics v0.0.0-20230215135002-90fa97165491/go.mod h1:SVV0nIAfeqYDVhf/P3WGOBKYGZba2VvHEf0UPg+pAt4= -k8s.io/metrics v0.26.1 h1:iB+QdMLa2V70a7zb0XYEcaUpPM0y+p4fZN0UtxcPHLk= -k8s.io/metrics v0.26.1/go.mod h1:fMeLXmK/xgvckFG63GJ0kDjFiQH7P0Dpi5Lvhlo5DXE= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/trcchatproxy/.gitignore b/trcchatproxy/.gitignore new file mode 100644 index 000000000..f15ac3fc6 --- /dev/null +++ b/trcchatproxy/.gitignore @@ -0,0 +1,48 @@ +bin +obj +csx +.vs +edge +Publish + +*.user +*.suo +*.cscfg +*.Cache +project.lock.json + +/packages +/TestResults + +/tools/NuGet.exe +/App_Data +/secrets +/data +.secrets +appsettings.json +local.settings.json + +node_modules +dist + +# Local python packages +.python_packages/ + +# Python Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json \ No newline at end of file diff --git a/trcchatproxy/cmd/googlechatcmd.go b/trcchatproxy/cmd/googlechatcmd.go new file mode 100644 index 000000000..aac9075cf --- /dev/null +++ b/trcchatproxy/cmd/googlechatcmd.go @@ -0,0 +1,29 @@ +package main + +import ( + "flag" + "log" + "os" + + eUtils "github.com/trimble-oss/tierceron/utils" + + "github.com/trimble-oss/tierceron/trcchatproxy/pubsub" + "github.com/trimble-oss/tierceron/trcchatproxy/trcchat" +) + +func main() { + shutdown := make(chan bool) + logFilePtr := flag.String("log", "./googlechatcmd.log", "Output path for log file") + tokenPtr := flag.String("token", "", "Output path for log file") + callerTokenPtr := flag.String("callerToken", "", "Output path for log file") + flag.Parse() + + f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + log.SetOutput(f) + configDriver := &eUtils.DriverConfig{Log: log.Default(), ExitOnFailure: true} + eUtils.CheckError(configDriver, err, true) + pubsub.CommonInit(true) + trcchat.CommonInit(*tokenPtr, *callerTokenPtr) + + <-shutdown +} diff --git a/trcchatproxy/function.json b/trcchatproxy/function.json new file mode 100644 index 000000000..0e94ab105 --- /dev/null +++ b/trcchatproxy/function.json @@ -0,0 +1,18 @@ +{ + "bindings": [ + { + "authLevel": "Anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "res" + } + ] +} \ No newline at end of file diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod new file mode 100644 index 000000000..c2001cafb --- /dev/null +++ b/trcchatproxy/go.mod @@ -0,0 +1,30 @@ +module github.com/trimble-oss/tierceron/trcchatproxy + +go 1.21.3 + +require google.golang.org/api v0.145.0 + +require ( + cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +require github.com/trimble-oss/tierceron-nute v0.0.0-20230128181737-65043c9e434b + +replace github.com/trimble-oss/tierceron-nute => ../../tierceron-nute diff --git a/trcchatproxy/go.sum b/trcchatproxy/go.sum new file mode 100644 index 000000000..9c1066b54 --- /dev/null +++ b/trcchatproxy/go.sum @@ -0,0 +1,143 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.145.0 h1:kBjvf1A3/m30kUvnUX9jZJxTu3lJrpGFt5V/1YZrjwg= +google.golang.org/api v0.145.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/trcchatproxy/googlechat.go b/trcchatproxy/googlechat.go deleted file mode 100644 index 4773ef660..000000000 --- a/trcchatproxy/googlechat.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - - "github.com/trimble-oss/tierceron-nute/mashupsdk" -) - -// Prints out answer to google chat user and sets up google chat app -// to ask another response -// Once the google chat api is set up, should send response to -// correct endpoint for google chat -func ProcessGChatAnswer(msg *mashupsdk.MashupDetailedElement) { - log.Printf("Message is ready to send to Google Chat user") - var info [][]interface{} - - err := json.Unmarshal([]byte(msg.Data), &info) - if err != nil { - log.Println("Error in decoding data in recursiveBuildArgosies") - } - - tenant := info[0][0] - enterpriseId := info[0][1] - error_msg := info[0][2] - err_time := info[0][3] - // Stack trace was empty, so skipping index of 4 - snap_mode := info[0][5] - msg.Data = fmt.Sprintf("The tenant %v with enterprise ID %v is failing with error message %v at %v with snapshot mode %v", tenant, enterpriseId, error_msg, err_time, snap_mode) - fmt.Println(msg.Data) - element := mashupsdk.MashupDetailedElement{ - Id: msg.Id, - Name: "GChatAnswer", - Data: msg.Data, - } - offset := GetId() - if gchatApp.MashupDetailedElementLibrary == nil { - gchatApp.MashupDetailedElementLibrary = make(map[int64]*mashupsdk.MashupDetailedElement) - } - gchatApp.MashupDetailedElementLibrary[msg.Id+offset] = gchatApp.DetailedElements[0] - gchatApp.DetailedElements = []*mashupsdk.MashupDetailedElement{&element} -} diff --git a/trcchatproxy/googlechatapi.go b/trcchatproxy/googlechatapi.go new file mode 100644 index 000000000..7a628d615 --- /dev/null +++ b/trcchatproxy/googlechatapi.go @@ -0,0 +1,72 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + + "github.com/trimble-oss/tierceron/trcchatproxy/pubsub" + "github.com/trimble-oss/tierceron/trcchatproxy/trcchat" + "google.golang.org/api/chat/v1" +) + +func get_port() string { + port := ":8080" + if val, ok := os.LookupEnv("FUNCTIONS_CUSTOMHANDLER_PORT"); ok { + port = ":" + val + } + return port +} + +// Will be the google chat api implementation --> Send messages from googlechat.go to the specified server and port +// This is just an example of what the method should look like based on the above link +func main() { + tokenPtr := flag.String("token", "", "Output path for log file") + callerTokenPtr := flag.String("callerToken", "", "Output path for log file") + flag.Parse() + + pubsub.CommonInit(false) + trcchat.CommonInit(*tokenPtr, *callerTokenPtr) + f := func(w http.ResponseWriter, r *http.Request) { + + switch r.Method { + case http.MethodGet: + w.Header().Add("Content-Type", "application/json") + fmt.Fprintf(w, `{"text": "Ask Flume has been enabled."}`) + return + } + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + var event chat.DeprecatedEvent + if err := json.NewDecoder(r.Body).Decode(&event); err == nil { + w.Header().Add("Content-Type", "application/json") + switch event.Type { + case "ADDED_TO_SPACE": + if event.Space.Type != "ROOM" { + break + } + fmt.Fprint(w, `{"text":"Google Chat App has been successfully added to this space!"}`) + case "REMOVED_FROM_SPACE": + if event.Space.Type != "ROOM" { + break + } + fmt.Fprint(w, `{"text":"Google Chat App has been successfully removed from this space!"}`) + case "MESSAGE": + go pubsub.PubChatEvent(&event) + responseEvent := pubsub.SubChatAnswerEvent() + fmt.Fprintf(w, `{"text": "%s"}`, responseEvent.Message.Text) + default: + fmt.Fprintf(w, `{"text": "you said %s"}`, event.Message.Text) + } + return + } + + } + log.Fatal(http.ListenAndServe(get_port(), http.HandlerFunc(f))) +} diff --git a/trcchatproxy/googlechatapi/function.json b/trcchatproxy/googlechatapi/function.json new file mode 100644 index 000000000..91052aaf8 --- /dev/null +++ b/trcchatproxy/googlechatapi/function.json @@ -0,0 +1,19 @@ +{ + "bindings": [ + { + "authLevel": "Anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "res" + } + ] +} \ No newline at end of file diff --git a/trcchatproxy/googlechatapi/main.go b/trcchatproxy/googlechatapi/main.go deleted file mode 100644 index a6c6b4139..000000000 --- a/trcchatproxy/googlechatapi/main.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - - "google.golang.org/api/chat/v1" -) - -// https://medium.com/google-cloud/google-chat-bot-go-cc91c5311d7e - -// Will be the google chat api implementation --> Send messages from googlechat.go to the specified server and port -// This is just an example of what the method should look like based on the above link -func main() { - f := func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - - var event chat.DeprecatedEvent - if err := json.NewDecoder(r.Body).Decode(&event); err != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(err.Error())) - return - } - - switch event.Type { - case "ADDED_TO_SPACE": - if event.Space.Type != "ROOM" { - break - } - fmt.Fprint(w, `{"text":"Google Chat App has been successfully added to this space!"}`) - case "MESSAGE": - fmt.Fprintf(w, `{"text": "you said %s"}`, event.Message.Text) - } - } - log.Fatal(http.ListenAndServe(":8080", http.HandlerFunc(f))) -} diff --git a/trcchatproxy/host.json b/trcchatproxy/host.json new file mode 100644 index 000000000..94f7b179d --- /dev/null +++ b/trcchatproxy/host.json @@ -0,0 +1,23 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + }, + "customHandler": { + "enableForwardingHttpRequest": true, + "description": { + "defaultExecutablePath": "askflume", + "workingDirectory": "", + "arguments": [] + } + } +} diff --git a/trcchatproxy/pubsub/pubsub.go b/trcchatproxy/pubsub/pubsub.go new file mode 100644 index 000000000..f90e1eea6 --- /dev/null +++ b/trcchatproxy/pubsub/pubsub.go @@ -0,0 +1,38 @@ +package pubsub + +import "google.golang.org/api/chat/v1" + +var manualInteraction bool = true +var chatEvents chan *chat.DeprecatedEvent +var chatAnswerEvents chan *chat.DeprecatedEvent + +func CommonInit(manualInteract bool) { + manualInteraction = manualInteract + chatEvents = make(chan *chat.DeprecatedEvent, 30) +} + +func IsManualInteractionEnabled() bool { + return manualInteraction +} + +func PubChatEvent(event *chat.DeprecatedEvent) { + chatEvents <- event +} + +func SubChatEvent() *chat.DeprecatedEvent { + select { + case event := <-chatEvents: + return event + } +} + +func PubChatAnswerEvent(event *chat.DeprecatedEvent) { + chatAnswerEvents <- event +} + +func SubChatAnswerEvent() *chat.DeprecatedEvent { + select { + case event := <-chatAnswerEvents: + return event + } +} diff --git a/trcchatproxy/chatworld.go b/trcchatproxy/trcchat/chatworld.go similarity index 99% rename from trcchatproxy/chatworld.go rename to trcchatproxy/trcchat/chatworld.go index 34ee0fdfa..c6ae7b7b7 100644 --- a/trcchatproxy/chatworld.go +++ b/trcchatproxy/trcchat/chatworld.go @@ -1,4 +1,4 @@ -package main +package trcchat import ( "log" diff --git a/trcchatproxy/main.go b/trcchatproxy/trcchat/common.go similarity index 70% rename from trcchatproxy/main.go rename to trcchatproxy/trcchat/common.go index 7f13c2ba5..08ccc97fb 100644 --- a/trcchatproxy/main.go +++ b/trcchatproxy/trcchat/common.go @@ -1,7 +1,8 @@ -package main +package trcchat import ( "bufio" + "crypto/sha256" "embed" "encoding/json" "flag" @@ -12,30 +13,30 @@ import ( "github.com/trimble-oss/tierceron-nute/mashupsdk" "github.com/trimble-oss/tierceron-nute/mashupsdk/server" + "github.com/trimble-oss/tierceron/trcchatproxy/pubsub" ) +var gchatApp GChatApp +var id int64 + //go:embed tls/mashup.crt var mashupCert embed.FS //go:embed tls/mashup.key var mashupKey embed.FS -var gchatApp GChatApp -var id int64 - func (w *GChatApp) InitServer(callerCreds string, insecure bool, maxMessageLength int) { if callerCreds != "" { server.InitServer(callerCreds, insecure, maxMessageLength, w.MashupSdkApiHandler, w.WClientInitHandler) } } -func main() { +func CommonInit(authToken string, callerToken string) { gchatApp = GChatApp{ MashupSdkApiHandler: &GoogleChatHandler{}, GoogleChatContext: &GoogleChatContext{}, WClientInitHandler: &WorldClientInitHandler{}, } - shutdown := make(chan bool) // Initialize local server. mashupsdk.InitCertKeyPair(mashupCert, mashupKey) @@ -54,8 +55,8 @@ func main() { } configs := mashupsdk.MashupConnectionConfigs{ - AuthToken: "", - CallerToken: "", + AuthToken: authToken, + CallerToken: callerToken, Server: "", Port: configPort, } @@ -69,7 +70,6 @@ func main() { id = 0 server.RemoteInitServer(*callerCreds, true, -2, gchatworld.MashupSdkApiHandler, gchatworld.WClientInitHandler) - <-shutdown } // Processes upserted query from client @@ -85,13 +85,15 @@ func ProcessQuery(msg *mashupsdk.MashupDetailedElement) { case "Get Message": gchatApp.DetailedElements = gchatApp.DetailedElements[:len(gchatApp.DetailedElements)-1] input := "" + messageId := "" for input == "" { - input = getUserInput() + messageId, input = getUserInput() if input != "" { gchatApp.DetailedElements = append(gchatApp.DetailedElements, &mashupsdk.MashupDetailedElement{ - Name: "GChatQuery", - Id: int64(len(gchatApp.DetailedElements)), // Make sure id matches index in elements - Data: input, + Name: "GChatQuery", + Id: int64(len(gchatApp.DetailedElements)), // Make sure id matches index in elements + Data: input, + Genre: messageId, }) } else { fmt.Println("An error occurred with reading the input. Please input your question in the command line and press enter!") @@ -105,15 +107,26 @@ func ProcessQuery(msg *mashupsdk.MashupDetailedElement) { // Asks user input // This is a stub version --> potentially shouldn't be needed if user can @askflume in google chat // However, maybe use this as a way to ask user if there is anything else they would like to ask -func getUserInput() string { - fmt.Println("This is a simulation of the Flume Chat App. Please type your question below and press enter: ") - reader := bufio.NewReader(os.Stdin) - input, err := reader.ReadString('\n') - if err != nil { - log.Printf("Error reading input from user: %v", err) - return "" +func getUserInput() (string, string) { + var messageId string + var input string + var err error + if pubsub.IsManualInteractionEnabled() { + fmt.Println("This is a simulation of the Flume Chat App. Please type your question below and press enter: ") + reader := bufio.NewReader(os.Stdin) + input, err = reader.ReadString('\n') + messageId = fmt.Sprintf("%x", sha256.Sum256([]byte(input))) // Hacky alias... + + if err != nil { + log.Printf("Error reading input from user: %v", err) + return "", "" + } + } else { + event := pubsub.SubChatEvent() + messageId = event.Message.ClientAssignedMessageId + input = event.Message.Text } - return input + return messageId, input } // Updates ID and returns value diff --git a/trcchatproxy/dialogflow.go b/trcchatproxy/trcchat/dialogflow.go similarity index 86% rename from trcchatproxy/dialogflow.go rename to trcchatproxy/trcchat/dialogflow.go index 29eb69a0e..770f82bad 100644 --- a/trcchatproxy/dialogflow.go +++ b/trcchatproxy/trcchat/dialogflow.go @@ -1,4 +1,4 @@ -package main +package trcchat import ( "log" @@ -23,6 +23,9 @@ func ProcessDFQuery(msg *mashupsdk.MashupDetailedElement) { } else if strings.Contains(strings.ToLower(msg.Data), "active") { msg.Data = "Active" msg.Alias = "2" + } else if strings.Contains(strings.ToLower(msg.Data), "ninja") || strings.Contains(strings.ToLower(msg.Data), "tests") { + msg.Data = "DataFlowState" + msg.Alias = "3" } // Change message type @@ -49,6 +52,9 @@ func ProcessDFResponse(msg *mashupsdk.MashupDetailedElement) { case response_type == "Active": // Report activitiy status gchatApp.DetailedElements[msg.Id].Data = "Activity report message response" + case response_type == "DataFlowState": + // Report activitiy status + gchatApp.DetailedElements[msg.Id].Data = "Ninja Test status" default: // Unable to process response... please try asking your question again error gchatApp.DetailedElements[msg.Id].Data = "Unable to process question" diff --git a/trcchatproxy/trcchat/googlechat.go b/trcchatproxy/trcchat/googlechat.go new file mode 100644 index 000000000..393874c0b --- /dev/null +++ b/trcchatproxy/trcchat/googlechat.go @@ -0,0 +1,74 @@ +package trcchat + +import ( + "encoding/json" + "fmt" + "log" + + "github.com/trimble-oss/tierceron-nute/mashupsdk" + "github.com/trimble-oss/tierceron/trcchatproxy/pubsub" + "google.golang.org/api/chat/v1" +) + +// Prints out answer to google chat user and sets up google chat app +// to ask another response +// Once the google chat api is set up, should send response to +// correct endpoint for google chat +func ProcessGChatAnswer(msg *mashupsdk.MashupDetailedElement) { + log.Printf("Message is ready to send to Google Chat user") + var infos [][]interface{} + + err := json.Unmarshal([]byte(msg.Data), &infos) + if err != nil { + log.Println("Error in decoding data in recursiveBuildArgosies") + } + + for _, info := range infos { + switch { + case msg.Alias == "Active": + tenant := info[0] + enterpriseId := info[1] + error_msg := info[2] + err_time := info[3] + // Stack trace was empty, so skipping index of 4 + snap_mode := info[5] + msg.Data = fmt.Sprintf("The pipeline tenant %v with enterprise ID %v is running with last error message %v at %v and snapshot mode %v", tenant, enterpriseId, error_msg, err_time, snap_mode) + case msg.Alias == "Error": + tenant := info[0] + enterpriseId := info[1] + error_msg := info[2] + err_time := info[3] + // Stack trace was empty, so skipping index of 4 + snap_mode := info[5] + msg.Data = fmt.Sprintf("The pipeline tenant %v with enterprise ID %v is failing with error message %v at %v with snapshot mode %v", tenant, enterpriseId, error_msg, err_time, snap_mode) + + case msg.Alias == "Tenant": + case msg.Alias == "DataFlowState": + tenant := info[0] + flowName := info[1] + //_ := info[2] + stateName := info[3] + lastTestedDate := info[4] + msg.Data = fmt.Sprintf("The pipeline tenant %v and flow name %v last failed with error: %v on %v", tenant, flowName, stateName, lastTestedDate) + + } + if pubsub.IsManualInteractionEnabled() { + fmt.Println(msg.Data) + } else { + go pubsub.PubChatAnswerEvent(&chat.DeprecatedEvent{Message: &chat.Message{ClientAssignedMessageId: msg.Genre, Text: msg.Data}}) + log.Printf("Message published to answer channel for delivery to Google Chat user") + } + } + + element := mashupsdk.MashupDetailedElement{ + Id: msg.Id, + Name: "GChatAnswer", + Data: msg.Data, + } + offset := GetId() + if gchatApp.MashupDetailedElementLibrary == nil { + gchatApp.MashupDetailedElementLibrary = make(map[int64]*mashupsdk.MashupDetailedElement) + } + gchatApp.MashupDetailedElementLibrary[msg.Id+offset] = gchatApp.DetailedElements[0] + gchatApp.DetailedElements = []*mashupsdk.MashupDetailedElement{&element} +} diff --git a/trcchatproxy/tls/mashup.crt b/trcchatproxy/trcchat/tls/mashup.crt similarity index 100% rename from trcchatproxy/tls/mashup.crt rename to trcchatproxy/trcchat/tls/mashup.crt diff --git a/trcchatproxy/tls/mashup.key b/trcchatproxy/trcchat/tls/mashup.key similarity index 100% rename from trcchatproxy/tls/mashup.key rename to trcchatproxy/trcchat/tls/mashup.key diff --git a/trcflow/core/askflume/process_askflume.go b/trcflow/core/askflume/process_askflume.go index 68a0e75b6..9f3e28887 100644 --- a/trcflow/core/askflume/process_askflume.go +++ b/trcflow/core/askflume/process_askflume.go @@ -47,7 +47,7 @@ func ProcessAskFlumeController(tfmContext *flowcore.TrcFlowMachineContext, trcFl go askFlumeFlowReceiver(askFlumeContext, &askFlumeWg) go askFlumeFlowSender(askFlumeContext) - go InitGoogleChat() + go InitGoogleChat("TODO_LOOKUP_AUTH_TOKEN") askFlumeWg.Wait() return err @@ -152,10 +152,12 @@ func handleGChatQuery(askFlumeContext *flowcore.AskFlumeContext) error { if askFlumeContext.Query.Message != "" { askFlumeContext.Queries = append(askFlumeContext.Queries, askFlumeContext.Query) fmt.Println("Received query from google chat channel in Flume: ", askFlumeContext.Query.Message, " with ID: ", askFlumeContext.Query.Id) - Flumeworld.DetailedElements[askFlumeContext.Query.Id].Name = "DialogFlow" - Flumeworld.DetailedElements[askFlumeContext.Query.Id].Data = askFlumeContext.Query.Message - askFlumeContext.Upsert <- &mashupsdk.MashupDetailedElementBundle{ - DetailedElements: Flumeworld.DetailedElements, + if Flumeworld.DetailedElements[askFlumeContext.Query.Id] != nil { + Flumeworld.DetailedElements[askFlumeContext.Query.Id].Name = "DialogFlow" + Flumeworld.DetailedElements[askFlumeContext.Query.Id].Data = askFlumeContext.Query.Message + askFlumeContext.Upsert <- &mashupsdk.MashupDetailedElementBundle{ + DetailedElements: Flumeworld.DetailedElements, + } } } @@ -176,6 +178,7 @@ func handleDFQuery(askFlumeContext *flowcore.AskFlumeContext) error { Flumeworld.DetailedElements[askFlumeContext.Query.Id].Alias = response.Type // Determines which category message belongs Flumeworld.DetailedElements[askFlumeContext.Query.Id].Name = "DialogFlowResponse" Flumeworld.DetailedElements[askFlumeContext.Query.Id].Data = response.Message + // Flumeworld.DetailedElements[askFlumeContext.Query.Id].Genre = response.Genre askFlumeContext.Upsert <- &mashupsdk.MashupDetailedElementBundle{ DetailedElements: Flumeworld.DetailedElements, } @@ -221,11 +224,11 @@ func handleGchatAnswer(askFlumeContext *flowcore.AskFlumeContext) error { // Initializes client of server running in cloud // Sets up polling loop for upserting elements to cloud server // Processes returned and updated elements -func InitGoogleChat() error { +func InitGoogleChat(authToken string) error { // Create client of known server running on cloud var params []string params = append(params, "flume") - params = append(params, "") + params = append(params, authToken) var envParams []string envParams = append(envParams, "localhost") // "Remote" server name @@ -245,7 +248,7 @@ func InitGoogleChat() error { for { updated_elements, upsertErr := Flumeworld.FlumeWorldContext.Client.UpsertElements(Flumeworld.FlumeWorldContext, &mashupsdk.MashupDetailedElementBundle{ - AuthToken: "", + AuthToken: authToken, DetailedElements: Flumeworld.DetailedElements, }) if upsertErr != nil { diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index 5da6cfd5c..b23d0cf27 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -14,9 +14,11 @@ import ( "strings" "sync" - "github.com/trimble-oss/tierceron/trcvault/carrierfactory/capauth" + "github.com/trimble-oss/tierceron/buildopts/coreopts" + "github.com/trimble-oss/tierceron/trcvault/carrierfactory/servercapauth" "github.com/trimble-oss/tierceron/trcvault/factory" "github.com/trimble-oss/tierceron/trcvault/opts/prod" + trcplgtool "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" trcvutils "github.com/trimble-oss/tierceron/trcvault/util" "github.com/trimble-oss/tierceron/trcvault/util/repository" sys "github.com/trimble-oss/tierceron/vaulthelper/system" @@ -31,50 +33,61 @@ func init() { } var onceAuth sync.Once +var gCapInitted bool = false + +func IsCapInitted() bool { return gCapInitted } func PluginDeployEnvFlow(pluginConfig map[string]interface{}, logger *log.Logger) error { logger.Println("PluginDeployInitFlow begun.") var err error + var config *eUtils.DriverConfig + var goMod *helperkv.Modifier + var vault *sys.Vault - onceAuth.Do(func() { - logger.Println("Cap auth init. ") - var config *eUtils.DriverConfig - var goMod *helperkv.Modifier - var vault *sys.Vault - - //Grabbing configs - tempAddr := pluginConfig["vaddress"] - tempToken := pluginConfig["token"] - pluginConfig["vaddress"] = pluginConfig["caddress"] - pluginConfig["token"] = pluginConfig["ctoken"] - config, goMod, vault, err = eUtils.InitVaultModForPlugin(pluginConfig, logger) - if vault != nil { - defer vault.Close() - } + //Grabbing configs + tempAddr := pluginConfig["vaddress"] + tempToken := pluginConfig["token"] + pluginConfig["vaddress"] = pluginConfig["caddress"] + pluginConfig["token"] = pluginConfig["ctoken"] + config, goMod, vault, err = eUtils.InitVaultModForPlugin(pluginConfig, logger) + if vault != nil { + defer vault.Close() + } - if goMod != nil { - defer goMod.Release() - } - pluginConfig["vaddress"] = tempAddr - pluginConfig["token"] = tempToken + if goMod != nil { + defer goMod.Release() + } + pluginConfig["vaddress"] = tempAddr + pluginConfig["token"] = tempToken - if err != nil { - eUtils.LogErrorMessage(config, "Could not access vault. Failure to start.", false) - return - } + if err != nil { + eUtils.LogErrorMessage(config, "Could not access vault. Failure to start.", false) + return err + } - err = capauth.Init(goMod, pluginConfig, logger) - if err != nil { - eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) - return - } + if ok, err := servercapauth.ValidatePathSha(goMod, pluginConfig, logger); true || ok { + onceAuth.Do(func() { + logger.Printf("Cap auth init for env: %s\n", pluginConfig["env"].(string)) + var featherAuth *servercapauth.FeatherAuth + featherAuth, err = servercapauth.Init(goMod, pluginConfig, logger) + if err != nil { + eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) + return + } + gCapInitted = true + + pluginConfig["trcHatSecretsPort"] = featherAuth.SecretsPort - capauth.Memorize(pluginConfig, logger) + servercapauth.Memorize(pluginConfig, logger) - // TODO: Support variables for different environments... - go capauth.Start(logger) - logger.Println("Cap auth init complete.") - }) + // TODO: Support variables for different environments... + // Not really clear how cap auth would do this... + go servercapauth.Start(featherAuth, pluginConfig["env"].(string), logger) + logger.Printf("Cap auth init complete for env: %s\n", pluginConfig["env"].(string)) + }) + } else { + eUtils.LogErrorMessage(config, fmt.Sprintf("Mismatched sha256 cap auth for env: %s. Skipping.", pluginConfig["env"].(string)), false) + } logger.Println("PluginDeployInitFlow complete.") @@ -83,8 +96,6 @@ func PluginDeployEnvFlow(pluginConfig map[string]interface{}, logger *log.Logger func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) error { logger.Println("PluginDeployFlow begun.") - var config *eUtils.DriverConfig - var vault *sys.Vault var err error var pluginName string @@ -99,64 +110,56 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e if hostNameErr != nil { return hostNameErr } else if hostName == "" { - return errors.New("Could not find hostname.") + return errors.New("could not find hostname") } //Grabbing certification from vault if pluginConfig["caddress"].(string) == "" { //if no certification address found, it will try to certify against itself. - return errors.New("Could not find certification address.") + return errors.New("could not find certification address") } if pluginConfig["ctoken"].(string) == "" { //if no certification address found, it will try to certify against itself. - return errors.New("Could not find certification token.") + return errors.New("could not find certification token") } - tempAddr := pluginConfig["vaddress"] - tempToken := pluginConfig["token"] + addrPtr := pluginConfig["vaddress"].(string) + tokPtr := pluginConfig["token"].(string) pluginConfig["vaddress"] = pluginConfig["caddress"] pluginConfig["token"] = pluginConfig["ctoken"] cConfig, cGoMod, _, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) cConfig.SubSectionValue = pluginName if err != nil { - eUtils.LogErrorMessage(config, "Could not access vault. Failure to start.", false) + eUtils.LogErrorMessage(cConfig, "Could not access vault. Failure to start.", false) return err } - vaultPluginSignature, ptcErr := trcvutils.GetPluginToolConfig(cConfig, cGoMod, pluginConfig, hostName) + vaultPluginSignature, ptcErr := trcvutils.GetPluginToolConfig(cConfig, cGoMod, pluginConfig, false) + + defer func(vaddrPtr *string, tPtr *string) { + pluginConfig["vaddress"] = *vaddrPtr + pluginConfig["token"] = *tPtr + }(&addrPtr, &tokPtr) + if ptcErr != nil { - eUtils.LogErrorMessage(config, "PluginDeployFlow failure: plugin load failure: "+ptcErr.Error(), false) - } - pluginConfig["vaddress"] = tempAddr - pluginConfig["token"] = tempToken - //grabbing configs - config, _, vault, err = eUtils.InitVaultModForPlugin(pluginConfig, logger) - config.SubSectionValue = pluginName - if vault != nil { - defer vault.Close() - } - if err != nil { - eUtils.LogErrorMessage(config, "Could not access vault. Failure to start.", false) - return err - } - logger.Println("PluginDeployFlow begun for plugin: " + pluginName) - insecure := false - if ok, _ := pluginConfig["insecure"].(bool); ok { - insecure = pluginConfig["insecure"].(bool) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: env: %s plugin load failure: %s", cConfig.Env, ptcErr.Error()), false) + return nil } - config = &eUtils.DriverConfig{Insecure: insecure, Log: logger, ExitOnFailure: false, StartDir: []string{}, SubSectionValue: pluginName} + //grabbing configs if _, ok := vaultPluginSignature["trcplugin"]; !ok { // TODO: maybe delete plugin if it exists since there was no entry in vault... - eUtils.LogErrorMessage(config, "PluginDeployFlow failure: plugin status load failure.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: env: %s plugin status load failure.", cConfig.Env), false) + return nil } if _, ok := vaultPluginSignature["acrrepository"].(string); !ok { // TODO: maybe delete plugin if it exists since there was no entry in vault... - eUtils.LogErrorMessage(config, "PluginDeployFlow failure: plugin status load failure - no certification entry found.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: env: %s plugin status load failure - no certification entry found.", cConfig.Env), false) + return nil } //Checks if this instance of carrier is allowed to deploy that certain plugin. if instanceList, ok := vaultPluginSignature["instances"].(string); !ok { - eUtils.LogErrorMessage(config, "Plugin has no valid instances: "+vaultPluginSignature["trcplugin"].(string), false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: env: %s Plugin has no valid instances: %s", cConfig.Env, vaultPluginSignature["trcplugin"].(string)), false) return nil } else { hostName, hostNameErr := os.Hostname() @@ -185,17 +188,19 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e vaultPluginSignature["deployed"] = false vaultPluginSignature["copied"] = false //Resets copied & deployed in memory to reset deployment for this instance. } else { - eUtils.LogErrorMessage(config, "Plugin not found for this instance: "+vaultPluginSignature["trcplugin"].(string), false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("Plugin %s not found for env: %s and this instance: %s\n", vaultPluginSignature["trcplugin"].(string), cConfig.Env, instanceIndex), false) + vaultPluginSignature["trcsha256"] = "notfound" + factory.PushPluginSha(cConfig, pluginConfig, vaultPluginSignature) return nil } } else { - eUtils.LogErrorMessage(config, "Unable to determine this instance's index for deployment: "+vaultPluginSignature["trcplugin"].(string)+" Error:"+hostNameErr.Error(), false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("Unable to determine for env: %s this instance: %s index for deployment. Error: %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string), hostNameErr.Error()), false) return nil } } if deployedVal, ok := vaultPluginSignature["deployed"].(bool); ok && deployedVal { - eUtils.LogErrorMessage(config, "Plugin has already been deployed and copied: "+vaultPluginSignature["trcplugin"].(string), false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("Plugin has already been deployed env: %s and copied: %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string)), false) return nil } @@ -208,50 +213,53 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e pluginExtension = "-prod" } + // trcsh is always type agent... even if it somehow ends up incorrect in vault... + if vaultPluginSignature["trcplugin"].(string) == "trcsh" { + vaultPluginSignature["trctype"] = "agent" + } + switch vaultPluginSignature["trctype"] { case "agent": agentPath = "/home/azuredeploy/bin/" + vaultPluginSignature["trcplugin"].(string) - case "service": - agentPath = vaultPluginSignature["trcpluginpath"].(string) + vaultPluginSignature["trcplugin"].(string) default: agentPath = "/etc/opt/vault/plugins/" + vaultPluginSignature["trcplugin"].(string) + pluginExtension } if _, err := os.Stat(agentPath); errors.Is(err, os.ErrNotExist) { pluginDownloadNeeded = true - logger.Println("Attempting to download new image.") + logger.Printf(fmt.Sprintf("Attempting to download new image for env: %s and plugin %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string))) } else { if imageFile, err := os.Open(agentPath); err == nil { - logger.Println("Found image for: " + vaultPluginSignature["trcplugin"].(string)) + logger.Printf(fmt.Sprintf("Found image for env: %s and plugin %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string))) sha256 := sha256.New() defer imageFile.Close() if _, err := io.Copy(sha256, imageFile); err != nil { - eUtils.LogErrorMessage(config, "PluginDeployFlow failure: Could not sha256 image from file system.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: Could not sha256 image from file system for env: %s and plugin %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string)), false) } filesystemsha256 := fmt.Sprintf("%x", sha256.Sum(nil)) if filesystemsha256 != vaultPluginSignature["trcsha256"] { //Sha256 from file system matches in vault pluginDownloadNeeded = true } else { - eUtils.LogErrorMessage(config, "Certified plugin already exists in file system - continuing with vault plugin status update", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("Certified plugin already exists in file system - continuing with vault plugin status update for env: %s and plugin %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string)), false) } } else { pluginDownloadNeeded = true - logger.Println("Attempting to download new image.") + logger.Printf(fmt.Sprintf("Attempting to download new image for env: %s and plugin %s\n", cConfig.Env, vaultPluginSignature["trcplugin"].(string))) } } if pluginDownloadNeeded { - logger.Println("PluginDeployFlow new plugin image found: " + pluginName) + logger.Printf(fmt.Sprintf("PluginDeployFlow new plugin image found for env: %s and plugin %s\n", cConfig.Env, pluginName)) // 1.c.i. Download new image from ECR. // 1.c.ii. Sha256 of new executable. // 1.c.ii.- if Sha256 of new executable === sha256 from vault. - downloadErr := repository.GetImageAndShaFromDownload(config, vaultPluginSignature) + downloadErr := repository.GetImageAndShaFromDownload(cConfig, vaultPluginSignature) if downloadErr != nil { - eUtils.LogErrorMessage(config, pluginName+": Could not get download image: "+downloadErr.Error(), false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("Could not get download image for env: %s and plugin %s error: %s\n", cConfig.Env, pluginName, downloadErr.Error()), false) vaultPluginSignature["imagesha256"] = "invalidurl" } if vaultPluginSignature["imagesha256"] == vaultPluginSignature["trcsha256"] { //Sha256 from download matches in vault @@ -259,7 +267,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e vaultPluginSignature["rawImageFile"] = nil if err != nil { - eUtils.LogErrorMessage(config, pluginName+": PluginDeployFlow failure: Could not write out download image.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: Could not write out download image for env: %s and plugin %s error: %s\n", cConfig.Env, pluginName, downloadErr.Error()), false) } if vaultPluginSignature["trctype"] == "agent" { @@ -277,11 +285,11 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e if imageFile, err := os.Open(agentPath); err == nil { chdModErr := imageFile.Chmod(0750) if chdModErr != nil { - eUtils.LogErrorMessage(config, pluginName+": PluginDeployFlow failure: Could not give permission to image in file system. Bailing..", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: Could not give permission to image in file system. Bailing.. for env: %s and plugin %s\n", cConfig.Env, pluginName), false) return nil } } else { - eUtils.LogErrorMessage(config, pluginName+": PluginDeployFlow failure: Could not open image in file system to give permissions.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: Could not open image in file system to give permissions for env: %s and plugin %s\n", cConfig.Env, pluginName), false) return nil } @@ -294,61 +302,66 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e cmd := exec.Command("setcap", "cap_ipc_lock=+ep", agentPath) output, err := cmd.CombinedOutput() if err != nil { - eUtils.LogErrorMessage(config, fmt.Sprint(err)+": "+string(output), false) - eUtils.LogErrorMessage(config, pluginName+": PluginDeployFlow failure: Could not set needed capabilities.", false) + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("PluginDeployFlow failure: Could not set needed capabilities for env: %s and plugin %s error: %s: %s\n", cConfig.Env, pluginName, err.Error(), string(output)), false) } pluginCopied = true - eUtils.LogInfo(config, pluginName+": Image has been copied.") + eUtils.LogInfo(cConfig, fmt.Sprintf("Image has been copied for env: %s and plugin %s\n", cConfig.Env, pluginName)) } else { - eUtils.LogErrorMessage(config, fmt.Sprintf("%s: PluginDeployFlow failure: Refusing to copy since vault certification does not match plugin sha256 signature. Downloaded: %s, Expected: %s", pluginName, vaultPluginSignature["imagesha256"], vaultPluginSignature["trcsha256"]), false) + imgsha := "notlatest or notfound" + if _, okImg := vaultPluginSignature["imagesha256"]; okImg { + imgsha = vaultPluginSignature["imagesha256"].(string) + } + eUtils.LogErrorMessage(cConfig, fmt.Sprintf("env: %s plugin: %s: PluginDeployFlow failure: Refusing to copy since vault certification does not match plugin sha256 signature. Downloaded: %s, Expected: %s", cConfig.Env, pluginName, imgsha, vaultPluginSignature["trcsha256"]), false) } } if (!pluginDownloadNeeded && !pluginCopied) || (pluginDownloadNeeded && pluginCopied) { // No download needed because it's already there, but vault may be wrong. if vaultPluginSignature["copied"].(bool) && !vaultPluginSignature["deployed"].(bool) { //If status hasn't changed, don't update - eUtils.LogInfo(config, pluginName+": Not updating plugin image to vault as status is the same for plugin: "+pluginName) + eUtils.LogInfo(cConfig, fmt.Sprintf("Not updating plugin image to vault as status is the same for env: %s and plugin: %s\n", cConfig.Env, pluginName)) } - eUtils.LogInfo(config, pluginName+": Updating plugin image to vault.") - factory.PushPluginSha(config, pluginConfig, vaultPluginSignature) - writeMap := make(map[string]interface{}) - writeMap["trcplugin"] = vaultPluginSignature["trcplugin"].(string) - writeMap["trcsha256"] = vaultPluginSignature["trcsha256"].(string) - writeMap["instances"] = vaultPluginSignature["instances"].(string) - if trcType, trcTypeOk := vaultPluginSignature["trctype"]; trcTypeOk { - writeMap["trctype"] = trcType.(string) - } else { - writeMap["trctype"] = "vault" + eUtils.LogInfo(cConfig, pluginName+": Updating plugin image to vault.") + if pluginSHA, pluginSHAOk := vaultPluginSignature["trcsha256"]; !pluginSHAOk || pluginSHA.(string) == "" { + eUtils.LogInfo(cConfig, fmt.Sprintf("Plugin is not registered with carrier for env: %s and plugin: %s\n", cConfig.Env, pluginName)) + return nil } + eUtils.LogInfo(cConfig, pluginName+": Checkpush sha256") + factory.PushPluginSha(cConfig, pluginConfig, vaultPluginSignature) + eUtils.LogInfo(cConfig, pluginName+": End checkpush sha256") + + writeMap, err := cGoMod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + pluginName + "/Certify") - _, err = cGoMod.Write("super-secrets/Index/TrcVault/trcplugin/"+writeMap["trcplugin"].(string)+"/Certify", writeMap, config.Log) if err != nil { - logger.Println(pluginName + ": PluginDeployFlow failure: Failed to write plugin state: " + err.Error()) + eUtils.LogInfo(cConfig, pluginName+": Initializing certification") + writeMap = make(map[string]interface{}) + } else { + eUtils.LogInfo(cConfig, pluginName+": Updating certification status") } - writeMap["copied"] = false - writeMap["deployed"] = false + if trcType, trcTypeOk := vaultPluginSignature["trctype"]; trcTypeOk { + writeMap["trctype"] = trcType.(string) + } else { + writeMap["trctype"] = "vault" + } + writeMap = trcplgtool.WriteMapUpdate(writeMap, vaultPluginSignature, false, writeMap["trctype"].(string)) if writeMap["trctype"].(string) == "agent" { writeMap["deployed"] = true } - - overridePath := "super-secrets/Index/TrcVault/trcplugin/overrides/" + hostName + "/" + writeMap["trcplugin"].(string) + "/Certify" - _, err = cGoMod.Write("super-secrets/Index/TrcVault/trcplugin/"+overridePath, writeMap, config.Log) + _, err = cGoMod.Write("super-secrets/Index/TrcVault/trcplugin/"+writeMap["trcplugin"].(string)+"/Certify", writeMap, cConfig.Log) if err != nil { - logger.Println(pluginName + ": PluginDeployFlow failure: Failed to write plugin state: " + err.Error()) + logger.Printf(fmt.Sprintf("PluginDeployFlow failure: Failed to write plugin state for env: %s and plugin: %s error: %s\n", cConfig.Env, pluginName, err.Error())) } - - eUtils.LogInfo(config, pluginName+": Plugin image config in vault has been updated.") + eUtils.LogInfo(cConfig, fmt.Sprintf("Plugin image config in vault has been updated for env: %s and plugin: %s\n", cConfig.Env, pluginName)) } else { if !pluginDownloadNeeded && pluginCopied { - eUtils.LogInfo(config, pluginName+": Not updating plugin image to vault as status is the same for plugin: "+pluginName) + eUtils.LogInfo(cConfig, fmt.Sprintf("Not updating plugin image to vault as status is the same for for env: %s and plugin: %s\n", cConfig.Env, pluginName)) // Already copied... Just echo back the sha256... } } // ALways set this so it completes if there is a sha256 available... // This will also release any clients attempting to communicate with carrier. - factory.PushPluginSha(config, pluginConfig, vaultPluginSignature) + factory.PushPluginSha(cConfig, pluginConfig, vaultPluginSignature) logger.Println("PluginDeployFlow complete.") @@ -356,93 +369,87 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e } // Updated deployed to true for any plugin -func PluginDeployedUpdate(mod *helperkv.Modifier, pluginNameList []string, logger *log.Logger) error { +func PluginDeployedUpdate(config *eUtils.DriverConfig, mod *helperkv.Modifier, vault *sys.Vault, pluginNameList []string, cPath []string, logger *log.Logger) error { logger.Println("PluginDeployedUpdate start.") hostName, hostNameErr := os.Hostname() if hostNameErr != nil { return hostNameErr } else if hostName == "" { - return errors.New("Could not find hostname.") + return errors.New("could not find hostname") } + hostRegion := coreopts.GetRegion(hostName) + mod.Regions = append(mod.Regions, hostRegion) + projects, services, _ := eUtils.GetProjectServices(cPath) for _, pluginName := range pluginNameList { - pluginData, err := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + pluginName + "/Certify") - if err != nil { - return err - } - mod.SectionPath = "super-secrets/Index/TrcVault/trcplugin/overrides/" + hostName + "/" + pluginName + "/Certify" - pluginStatusData, statusErr := mod.ReadData(mod.SectionPath) - if statusErr != nil { - return statusErr - } - - for k, v := range pluginStatusData { - pluginData[k] = v - } - - if pluginData == nil { - pluginData = make(map[string]interface{}) - pluginData["trcplugin"] = pluginName - - var agentPath string - pluginExtension := "" - if prod.IsProd() { - pluginExtension = "-prod" - } - - if pluginData["trctype"] == "agent" { - agentPath = "/home/azuredeploy/bin/" + pluginName - } else { - agentPath = "/etc/opt/vault/plugins/" + pluginName + pluginExtension - } + for i := 0; i < len(projects); i++ { + if services[i] == "Certify" { + mod.SectionName = "trcplugin" + mod.SectionKey = "/Index/" + mod.SubSectionValue = pluginName + + properties, err := trcvutils.NewProperties(config, vault, mod, config.Env, projects[i], services[i]) + if err != nil { + return err + } - logger.Println("Checking file.") - if imageFile, err := os.Open(agentPath); err == nil { - sha256 := sha256.New() + pluginData, replacedFields := properties.GetPluginData(hostRegion, services[i], "config", config.Log) + if pluginData == nil { + pluginData = make(map[string]interface{}) + pluginData["trcplugin"] = pluginName + + var agentPath string + pluginExtension := "" + if prod.IsProd() { + pluginExtension = "-prod" + } + + if pluginData["trctype"] == "agent" { + agentPath = "/home/azuredeploy/bin/" + pluginName + } else { + agentPath = "/etc/opt/vault/plugins/" + pluginName + pluginExtension + } + + logger.Println("Checking file.") + if imageFile, err := os.Open(agentPath); err == nil { + sha256 := sha256.New() + + defer imageFile.Close() + if _, err := io.Copy(sha256, imageFile); err != nil { + continue + } + + filesystemsha256 := fmt.Sprintf("%x", sha256.Sum(nil)) + pluginData["trcsha256"] = filesystemsha256 + pluginData["copied"] = false + pluginData["instances"] = "0" + + if pluginData["trctype"].(string) == "agent" { + pluginData["deployed"] = false + } + } + } - defer imageFile.Close() - if _, err := io.Copy(sha256, imageFile); err != nil { + if copied, okCopied := pluginData["copied"]; !okCopied || !copied.(bool) { + logger.Println("Cannot certify plugin. Plugin not copied: " + pluginName) continue } - filesystemsha256 := fmt.Sprintf("%x", sha256.Sum(nil)) - pluginData["trcsha256"] = filesystemsha256 - pluginData["copied"] = false - pluginData["instances"] = "0" - - if pluginData["trctype"].(string) == "agent" { - pluginData["deployed"] = false + if deployed, okDeployed := pluginData["deployed"]; !okDeployed || deployed.(bool) { + continue } - } - } - - if copied, okCopied := pluginData["copied"]; !okCopied || !copied.(bool) { - logger.Println("Cannot certify plugin. Plugin not copied: " + pluginName) - continue - } - if deployed, okDeployed := pluginData["deployed"]; !okDeployed || deployed.(bool) { - continue - } - - writeMap := make(map[string]interface{}) - writeMap["trcplugin"] = pluginData["trcplugin"] - writeMap["trctype"] = pluginData["trctype"] - writeMap["trcsha256"] = pluginData["trcsha256"] - writeMap["instances"] = pluginData["instances"] + if hostRegion != "" { + pluginData["deployed"] = true //Update deploy status if region exist otherwise this will block regionless deploys if set for regionless status + } - _, err = mod.Write("super-secrets/Index/TrcVault/trcplugin/"+pluginName+"/Certify", writeMap, logger) - if err != nil { - return err - } + statusUpdateErr := properties.WritePluginData(pluginData, replacedFields, mod, config.Log, hostRegion, pluginName) + if err != nil { + return statusUpdateErr + } - writeMap = make(map[string]interface{}) - writeMap["copied"] = pluginData["copied"] - writeMap["deployed"] = false - _, err = mod.Write("super-secrets/Index/TrcVault/trcplugin/overrides/"+hostName+"/"+pluginName+"/Certify", writeMap, logger) - if err != nil { - return err + } } } logger.Println("PluginDeployedUpdate complete.") diff --git a/trcflow/flows/process_dataflowstatistics.go b/trcflow/flows/process_dataflowstatistics.go index 384733b07..d7570c628 100644 --- a/trcflow/flows/process_dataflowstatistics.go +++ b/trcflow/flows/process_dataflowstatistics.go @@ -23,6 +23,9 @@ import ( const flowGroupName = "Ninja" +var refresh = false +var endRefreshChan = make(chan bool, 1) + func GetDataflowStatIndexedPathExt(engine interface{}, rowDataMap map[string]interface{}, indexColumnNames interface{}, databaseName string, tableName string, dbCallBack func(interface{}, map[string]interface{}) (string, []string, [][]interface{}, error)) (string, error) { tenantIndexPath, _ := utilcore.GetDFSPathName() if _, ok := rowDataMap[dfssql.DataflowTestIdColumn].(string); ok { @@ -204,6 +207,9 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex stateUpdate.SyncFilter = "N/A" if previousState.State == stateUpdate.State && previousState.SyncMode == stateUpdate.SyncMode && previousState.SyncFilter == stateUpdate.SyncFilter && previousState.FlowAlias == stateUpdate.FlowAlias { continue + } else if previousState.SyncMode == "refreshingDaily" && stateUpdate.SyncMode != "refreshEnd" && stateUpdate.State == 2 && int(previousState.State) != utilcore.PreviousStateCheck(int(stateUpdate.State)) { + sPC <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: strconv.Itoa(int(stateUpdate.State)), SyncFilter: stateUpdate.SyncFilter, SyncMode: previousState.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} + break } else if int(previousState.State) != utilcore.PreviousStateCheck(int(stateUpdate.State)) && stateUpdate.State != previousState.State { sPC <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: strconv.Itoa(int(previousState.State)), SyncFilter: stateUpdate.SyncFilter, SyncMode: stateUpdate.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} continue @@ -243,6 +249,10 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex } else if tfContext.FlowState.State == 0 { tfContext.FlowLock.Unlock() tfmContext.Log("DataFlowStatistics flow is currently offline...", nil) + if tfContext.FlowState.SyncMode == "refreshingDaily" { + refresh = true + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "1", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: tfContext.FlowState.SyncMode, FlowAlias: tfContext.FlowState.FlowAlias} + } continue } else if tfContext.FlowState.State == 1 { tfContext.FlowLock.Unlock() @@ -261,6 +271,22 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex tfmContext.Log("Ignoring invalid flow.", nil) continue } + + tfContext.FlowLock.Lock() + if strings.HasPrefix(tfContext.FlowState.SyncMode, "refresh") { //This is to refresh from vault - different from pulling/pushing. + refreshSuffix, _ := strings.CutPrefix(tfContext.FlowState.SyncMode, "refresh") + if tfContext.FlowState.SyncMode == "refreshingDaily" { + if !refresh { //This is for if trcdb loads up in "refreshingDaily" -> need to kick off refresh again. + KickOffTimedRefresh(tfContext, stateUpdateChannel, "Daily") + } + } else if !KickOffTimedRefresh(tfContext, stateUpdateChannel, refreshSuffix) { + tfmContext.Log("DataFlowStatistics has an invalid refresh timing"+flowcorehelper.SyncCheck(tfContext.FlowState.SyncMode)+".", nil) + tfContext.FlowState.SyncMode = "InvalidRefreshMode" + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "InvalidRefreshMode", FlowAlias: tfContext.FlowState.FlowAlias} + } + } + tfContext.FlowLock.Unlock() + tfContext.FlowLock.Lock() if tfContext.FlowState.SyncMode == "pullonce" { tfContext.FlowState.SyncMode = "pullsynccomplete" @@ -277,3 +303,41 @@ func ProcessDataFlowStatConfigurations(tfmContext *flowcore.TrcFlowMachineContex tfContext.CancelContext() return nil } + +func KickOffTimedRefresh(tfContext *flowcore.TrcFlowContext, stateUpdateChannel chan flowcorehelper.FlowStateUpdate, timing string) bool { + switch { //Always at midnight + case timing == "Daily": + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "refreshingDaily", FlowAlias: tfContext.FlowState.FlowAlias} + loc, _ := time.LoadLocation("America/Los_Angeles") + now := time.Now().In(loc) + midnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, loc) + timeTilMidnight := midnight.Sub(now) + go func(tfc *flowcore.TrcFlowContext, tilMidnight time.Duration) { + refresh = true + time.Sleep(tilMidnight) + refreshTime := time.Duration(time.Second * 0) + for { + select { + case <-endRefreshChan: + tfContext.Log.Println("Daily Refresh Ended - no longer refreshing DFS") + return + case <-time.After(refreshTime): + tfContext.Log.Println("Daily Refresh Triggered - refreshing DFS") + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfc.Flow.TableName(), StateUpdate: "3", SyncFilter: tfc.FlowState.SyncFilter, SyncMode: "refreshingDaily", FlowAlias: tfc.FlowState.FlowAlias} + refreshTime = time.Duration(time.Hour * 24) + } + + } + }(tfContext, timeTilMidnight) + case timing == "End": + endRefreshChan <- true + refresh = false + stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "refreshEnded", FlowAlias: tfContext.FlowState.FlowAlias} + case timing == "Ended": + return true + default: + return false + } + + return true +} diff --git a/trcflow/flumen/process.go b/trcflow/flumen/process.go index 1f38f8a7f..23cbebdba 100644 --- a/trcflow/flumen/process.go +++ b/trcflow/flumen/process.go @@ -46,19 +46,19 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error //Need new function writing to that path using pluginName -> //if not copied -> this plugin should fail to start up //Update deployed status & return if - if pluginNameList, ok := pluginConfig["pluginNameList"].([]string); ok { + if pluginNameList, ok := pluginConfig["pluginNameList"].([]string); ok || true { tempAddr := pluginConfig["vaddress"] tempToken := pluginConfig["token"] pluginConfig["vaddress"] = pluginConfig["caddress"] pluginConfig["token"] = pluginConfig["ctoken"] pluginConfig["exitOnFailure"] = true - _, cGoMod, _, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) + cConfig, cGoMod, cVault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) if err != nil { eUtils.LogErrorMessage(config, "Could not access vault. Failure to start.", false) return err } - deployedUpdateErr := deploy.PluginDeployedUpdate(cGoMod, pluginNameList, logger) + deployedUpdateErr := deploy.PluginDeployedUpdate(cConfig, cGoMod, cVault, pluginNameList, pluginConfig["certifyPath"].([]string), logger) if deployedUpdateErr != nil { eUtils.LogErrorMessage(config, deployedUpdateErr.Error(), false) eUtils.LogErrorMessage(config, "Could not update plugin deployed status in vault.", false) diff --git a/trcinit/initlib/vsub.go b/trcinit/initlib/vsub.go index 834de4a14..b4adf458d 100644 --- a/trcinit/initlib/vsub.go +++ b/trcinit/initlib/vsub.go @@ -12,6 +12,77 @@ import ( helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" ) +func DownloadTemplates(config *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string, logger *log.Logger, templatePaths *string) { + var filterTemplatePathSlice []string + if len(*templatePaths) > 0 { + filterTemplatePathSlice = strings.Split(*templatePaths, ",") + } + + for _, filterTemplatePath := range filterTemplatePathSlice { + path := filterTemplatePath + ext := "" + if !strings.HasSuffix(filterTemplatePath, "/") { + path = filterTemplatePath + "/" + } + tfMap, err := mod.ReadData(fmt.Sprintf("templates/%stemplate-file", path)) //Grab extention of file + if err != nil { + eUtils.LogErrorMessage(config, "Skipping template: "+path+" Error: "+err.Error(), false) + continue + } + if _, extOk := tfMap["ext"]; extOk { + ext = tfMap["ext"].(string) + } + + var data string + if _, dataOk := tfMap["data"]; dataOk { + data = tfMap["data"].(string) + } else { + // TODO: In recent run in prod, sub was printing an annoying warning here + // and yet correct templates seem to have gotten created... + fmt.Println("No data found for: " + filterTemplatePath + "template-file") + continue + } + templateBytes, decodeErr := base64.StdEncoding.DecodeString(data) + if decodeErr != nil { + eUtils.LogErrorMessage(config, "Couldn't decode data for: "+path+"template-file", false) + continue + } + //Ensure directory has been created + filePath := strings.Trim(path, "/") + templateAndFilePath := fmt.Sprintf("%s/%s", dirName, filePath) + dirPath := filepath.Dir(templateAndFilePath) + file := filepath.Base(templateAndFilePath) + fmt.Printf("templateDir: %s\n", templateAndFilePath) + fmt.Printf("Dir: %s\n", dirPath) + fmt.Printf("file: %s\n", file) + err = os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + eUtils.LogErrorMessage(config, "Couldn't make directory: "+dirName+filePath, false) + continue + } + //create new file + templateFile := fmt.Sprintf("%s/%s%s.tmpl", dirPath, file, ext) + newFile, err := os.Create(templateFile) + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't create file: %s", templateFile), false) + continue + } + //write to file + _, err = newFile.Write(templateBytes) + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't write file: %s", templateFile), false) + continue + } + err = newFile.Sync() + if err != nil { + eUtils.LogErrorMessage(config, fmt.Sprintf("Couldn't sync file: %s", templateFile), false) + continue + } + newFile.Close() + fmt.Printf("File has been writen to %s\n", templateFile) + } +} + func DownloadTemplateDirectory(config *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string, logger *log.Logger, templateFilter *string) ([]string, error) { var filterTemplateSlice []string diff --git a/trcsh/kube/native/trckube.go b/trcsh/kube/native/trckube.go index b900b462d..a6450b528 100644 --- a/trcsh/kube/native/trckube.go +++ b/trcsh/kube/native/trckube.go @@ -26,9 +26,9 @@ import ( kubectlutil "k8s.io/kubectl/pkg/util" "github.com/go-git/go-billy/v5" + "github.com/trimble-oss/tierceron/capauth" "github.com/trimble-oss/tierceron/trcsh/kube/native/path" "github.com/trimble-oss/tierceron/trcsh/kube/native/trccreate" - "github.com/trimble-oss/tierceron/trcsh/trcshauth" eUtils "github.com/trimble-oss/tierceron/utils" ) @@ -108,8 +108,8 @@ func LoadFromKube(kubeConfigBytes []byte, config *eUtils.DriverConfig) (*TrcKube return trcConfig, nil } -func InitTrcKubeConfig(trcshConfig *trcshauth.TrcShConfig, config *eUtils.DriverConfig) (*TrcKubeConfig, error) { - kubeConfigBytes, decodeErr := base64.StdEncoding.DecodeString(trcshConfig.KubeConfig) +func InitTrcKubeConfig(trcshConfig *capauth.TrcShConfig, config *eUtils.DriverConfig) (*TrcKubeConfig, error) { + kubeConfigBytes, decodeErr := base64.StdEncoding.DecodeString(*trcshConfig.KubeConfig) if decodeErr != nil { fmt.Println("Decoding error") eUtils.LogErrorObject(config, decodeErr, false) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 6f4463c39..47a64bfbb 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -2,30 +2,39 @@ package main import ( "bytes" + "errors" "flag" "fmt" "io" "log" "os" + "os/signal" + "runtime" "strings" "sync" + "syscall" "time" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" + "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" + "github.com/trimble-oss/tierceron/capauth" "github.com/trimble-oss/tierceron/trcconfigbase" "github.com/trimble-oss/tierceron/trcpubbase" kube "github.com/trimble-oss/tierceron/trcsh/kube/native" "github.com/trimble-oss/tierceron/trcsh/trcshauth" - "github.com/trimble-oss/tierceron/trcvault/carrierfactory/capauth" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" + "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" ) +var gAgentConfig *capauth.AgentConfigs = nil +var gTrcshConfig *capauth.TrcShConfig + // This is a controller program that can act as any command line utility. // The Tierceron Shell runs tierceron and kubectl commands in a secure shell. func main() { @@ -33,41 +42,380 @@ func main() { memprotectopts.MemProtectInit(nil) } eUtils.InitHeadless(true) - fmt.Println("trcsh Version: " + "1.11") + fmt.Println("trcsh Version: " + "1.20") + var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string - if os.Geteuid() == 0 { - fmt.Println("Trcsh cannot be run as root.") - os.Exit(-1) + if runtime.GOOS != "windows" { + if os.Geteuid() == 0 { + fmt.Println("Trcsh cannot be run as root.") + os.Exit(-1) + } else { + util.CheckNotSudo() + } + if len(os.Args) > 1 { + if strings.Contains(os.Args[1], "trc") && !strings.Contains(os.Args[1], "-c") { + // Running as shell. + os.Args[1] = "-c=" + os.Args[1] + // Initiate signal handling. + var ic chan os.Signal = make(chan os.Signal) + signal.Notify(ic, os.Interrupt, syscall.SIGTERM) + go func() { + x := <-ic + interruptChan <- x + }() + } + } + envPtr = flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. + regionPtr = flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + trcPathPtr = flag.String("c", "", "Optional script to execute.") //If this is blank -> use context otherwise override context. + appRoleIDPtr = flag.String("appRoleID", "", "Public app role ID") + secretIDPtr = flag.String("secretID", "", "App role secret") + flag.Parse() + + if len(*appRoleIDPtr) == 0 { + *appRoleIDPtr = os.Getenv("DEPLOY_ROLE") + } + + if len(*secretIDPtr) == 0 { + *secretIDPtr = os.Getenv("DEPLOY_SECRET") + } + memprotectopts.MemProtect(nil, secretIDPtr) + memprotectopts.MemProtect(nil, appRoleIDPtr) + + //Open deploy script and parse it. + ProcessDeploy(*envPtr, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) + } else { + gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 1)} + deployments := os.Getenv("DEPLOYMENTS") + agentToken := os.Getenv("AGENT_TOKEN") + agentEnv := os.Getenv("AGENT_ENV") + address := os.Getenv("VAULT_ADDR") + + envPtr = flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. + regionPtr = flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + trcPathPtr = flag.String("c", "", "Optional script to execute.") //If this is blank -> use context otherwise override context. + appRoleIDPtr = flag.String("appRoleID", "", "Public app role ID") + secretIDPtr = flag.String("secretID", "", "App role secret") + flag.Parse() + + if len(deployments) == 0 { + fmt.Println("trcsh on windows requires a DEPLOYMENTS.") + os.Exit(-1) + } + + if len(agentToken) == 0 { + fmt.Println("trcsh on windows requires AGENT_TOKEN.") + os.Exit(-1) + } + + if len(agentEnv) == 0 { + fmt.Println("trcsh on windows requires AGENT_ENV.") + os.Exit(-1) + } + + if len(address) == 0 { + fmt.Println("trcsh on windows requires VAULT_ADDR address.") + os.Exit(-1) + } + + memprotectopts.MemProtect(nil, &agentToken) + memprotectopts.MemProtect(nil, &address) + shutdown := make(chan bool) + + // Preload agent synchronization configs + gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) + for { + if deployFlapMode, deployEmitErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); deployEmitErr == nil && strings.HasPrefix(deployFlapMode, cap.MODE_GAZE) { + go func() { + for { + perching: + select { + case <-time.After(120 * time.Second): + ctlMsg := "Deployment timed out after 120 seconds" + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + case ctlMsg := <-gAgentConfig.CtlMessage: + if ctlMsg != capauth.TrcCtlComplete { + deployLineFlapMode := cap.MODE_FLAP + "_" + ctlMsg + ctlFlapMode := deployLineFlapMode + var err error + for { + if err == nil && ctlFlapMode == cap.MODE_PERCH { + // Acknowledge perching... + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) + ctlFlapMode = cap.MODE_PERCH + goto perching + } + + // Notify deployer of command run and wait for confirmation of msg received. + if err == nil && deployLineFlapMode != ctlFlapMode { + // Flap, Gaze, etc... + break + } else { + callFlap := deployLineFlapMode + if err == nil { + time.Sleep(200 * time.Millisecond) + } else { + if err.Error() != "init" { + time.Sleep(1 * time.Second) + } + } + deployLineFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, deployments+"."+*gAgentConfig.Env) + } + } + } else { + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) + } + } + } + }() + ProcessDeploy(*gAgentConfig.Env, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) + + } else { + time.Sleep(500 * time.Millisecond) + } + } + <-shutdown + } + +} + +var interruptChan chan os.Signal = make(chan os.Signal) +var twoHundredMilliInterruptTicker *time.Ticker = time.NewTicker(200 * time.Millisecond) +var multiSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second) + +func interruptFun(tickerInterrupt *time.Ticker) { + select { + case <-interruptChan: + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env) + os.Exit(1) + case <-tickerInterrupt.C: + } +} + +func featherCtlCb(agentName string) error { + + if gAgentConfig == nil { + return errors.New("Incorrect agent initialization") } else { - capauth.CheckNotSudo() + gAgentConfig.Deployments = &agentName } - if len(os.Args) > 1 { - if strings.Contains(os.Args[1], "trc") && !strings.Contains(os.Args[1], "-c") { - // Running as shell. - os.Args[1] = "-c=" + os.Args[1] + callFlap := cap.MODE_GAZE + for { + // Azure deployment agent kicks off a deploy with a flap command... + if ctlFlapMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, agentName+"."+*gAgentConfig.Env); featherErr != nil || ctlFlapMode == cap.MODE_PERCH || ctlFlapMode == cap.MODE_GLIDE { + if featherErr != nil { + fmt.Printf("\nDeployment error.\n") + } else { + fmt.Printf("\nDeployment complete.\n") + } + os.Exit(0) + } else { + if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { + if strings.Contains(ctlFlapMode, "_") { + ctlMessage := strings.Split(ctlFlapMode, "_") + if len(ctlMessage) > 1 { + fmt.Printf("%s\n", ctlMessage[1]) + } + } + } + callFlap = cap.MODE_GAZE + interruptFun(multiSecondInterruptTicker) } } - envPtr := flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. - regionPtr := flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. - trcPathPtr := flag.String("c", "", "Optional script to execute.") //If this is blank -> use context otherwise override context. - appRoleIDPtr := flag.String("appRoleID", "", "Public app role ID") - secretIDPtr := flag.String("secretID", "", "App role secret") - flag.Parse() +} - if len(*appRoleIDPtr) == 0 { - *appRoleIDPtr = os.Getenv("DEPLOY_ROLE") +func configCmd(env string, + trcshConfig *capauth.TrcShConfig, + region string, + config *eUtils.DriverConfig, + agentToken bool, + token string, + argsOrig []string, + deployArgLines []string, + configCount *int) { + *configCount -= 1 + if *configCount != 0 { //This is to keep result channel open - closes on the final config call of the script. + config.EndDir = "deploy" + } + config.AppRoleConfig = "config.yml" + config.FileFilter = nil + config.EnvRaw = env + config.WantCerts = false + config.IsShellSubProcess = true + + configRoleSlice := strings.Split(*trcshConfig.ConfigRole, ":") + tokenName := "config_token_" + env + tokenConfig := "" + configEnv := env + + trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + + if !agentToken { + token = "" + config.Token = token } +} + +func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, + onceKubeInit *sync.Once, + PipeOS billy.File, + env string, + trcshConfig *capauth.TrcShConfig, + region string, + config *eUtils.DriverConfig, + control string, + agentToken bool, + token string, + argsOrig []string, + deployArgLines []string, + configCount *int, + logger *log.Logger) { + switch control { + case "trcpub": + config.AppRoleConfig = "configpub.yml" + config.EnvRaw = env + config.IsShellSubProcess = true - if len(*secretIDPtr) == 0 { - *secretIDPtr = os.Getenv("DEPLOY_SECRET") + pubRoleSlice := strings.Split(*gTrcshConfig.PubRole, ":") + tokenName := "vault_pub_token_" + env + tokenPub := "" + pubEnv := env + + trcpubbase.CommonMain(&pubEnv, &config.VaultAddress, &tokenPub, &trcshConfig.EnvContext, &pubRoleSlice[1], &pubRoleSlice[0], &tokenName, config) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } + case "trcconfig": + configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + case "trcplgtool": + config.AppRoleConfig = "" + config.EnvRaw = env + config.IsShellSubProcess = true + + config.FeatherCtlCb = featherCtlCb + if gAgentConfig == nil { + gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 1)} + gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", "dev") // Feathering always in dev environmnent. + } + gAgentConfig.Env = &env + + trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + config.FeatherCtlCb = nil + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } + + case "kubectl": + onceKubeInit.Do(func() { + var kubeInitErr error + config.Log.Println("Setting up kube config") + *trcKubeDeploymentConfig, kubeInitErr = kube.InitTrcKubeConfig(trcshConfig, config) + if kubeInitErr != nil { + fmt.Println(kubeInitErr) + return + } + config.Log.Println("Setting kube config setup complete") + }) + config.Log.Println("Preparing for kubectl") + (*(*trcKubeDeploymentConfig)).PipeOS = PipeOS + + kubectlErrChan := make(chan error, 1) + + go func() { + config.Log.Println("Executing kubectl") + kubectlErrChan <- kube.KubeCtl(*trcKubeDeploymentConfig, config) + }() + + select { + case <-time.After(15 * time.Second): + fmt.Println("Agent is not yet ready..") + logger.Println("Timed out waiting for KubeCtl.") + os.Exit(-1) + case kubeErr := <-kubectlErrChan: + if kubeErr != nil { + logger.Println(kubeErr) + os.Exit(-1) + } + } } +} - memprotectopts.MemProtect(nil, secretIDPtr) - memprotectopts.MemProtect(nil, appRoleIDPtr) +func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, + onceKubeInit *sync.Once, + PipeOS billy.File, + env string, + trcshConfig *capauth.TrcShConfig, + region string, + config *eUtils.DriverConfig, + control string, + agentToken bool, + token string, + argsOrig []string, + deployArgLines []string, + configCount *int, + logger *log.Logger) { + switch control { + case "trcplgtool": + config.AppRoleConfig = "" + config.EnvRaw = env + config.IsShellSubProcess = true - //Open deploy script and parse it. - ProcessDeploy(*envPtr, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr) + trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + ResetModifier(config) + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } + case "trcconfig": + configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } + } } // ProcessDeploy @@ -83,7 +431,7 @@ func main() { // Returns: // // Nothing. -func ProcessDeploy(env string, region string, token string, trcPath string, secretId *string, approleId *string) { +func ProcessDeploy(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { var err error agentToken := false if token != "" { @@ -125,10 +473,11 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr f, _ := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) logger := log.New(f, "[DEPLOY]", log.LstdFlags) config := &eUtils.DriverConfig{Insecure: true, + EnvRaw: env, Log: logger, IsShell: true, IsShellSubProcess: false, - OutputMemCache: true, + OutputMemCache: outputMemCache, MemFs: memfs.New(), Regions: regions, ExitOnFailure: true} @@ -139,24 +488,63 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr fmt.Println("Logging initialized.") logger.Printf("Logging initialized for env:%s\n", env) - // trcshConfig := &trcshauth.TrcShConfig{Env: "", - // EnvContext: "", - // ConfigRole: "", - // PubRole: "", - // } - trcshConfig, err := trcshauth.TrcshAuth(config) - if err != nil { - fmt.Println("Tierceron bootstrap failure.") - logger.Println(err) - os.Exit(-1) + // Chewbacca: scrub before checkin + // This data is generated by TrcshAuth + // cToken := "" + // configRole := "" + // pubRole := "" + // fileBytes, err := ioutil.ReadFile("") + // kc := base64.StdEncoding.EncodeToString(fileBytes) + // gTrcshConfig = &trcshauth.TrcShConfig{Env: "dev", + // EnvContext: "dev", + // CToken: &cToken, + // ConfigRole: &configRole, + // PubRole: &pubRole, + // KubeConfig: &kc, + // } + // config.VaultAddress = "" + // config.Token = "" + // Chewbacca: end scrub + if gTrcshConfig == nil { + gTrcshConfig, err = trcshauth.TrcshAuth(gAgentConfig, config) + if err != nil { + fmt.Println("Tierceron bootstrap failure.") + fmt.Println(err.Error()) + logger.Println(err) + os.Exit(-1) + } + fmt.Printf("Auth loaded %s\n", env) } - fmt.Println("Auth loaded" + env) - // Begin dbg comment + // Chewbacca: Begin dbg comment var auth string + mergedVaultAddress := config.VaultAddress + mergedEnvRaw := config.EnvRaw + + if (approleId != nil && len(*approleId) == 0) || (secretId != nil && len(*secretId) == 0) { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil && gTrcshConfig.CToken != nil { + auth = *gTrcshConfig.CToken + } + } + + if len(mergedVaultAddress) == 0 { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil && gTrcshConfig.VaultAddress != nil { + mergedVaultAddress = *gTrcshConfig.VaultAddress + } + } + + if len(mergedEnvRaw) == 0 { + // If in context of trcsh, utilize CToken to auth... + if gTrcshConfig != nil { + mergedEnvRaw = gTrcshConfig.EnvContext + } + } + authTokenName := "vault_token_azuredeploy" authTokenEnv := "azuredeploy" - autoErr := eUtils.AutoAuth(config, secretId, approleId, &auth, &authTokenName, &authTokenEnv, &config.VaultAddress, &config.EnvRaw, "deployauth", false) + autoErr := eUtils.AutoAuth(config, secretId, approleId, &auth, &authTokenName, &authTokenEnv, &mergedVaultAddress, &mergedEnvRaw, "deployauth", false) if autoErr != nil || auth == "" { fmt.Println("Unable to auth.") fmt.Println(autoErr) @@ -168,13 +556,13 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr // Generate .trc code... trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} - configRoleSlice := strings.Split(trcshConfig.ConfigRole, ":") + configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") tokenName := "config_token_" + env configEnv := env config.EnvRaw = env config.EndDir = "deploy" config.OutputMemCache = true - trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &token, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. var memFile billy.File @@ -225,7 +613,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") - if strings.HasPrefix(deployPipeline, "#") { + if strings.HasPrefix(deployPipeline, "#") || deployPipeline == "" { continue } // Print current process line. @@ -276,81 +664,45 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr os.Args = deployArgs } } - - switch control { - case "trcpub": - config.AppRoleConfig = "configpub.yml" - config.EnvRaw = env - config.IsShellSubProcess = true - - pubRoleSlice := strings.Split(trcshConfig.PubRole, ":") - tokenName := "vault_pub_token_" + env - tokenPub := "" - pubEnv := env - - trcpubbase.CommonMain(&pubEnv, &config.VaultAddress, &tokenPub, &trcshConfig.EnvContext, &pubRoleSlice[1], &pubRoleSlice[0], &tokenName, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. - if !agentToken { - token = "" - config.Token = token - } - case "trcconfig": - configCount -= 1 - if configCount != 0 { //This is to keep result channel open - closes on the final config call of the script. - config.EndDir = "deploy" - } - config.AppRoleConfig = "config.yml" - config.FileFilter = nil - config.EnvRaw = env - config.WantCerts = false - config.IsShellSubProcess = true - - configRoleSlice := strings.Split(trcshConfig.ConfigRole, ":") - tokenName := "config_token_" + env - tokenConfig := "" - configEnv := env - - trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. - - if !agentToken { - token = "" - config.Token = token - } - case "trcpluginctl": - case "kubectl": - onceKubeInit.Do(func() { - var kubeInitErr error - trcKubeDeploymentConfig, kubeInitErr = kube.InitTrcKubeConfig(trcshConfig, config) - if kubeInitErr != nil { - fmt.Println(kubeInitErr) - return - } - }) - trcKubeDeploymentConfig.PipeOS = PipeOS - - kubectlErrChan := make(chan error, 1) - - go func() { - kubectlErrChan <- kube.KubeCtl(trcKubeDeploymentConfig, config) - }() - - select { - case <-time.After(15 * time.Second): - fmt.Println("Agent is not yet ready..") - logger.Println("Timed out waiting for KubeCtl.") - os.Exit(-1) - case kubeErr := <-kubectlErrChan: - if kubeErr != nil { - logger.Println(kubeErr) - os.Exit(-1) - } - } + if runtime.GOOS == "windows" { + processWindowsCmds( + trcKubeDeploymentConfig, + &onceKubeInit, + PipeOS, + env, + gTrcshConfig, + region, + config, + control, + agentToken, + token, + argsOrig, + deployArgLines, + &configCount, + logger) + gAgentConfig.CtlMessage <- deployLine + } else { + processPluginCmds( + &trcKubeDeploymentConfig, + &onceKubeInit, + PipeOS, + env, + gTrcshConfig, + region, + config, + control, + agentToken, + token, + argsOrig, + deployArgLines, + &configCount, + logger) } } } + if runtime.GOOS == "windows" { + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + } //Make the arguments in the script -> os.args. } diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index 4f55ff346..74d518ad2 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -1,56 +1,19 @@ package trcshauth import ( - "context" - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "embed" "encoding/base64" - "encoding/hex" - "encoding/pem" "errors" "fmt" "math/rand" "os" "regexp" "strings" - "time" - "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" + "github.com/trimble-oss/tierceron/capauth" eUtils "github.com/trimble-oss/tierceron/utils" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) -//go:embed tls/mashup.crt -var MashupCert embed.FS - -//go:embed tls/mashup.key -var MashupKey embed.FS - -var mashupCertPool *x509.CertPool - -func init() { - rand.Seed(time.Now().UnixNano()) - mashupCertBytes, err := MashupCert.ReadFile("tls/mashup.crt") - if err != nil { - fmt.Println("Cert read failure.") - return - } - - mashupBlock, _ := pem.Decode([]byte(mashupCertBytes)) - - mashupClientCert, parseErr := x509.ParseCertificate(mashupBlock.Bytes) - if parseErr != nil { - fmt.Println("Cert parse read failure.") - return - } - mashupCertPool = x509.NewCertPool() - mashupCertPool.AddCert(mashupClientCert) -} - var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") func randomString(n int) string { @@ -61,14 +24,6 @@ func randomString(n int) string { return string(b) } -type TrcShConfig struct { - Env string - EnvContext string // Current env context... - ConfigRole string - PubRole string - KubeConfig string -} - const configDir = "/.tierceron/config.yml" const envContextPrefix = "envContext: " @@ -131,11 +86,10 @@ func GetSetEnvAddrContext(env string, envContext string, addrPort string) (strin } // Helper function for obtaining auth components. -func TrcshAuth(config *eUtils.DriverConfig) (*TrcShConfig, error) { - trcshConfig := &TrcShConfig{} +func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { + trcshConfig := &capauth.TrcShConfig{} var err error - fmt.Println("Auth phase 1") if config.EnvRaw == "staging" || config.EnvRaw == "prod" || len(config.TrcShellRaw) > 0 { dir, err := os.UserHomeDir() if err != nil { @@ -147,88 +101,96 @@ func TrcshAuth(config *eUtils.DriverConfig) (*TrcShConfig, error) { fmt.Println("No local kube config found...") os.Exit(1) } - trcshConfig.KubeConfig = base64.StdEncoding.EncodeToString(fileBytes) + kc := base64.StdEncoding.EncodeToString(fileBytes) + trcshConfig.KubeConfig = &kc if len(config.TrcShellRaw) > 0 { return trcshConfig, nil } } else { - trcshConfig.KubeConfig, err = PenseQuery("kubeconfig") + if agentConfig == nil { + fmt.Println("Auth phase 1") + trcshConfig.KubeConfig, err = capauth.PenseQuery(config, "kubeconfig") + } + } + + if err != nil { + return trcshConfig, err + } + if trcshConfig.KubeConfig != nil { + memprotectopts.MemProtect(nil, trcshConfig.KubeConfig) } + if agentConfig != nil { + trcshConfig.VaultAddress, err = agentConfig.PenseFeatherQuery("caddress") + } else { + fmt.Println("Auth phase 2") + trcshConfig.VaultAddress, err = capauth.PenseQuery(config, "caddress") + } if err != nil { return trcshConfig, err } - memprotectopts.MemProtect(nil, &trcshConfig.KubeConfig) - fmt.Println("Auth phase 2") - addr, vAddressErr := PenseQuery("vaddress") - if vAddressErr != nil { + memprotectopts.MemProtect(nil, trcshConfig.VaultAddress) + + if err != nil { var addrPort string var env, envContext string - fmt.Println(vAddressErr) + fmt.Println(err) //Env should come from command line - not context here. but addr port is needed. trcshConfig.Env, trcshConfig.EnvContext, addrPort, err = GetSetEnvAddrContext(env, envContext, addrPort) if err != nil { fmt.Println(err) return trcshConfig, err } - addr = "https://127.0.0.1:" + addrPort + vAddr := "https://127.0.0.1:" + addrPort + trcshConfig.VaultAddress = &vAddr config.Env = env config.EnvRaw = env } - config.VaultAddress = addr + config.VaultAddress = *trcshConfig.VaultAddress memprotectopts.MemProtect(nil, &config.VaultAddress) - fmt.Println("Auth phase 3") - trcshConfig.ConfigRole, err = PenseQuery("configrole") - if err != nil { - return trcshConfig, err + if agentConfig != nil { + trcshConfig.ConfigRole, err = agentConfig.PenseFeatherQuery("configrole") + } else { + fmt.Println("Auth phase 3") + trcshConfig.ConfigRole, err = capauth.PenseQuery(config, "configrole") } - memprotectopts.MemProtect(nil, &trcshConfig.ConfigRole) - - fmt.Println("Auth phase 4") - trcshConfig.PubRole, err = PenseQuery("pubrole") if err != nil { return trcshConfig, err } - memprotectopts.MemProtect(nil, &trcshConfig.PubRole) - fmt.Println("Auth complete.") - return trcshConfig, err -} + memprotectopts.MemProtect(nil, trcshConfig.ConfigRole) -func PenseQuery(pense string) (string, error) { - penseCode := randomString(7 + rand.Intn(7)) - penseArray := sha256.Sum256([]byte(penseCode)) - penseSum := hex.EncodeToString(penseArray[:]) - - capWriteErr := cap.TapWriter(penseSum) - if capWriteErr != nil { - fmt.Println("Code 54 failure...") - // 2023-06-30T01:29:21.7020686Z read unix @->/tmp/trccarrier/trcsnap.sock: read: connection reset by peer - os.Exit(-1) // restarting carrier will rebuild necessary resources... - return "", errors.Join(errors.New("Tap writer error"), capWriteErr) + if agentConfig == nil { + fmt.Println("Auth phase 4") + trcshConfig.PubRole, err = capauth.PenseQuery(config, "pubrole") + if err != nil { + return trcshConfig, err + } + memprotectopts.MemProtect(nil, trcshConfig.PubRole) } - conn, err := grpc.Dial("127.0.0.1:12384", grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ServerName: "", RootCAs: mashupCertPool, InsecureSkipVerify: true}))) + if agentConfig != nil { + trcshConfig.CToken, err = agentConfig.PenseFeatherQuery("ctoken") + } else { + fmt.Println("Auth phase 5") + trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") + if err != nil { + return trcshConfig, err + } + } if err != nil { - return "", err + return trcshConfig, err } - defer conn.Close() - c := cap.NewCapClient(conn) - // Contact the server and print out its response. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + memprotectopts.MemProtect(nil, trcshConfig.CToken) - r, penseErr := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) - if penseErr != nil { - return "", errors.Join(errors.New("pense error"), penseErr) - } + fmt.Println("Auth complete.") - return r.GetPense(), nil + return trcshConfig, err } diff --git a/trcsh/trcshauth/tls/mashup.crt b/trcsh/trcshauth/tls/mashup.crt deleted file mode 100644 index 6fef782a8..000000000 --- a/trcsh/trcshauth/tls/mashup.crt +++ /dev/null @@ -1,24 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEAzCCAuugAwIBAgIUaov2FENarRF8/Y4sSLo7MUORteYwDQYJKoZIhvcNAQEL -BQAwgZAxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQH -DAdTZWF0dGxlMRAwDgYDVQQKDAdUcmltYmxlMRIwEAYDVQQLDAlWaWV3cG9pbnQx -DTALBgNVBAMMBEpvZWwxJTAjBgkqhkiG9w0BCQEWFmpvZWxfcmlla2VAdHJpbWJs -ZS5jb20wHhcNMjIwNjI3MTYzMDQ3WhcNMjMwNjI3MTYzMDQ3WjCBkDELMAkGA1UE -BhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxEDAO -BgNVBAoMB1RyaW1ibGUxEjAQBgNVBAsMCVZpZXdwb2ludDENMAsGA1UEAwwESm9l -bDElMCMGCSqGSIb3DQEJARYWam9lbF9yaWVrZUB0cmltYmxlLmNvbTCCASIwDQYJ -KoZIhvcNAQEBBQADggEPADCCAQoCggEBAK4kYJ1uOPW1HJ5Ly40sjTnjV31NQMdA -x+G00nI3jgBw0A3+SAGiKCZazbuPdp48Ag7aAzv6C1S/EWs/pTP/00oJeJptOkB+ -Duk+iPpTFgZLl5sEhqUFxGld6IUfFs8S0zVrktioZxsn/PMBdf0HRqZeU2j/HDOq -OupGUvltQ7lleTBpmyjTmTcP3a/2GvXKRBi6GLpx07PYIl7sd5TBdfXTcq5Gf4Mm -3+39zNXCCuUW5YFtWMClYn6NE4wpC5s2NRC/2WopKgwQ4FyqkGBJ1s4PVGC3WBdz -dvdRNy5/y1UanKqxRiN3IcG9uvlyLN2KLO8fd9ujy9UbM3+AEBnJUe0CAwEAAaNT -MFEwHQYDVR0OBBYEFJtg/VCeqfOZNTx3XHIcZ4ywz3lsMB8GA1UdIwQYMBaAFJtg -/VCeqfOZNTx3XHIcZ4ywz3lsMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggEBADrvLT4/VAE88B6gMJyxtMJoLEcXrhczQ4BGlcJ8HAc5udRsjwMcaAZ2 -WDyCB+zjeszEpA0dCsDIEvQKVvdkQ7OdRHqFipPIkVV8H2R3n8+e4FXqVtQFRxWj -sv0ByvwGsrTIUZkY1g6b0EPfbBkjrXtG2riX1Q6c2C0M1E3QSBDif60TQcPaD3vX -o9XuY14/qmlvsGtGwNePMXNZADXmobPGT/DiUIG6p6bPm9kbcTM4m2071l6jZEc3 -oTrzEcZR8Zwn1sdUFPVU6r/Y4Y+ASReEjQO5T5lgIRwQj0j9w12ItbQUnqX4V6Lu -4MLSXbBpX5KbToDJj19t1jOdpikPYxo= ------END CERTIFICATE----- diff --git a/trcsh/trcshauth/tls/mashup.key b/trcsh/trcshauth/tls/mashup.key deleted file mode 100644 index fcbaeaa5c..000000000 --- a/trcsh/trcshauth/tls/mashup.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCuJGCdbjj1tRye -S8uNLI0541d9TUDHQMfhtNJyN44AcNAN/kgBoigmWs27j3aePAIO2gM7+gtUvxFr -P6Uz/9NKCXiabTpAfg7pPoj6UxYGS5ebBIalBcRpXeiFHxbPEtM1a5LYqGcbJ/zz -AXX9B0amXlNo/xwzqjrqRlL5bUO5ZXkwaZso05k3D92v9hr1ykQYuhi6cdOz2CJe -7HeUwXX103KuRn+DJt/t/czVwgrlFuWBbVjApWJ+jROMKQubNjUQv9lqKSoMEOBc -qpBgSdbOD1Rgt1gXc3b3UTcuf8tVGpyqsUYjdyHBvbr5cizdiizvH3fbo8vVGzN/ -gBAZyVHtAgMBAAECggEBAJfxCZOTPoO0eLntbDdLbhXaNNYRd7ni3gQjX5NkwbPE -uuPawD0cnWeJxwQl0tuRhEItGrFUJFrXM32eVKigSFzLEr27HB51XtFITQANSqRH -hIovUsPd2lgNJfow/BfFVx+/MqtzlDLvmxb0aVxS4uP6XVC5doPlve33WiJjobm6 -TX3MLK3qhvSStWJ6TUXBmfdm76ocmg8NCE3+B6GpGdXdvkBpKkmWtO5fGraGQ5cr -vcbOgzCrao7X/zeKbw+EMJQhv9dgChdOhD5IfLNNj5lqIssZysfnaEh312tirlPq -C6Mt06JEF85C205yF+1LurnVtmDw4rMlY5r7xuurkokCgYEA2VzNowYvB8KxLfcJ -hOzrM3DoZ2I3+1hbrsceSq+6LrrY1ciHcFFC/x4MBUiNO0t5Xs28Koq85tSSYp92 -G/jLkLMfGwsrKw4BLIIfcWxZss345pC2hgAVzmFrTf0/EUIClG3Q1P8rYLyrNVRk -ssnFJHhysupsqnNZx/+mjdePeMsCgYEAzRjOZTO0GWe4NRW7fj500dk/1qRJsVaF -U+RHYrMm8P3KRTq9MZ5/AGfYvViJB85WtK/ASSN14IToY552Lm/krIMm73N89JBo -2WldAQxDuI77yk8y/6irGYVublcAe/DoBgeYO/3AMJMPTz6Bttvas4nA7Gq7SDk9 -KLOvtCJbYScCgYEArTYgBJaL6dJDIQ7ofg3STBbgjGoflGLQBNJWGEw6wLxSyOHY -1C9dhTFrgNZU/XhiiqLGHNycNhIVaRQLqbS0e2XSFtnql4+8bbxv1BH01OEARLPz -T4tH1UOan/Z8SINizOdJ8EI7TgZrJL9Q04yoVizJsjvPRnFPjWnzTrsFcMsCgYBw -8jc/fkCxOPA1s7kkJcr5gZVpY+H9WjNkZrN1REmur2TDyTxFZg50r7upUmdsOBKB -wMO4w2F3+llr+ukEW7Rn2HYJqeTnXduP1zlTiWiF65/34rOoHT/UdynAdAMdPibY -ozgP2YcMeQAWWMrgY6TF+JeY2pI2jOcEUifxoHYIQwKBgQCbqzTBkZtZ0cikElmU -DIn+lluMLb8fh5X1VdLMTouWOIuYacKDXIhGTTQabLVAvmttVtLvixf1ABQgvrO6 -3KM4/SSCoMbrIiOTFbMFdGbe270lNzLtcTgYBQllZ7bHzmIw7Tp0xBD7mx9M8Vz4 -hySFolxFcPmQsrIfm403erpoVg== ------END PRIVATE KEY----- diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index aa2d34c72..3c6b6d162 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -36,10 +36,11 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"sub.log", "Output path for log files") projectInfoPtr := flag.Bool("projectInfo", false, "Lists all project info") filterTemplatePtr := flag.String("templateFilter", "", "Specifies which templates to filter") + templatePathsPtr := flag.String("templatePaths", "", "Specifies which specific templates to download.") flag.Parse() - if len(*filterTemplatePtr) == 0 && !*projectInfoPtr { + if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") os.Exit(1) } @@ -80,7 +81,11 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { eUtils.CheckError(config, err, true) mod.Env = *envPtr - if *projectInfoPtr { + if *templatePathsPtr != "" { + fmt.Printf("Downloading templates from vault to %s\n", *dirPtr) + // The actual download templates goes here. + il.DownloadTemplates(config, mod, *dirPtr, logger, templatePathsPtr) + } else if *projectInfoPtr { templateList, err := mod.List("templates/", logger) if err != nil { eUtils.CheckError(config, err, true) diff --git a/trcvault/carrierfactory/backend.go b/trcvault/carrierfactory/backend.go index 6687aec73..27bf89afb 100644 --- a/trcvault/carrierfactory/backend.go +++ b/trcvault/carrierfactory/backend.go @@ -10,13 +10,14 @@ import ( "strings" "sync" + "github.com/trimble-oss/tierceron/buildopts" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" + "github.com/trimble-oss/tierceron/trcflow/deploy" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/opts/prod" trcvutils "github.com/trimble-oss/tierceron/trcvault/util" eUtils "github.com/trimble-oss/tierceron/utils" - helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" kv "github.com/hashicorp/vault-plugin-secrets-kv" @@ -91,12 +92,22 @@ func Init(processFlowConfig trcvutils.ProcessFlowConfig, processFlowInit trcvuti supportedPluginNames = pluginEnvConfig["pluginNameList"].([]string) } } + // Range over all plugins and init them... but only once! for _, pluginName := range pluginEnvConfig["pluginNameList"].([]string) { pluginEnvConfigClone := make(map[string]interface{}) + logger.Printf("Cloning %d..\n", len(pluginEnvConfig)) for k, v := range pluginEnvConfig { - pluginEnvConfigClone[k] = v + if _, okStr := v.(string); okStr { + v2 := strings.Clone(v.(string)) + memprotectopts.MemProtect(nil, &v2) + pluginEnvConfigClone[k] = v2 + } else if _, okBool := v.(bool); okBool { + pluginEnvConfigClone[k] = v + } } + logger.Printf("Cloned %d..\n", len(pluginEnvConfigClone)) + pluginEnvConfigClone["trcplugin"] = pluginName logger.Println("*****Env: " + pluginEnvConfig["env"].(string) + " plugin: " + pluginEnvConfigClone["trcplugin"].(string)) pecError := ProcessPluginEnvConfig(processFlowConfig, processFlow, pluginEnvConfigClone, configCompleteChan) @@ -357,7 +368,7 @@ func ProcessPluginEnvConfig(processFlowConfig trcvutils.ProcessFlowConfig, configCompleteChan <- true } if flowErr != nil { - l.Println("Flow had an error: " + flowErr.Error()) + l.Printf("Flow %s had an error: %s\n", pec["trcplugin"].(string), flowErr.Error()) } }(pluginEnvConfig, logger) @@ -490,12 +501,6 @@ func TrcCreate(ctx context.Context, req *logical.Request, data *framework.FieldD return nil, errors.New("vault Certify token required") } - if caddr, addressOk := data.GetOk("caddress"); addressOk { - tokenEnvMap["caddress"] = caddr.(string) - } else { - return nil, errors.New("Vault Certify Url required.") - } - tokenEnvMap["env"] = req.Path // Check that some fields are given @@ -568,23 +573,58 @@ func TrcUpdate(ctx context.Context, req *logical.Request, reqData *framework.Fie } logger.Println("Creating modifier for env: " + req.Path) - // Plugins - mod, err := helperkv.NewModifier(true, tokenEnvMap["ctoken"].(string), tokenEnvMap["caddress"].(string), req.Path, nil, true, logger) - if mod != nil { - defer mod.Release() + pluginConfig := map[string]interface{}{} + pluginConfig = buildopts.ProcessPluginEnvConfig(pluginConfig) //contains logNamespace for InitVaultMod + if pluginConfig == nil { + logger.Println("Error: " + errors.New("Could not find plugin config").Error()) + return logical.ErrorResponse("Failed to find config for TrcUpdate."), nil + } + if strings.HasPrefix(plugin.(string), "trc-") && strings.HasPrefix(plugin.(string), "-plugin") { + ns := strings.Replace(plugin.(string), "trc-", "", 1) + namespace := strings.Replace(ns, "-plugin", "", 1) + pluginConfig["logNamespace"] = namespace } + + hostName, hostNameErr := os.Hostname() + if hostNameErr != nil { + logger.Println("Error: " + hostNameErr.Error() + " - 1") + return logical.ErrorResponse("Failed to find hostname"), nil + } else if hostName == "" { + logger.Println("Error: " + errors.New("Found empty hostname").Error()) + return logical.ErrorResponse("Found empty hostname"), nil + } + hostRegion := coreopts.GetRegion(hostName) + + pluginConfig["env"] = req.Path + pluginConfig["vaddress"] = tokenEnvMap["caddress"].(string) + pluginConfig["token"] = tokenEnvMap["ctoken"].(string) + pluginConfig["regions"] = []string{hostRegion} + cConfig, cMod, cVault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) if err != nil { + logger.Println("Error: " + err.Error() + " - 1") logger.Println("Failed to init mod for deploy update") - logger.Println("Error: " + err.Error()) - return logical.ErrorResponse("Failed to init mod for deploy update"), nil + return logical.ErrorResponse("Failed to create config for TrcUpdate."), nil + } + if cMod != nil { + defer cMod.Release() } - mod.Env = req.Path + cMod.Env = req.Path logger.Println("TrcCarrierUpdate getting plugin settings for env: " + req.Path) // The following confirms that this version of carrier has been certified to run... // It will bail if it hasn't. - writeMap, err := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + tokenEnvMap["trcplugin"].(string) + "/Certify") + cMod.SectionName = "trcplugin" + cMod.SectionKey = "/Index/" + cMod.SubSectionValue = plugin.(string) + + properties, err := trcvutils.NewProperties(cConfig, cVault, cMod, cMod.Env, "TrcVault", "Certify") if err != nil { + logger.Println("Error: " + err.Error()) + return logical.ErrorResponse("Failed to read previous plugin status from vault - 1"), nil + } + + writeMap, _ := properties.GetPluginData(hostRegion, "Certify", "config", logger) + if writeMap == nil { logger.Println("Failed to read previous plugin status from vault") logger.Println("Error: " + err.Error()) return logical.ErrorResponse("Failed to read previous plugin status from vault"), nil @@ -625,6 +665,11 @@ func TrcUpdate(ctx context.Context, req *logical.Request, reqData *framework.Fie return nil, errors.New("input data validation error") } + if !deploy.IsCapInitted() { + // Keep trying to initialize capauth whenever there is a refresh... + deploy.PluginDeployEnvFlow(tokenEnvMap, logger) + } + logger.Println("TrcCarrierUpdate merging tokens.") if key == "" { return logical.ErrorResponse("missing path"), nil @@ -728,6 +773,10 @@ func TrcFactory(ctx context.Context, conf *logical.BackendConfig) (logical.Backe Type: framework.TypeString, Description: "Vault Url for plugin certification purposes.", }, + "ctoken": { + Type: framework.TypeString, + Description: "Token for plugin certification purposes.", + }, "plugin": { Type: framework.TypeString, Description: "Optional plugin name.", diff --git a/trcvault/carrierfactory/capauth/capauth.go b/trcvault/carrierfactory/capauth/capauth.go deleted file mode 100644 index 7597d1d55..000000000 --- a/trcvault/carrierfactory/capauth/capauth.go +++ /dev/null @@ -1,115 +0,0 @@ -package capauth - -import ( - "crypto/tls" - "fmt" - "log" - "os" - "os/user" - "strconv" - "sync" - - "github.com/trimble-oss/tierceron/trcsh/trcshauth" - - "github.com/trimble-oss/tierceron-hat/cap" - "github.com/trimble-oss/tierceron/vaulthelper/kv" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" -) - -var onceMemo sync.Once - -// CheckNotSudo -- checks if current user is sudoer and exits if they are. -func CheckNotSudo() { - sudoer, sudoErr := user.LookupGroup("sudo") - if sudoErr != nil { - fmt.Println("Trcsh unable to definitively identify sudoers.") - os.Exit(-1) - } - sudoerGid, sudoConvErr := strconv.Atoi(sudoer.Gid) - if sudoConvErr != nil { - fmt.Println("Trcsh unable to definitively identify sudoers. Conversion error.") - os.Exit(-1) - } - groups, groupErr := os.Getgroups() - if groupErr != nil { - fmt.Println("Trcsh unable to definitively identify sudoers. Missing groups.") - os.Exit(-1) - } - for _, groupId := range groups { - if groupId == sudoerGid { - fmt.Println("Trcsh cannot be run with user having sudo privileges.") - os.Exit(-1) - } - } - -} - -func Init(mod *kv.Modifier, pluginConfig map[string]interface{}, logger *log.Logger) error { - - certifyMap, err := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/trcsh/Certify") - if err != nil { - return err - } - - if _, ok := certifyMap["trcsha256"]; ok { - logger.Println("Registering cap auth.") - go func() { - retryCap := 0 - for retryCap < 5 { - //err := cap.Tap("/home/jrieke/workspace/Github/tierceron/trcvault/deploy/target/trcsh", certifyMap["trcsha256"].(string), "azuredeploy", true) - //err := cap.Tap("/home/jrieke/workspace/Github/tierceron/trcsh/__debug_bin", certifyMap["trcsha256"].(string), "azuredeploy", true) - err := cap.Tap("/home/azuredeploy/bin/trcsh", certifyMap["trcsha256"].(string), "azuredeploy", false) - if err != nil { - logger.Println("Cap failure with error: " + err.Error()) - retryCap++ - } else { - retryCap = 0 - } - } - logger.Println("Mad hat cap failure.") - }() - } - return nil -} - -func Memorize(memorizeFields map[string]interface{}, logger *log.Logger) { - for key, value := range memorizeFields { - switch key { - case "vaddress", "pubrole", "configrole", "kubeconfig": - logger.Println("Memorizing: " + key) - cap.TapMemorize(key, value.(string)) - default: - logger.Println("Skipping key: " + key) - } - } -} - -// Things to make available to trusted agent. -func Start(logger *log.Logger) error { - mashupCertBytes, err := trcshauth.MashupCert.ReadFile("tls/mashup.crt") - if err != nil { - logger.Printf("Couldn't load cert: %v\n", err) - return err - } - - mashupKeyBytes, err := trcshauth.MashupKey.ReadFile("tls/mashup.key") - if err != nil { - logger.Printf("Couldn't load key: %v\n", err) - return err - } - - cert, err := tls.X509KeyPair(mashupCertBytes, mashupKeyBytes) - if err != nil { - logger.Printf("Couldn't load cert: %v\n", err) - return err - } - creds := credentials.NewServerTLSFromCert(&cert) - logger.Println("Tapping server.") - - // TODO: make port configured and stored in vault. - cap.TapServer("127.0.0.1:12384", grpc.Creds(creds)) - logger.Println("Server tapped.") - - return nil -} diff --git a/trcvault/carrierfactory/servercapauth/capauth.go b/trcvault/carrierfactory/servercapauth/capauth.go new file mode 100644 index 000000000..cd2320556 --- /dev/null +++ b/trcvault/carrierfactory/servercapauth/capauth.go @@ -0,0 +1,166 @@ +package servercapauth + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "sync" + + "github.com/trimble-oss/tierceron-hat/cap" + "github.com/trimble-oss/tierceron-hat/cap/tap" + "github.com/trimble-oss/tierceron/capauth" + "github.com/trimble-oss/tierceron/vaulthelper/kv" + "google.golang.org/grpc" +) + +var onceMemo sync.Once + +type FeatherAuth struct { + EncryptPass string + EncryptSalt string + HandshakePort string + SecretsPort string + HandshakeCode string +} + +var trcshaPath string = "/home/azuredeploy/bin/trcsh" + +func ValidatePathSha(mod *kv.Modifier, pluginConfig map[string]interface{}, logger *log.Logger) (bool, error) { + + certifyMap, err := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/trcsh/Certify") + if err != nil { + return false, err + } + + if _, ok := certifyMap["trcsha256"]; ok { + h := sha256.New() + + peerExe, err := os.Open(trcshaPath) + if err != nil { + return false, err + } + defer peerExe.Close() + + if _, err := io.Copy(h, peerExe); err != nil { + return false, err + } + + if certifyMap["trcsha256"].(string) == hex.EncodeToString(h.Sum(nil)) { + return true, nil + } + } + return false, errors.New("missing certification") +} + +func Init(mod *kv.Modifier, pluginConfig map[string]interface{}, logger *log.Logger) (*FeatherAuth, error) { + + certifyMap, err := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/trcsh/Certify") + if err != nil { + return nil, err + } + + if _, ok := certifyMap["trcsha256"]; ok { + logger.Println("Registering cap auth.") + go func() { + retryCap := 0 + for retryCap < 5 { + //err := cap.Tap("/home/jrieke/workspace/Github/tierceron/trcvault/deploy/target/trcsh", certifyMap["trcsha256"].(string), "azuredeploy", true) + //err := tap.Tap("/home/jrieke/workspace/Github/tierceron/trcsh/__debug_bin", certifyMap["trcsha256"].(string), "azuredeploy", true) + err := tap.Tap("/home/azuredeploy/bin/trcsh", certifyMap["trcsha256"].(string), "azuredeploy", false) + if err != nil { + logger.Println("Cap failure with error: " + err.Error()) + retryCap++ + } else { + retryCap = 0 + } + } + logger.Println("Mad hat cap failure.") + }() + } + if pluginConfig["env"] == "staging" || pluginConfig["env"] == "prod" { + // Feathering not supported in staging/prod at this time. + return nil, nil + } + featherMap, _ := mod.ReadData("super-secrets/Restricted/TrcshAgent/config") + // TODO: enable error validation when secrets are stored... + // if err != nil { + // return nil, err + // } + if featherMap != nil { + if _, ok := featherMap["trcHatEncryptPass"]; ok { + if _, ok := featherMap["trcHatEncryptSalt"]; ok { + if _, ok := featherMap["trcHatHandshakePort"]; ok { + if _, ok := featherMap["trcHatHandshakeCode"]; ok { + if _, ok := featherMap["trcHatSecretsPort"]; ok { + featherAuth := &FeatherAuth{EncryptPass: featherMap["trcHatEncryptPass"].(string), EncryptSalt: featherMap["trcHatEncryptSalt"].(string), HandshakePort: featherMap["trcHatHandshakePort"].(string), SecretsPort: featherMap["trcHatSecretsPort"].(string), HandshakeCode: featherMap["trcHatHandshakeCode"].(string)} + return featherAuth, nil + } + } + } + } + } + } + + return nil, nil +} + +func Memorize(memorizeFields map[string]interface{}, logger *log.Logger) { + for key, value := range memorizeFields { + switch key { + case "trcHatSecretsPort": + // Insecure things can be remembered here... + logger.Println("EyeRemember: " + key) + tap.TapEyeRemember(key, value.(string)) + case "vaddress", "caddress", "ctoken", "configrole": + cap.TapFeather(key, value.(string)) + fallthrough + case "pubrole", "kubeconfig": + logger.Println("Memorizing: " + key) + cap.TapMemorize(key, value.(string)) + default: + logger.Println("Skipping key: " + key) + } + } +} + +// Things to make available to trusted agent. +func Start(featherAuth *FeatherAuth, env string, logger *log.Logger) error { + logger.Println("Cap server.") + + creds, credErr := capauth.GetServerCredentials(logger) + if credErr != nil { + logger.Printf("Couldn't server creds: %v\n", creds) + return credErr + } + + logger.Println("Cap creds.") + + localip, err := capauth.LocalIp(env) + if err != nil { + logger.Printf("Couldn't load ip: %v\n", err) + return err + } + + if featherAuth != nil { + logger.Println("Feathering server.") + go cap.Feather(featherAuth.EncryptPass, + featherAuth.EncryptSalt, + fmt.Sprintf("%s:%s", localip, featherAuth.HandshakePort), + featherAuth.HandshakeCode, + func(int, string) bool { + return true + }, + ) + logger.Println("Feathered server.") + } + + logger.Println("Tapping server.") + cap.TapServer(fmt.Sprintf("%s:%s", localip, featherAuth.SecretsPort), grpc.Creds(creds)) + logger.Println("Server tapped.") + + return nil +} diff --git a/trcvault/deploy/deploy.sh b/trcvault/deploy/deploy.sh index e09fdc583..ef31a6c1b 100755 --- a/trcvault/deploy/deploy.sh +++ b/trcvault/deploy/deploy.sh @@ -22,11 +22,13 @@ fi FILESHAVAL=$(cat $FILESHA) +echo "Enter agent environment: " +read VAULT_ENV + +if [[ -z "${VAULT_ADDR}" ]]; then echo "Enter agent vault host base url: " read VAULT_ADDR - -echo "Enter vault host name: " -read VAULT_HOSTNAME +fi if [[ -z "${SECRET_VAULT_ADDR}" ]]; then echo "Enter organization vault host base url including port: " @@ -34,20 +36,18 @@ read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_ENV_TOKEN}" ]]; then -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read SECRET_VAULT_ENV_TOKEN fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then -echo "Enter organization vault plugin token for certification: " +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " read SECRET_VAULT_PLUGIN_TOKEN fi echo "Enter agent root token: " read VAULT_TOKEN -echo "Enter agent environment: " -read VAULT_ENV echo "Is this plugin an agent deployment tool (Y or N): " read VAULT_AGENT @@ -55,13 +55,16 @@ read VAULT_AGENT if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then PRE_CERTIFY="Y" else - echo "Enter trc plugin runtime environment token with write permissions unrestricted: " - read VAULT_ENV_TOKEN + if [[ -z "${VAULT_ENV_TOKEN}" ]]; then + echo "Enter trc plugin runtime environment token with write permissions unrestricted(config_token_"$VAULT_ENV"_unrestricted): " + read VAULT_ENV_TOKEN + fi if [ "$VAULT_PLUGIN_DIR" ] then echo "Deploying using local vault strategy." - PRE_CERTIFY="N" + echo "Precertify plugin (Y or N): " + read PRE_CERTIFY else echo "Precertify plugin (Y or N): " read PRE_CERTIFY @@ -71,18 +74,26 @@ fi if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = "y" ]; then if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then - echo "Deploying agent deploy tool" - trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent -hostName=$VAULT_HOSTNAME + echo "Certifying agent deployment tool plugin..." + trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent certifystatus=$? if [ $certifystatus -eq 0 ]; then - echo "No problems encountered." + echo "No certification problems encountered." exit $certifystatus else - echo "Unexpected certifyication errorerror." + echo "Unexpected certification error." exit $certifystatus fi else - trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -hostName=$VAULT_HOSTNAME + echo "Certifying vault type plugin..." + trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) + certifystatus=$? + if [ $certifystatus -eq 0 ]; then + echo "No certification problems encountered." + else + echo "Unexpected certification error." + exit $certifystatus + fi fi fi @@ -92,13 +103,13 @@ fi if [ "$VAULT_PLUGIN_DIR" ] then - echo "Local plugin registration skipping certification." + echo "Local plugin registration skipping certified check." else echo "Checking plugin deploy status." - echo "Certifying plugin for env $VAULT_ENV." - trcplgtool -env=$VAULT_ENV -checkDeployed -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -hostName=$VAULT_HOSTNAME + echo "Checking deployment status on plugin for env $VAULT_ENV." + trcplgtool -env=$VAULT_ENV -checkDeployed -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) status=$? - echo "Plugin certified with result $status." + echo "Plugin deployment had status result $status." if [ $status -eq 0 ]; then echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." @@ -163,16 +174,10 @@ if [ "$SHAVAL" != "$FILESHAVAL" ]; then exit -1 fi -PROD_EXT="" -for x in "staging" "prod"; do - if [ $x = $VAULT_ENV ]; then - PROD_EXT="-prod" - fi -done echo "Registering new plugin." vault plugin register \ - -command=$TRC_PLUGIN_NAME$PROD_EXT \ + -command=$TRC_PLUGIN_NAME \ -sha256=$(echo $SHAVAL) \ -args=`backendUUID=789` \ $TRC_PLUGIN_NAME @@ -186,6 +191,10 @@ vault secrets enable \ #Activates/starts the deployed plugin. # Note: plugin should update deployed flag for itself. + +echo "Activating plugin." vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN -vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME \ No newline at end of file +echo "Notifying carrier." +vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME +echo "Deployment complete." diff --git a/trcvault/deploy/deployupdate.sh b/trcvault/deploy/deployupdate.sh index 0be4ed0ab..f65bff8db 100755 --- a/trcvault/deploy/deployupdate.sh +++ b/trcvault/deploy/deployupdate.sh @@ -29,10 +29,10 @@ read VAULT_TOKEN echo "Enter environment: " read VAULT_ENV -echo "Enter trc plugin runtime environment token with write permissions unrestricted: " +echo "Enter trc plugin runtime environment token with write permissions unrestricted(config_token_"$VAULT_ENV"_unrestricted): " read VAULT_ENV_TOKEN -echo "Enter carrier deployment runtime token pluginEnv: " +echo "Enter carrier deployment runtime token pluginEnv(config_token_plugin$VAULT_ENV): " read VAULT_CARRIER_DEPLOY_TOKEN if [ "$VAULT_PLUGIN_DIR" ] diff --git a/trcvault/deploy/refreshcarriertoken.sh b/trcvault/deploy/refreshcarriertoken.sh index 68f7f67c3..5e9e6d9f6 100755 --- a/trcvault/deploy/refreshcarriertoken.sh +++ b/trcvault/deploy/refreshcarriertoken.sh @@ -1,7 +1,12 @@ #!/bin/bash -if [[ -z "${VAULT_ADDR}" ]]; then +if [[ -z "${VAULT_ENV}" ]]; then +echo "Enter vault environment being configured: " +read VAULT_ENV +fi + +if [[ -z "${AGENT_VAULT_ADDR}" ]]; then echo "Enter agent vault host base url including port: " -read VAULT_ADDR +read AGENT_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_ADDR}" ]]; then @@ -9,18 +14,18 @@ echo "Enter organization vault host base url including port (hit enter if just r read SECRET_VAULT_ADDR fi +if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +read SECRET_VAULT_PLUGIN_TOKEN +fi + if [[ -z "${VAULT_TOKEN}" ]]; then echo "Enter agent vault root token: " read VAULT_TOKEN fi -if [[ -z "${VAULT_ENV}" ]]; then -echo "Enter vault environment being configured: " -read VAULT_ENV -fi - if [[ -z "${VAULT_ENV_TOKEN}" ]]; then -echo "Enter organization vault *plugin* environment token with tightly confined write permissions: " +echo "Enter agent vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " read VAULT_ENV_TOKEN fi @@ -49,4 +54,4 @@ export VAULT_API_ADDR echo $VAULT_ADDR -vault write vaultcarrier/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$VAULT_ENV_TOKEN pubrole=$SECRET_VAULT_PUB_ROLE configrole=$SECRET_VAULT_CONFIG_ROLE kubeconfig=$TRC_KUBE_CONFIG +vault write vaultcarrier/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$AGENT_VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN pubrole=$SECRET_VAULT_PUB_ROLE configrole=$SECRET_VAULT_CONFIG_ROLE kubeconfig=$TRC_KUBE_CONFIG diff --git a/trcvault/deploy/refreshtoken.sh b/trcvault/deploy/refreshtoken.sh index 66c2cd193..091426635 100755 --- a/trcvault/deploy/refreshtoken.sh +++ b/trcvault/deploy/refreshtoken.sh @@ -8,32 +8,49 @@ if [ "$TRC_PLUGIN_NAME" = 'trc-vault-carrier-plugin' ] ; then exit 1 fi +echo "Enter environment: " +read VAULT_ENV + if [[ -z "${SECRET_VAULT_ADDR}" ]]; then echo "Enter organization vault host base url including port (hit enter if just refreshing org tokens): " read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_ENV_TOKEN}" ]]; then -echo "Enter organization vault *plugin* environment token with tightly confined write permissions: " +echo "Enter organization vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " read SECRET_ENV_TOKEN fi -echo "Enter organization vault host base url including port: " +if [[ -z "${VAULT_ADDR}" ]]; then +echo "Enter agent vault host base url including port: " read VAULT_ADDR +fi -echo "Enter organization vault root token: " +if [[ -z "${VAULT_TOKEN}" ]]; then +echo "Enter agent vault root token: " read VAULT_TOKEN +fi -echo "Enter environment: " -read VAULT_ENV - -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read VAULT_ENV_TOKEN -VAULT_API_ADDR=VAULT_ADDR +VAULT_API_ADDR=$VAULT_ADDR +VAULT_API_TOKEN=$VAULT_TOKEN export VAULT_ADDR export VAULT_TOKEN export VAULT_API_ADDR +export VAULT_API_TOKEN + +echo "Secret vault: " $SECRET_VAULT_ADDR +echo "Agent vault: " $VAULT_ADDR + +echo "Agent and secrets stored in secrets vault only? (Y or N): " +read SECRETS_ONLY + +if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_API_ADDR=$SECRET_VAULT_ADDR + VAULT_API_TOKEN=$SECRET_ENV_TOKEN +fi -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_API_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME diff --git a/trcvault/factory/backend.go b/trcvault/factory/backend.go index 39583f955..9a2aca1e1 100644 --- a/trcvault/factory/backend.go +++ b/trcvault/factory/backend.go @@ -11,12 +11,13 @@ import ( "strings" "time" + "github.com/trimble-oss/tierceron/buildopts" + "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/opts/prod" trcvutils "github.com/trimble-oss/tierceron/trcvault/util" eUtils "github.com/trimble-oss/tierceron/utils" - helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" kv "github.com/hashicorp/vault-plugin-secrets-kv" @@ -136,6 +137,7 @@ func StartPluginSettingEater() { }() } +// Only push to sha256 chan if one is present. Non blocking otherwise. func PushPluginSha(config *eUtils.DriverConfig, pluginConfig map[string]interface{}, vaultPluginSignature map[string]interface{}) { if _, trcShaChanOk := pluginConfig["trcsha256chan"]; trcShaChanOk { if vaultPluginSignature != nil { @@ -231,14 +233,14 @@ func ProcessPluginEnvConfig(processFlowConfig trcvutils.ProcessFlowConfig, return errors.New("missing address") } - ctoken, tOk := pluginEnvConfig["ctoken"] - if !tOk || ctoken.(string) == "" { + ctoken, cTOk := pluginEnvConfig["ctoken"] + if !cTOk || ctoken.(string) == "" { logger.Println("Bad configuration data for env: " + env.(string) + ". Missing ctoken.") - return errors.New("missing token") + return errors.New("missing ctoken") } - caddress, aOk := pluginEnvConfig["caddress"] - if !aOk || caddress.(string) == "" { + caddress, caOk := pluginEnvConfig["caddress"] + if !caOk || caddress.(string) == "" { logger.Println("Bad configuration data for env: " + env.(string) + ". Missing caddress.") return errors.New("missing address") } @@ -388,16 +390,9 @@ func TrcRead(ctx context.Context, req *logical.Request, data *framework.FieldDat return nil, mTokenErr } tokenEnvMap["token"] = vData["token"] - if cAddr, tokenOK := data.GetOk("caddr"); tokenOK { - tokenEnvMap["caddr"] = cAddr - } else { - return nil, errors.New("caddr required.") - } - if cToken, tokenOK := data.GetOk("ctoken"); tokenOK { - tokenEnvMap["ctoken"] = cToken - } else { - return nil, errors.New("ctoken required.") - } + tokenEnvMap["caddress"] = vData["caddress"] + tokenEnvMap["ctoken"] = vData["ctoken"] + logger.Println("Read Pushing env: " + tokenEnvMap["env"].(string)) PushEnv(tokenEnvMap) //ctx.Done() @@ -426,10 +421,10 @@ func TrcCreate(ctx context.Context, req *logical.Request, data *framework.FieldD return nil, errors.New("Token required.") } - if cAddr, tokenOK := data.GetOk("caddr"); tokenOK { - tokenEnvMap["caddr"] = cAddr + if cAddr, tokenOK := data.GetOk("caddress"); tokenOK { + tokenEnvMap["caddress"] = cAddr } else { - return nil, errors.New("caddr required.") + return nil, errors.New("caddress required.") } if cToken, tokenOK := data.GetOk("ctoken"); tokenOK { tokenEnvMap["ctoken"] = cToken @@ -474,7 +469,7 @@ func TrcCreate(ctx context.Context, req *logical.Request, data *framework.FieldD } logger.Println("Create Pushing env: " + tokenEnvMap["env"].(string)) - tokenEnvChan <- tokenEnvMap + PushEnv(tokenEnvMap) //ctx.Done() logger.Println("TrcCreateUpdate complete.") @@ -488,7 +483,6 @@ func TrcCreate(ctx context.Context, req *logical.Request, data *framework.FieldD func TrcUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { logger.Println("TrcUpdate") tokenEnvMap := map[string]interface{}{} - var plugin interface{} pluginOk := false if plugin, pluginOk = data.GetOk("plugin"); pluginOk { @@ -504,33 +498,19 @@ func TrcUpdate(ctx context.Context, req *logical.Request, data *framework.FieldD if _, tokenOk := data.GetOk("token"); tokenOk { logger.Println("TrcUpdate stage 1") - if GetVaultPort() == "" { - logger.Println("TrcUpdate stage 1.1") - if vaddr, addressOk := data.GetOk("vaddress"); addressOk { - logger.Println("TrcUpdate stage 1.1.1") - vaultUrl, err := url.Parse(vaddr.(string)) + if vaddr, addressOk := data.GetOk("vaddress"); addressOk { + if _, err := url.Parse(vaddr.(string)); err == nil { tokenEnvMap["vaddress"] = vaddr.(string) - if err == nil { - logger.Println("TrcUpdate stage 1.1.1.1") - vaultPort = vaultUrl.Port() - } else { - logger.Println("Bad address: " + vaddr.(string)) - } - } else { - return nil, errors.New("Vault Update Url required.") + vaultHost = vaddr.(string) } - } - - if !strings.HasSuffix(vaultHost, GetVaultPort()) { - // Missing port. - vaultHost = vaultHost + ":" + GetVaultPort() + } else { + return nil, errors.New("Certification Vault Url required.") } if caddr, addressOk := data.GetOk("caddress"); addressOk { - vaultUrl, err := url.Parse(caddr.(string)) - if err == nil { - cVaultPort := vaultUrl.Port() - tokenEnvMap["caddress"] = vaultUrl.Host + cVaultPort + + if _, err := url.Parse(caddr.(string)); err == nil { + tokenEnvMap["caddress"] = caddr } } else { return nil, errors.New("Certification Vault Url required.") @@ -542,29 +522,54 @@ func TrcUpdate(ctx context.Context, req *logical.Request, data *framework.FieldD return nil, errors.New("Certification Vault token required.") } - if !strings.HasSuffix(vaultHost, GetVaultPort()) { - // Missing port. - vaultHost = vaultHost + ":" + GetVaultPort() + pluginConfig := map[string]interface{}{} + pluginConfig = buildopts.ProcessPluginEnvConfig(pluginConfig) //contains logNamespace for InitVaultMod + if pluginConfig == nil { + logger.Println("Error: " + errors.New("Could not find plugin config").Error()) + return logical.ErrorResponse("Failed to find config for TrcUpdate."), nil } - // Plugins - cMod, err := helperkv.NewModifier(true, tokenEnvMap["ctoken"].(string), tokenEnvMap["caddress"].(string), req.Path, nil, true, logger) - if cMod != nil { - defer cMod.Release() + hostName, hostNameErr := os.Hostname() + if hostNameErr != nil { + logger.Println("Error: " + hostNameErr.Error() + " - 1") + return logical.ErrorResponse("Failed to find hostname"), nil + } else if hostName == "" { + logger.Println("Error: " + errors.New("Found empty hostname").Error()) + return logical.ErrorResponse("Found empty hostname"), nil } + hostRegion := coreopts.GetRegion(hostName) + + pluginConfig["env"] = req.Path + pluginConfig["vaddress"] = tokenEnvMap["caddress"].(string) + pluginConfig["token"] = tokenEnvMap["ctoken"].(string) + pluginConfig["regions"] = []string{hostRegion} + cConfig, cMod, cVault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) if err != nil { + logger.Println("Error: " + err.Error() + " - 1") logger.Println("Failed to init mod for deploy update") - //ctx.Done() - logger.Println("Error: " + err.Error()) - return logical.ErrorResponse("Failed to init mod for deploy update"), nil + return logical.ErrorResponse("Failed to create config for TrcUpdate."), nil + } + if cMod != nil { + defer cMod.Release() } cMod.Env = req.Path logger.Println("TrcUpdate getting plugin settings for env: " + req.Path) - writeMap, err := cMod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + tokenEnvMap["trcplugin"].(string) + "/Certify") + + cMod.SectionName = "trcplugin" + cMod.SectionKey = "/Index/" + cMod.SubSectionValue = plugin.(string) + + properties, err := trcvutils.NewProperties(cConfig, cVault, cMod, cMod.Env, "TrcVault", "Certify") if err != nil { + logger.Println("Error: " + err.Error()) + return logical.ErrorResponse("Failed to read previous plugin status from vault - 1"), nil + } + + writeMap, _ := properties.GetPluginData(hostRegion, "Certify", "config", logger) + if writeMap == nil { logger.Println("Failed to read previous plugin status from vault") logger.Println("Error: " + err.Error()) - return logical.ErrorResponse("Failed to read previous plugin status from vault"), nil + return logical.ErrorResponse("Failed to read previous plugin status from vault - 2"), nil } logger.Println("TrcUpdate Checking sha") @@ -574,90 +579,87 @@ func TrcUpdate(ctx context.Context, req *logical.Request, data *framework.FieldD } cMod.Close() } - } - - // TODO: Verify token and env... - // Path includes Env and token will only work if it has right permissions. - tokenEnvMap["env"] = req.Path + // TODO: Verify token and env... + // Path includes Env and token will only work if it has right permissions. + tokenEnvMap["env"] = req.Path - if token, tokenOk := data.GetOk("token"); tokenOk { - tokenEnvMap["token"] = token - } else { - //ctx.Done() - return nil, errors.New("Token required.") - } + if token, tokenOk := data.GetOk("token"); tokenOk { + tokenEnvMap["token"] = token + } else { + //ctx.Done() + return nil, errors.New("Token required.") + } - if vaddr, addressOk := data.GetOk("vaddress"); addressOk { - vaultUrl, err := url.Parse(vaddr.(string)) - tokenEnvMap["vaddress"] = vaddr.(string) - if err == nil { - vaultPort = vaultUrl.Port() + if vaddr, addressOk := data.GetOk("vaddress"); addressOk { + vaultUrl, err := url.Parse(vaddr.(string)) + tokenEnvMap["vaddress"] = vaddr.(string) + if err == nil { + vaultPort = vaultUrl.Port() + } + } else { + return nil, errors.New("Vault Create Url required.") } - } else { - return nil, errors.New("Vault Create Url required.") - } - tokenEnvMap["vaddress"] = vaultHost + tokenEnvMap["vaddress"] = vaultHost - key := req.Path - if key == "" { - //ctx.Done() - return logical.ErrorResponse("missing path"), nil - } + key := req.Path + if key == "" { + //ctx.Done() + return logical.ErrorResponse("missing path"), nil + } - // Check that some fields are given - if len(req.Data) == 0 { - //ctx.Done() - return logical.ErrorResponse("missing data fields"), nil - } + // Check that some fields are given + if len(req.Data) == 0 { + //ctx.Done() + return logical.ErrorResponse("missing data fields"), nil + } - // JSON encode the data - buf, err := json.Marshal(req.Data) - if err != nil { - //ctx.Done() - return nil, fmt.Errorf("json encoding failed: %v", err) - } + // JSON encode the data + buf, err := json.Marshal(req.Data) + if err != nil { + //ctx.Done() + return nil, fmt.Errorf("json encoding failed: %v", err) + } - // Write out a new key - entry := &logical.StorageEntry{ - Key: key, - Value: buf, - } - if err := req.Storage.Put(ctx, entry); err != nil { - //ctx.Done() - return nil, fmt.Errorf("failed to write: %v", err) - } + // Write out a new key + entry := &logical.StorageEntry{ + Key: key, + Value: buf, + } + if err := req.Storage.Put(ctx, entry); err != nil { + //ctx.Done() + return nil, fmt.Errorf("failed to write: %v", err) + } - // This will kick off the main flow for the plugin.. - logger.Println("Update Pushing env: " + tokenEnvMap["env"].(string)) - tokenEnvChan <- tokenEnvMap + // This will kick off the main flow for the plugin.. + logger.Println("Update Pushing env: " + tokenEnvMap["env"].(string)) + PushEnv(tokenEnvMap) - if pluginOk { - // Listen on sha256 channel.... - var sha256 string - sha256, shaOk := pluginShaMap[tokenEnvMap["trcplugin"].(string)] + if pluginOk { + // Listen on sha256 channel.... + var sha256 string + sha256, shaOk := pluginShaMap[tokenEnvMap["trcplugin"].(string)] - select { - case <-pluginSettingsChan[tokenEnvMap["trcplugin"].(string)]: - sha256 = pluginShaMap[tokenEnvMap["trcplugin"].(string)] - case <-time.After(time.Second * 7): - if !shaOk { - sha256 = "Failure to copy plugin." + select { + case <-pluginSettingsChan[tokenEnvMap["trcplugin"].(string)]: + sha256 = pluginShaMap[tokenEnvMap["trcplugin"].(string)] + case <-time.After(time.Second * 7): + if !shaOk { + sha256 = "Failure to copy plugin." + } } + //ctx.Done() + + return &logical.Response{ + Data: map[string]interface{}{ + "message": sha256, + }, + }, nil } //ctx.Done() - - return &logical.Response{ - Data: map[string]interface{}{ - "message": sha256, - }, - }, nil } - //ctx.Done() - logger.Println("TrcUpdate complete.") - return &logical.Response{ Data: map[string]interface{}{ "message": "Token updated.", diff --git a/trcvault/plugins/carrier/main.go b/trcvault/plugins/carrier/main.go index 2f3ae4565..2abd1715b 100644 --- a/trcvault/plugins/carrier/main.go +++ b/trcvault/plugins/carrier/main.go @@ -4,7 +4,6 @@ import ( "fmt" "log" "os" - "runtime/debug" "strings" "github.com/trimble-oss/tierceron/buildopts" @@ -47,15 +46,11 @@ func main() { } buildopts.SetLogger(logger.Writer()) - defer func() { - if e := recover(); e != nil { - logger.Printf("%s: %s", e, debug.Stack()) - } - }() + carrierfactory.InitLogger(logger) e := os.Remove("/tmp/trccarrier/trcsnap.sock") if e != nil { - logger.Println("Unable to refresh socket") + logger.Println("Unable to refresh socket. Uneccessary.") } carrierfactory.Init(coreopts.ProcessDeployPluginEnvConfig, deploy.PluginDeployEnvFlow, deploy.PluginDeployFlow, true, logger) diff --git a/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl b/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl index a8d6260eb..873e43ad3 100644 --- a/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl +++ b/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl @@ -5,4 +5,6 @@ copied: {{.copied}} deployed: {{.deployed}} instances: {{.instances}} trctype: {{.trctype}} -trcpluginpath: {{.trcpluginpath}} \ No newline at end of file +trcdeployroot: {{.trcdeployroot}} +trccodebundle: {{.trccodebundle}} +trcservicename: {{.trcservicename}} \ No newline at end of file diff --git a/trcvault/trc_templates/TrcVault/TrcshAgent/config.yml.tmpl b/trcvault/trc_templates/TrcVault/TrcshAgent/config.yml.tmpl new file mode 100644 index 000000000..faa81b598 --- /dev/null +++ b/trcvault/trc_templates/TrcVault/TrcshAgent/config.yml.tmpl @@ -0,0 +1,7 @@ +trcHatEncryptPass {{.trcHatEncryptPass}} +trcHatEncryptSalt {{.trcHatEncryptSalt}} +trcHatEnv {{.trcHatEnv}} +trcHatHandshakeCode {{.trcHatHandshakeCode}} +trcHatHandshakePort {{.trcHatHandshakePort}} +trcHatHost {{.trcHatHost}} +trcHatSecretsPort {{.trcHatSecretsPort}} \ No newline at end of file diff --git a/trcvault/trcplgtool/trcplgtool.go b/trcvault/trcplgtool/trcplgtool.go index 0717c77b0..de529cfe0 100644 --- a/trcvault/trcplgtool/trcplgtool.go +++ b/trcvault/trcplgtool/trcplgtool.go @@ -1,13 +1,20 @@ package main import ( + "flag" "fmt" - plgt "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" + plgtbase "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" ) // This executable automates the cerification of a plugin docker image. func main() { fmt.Println("Version: " + "1.01") - plgt.PluginMain() + + envPtr := flag.String("env", "dev", "Environment to configure") + addrPtr := flag.String("addr", "", "API endpoint for the vault") + tokenPtr := flag.String("token", "", "Vault access token") + regionPtr := flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + + plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, nil) } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 46a173d2d..860d3c023 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -1,49 +1,78 @@ package trcplgtoolbase import ( + "bufio" + "crypto/sha256" "flag" "fmt" + "io" "log" "os" + "os/exec" + "strings" + "github.com/trimble-oss/tierceron/buildopts" "github.com/trimble-oss/tierceron/buildopts/coreopts" trcvutils "github.com/trimble-oss/tierceron/trcvault/util" "github.com/trimble-oss/tierceron/trcvault/util/repository" eUtils "github.com/trimble-oss/tierceron/utils" - helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" ) -func PluginMain() { - addrPtr := flag.String("addr", "", "API endpoint for the vault") - tokenPtr := flag.String("token", "", "Vault access token") - envPtr := flag.String("env", "dev", "Environement in vault") +func CommonMain(envPtr *string, + addrPtr *string, + tokenPtr *string, + regionPtr *string, + c *eUtils.DriverConfig) { + + // Main functions are as follows: + defineServicePtr := flag.Bool("defineService", false, "Service is defined.") + certifyImagePtr := flag.Bool("certify", false, "Used to certifies vault plugin.") + // These functions only valid for pluginType trcshservice + winservicestopPtr := flag.Bool("winservicestop", false, "To stop a windows service for a particular plugin.") + winservicestartPtr := flag.Bool("winservicestart", false, "To start a windows service for a particular plugin.") + codebundledeployPtr := flag.Bool("codebundledeploy", false, "To deploy a code bundle.") + agentdeployPtr := flag.Bool("agentdeploy", false, "To initiate deployment on agent.") + + // Common flags... startDirPtr := flag.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"plgtool.log", "Output path for log files") - certifyImagePtr := flag.Bool("certify", false, "Used to certifies vault plugin.") + + // defineService flags... + deployrootPtr := flag.String("deployroot", "", "Optional path for deploying services to.") + serviceNamePtr := flag.String("serviceName", "", "Optional name of service to use in managing service.") + codeBundlePtr := flag.String("codeBundle", "", "Code bundle to deploy.") + + // Common plugin flags... pluginNamePtr := flag.String("pluginName", "", "Used to certify vault plugin") pluginTypePtr := flag.String("pluginType", "vault", "Used to indicate type of plugin. Default is vault.") - pluginPathPtr := flag.String("pluginPathPtr", "", "Optional path for deploying services to.") + + // Certify flags... sha256Ptr := flag.String("sha256", "", "Used to certify vault plugin") //This has to match the image that is pulled -> then we write the vault. checkDeployedPtr := flag.Bool("checkDeployed", false, "Used to check if plugin has been copied, deployed, & certified") checkCopiedPtr := flag.Bool("checkCopied", false, "Used to check if plugin has been copied & certified") - hostNamePtr := flag.String("hostName", "", "Used to find overrides for plugin certification.") //This is used for plugin copied/deploy status. + certifyInit := false - args := os.Args[1:] - for i := 0; i < len(args); i++ { - s := args[i] - if s[0] != '-' { - fmt.Println("Wrong flag syntax: ", s) - os.Exit(1) + if c == nil || !c.IsShellSubProcess { + args := os.Args[1:] + for i := 0; i < len(args); i++ { + s := args[i] + if s[0] != '-' { + fmt.Println("Wrong flag syntax: ", s) + os.Exit(1) + } } - } - flag.Parse() + flag.Parse() - // Prints usage if no flags are specified - if flag.NFlag() == 0 { - flag.Usage() - os.Exit(1) + // Prints usage if no flags are specified + if flag.NFlag() == 0 { + flag.Usage() + os.Exit(1) + } + } else { + flag.CommandLine.Parse(os.Args) + flag.Parse() } if *certifyImagePtr && (len(*pluginNamePtr) == 0 || len(*sha256Ptr) == 0) { @@ -56,94 +85,324 @@ func PluginMain() { os.Exit(1) } + if *defineServicePtr && (len(*pluginNamePtr) == 0) { + fmt.Println("Must use -pluginName flag to use -defineService flag") + os.Exit(1) + } + + if strings.Contains(*pluginNamePtr, ".") { + fmt.Println("-pluginName cannot contain reserved character '.'") + os.Exit(1) + } + + if *agentdeployPtr || *winservicestopPtr || *winservicestartPtr || *codebundledeployPtr { + *pluginTypePtr = "trcshservice" + } + switch *pluginTypePtr { - case "vault": - case "agent": + case "vault": // A vault plugin + if c != nil { + // TODO: do we want to support Deployment certifications in the pipeline at some point? + // If so this is a config check to remove. + fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) + os.Exit(-1) + } + case "agent": // A deployment agent tool. + if c != nil { + // TODO: do we want to support Deployment certifications in the pipeline at some point? + // If so this is a config check to remove. + fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) + os.Exit(-1) + } + case "trcshservice": // A trcshservice managed microservice default: - fmt.Println("Unsupported plugin type: " + *pluginTypePtr) - os.Exit(1) + if !*agentdeployPtr { + fmt.Println("Unsupported plugin type: " + *pluginTypePtr) + os.Exit(1) + } } - // If logging production directory does not exist and is selected log to local directory - if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && *logFilePtr == "/var/log/"+coreopts.GetFolderPrefix(nil)+"plgtool.log" { - *logFilePtr = "./" + coreopts.GetFolderPrefix(nil) + "plgtool.log" + if *pluginTypePtr != "vault" { + *regionPtr = "" } - f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - logger := log.New(f, "[INIT]", log.LstdFlags) - config := &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true, StartDir: []string{*startDirPtr}, SubSectionValue: *pluginNamePtr} + var configBase *eUtils.DriverConfig + var logger *log.Logger + if c != nil { + configBase = c + logger = c.Log + configBase.SubSectionValue = *pluginNamePtr + *insecurePtr = configBase.Insecure + } else { + if *agentdeployPtr { + fmt.Println("Unsupported agentdeploy outside trcsh") + os.Exit(1) + } - eUtils.CheckError(config, err, true) + // If logging production directory does not exist and is selected log to local directory + if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && *logFilePtr == "/var/log/"+coreopts.GetFolderPrefix(nil)+"plgtool.log" { + *logFilePtr = "./" + coreopts.GetFolderPrefix(nil) + "plgtool.log" + } + f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + logger = log.New(f, "[INIT]", log.LstdFlags) - //Grabbing configs - mod, err := helperkv.NewModifier(*insecurePtr, *tokenPtr, *addrPtr, *envPtr, nil, true, logger) - if mod != nil { - defer mod.Release() + configBase = &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true, StartDir: []string{*startDirPtr}, SubSectionValue: *pluginNamePtr} + eUtils.CheckError(configBase, err, true) } + + regions := []string{} + + pluginConfig := map[string]interface{}{} + pluginConfig = buildopts.ProcessPluginEnvConfig(pluginConfig) //contains logNamespace for InitVaultMod + if pluginConfig == nil { + fmt.Println("Error: Could not find plugin config") + os.Exit(1) + } + pluginConfig["env"] = *envPtr + pluginConfig["vaddress"] = *addrPtr + if tokenPtr != nil { + pluginConfig["token"] = *tokenPtr + } + pluginConfig["ExitOnFailure"] = true + if *regionPtr != "" { + pluginConfig["regions"] = []string{*regionPtr} + } + config, mod, vault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) + config.FeatherCtlCb = configBase.FeatherCtlCb if err != nil { - eUtils.CheckError(config, err, true) + logger.Println("Error: " + err.Error() + " - 1") + logger.Println("Failed to init mod for deploy update") + os.Exit(1) } + config.StartDir = []string{*startDirPtr} + config.SubSectionValue = *pluginNamePtr mod.Env = *envPtr + eUtils.CheckError(config, err, true) + + if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") { + supportedRegions := eUtils.GetSupportedProdRegions() + if *regionPtr != "" { + for _, supportedRegion := range supportedRegions { + if *regionPtr == supportedRegion { + regions = append(regions, *regionPtr) + break + } + } + if len(regions) == 0 { + fmt.Println("Unsupported region: " + *regionPtr) + os.Exit(1) + } + } + configBase.Regions = regions + } + // Get existing configs if they exist... - pluginToolConfig, plcErr := trcvutils.GetPluginToolConfig(config, mod, coreopts.ProcessDeployPluginEnvConfig(map[string]interface{}{}), *hostNamePtr) + pluginToolConfig, plcErr := trcvutils.GetPluginToolConfig(configBase, mod, coreopts.ProcessDeployPluginEnvConfig(map[string]interface{}{}), *defineServicePtr) if plcErr != nil { fmt.Println(plcErr.Error()) os.Exit(1) } + if len(*sha256Ptr) > 0 { + fileInfo, statErr := os.Stat(*sha256Ptr) + if statErr == nil { + if fileInfo.Mode().IsRegular() { + file, fileOpenErr := os.Open(*sha256Ptr) + if fileOpenErr != nil { + fmt.Println(fileOpenErr) + return + } + + // Close the file when we're done. + defer file.Close() + + // Create a reader to the file. + reader := bufio.NewReader(file) + + pluginImage, imageErr := io.ReadAll(reader) + if imageErr != nil { + fmt.Println("Failed to read image:" + imageErr.Error()) + return + } + sha256Bytes := sha256.Sum256(pluginImage) + *sha256Ptr = fmt.Sprintf("%x", sha256Bytes) + } + } + } + pluginToolConfig["trcsha256"] = *sha256Ptr pluginToolConfig["pluginNamePtr"] = *pluginNamePtr + pluginToolConfig["deployrootPtr"] = *deployrootPtr + pluginToolConfig["serviceNamePtr"] = *serviceNamePtr + pluginToolConfig["codeBundlePtr"] = *codeBundlePtr if _, ok := pluginToolConfig["trcplugin"].(string); !ok { pluginToolConfig["trcplugin"] = pluginToolConfig["pluginNamePtr"].(string) if *certifyImagePtr { certifyInit = true } + + if *defineServicePtr && + !*winservicestopPtr && + !*winservicestartPtr && + !*codebundledeployPtr && + !*certifyImagePtr { + + if c != nil { + fmt.Println("Service definition not supported in trcsh.") + os.Exit(-1) + } + if _, ok := pluginToolConfig["deployrootPtr"].(string); ok { + pluginToolConfig["trcdeployroot"] = pluginToolConfig["deployrootPtr"].(string) + } + + if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { + pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) + } + if _, ok := pluginToolConfig["codeBundlePtr"].(string); ok { + pluginToolConfig["trccodebundle"] = pluginToolConfig["codeBundlePtr"].(string) + } + } } - //Certify Image - if *certifyImagePtr { - if !certifyInit { - err := repository.GetImageAndShaFromDownload(config, pluginToolConfig) + + //Define Service Image + if *defineServicePtr { + fmt.Printf("Connecting to vault @ %s\n", *addrPtr) + writeMap := make(map[string]interface{}) + writeMap["trcplugin"] = *pluginNamePtr + writeMap["trctype"] = *pluginTypePtr + + if _, ok := pluginToolConfig["trcdeployroot"]; ok { + writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"] + } + if _, ok := pluginToolConfig["trcservicename"]; ok { + writeMap["trcservicename"] = pluginToolConfig["trcservicename"] + } + if _, ok := pluginToolConfig["trccodebundle"]; ok { + writeMap["trccodebundle"] = pluginToolConfig["trccodebundle"] + } + _, err = mod.Write(pluginToolConfig["pluginpath"].(string), writeMap, configBase.Log) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Deployment definition applied to vault and is ready for deployments.") + } else if *winservicestopPtr { + fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("sc", "stop", pluginToolConfig["trcservicename"].(string)) + err := cmd.Run() + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("Service stopped: %s\n", pluginToolConfig["trcservicename"].(string)) + + } else if *winservicestartPtr { + fmt.Printf("Starting service %s\n", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("sc", "start", pluginToolConfig["trcservicename"].(string)) + err := cmd.Run() + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("Service started: %s\n", pluginToolConfig["trcservicename"].(string)) + } else if *codebundledeployPtr { + if pluginToolConfig["trcsha256"] != nil && len(pluginToolConfig["trcsha256"].(string)) > 0 { + err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println(err.Error()) os.Exit(1) } } - if certifyInit || pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { //Comparing generated sha from image to sha from flag - fmt.Println("Valid image found.") - //SHA MATCHES - fmt.Printf("Connecting to vault @ %s\n", *addrPtr) - writeMap := make(map[string]interface{}) - writeMap["trcplugin"] = pluginToolConfig["trcplugin"].(string) - writeMap["trcpluginpath"] = *pluginPathPtr - writeMap["trctype"] = *pluginTypePtr - writeMap["trcsha256"] = pluginToolConfig["trcsha256"].(string) - if pluginToolConfig["instances"] == nil { - pluginToolConfig["instances"] = "0" - } - writeMap["instances"] = pluginToolConfig["instances"].(string) + if pluginToolConfig["trcsha256"] != nil && + pluginToolConfig["imagesha256"] != nil && + pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { + // Write the image to the destination... + deployPath := fmt.Sprintf("%s\\%s", pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) + fmt.Printf("Deploying image to: %s\n", deployPath) - _, err = mod.Write(pluginToolConfig["pluginpath"].(string), writeMap, config.Log) + err = os.WriteFile(deployPath, pluginToolConfig["rawImageFile"].([]byte), 0644) if err != nil { - fmt.Println(err) + fmt.Println(err.Error()) os.Exit(1) } - writeMap = make(map[string]interface{}) - writeMap["copied"] = false - writeMap["deployed"] = false - _, err = mod.Write(pluginToolConfig["overridepath"].(string), writeMap, config.Log) - if err != nil { - fmt.Println(err) + fmt.Println("Image deployed.") + } else { + fmt.Printf("Image not certified. Cannot deploy image for %s\n", pluginToolConfig["trcplugin"]) + } + } else if *certifyImagePtr { + //Certify Image + if !certifyInit { + // Already certified... + fmt.Println("Checking for existing image.") + err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) + if _, ok := pluginToolConfig["imagesha256"].(string); err != nil || !ok { + fmt.Println("Invalid or nonexistent image.") + if err != nil { + fmt.Println(err.Error()) + } os.Exit(1) } + } + + if certifyInit || + pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { // Comparing generated sha from image to sha from flag + // || + //(pluginToolConfig["imagesha256"].(string) != "" && pluginToolConfig["trctype"].(string) == "trcshservice") { + fmt.Println("Valid image found.") + //SHA MATCHES + fmt.Printf("Connecting to vault @ %s\n", *addrPtr) + logger.Println("TrcCarrierUpdate getting plugin settings for env: " + mod.Env) + // The following confirms that this version of carrier has been certified to run... + // It will bail if it hasn't. + if _, pluginPathOk := pluginToolConfig["pluginpath"].(string); !pluginPathOk { //If region is set + mod.SectionName = "trcplugin" + mod.SectionKey = "/Index/" + mod.SubSectionValue = pluginToolConfig["trcplugin"].(string) + + properties, err := trcvutils.NewProperties(config, vault, mod, mod.Env, "TrcVault", "Certify") + if err != nil { + fmt.Println("Couldn't create properties for regioned certify:" + err.Error()) + os.Exit(1) + } + + writeMap, replacedFields := properties.GetPluginData(*regionPtr, "Certify", "config", logger) - fmt.Println("Image certified in vault and is ready for release.") + writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginToolConfig["trcplugin"].(string)) + if writeErr != nil { + fmt.Println(writeErr) + os.Exit(1) + } + } else { //Non region certify + writeMap, readErr := mod.ReadData(pluginToolConfig["pluginpath"].(string)) + if readErr != nil { + fmt.Println(readErr) + os.Exit(1) + } + _, err = mod.Write(pluginToolConfig["pluginpath"].(string), WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), configBase.Log) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Image certified in vault and is ready for release.") + } } else { fmt.Println("Invalid or nonexistent image.") os.Exit(1) } + } else if *agentdeployPtr { + if config.FeatherCtlCb != nil { + err := config.FeatherCtlCb(*pluginNamePtr) + if err != nil { + fmt.Println("Incorrect installation") + os.Exit(1) + } + } else { + fmt.Println("Incorrect trcplgtool utilization") + os.Exit(1) + } } //Checks if image has been copied & deployed @@ -155,7 +414,7 @@ func PluginMain() { os.Exit(0) } - err := repository.GetImageAndShaFromDownload(config, pluginToolConfig) + err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -178,7 +437,7 @@ func PluginMain() { os.Exit(0) } - err := repository.GetImageAndShaFromDownload(config, pluginToolConfig) + err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -195,3 +454,27 @@ func PluginMain() { os.Exit(2) } } + +func WriteMapUpdate(writeMap map[string]interface{}, pluginToolConfig map[string]interface{}, defineServicePtr bool, pluginTypePtr string) map[string]interface{} { + if pluginTypePtr != "trcshservice" { + writeMap["trcplugin"] = pluginToolConfig["trcplugin"].(string) + writeMap["trctype"] = pluginTypePtr + if pluginToolConfig["instances"] == nil { + pluginToolConfig["instances"] = "0" + } + writeMap["instances"] = pluginToolConfig["instances"].(string) + } + if defineServicePtr { + writeMap["trccodebundle"] = pluginToolConfig["trccodebundle"].(string) + writeMap["trcservicename"] = pluginToolConfig["trcservicename"].(string) + writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"].(string) + } + if _, imgShaOk := pluginToolConfig["imagesha256"].(string); imgShaOk { + writeMap["trcsha256"] = pluginToolConfig["imagesha256"].(string) // Pull image sha from registry... + } else { + writeMap["trcsha256"] = pluginToolConfig["trcsha256"].(string) // Pull image sha from registry... + } + writeMap["copied"] = false + writeMap["deployed"] = false + return writeMap +} diff --git a/trcvault/util/authutil.go b/trcvault/util/authutil.go new file mode 100644 index 000000000..587271b26 --- /dev/null +++ b/trcvault/util/authutil.go @@ -0,0 +1,34 @@ +package util + +import ( + "fmt" + "os" + "os/user" + "strconv" +) + +// CheckNotSudo -- checks if current user is sudoer and exits if they are. +func CheckNotSudo() { + sudoer, sudoErr := user.LookupGroup("sudo") + if sudoErr != nil { + fmt.Println("Trcsh unable to definitively identify sudoers.") + os.Exit(-1) + } + sudoerGid, sudoConvErr := strconv.Atoi(sudoer.Gid) + if sudoConvErr != nil { + fmt.Println("Trcsh unable to definitively identify sudoers. Conversion error.") + os.Exit(-1) + } + groups, groupErr := os.Getgroups() + if groupErr != nil { + fmt.Println("Trcsh unable to definitively identify sudoers. Missing groups.") + os.Exit(-1) + } + for _, groupId := range groups { + if groupId == sudoerGid { + fmt.Println("Trcsh cannot be run with user having sudo privileges.") + os.Exit(-1) + } + } + +} diff --git a/trcvault/util/properties.go b/trcvault/util/properties.go index 4c21cb44d..a956ef9b4 100644 --- a/trcvault/util/properties.go +++ b/trcvault/util/properties.go @@ -1,6 +1,10 @@ package util import ( + "log" + "strconv" + "strings" + vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" eUtils "github.com/trimble-oss/tierceron/utils" @@ -80,3 +84,100 @@ func ResolveTokenName(env string) string { } return tokenNamePtr } + +func (p *Properties) GetPluginData(region string, service string, config string, log *log.Logger) (map[string]interface{}, map[string]interface{}) { + valueMap, _ := p.GetConfigValues(service, config) + replacedDefaultFields := make(map[string]interface{}) + //Grabs region fields and replaces into base fields if region is available. + if region != "" { + regionFields := make(map[string]interface{}) + region = "~" + region + for field, value := range valueMap { + if !strings.Contains(field, region) { + continue + } else { + regionFields[field] = value + if _, valueOK := valueMap[strings.TrimSuffix(field, region)]; valueOK { + replacedDefaultFields[strings.TrimSuffix(field, region)] = valueMap[strings.TrimSuffix(field, region)] + } + } + } + + if len(regionFields) == 0 { + log.Println("Region was provided, but no regional data. Continuing with base data.") + regionDefaultList := []string{"trcsha256", "copied", "deployed", "trcservicename"} + for _, field := range regionDefaultList { //Create regioned defaults + if _, valueOK := valueMap[field]; valueOK { + replacedDefaultFields[field] = valueMap[field] + valueMap[field+region] = valueMap[field] + } else { + if field == "copied" || field == "deployed" { + replacedDefaultFields[field] = "false" + valueMap[field+region] = "false" + } else { + replacedDefaultFields[field] = "" + valueMap[field+region] = "" + } + } + } + + } else { + for field, value := range regionFields { + valueMap[strings.TrimSuffix(field, region)] = value + } + + } + } + + //String to bool conversion + //Bools come in as strings from GetConfigValues + boolValuesList := []string{"copied", "deployed"} + for _, boolVal := range boolValuesList { + if copiedInterface, valueOK := valueMap[boolVal]; valueOK { + switch copiedVal := copiedInterface.(type) { + case string: + if region != "" { //save defaults if regioned before bool conversion + if _, valueOK := valueMap[strings.TrimSuffix(boolVal, region)]; valueOK { + replacedDefaultFields[strings.TrimSuffix(boolVal, region)] = valueMap[strings.TrimSuffix(boolVal, region)] + } + } + boolValue, _ := strconv.ParseBool(copiedVal) //throws false so it's ok to ignore error. + valueMap[boolVal] = boolValue + default: + } + } + } + + return valueMap, replacedDefaultFields +} + +func (p *Properties) WritePluginData(pluginData map[string]interface{}, replacedFields map[string]interface{}, mod *helperkv.Modifier, log *log.Logger, hostRegion string, pluginName string) error { + //writeMap := make(map[string]interface{}) + regionSuffix := "" + if hostRegion != "" { + regionSuffix = "~" + hostRegion + } + + for field, value := range replacedFields { + if _, valueOK := pluginData[field]; valueOK { + pluginData[field+regionSuffix] = pluginData[field] + pluginData[field] = value + } + } + + writeMap, readErr := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + pluginName + "/Certify") //This read is need to avoid overwritting un-used region data. + if readErr != nil { + return readErr + } + + for field, value := range pluginData { + writeMap[field] = value + } + + _, writeErr := mod.Write("super-secrets/Index/TrcVault/trcplugin/"+pluginName+"/Certify", writeMap, log) + if writeErr != nil { + return writeErr + } + + return nil +} diff --git a/trcvault/util/repository/azrcr.go b/trcvault/util/repository/azrcr.go index 163cc083b..7c25c52af 100644 --- a/trcvault/util/repository/azrcr.go +++ b/trcvault/util/repository/azrcr.go @@ -37,7 +37,6 @@ func getImageSHA(config *eUtils.DriverConfig, svc *azidentity.ClientSecretCreden if err != nil { config.Log.Printf("Failed to advance page for tags: %v", err) return err - } for _, v := range page.Tags { latestTag = *v.Name //Always only returns 1 tag due to MaxNum being set @@ -50,6 +49,7 @@ func getImageSHA(config *eUtils.DriverConfig, svc *azidentity.ClientSecretCreden config.Log.Printf("failed to get manifest: %v", err) return err } + reader, err := azcontainerregistry.NewDigestValidationReader(*manifestRes.DockerContentDigest, manifestRes.ManifestData) if err != nil { config.Log.Printf("failed to create validation reader: %v", err) @@ -69,61 +69,91 @@ func getImageSHA(config *eUtils.DriverConfig, svc *azidentity.ClientSecretCreden return err } // Get layers - var layerDigest string layers := manifestJSON["layers"].([]any) - for _, layer := range layers { - layerDigest = layer.(map[string]any)["digest"].(string) + blobClient, err := azcontainerregistry.NewBlobClient(pluginToolConfig["acrrepository"].(string), svc, nil) + if err != nil { + return errors.New("Failed to create blob client:" + err.Error()) + } + + for i := len(layers) - 1; i >= 0; i-- { + if layer, layerOk := layers[i].(map[string]any)["digest"]; layerOk { + if layerD, ok := layer.(string); ok { + sha256, shaErr := GetImageShaFromLayer(blobClient, pluginToolConfig["trcplugin"].(string), layerD, pluginToolConfig) + if shaErr != nil { + return errors.New("Failed to load image sha from layer:" + shaErr.Error()) + } + if _, ok := pluginToolConfig["trcsha256"]; !ok { + // Not looking for anything in particular so just grab the last image. + break + } else { + if pluginToolConfig["trcsha256"].(string) == sha256 { + pluginToolConfig["imagesha256"] = sha256 + break + } + } + } + } } - pluginToolConfig["layerDigest"] = layerDigest return nil } -// Return url to the image to be used for download. -func GetImageAndShaFromDownload(config *eUtils.DriverConfig, pluginToolConfig map[string]interface{}) error { - svc, err := azidentity.NewClientSecretCredential( - pluginToolConfig["azureTenantId"].(string), - pluginToolConfig["azureClientId"].(string), - pluginToolConfig["azureClientSecret"].(string), - nil) - if err != nil { - return err - } - - imageErr := getImageSHA(config, svc, pluginToolConfig) - if imageErr != nil { - return imageErr - } - blobClient, err := azcontainerregistry.NewBlobClient(pluginToolConfig["acrrepository"].(string), svc, nil) - if err != nil { - return errors.New("Failed to create blob client:" + err.Error()) - } +func GetImageShaFromLayer(blobClient *azcontainerregistry.BlobClient, name string, digest string, pluginToolConfig map[string]interface{}) (string, error) { - configRes, err := blobClient.GetBlob(context.Background(), pluginToolConfig["trcplugin"].(string), pluginToolConfig["layerDigest"].(string), nil) + configRes, err := blobClient.GetBlob(context.Background(), name, digest, nil) if err != nil { - return errors.New("Failed to get config:" + err.Error()) + return "", errors.New("Failed to get config:" + err.Error()) } - reader, readErr := azcontainerregistry.NewDigestValidationReader(pluginToolConfig["layerDigest"].(string), configRes.BlobData) + reader, readErr := azcontainerregistry.NewDigestValidationReader(digest, configRes.BlobData) if readErr != nil { - return errors.New("Failed to create validation reader" + readErr.Error()) + return "", errors.New("Failed to create validation reader" + readErr.Error()) } layerData, configErr := io.ReadAll(reader) if configErr != nil { - return errors.New("Failed to read config data:" + configErr.Error()) + return "", errors.New("Failed to read config data:" + configErr.Error()) } pluginTarredData, gUnZipError := gUnZipData(layerData) if gUnZipError != nil { - return errors.New("gunzip failed") + return "", errors.New("gunzip failed") } pluginImage, gUnTarError := untarData(pluginTarredData) if gUnTarError != nil { - return errors.New("untarring failed") + return "", errors.New("untarring failed") } pluginSha := sha256.Sum256(pluginImage) - pluginToolConfig["rawImageFile"] = pluginImage - pluginToolConfig["imagesha256"] = fmt.Sprintf("%x", pluginSha) + sha256 := fmt.Sprintf("%x", pluginSha) + if pluginToolConfig != nil { + if _, ok := pluginToolConfig["trcsha256"]; !ok { + // Not looking for anything in particular so just grab the last image. + pluginToolConfig["rawImageFile"] = pluginImage + } else { + if pluginToolConfig["trcsha256"].(string) == sha256 { + pluginToolConfig["rawImageFile"] = pluginImage + } + } + } + + return sha256, nil +} + +// Return url to the image to be used for download. +func GetImageAndShaFromDownload(config *eUtils.DriverConfig, pluginToolConfig map[string]interface{}) error { + svc, err := azidentity.NewClientSecretCredential( + pluginToolConfig["azureTenantId"].(string), + pluginToolConfig["azureClientId"].(string), + pluginToolConfig["azureClientSecret"].(string), + nil) + if err != nil { + return err + } + + imageErr := getImageSHA(config, svc, pluginToolConfig) + if imageErr != nil { + return imageErr + } + return nil } diff --git a/trcvault/util/util.go b/trcvault/util/util.go index 1534d7f94..1e4238f2b 100644 --- a/trcvault/util/util.go +++ b/trcvault/util/util.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/trimble-oss/tierceron/buildopts/memprotectopts" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -248,31 +249,56 @@ func SeedVaultById(config *eUtils.DriverConfig, goMod *helperkv.Modifier, servic return nil } -func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pluginConfig map[string]interface{}, hostName string) (map[string]interface{}, error) { +func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pluginConfig map[string]interface{}, defineService bool) (map[string]interface{}, error) { config.Log.Println("GetPluginToolConfig begin processing plugins.") //templatePaths indexFound := false templatePaths := pluginConfig["templatePath"].([]string) + config.Log.Println("GetPluginToolConfig reading base configurations.") pluginToolConfig, err := mod.ReadData("super-secrets/Restricted/PluginTool/config") if err != nil { + config.Log.Println("GetPluginToolConfig errored with missing base PluginTool configurations.") return nil, err } else { if len(pluginToolConfig) == 0 { + config.Log.Println("GetPluginToolConfig empty base PluginTool configurations.") return nil, errors.New("Tierceron plugin management presently not configured for env: " + mod.Env) } } + pluginEnvConfigClone := make(map[string]interface{}) + + for k, v := range pluginToolConfig { + if _, okStr := v.(string); okStr { + v2 := strings.Clone(v.(string)) + memprotectopts.MemProtect(nil, &v2) + pluginEnvConfigClone[k] = v2 + } else { + // Safe to share... + pluginEnvConfigClone[k] = v + } + } + for k, v := range pluginConfig { - pluginToolConfig[k] = v + if _, okStr := v.(string); okStr { + v2 := strings.Clone(v.(string)) + memprotectopts.MemProtect(nil, &v2) + pluginEnvConfigClone[k] = v2 + } else { + // Safe to share... + pluginEnvConfigClone[k] = v + } } var ptc1 map[string]interface{} + + config.Log.Println("GetPluginToolConfig loading plugin data.") for _, templatePath := range templatePaths { project, service, _ := eUtils.GetProjectService(templatePath) config.Log.Println("GetPluginToolConfig project: " + project + " plugin: " + config.SubSectionValue + " service: " + service) - overridePath := "" - if pluginPath, pathOk := pluginToolConfig["pluginpath"]; pathOk && len(pluginPath.(string)) != 0 && pluginPath != "n/a" { + + if pluginPath, pathOk := pluginToolConfig["pluginpath"]; pathOk && len(pluginPath.(string)) != 0 { mod.SectionPath = "super-secrets/Index/" + project + pluginPath.(string) + config.SubSectionValue + "/" + service } else { mod.SectionPath = "super-secrets/Index/" + project + "/trcplugin/" + config.SubSectionValue + "/" + service @@ -280,49 +306,47 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl ptc1, err = mod.ReadData(mod.SectionPath) pluginToolConfig["pluginpath"] = mod.SectionPath if err != nil || ptc1 == nil { - config.Log.Println("No data found.") + config.Log.Println("No data found for project: " + project + " plugin: " + config.SubSectionValue + " service: " + service) continue } indexFound = true - for k, v := range ptc1 { - pluginToolConfig[k] = v - } - //Override - if pluginPath, pathOk := pluginToolConfig["pluginpath"]; pathOk && len(pluginPath.(string)) != 0 && hostName != "" { - overridePath = "super-secrets/Index/" + project + "/trcplugin/overrides/" + hostName + "/" + config.SubSectionValue + "/" + service - mod.SectionPath = overridePath - ptc2, err := mod.ReadData(mod.SectionPath) - pluginToolConfig["overridepath"] = overridePath - if err != nil || ptc2 == nil { - pluginToolConfig["copied"] = false - pluginToolConfig["deployed"] = false - config.Log.Println("No override found for plugin.") - continue - } - for k, v := range ptc2 { - pluginToolConfig[k] = v + for k, v := range ptc1 { + if _, okStr := v.(string); okStr { + v2 := strings.Clone(v.(string)) + memprotectopts.MemProtect(nil, &v2) + pluginEnvConfigClone[k] = v2 + } else { + // Safe to share... + pluginEnvConfigClone[k] = v } } break } mod.SectionPath = "" + config.Log.Println("GetPluginToolConfig plugin data load process complete.") - if pluginToolConfig == nil { + if pluginEnvConfigClone == nil { config.Log.Println("No data found for plugin.") if err == nil { - err = errors.New("No data and unexpected error.") + err = errors.New("no data and unexpected error") } - return pluginToolConfig, err + return pluginEnvConfigClone, err } else if !indexFound { - return pluginToolConfig, nil + if defineService { + pluginEnvConfigClone["pluginpath"] = pluginToolConfig["pluginpath"] + } + return pluginEnvConfigClone, nil + } else { + if _, ok := pluginEnvConfigClone["trcplugin"]; ok { + if strings.ContainsAny(pluginEnvConfigClone["trcplugin"].(string), "./") { + err = errors.New("Invalid plugin configuration: " + pluginEnvConfigClone["trcplugin"].(string)) + return nil, err + } + } } config.Log.Println("GetPluginToolConfig end processing plugins.") - if strings.ContainsAny(pluginToolConfig["trcplugin"].(string), "./") { - err = errors.New("Invalid plugin configuration: " + pluginToolConfig["trcplugin"].(string)) - return nil, err - } - return pluginToolConfig, nil + return pluginEnvConfigClone, nil } diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 4269fffa6..0c7db2062 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -43,6 +43,7 @@ type DriverConfig struct { EndDir string OutputMemCache bool MemFs billy.Filesystem + FeatherCtlCb func(string) error // Config modes.... WantCerts bool diff --git a/utils/vaultUtil.go b/utils/vaultUtil.go index 519705eb4..445706b6d 100644 --- a/utils/vaultUtil.go +++ b/utils/vaultUtil.go @@ -5,6 +5,7 @@ import ( "log" "os" "strings" + "sync" "github.com/trimble-oss/tierceron/buildopts/coreopts" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -111,6 +112,8 @@ func GetAcceptedTemplatePaths(config *DriverConfig, modCheck *helperkv.Modifier, return templatePaths, nil } +var logMap sync.Map = sync.Map{} + // Helper to easiliy intialize a vault and a mod all at once. func InitVaultModForPlugin(pluginConfig map[string]interface{}, logger *log.Logger) (*DriverConfig, *helperkv.Modifier, *sys.Vault, error) { logger.Println("InitVaultModForPlugin log setup: " + pluginConfig["env"].(string)) @@ -120,25 +123,32 @@ func InitVaultModForPlugin(pluginConfig map[string]interface{}, logger *log.Logg logPrefix := fmt.Sprintf("[trcplugin%s-%s]", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)) if logger.Prefix() != logPrefix { - logger.Println("Checking log permissions..") logFile := fmt.Sprintf("/var/log/trcplugin%s-%s.log", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)) + if tLogger, logOk := logMap.Load(logFile); !logOk { + logger.Printf("Checking log permissions for logfile: %s\n", logFile) - f, logErr := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if logErr != nil { - logFile = fmt.Sprintf("trcplugin%s-%s.log", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)) - f, logErr = os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + f, logErr := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if logErr != nil { - logger.Println("Log permissions failure. Will exit.") + logFile = fmt.Sprintf("trcplugin%s-%s.log", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)) + f, logErr = os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if logErr != nil { + logger.Println("Log permissions failure. Will exit.") + } } - } - trcdbEnvLogger = log.New(f, fmt.Sprintf("[trcplugin%s-%s]", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)), log.LstdFlags) - CheckError(&DriverConfig{Insecure: true, Log: trcdbEnvLogger, ExitOnFailure: true}, logErr, true) - logger.Println("InitVaultModForPlugin log setup complete") + trcdbEnvLogger = log.New(f, fmt.Sprintf("[trcplugin%s-%s]", pluginConfig["logNamespace"].(string), pluginConfig["env"].(string)), log.LstdFlags) + CheckError(&DriverConfig{Insecure: true, Log: trcdbEnvLogger, ExitOnFailure: true}, logErr, true) + logMap.Store(logFile, trcdbEnvLogger) + logger.Println("InitVaultModForPlugin log setup complete") + } else { + logger.Printf("Utilizing existing logger for logfile: %s\n", logFile) + trcdbEnvLogger = tLogger.(*log.Logger) + } } else { trcdbEnvLogger = logger } } else { + logger.Printf("Utilizing default logger invalid namespace\n") trcdbEnvLogger = logger } diff --git a/vaulthelper/kv/Modifier.go b/vaulthelper/kv/Modifier.go index 5a34234fd..ee13f5c8e 100644 --- a/vaulthelper/kv/Modifier.go +++ b/vaulthelper/kv/Modifier.go @@ -127,7 +127,9 @@ func NewModifier(insecure bool, token string, address string, env string, region HttpClient: httpClient, }) if err != nil { - logger.Println("vaultHost: "+modClient.Address(), logger) + if logger != nil { + logger.Println("vaultHost: "+modClient.Address(), logger) + } return nil, err } From 193b3c3b3b7a05e74c058d0676a35667d11ecdf2 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:46:28 -0800 Subject: [PATCH 06/55] Fix broken build (#778) * Fix missing import. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --- trcsh/trcsh.go | 1 + 1 file changed, 1 insertion(+) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 47a64bfbb..d146ce348 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -27,6 +27,7 @@ import ( "github.com/trimble-oss/tierceron/trcsh/trcshauth" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" + "github.com/trimble-oss/tierceron/trcvault/util" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" From fea251b38d83529c63da498d94e1886340cdb37f Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:56:28 -0800 Subject: [PATCH 07/55] Fix broken build. (#779) * Fix broken build (#778) * Fix missing import. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: Karnveer Gill Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: meghan-bailey <107890930+meghan-bailey@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- capauth/creds.go | 11 +++++++++-- trcsh/trcsh.go | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/capauth/creds.go b/capauth/creds.go index 3234865f3..2c12b9267 100644 --- a/capauth/creds.go +++ b/capauth/creds.go @@ -9,6 +9,7 @@ import ( "math/rand" "net" "os" + "runtime" "strings" "time" @@ -27,8 +28,14 @@ func init() { rand.Seed(time.Now().UnixNano()) mashupCertBytes, err := os.ReadFile(ServCert) if err != nil { - mashupCertBytes, err = os.ReadFile(ServCertLocal) - if err != nil { + if runtime.GOOS == "windows" { + // But only on windows. + mashupCertBytes, err = os.ReadFile(ServCertLocal) + if err != nil { + fmt.Println("Cert read failure.") + return + } + } else { fmt.Println("Cert read failure.") return } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 47a64bfbb..d146ce348 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -27,6 +27,7 @@ import ( "github.com/trimble-oss/tierceron/trcsh/trcshauth" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" + "github.com/trimble-oss/tierceron/trcvault/util" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" From 244ba091a24b8b04ab14cc8137006ccbcebbe2ef Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:03:39 -0800 Subject: [PATCH 08/55] Fix for windows (#781) * Better support for server cert. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --- capauth/creds.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/capauth/creds.go b/capauth/creds.go index 2c12b9267..da50243b5 100644 --- a/capauth/creds.go +++ b/capauth/creds.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "errors" "fmt" "log" "math/rand" @@ -24,22 +25,25 @@ const ( var MashupCertPool *x509.CertPool -func init() { - rand.Seed(time.Now().UnixNano()) - mashupCertBytes, err := os.ReadFile(ServCert) - if err != nil { +func ReadServerCert() ([]byte, error) { + if _, err := os.Stat(ServCert); err == nil { + return os.ReadFile(ServCert) + } else { if runtime.GOOS == "windows" { - // But only on windows. - mashupCertBytes, err = os.ReadFile(ServCertLocal) - if err != nil { - fmt.Println("Cert read failure.") - return - } + return os.ReadFile(ServCertLocal) } else { - fmt.Println("Cert read failure.") - return + return nil, errors.New("file not found") } } +} + +func init() { + rand.Seed(time.Now().UnixNano()) + mashupCertBytes, err := ReadServerCert() + if err != nil { + fmt.Println("Cert read failure.") + return + } mashupBlock, _ := pem.Decode([]byte(mashupCertBytes)) @@ -73,7 +77,7 @@ func LocalIp(env string) (string, error) { func GetTransportCredentials() (credentials.TransportCredentials, error) { - mashupKeyBytes, err := os.ReadFile(ServCert) + mashupKeyBytes, err := ReadServerCert() if err != nil { return nil, err } From 011227822bda7627377579c6e95f026929543a1d Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:29:07 -0800 Subject: [PATCH 09/55] Prepare for trcsh release. (#783) * Update to go 1.21.4 * More robust state handling. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --- azure-pipelines-legacy.yml | 2 +- azure-pipelines-linux.yml | 2 +- azure-pipelines-plugin-develop.yml | 2 +- azure-pipelines-plugin.yml | 2 +- azure-pipelines.yml | 2 +- go.mod | 2 +- trcchatproxy/go.mod | 2 +- trcsh/trcsh.go | 124 ++++++++++++++++------------- 8 files changed, 76 insertions(+), 62 deletions(-) diff --git a/azure-pipelines-legacy.yml b/azure-pipelines-legacy.yml index 06757e0da..01f7bdf5d 100644 --- a/azure-pipelines-legacy.yml +++ b/azure-pipelines-legacy.yml @@ -75,7 +75,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.3' + version: '1.21.4' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-linux.yml b/azure-pipelines-linux.yml index c39a51a56..db38e7a48 100644 --- a/azure-pipelines-linux.yml +++ b/azure-pipelines-linux.yml @@ -71,7 +71,7 @@ steps: - task: GoTool@0 inputs: - version: '1.21.3' + version: '1.21.4' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin-develop.yml b/azure-pipelines-plugin-develop.yml index e4b420f62..9f1f4a34f 100644 --- a/azure-pipelines-plugin-develop.yml +++ b/azure-pipelines-plugin-develop.yml @@ -73,7 +73,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.3' + version: '1.21.4' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index c6cf7f7ea..dc738e255 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -78,7 +78,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.3' + version: '1.21.4' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 238a79e59..471bb6a22 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.3' + version: '1.21.4' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/go.mod b/go.mod index af90651f5..0714b02fc 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/trimble-oss/tierceron go 1.21 -toolchain go1.21.3 +toolchain go1.21.4 require ( VaultConfig.Bootstrap v0.0.0-00010101000000-000000000000 diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod index c2001cafb..ed147ef01 100644 --- a/trcchatproxy/go.mod +++ b/trcchatproxy/go.mod @@ -1,6 +1,6 @@ module github.com/trimble-oss/tierceron/trcchatproxy -go 1.21.3 +go 1.21.4 require google.golang.org/api v0.145.0 diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index d146ce348..745c280d6 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -125,29 +125,33 @@ func main() { // Preload agent synchronization configs gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) - for { - if deployFlapMode, deployEmitErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); deployEmitErr == nil && strings.HasPrefix(deployFlapMode, cap.MODE_GAZE) { - go func() { + + go func() { + // Timeout and CtlMessage subscriber + for { + select { + case <-time.After(120 * time.Second): + ctlMsg := "Deployment timed out after 120 seconds" + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + case modeCtl := <-gAgentConfig.CtlMessage: + modeCtlTrail := []string{modeCtl} for { perching: - select { - case <-time.After(120 * time.Second): - ctlMsg := "Deployment timed out after 120 seconds" - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - case ctlMsg := <-gAgentConfig.CtlMessage: - if ctlMsg != capauth.TrcCtlComplete { - deployLineFlapMode := cap.MODE_FLAP + "_" + ctlMsg - ctlFlapMode := deployLineFlapMode - var err error + if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { + for _, modeCtl := range modeCtlTrail { + flapMode := cap.MODE_FLAP + "_" + modeCtl + ctlFlapMode := flapMode + var err error = errors.New("init") + for { if err == nil && ctlFlapMode == cap.MODE_PERCH { // Acknowledge perching... @@ -160,45 +164,44 @@ func main() { goto perching } - // Notify deployer of command run and wait for confirmation of msg received. - if err == nil && deployLineFlapMode != ctlFlapMode { + if err == nil && flapMode != ctlFlapMode { // Flap, Gaze, etc... + interruptFun(twoHundredMilliInterruptTicker) break } else { - callFlap := deployLineFlapMode + callFlap := flapMode if err == nil { - time.Sleep(200 * time.Millisecond) + interruptFun(twoHundredMilliInterruptTicker) } else { if err.Error() != "init" { - time.Sleep(1 * time.Second) + interruptFun(multiSecondInterruptTicker) } } - deployLineFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, callFlap, deployments+"."+*gAgentConfig.Env) } } - } else { - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) } + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) + } else { + time.Sleep(1 * time.Second) } } - }() - ProcessDeploy(*gAgentConfig.Env, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) - - } else { - time.Sleep(500 * time.Millisecond) + } } - } + }() + // Process the script.... + // This will feed CtlMessages into the Timeout and CtlMessage subscriber + ProcessDeploy(*gAgentConfig.Env, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) <-shutdown } - } var interruptChan chan os.Signal = make(chan os.Signal) @@ -225,31 +228,42 @@ func featherCtlCb(agentName string) error { } else { gAgentConfig.Deployments = &agentName } - callFlap := cap.MODE_GAZE + flapMode := cap.MODE_GAZE + ctlFlapMode := flapMode + var err error = errors.New("init") + for { - // Azure deployment agent kicks off a deploy with a flap command... - if ctlFlapMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env); featherErr != nil || ctlFlapMode == cap.MODE_PERCH || ctlFlapMode == cap.MODE_GLIDE { - if featherErr != nil { + if err == nil && ctlFlapMode == cap.MODE_GLIDE { + if err != nil { fmt.Printf("\nDeployment error.\n") } else { fmt.Printf("\nDeployment complete.\n") } os.Exit(0) } else { - if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { - if strings.Contains(ctlFlapMode, "_") { - ctlMessage := strings.Split(ctlFlapMode, "_") - if len(ctlMessage) > 1 { - fmt.Printf("%s\n", ctlMessage[1]) + callFlap := flapMode + if err == nil { + if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { + ctl := strings.Split(ctlFlapMode, "_") + if len(ctl) > 1 { + fmt.Printf("%s\n", ctl[1]) } + callFlap = cap.MODE_GAZE + } else { + callFlap = cap.MODE_GAZE + } + interruptFun(twoHundredMilliInterruptTicker) + } else { + if err.Error() != "init" { + interruptFun(multiSecondInterruptTicker) + callFlap = cap.MODE_GAZE } } - callFlap = cap.MODE_GAZE - interruptFun(multiSecondInterruptTicker) + ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, agentName+"."+*gAgentConfig.Env) } } From fceeaca4560b9a82f9c60fb7a5f39831bb6b8fc3 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:26:43 -0800 Subject: [PATCH 10/55] Prepare for release. (#785) * Prevent premature exits. * Switch to ranging over channel. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> --- trcsh/trcsh.go | 100 +++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 745c280d6..36503f9f0 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -128,9 +128,8 @@ func main() { go func() { // Timeout and CtlMessage subscriber - for { - select { - case <-time.After(120 * time.Second): + go func() { + for range time.After(120 * time.Second) { ctlMsg := "Deployment timed out after 120 seconds" cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, @@ -138,64 +137,69 @@ func main() { *gAgentConfig.HandshakeCode, cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - case modeCtl := <-gAgentConfig.CtlMessage: - modeCtlTrail := []string{modeCtl} - for { - perching: - if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { - for _, modeCtl := range modeCtlTrail { - flapMode := cap.MODE_FLAP + "_" + modeCtl - ctlFlapMode := flapMode - var err error = errors.New("init") - - for { - if err == nil && ctlFlapMode == cap.MODE_PERCH { - // Acknowledge perching... - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) - ctlFlapMode = cap.MODE_PERCH - goto perching - } + break + } + }() - if err == nil && flapMode != ctlFlapMode { - // Flap, Gaze, etc... - interruptFun(twoHundredMilliInterruptTicker) - break - } else { - callFlap := flapMode - if err == nil { - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(multiSecondInterruptTicker) - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, deployments+"."+*gAgentConfig.Env) + for { + perching: + if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { + for modeCtl := range gAgentConfig.CtlMessage { + flapMode := cap.MODE_FLAP + "_" + modeCtl + ctlFlapMode := flapMode + var err error = errors.New("init") + + for { + if err == nil && ctlFlapMode == cap.MODE_PERCH { + // Acknowledge perching... + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) + ctlFlapMode = cap.MODE_PERCH + goto perching + } + + if err == nil && flapMode != ctlFlapMode { + // Flap, Gaze, etc... + interruptFun(twoHundredMilliInterruptTicker) + break + } else { + callFlap := flapMode + if err == nil { + interruptFun(twoHundredMilliInterruptTicker) + } else { + if err.Error() != "init" { + interruptFun(multiSecondInterruptTicker) } } + ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, deployments+"."+*gAgentConfig.Env) } + } + if modeCtl == capauth.TrcCtlComplete { + // Only exit with TrcCtlComplete. cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) - } else { - time.Sleep(1 * time.Second) + goto deploycomplete } } + } else { + time.Sleep(1 * time.Second) } } + deploycomplete: }() // Process the script.... // This will feed CtlMessages into the Timeout and CtlMessage subscriber From 1bc82ec62ebec2686b63d20d6f57e1db4e4cd331 Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:31:35 -0800 Subject: [PATCH 11/55] Merging Develop into Main (#787) --- trcflow/flows/process_dataflowstatistics.go | 4 +++ trcinit/initlib/vpub.go | 33 +++++++++++-------- trcpubbase/trcpub.go | 11 +++++-- .../Tierceron/TrcDeploy/setup.cmd.tmpl | 7 ++++ trcsh/trcsh.go | 30 +++++++++-------- trcsh/trcshauth/auth.go | 12 +++---- trcvault/trcplgtoolbase/trcplgtoolbase.go | 2 ++ utils/autoUtil.go | 6 +++- webapi/server/VaultInit.go | 2 +- 9 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl diff --git a/trcflow/flows/process_dataflowstatistics.go b/trcflow/flows/process_dataflowstatistics.go index d7570c628..b90c8df86 100644 --- a/trcflow/flows/process_dataflowstatistics.go +++ b/trcflow/flows/process_dataflowstatistics.go @@ -311,6 +311,7 @@ func KickOffTimedRefresh(tfContext *flowcore.TrcFlowContext, stateUpdateChannel loc, _ := time.LoadLocation("America/Los_Angeles") now := time.Now().In(loc) midnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, loc) + //midnight := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()+3, 0, loc) timeTilMidnight := midnight.Sub(now) go func(tfc *flowcore.TrcFlowContext, tilMidnight time.Duration) { refresh = true @@ -334,6 +335,9 @@ func KickOffTimedRefresh(tfContext *flowcore.TrcFlowContext, stateUpdateChannel refresh = false stateUpdateChannel <- flowcorehelper.FlowStateUpdate{FlowName: tfContext.Flow.TableName(), StateUpdate: "2", SyncFilter: tfContext.FlowState.SyncFilter, SyncMode: "refreshEnded", FlowAlias: tfContext.FlowState.FlowAlias} case timing == "Ended": + for len(endRefreshChan) > 0 { + <-endRefreshChan + } return true default: return false diff --git a/trcinit/initlib/vpub.go b/trcinit/initlib/vpub.go index 69a79d1ca..4a106b4e2 100644 --- a/trcinit/initlib/vpub.go +++ b/trcinit/initlib/vpub.go @@ -2,7 +2,6 @@ package initlib import ( "fmt" - "log" "os" "path/filepath" "strings" @@ -12,7 +11,7 @@ import ( helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" ) -func UploadTemplateDirectory(mod *helperkv.Modifier, dirName string, logger *log.Logger) ([]string, error) { +func UploadTemplateDirectory(c *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string) ([]string, error) { dirs, err := os.ReadDir(dirName) if err != nil { @@ -24,7 +23,7 @@ func UploadTemplateDirectory(mod *helperkv.Modifier, dirName string, logger *log for _, subDir := range dirs { if subDir.IsDir() { pathName := dirName + "/" + subDir.Name() - warn, err := UploadTemplates(mod, pathName, logger) + warn, err := UploadTemplates(c, mod, pathName) if err != nil || len(warn) > 0 { fmt.Printf("Upload templates couldn't be completed. %v", err) return warn, err @@ -34,7 +33,7 @@ func UploadTemplateDirectory(mod *helperkv.Modifier, dirName string, logger *log return nil, nil } -func UploadTemplates(mod *helperkv.Modifier, dirName string, logger *log.Logger) ([]string, error) { +func UploadTemplates(c *eUtils.DriverConfig, mod *helperkv.Modifier, dirName string) ([]string, error) { // Open directory files, err := os.ReadDir(dirName) if err != nil { @@ -49,7 +48,7 @@ func UploadTemplates(mod *helperkv.Modifier, dirName string, logger *log.Logger) for _, file := range files { // Extract extension and name if file.IsDir() { // Recurse folders - warn, err := UploadTemplates(mod, dirName+"/"+file.Name(), logger) + warn, err := UploadTemplates(c, mod, dirName+"/"+file.Name()) if err != nil || len(warn) > 0 { return warn, err } @@ -60,14 +59,20 @@ func UploadTemplates(mod *helperkv.Modifier, dirName string, logger *log.Logger) name = name[0 : len(name)-len(ext)] // Truncate extension if ext == ".tmpl" { // Only upload template files - fmt.Printf("Found template file %s for %s\n", file.Name(), mod.Env) - logger.Printf("Found template file %s for %s\n", file.Name(), mod.Env) + if c != nil && c.IsShell { + c.Log.Printf("Found template file %s for %s\n", file.Name(), mod.Env) + } else { + fmt.Printf("Found template file %s for %s\n", file.Name(), mod.Env) + if c != nil { + c.Log.Printf("Found template file %s for %s\n", file.Name(), mod.Env) + } + } // Seperate name and extension one more time for saving to vault ext = filepath.Ext(name) name = name[0 : len(name)-len(ext)] - logger.Printf("dirName: %s\n", dirName) - logger.Printf("file name: %s\n", file.Name()) + c.Log.Printf("dirName: %s\n", dirName) + c.Log.Printf("file name: %s\n", file.Name()) // Extract values extractedValues, err := eUtils.Parse(dirName+"/"+file.Name(), subDir, name) if err != nil { @@ -103,25 +108,25 @@ func UploadTemplates(mod *helperkv.Modifier, dirName string, logger *log.Logger) // Construct template path for vault templatePath := "templates/" + subDir + "/" + name + "/template-file" - logger.Printf("\tUploading template to path:\t%s\n", templatePath) + c.Log.Printf("\tUploading template to path:\t%s\n", templatePath) // Construct value path for vault valuePath := "values/" + subDir + "/" + name - logger.Printf("\tUploading values to path:\t%s\n", valuePath) + c.Log.Printf("\tUploading values to path:\t%s\n", valuePath) // Write templates to vault and output errors/warnings - warn, err := mod.Write(templatePath, map[string]interface{}{"data": fileBytes, "ext": ext}, logger) + warn, err := mod.Write(templatePath, map[string]interface{}{"data": fileBytes, "ext": ext}, c.Log) if err != nil || len(warn) > 0 { return warn, err } // Write values to vault and output any errors/warnings - warn, err = mod.Write(valuePath, extractedValues, logger) + warn, err = mod.Write(valuePath, extractedValues, c.Log) if err != nil || len(warn) > 0 { return warn, err } } else { - logger.Printf("\tSkippping template (templates must end in .tmpl):\t%s\n", file.Name()) + c.Log.Printf("\tSkippping template (templates must end in .tmpl):\t%s\n", file.Name()) } } return nil, nil diff --git a/trcpubbase/trcpub.go b/trcpubbase/trcpub.go index 6dd848e30..795d2fdea 100644 --- a/trcpubbase/trcpub.go +++ b/trcpubbase/trcpub.go @@ -70,8 +70,13 @@ func CommonMain(envPtr *string, eUtils.CheckError(configBase, err, true) } - fmt.Printf("Connecting to vault @ %s\n", *addrPtr) - fmt.Printf("Uploading templates in %s to vault\n", *dirPtr) + if c != nil && c.IsShell { + c.Log.Printf("Connecting to vault @ %s\n", *addrPtr) + c.Log.Printf("Uploading templates in %s to vault\n", *dirPtr) + } else { + fmt.Printf("Connecting to vault @ %s\n", *addrPtr) + fmt.Printf("Uploading templates in %s to vault\n", *dirPtr) + } mod, err := helperkv.NewModifier(*insecurePtr, *tokenPtr, *addrPtr, *envPtr, nil, true, configBase.Log) if mod != nil { @@ -80,7 +85,7 @@ func CommonMain(envPtr *string, eUtils.CheckError(configBase, err, true) mod.Env = *envPtr - warn, err := il.UploadTemplateDirectory(mod, *dirPtr, configBase.Log) + warn, err := il.UploadTemplateDirectory(configBase, mod, *dirPtr) if err != nil { if strings.Contains(err.Error(), "x509: certificate") { os.Exit(-1) diff --git a/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl b/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl new file mode 100644 index 000000000..350d251aa --- /dev/null +++ b/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl @@ -0,0 +1,7 @@ +setx /M AGENT_ENV "{{.trcDeployAgentEnv}}" +setx /M AGENT_TOKEN "{{.trcDeployAgentToken}}" +setx /M DEPLOYMENTS "{{.trcDeployDeployments}}" +setx /M VAULT_ADDR "{{.trcDeployVaultAddr}}" + +.\nssm.exe install TrcDeploy "{{.trcDeployDirectory}}" +.\nssm.exe set TrcDeploy AppDirectory "{{.trcDeployStartDirectory}}" diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 36503f9f0..bf0a1a1cd 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -135,7 +135,7 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env) + cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env, true) gAgentConfig.CtlMessage <- capauth.TrcCtlComplete break } @@ -147,7 +147,7 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { + cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env, false); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { for modeCtl := range gAgentConfig.CtlMessage { flapMode := cap.MODE_FLAP + "_" + modeCtl ctlFlapMode := flapMode @@ -160,7 +160,7 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env) + cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env, true) ctlFlapMode = cap.MODE_PERCH goto perching } @@ -182,7 +182,7 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - callFlap, deployments+"."+*gAgentConfig.Env) + callFlap, deployments+"."+*gAgentConfig.Env, true) } } if modeCtl == capauth.TrcCtlComplete { @@ -191,12 +191,12 @@ func main() { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env) + cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env, true) goto deploycomplete } } } else { - time.Sleep(1 * time.Second) + interruptFun(multiSecondInterruptTicker) } } deploycomplete: @@ -219,7 +219,7 @@ func interruptFun(tickerInterrupt *time.Ticker) { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env) + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true) os.Exit(1) case <-tickerInterrupt.C: } @@ -249,7 +249,7 @@ func featherCtlCb(agentName string) error { if err == nil { if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { ctl := strings.Split(ctlFlapMode, "_") - if len(ctl) > 1 { + if len(ctl) > 1 && ctl[1] != "trcctlcomplete" { fmt.Printf("%s\n", ctl[1]) } callFlap = cap.MODE_GAZE @@ -267,7 +267,7 @@ func featherCtlCb(agentName string) error { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env) + callFlap, agentName+"."+*gAgentConfig.Env, true) } } @@ -532,7 +532,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr logger.Println(err) os.Exit(-1) } - fmt.Printf("Auth loaded %s\n", env) + config.Log.Printf("Auth loaded %s\n", env) } // Chewbacca: Begin dbg comment @@ -570,7 +570,12 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr os.Exit(-1) } // End dbg comment - fmt.Println("Session Authorized") + if config.IsShell { + config.Log.Println("Session Authorized") + } else { + fmt.Println("Session Authorized") + } + if len(os.Args) > 1 || len(trcPath) > 0 { // Generate .trc code... trcPathParts := strings.Split(trcPath, "/") @@ -604,8 +609,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr config.OutputMemCache = false } os.Args = []string{os.Args[0]} - fmt.Println("Processing trcshell") - + config.Log.Println("Processing trcshell") } else { fmt.Println("Processing manual trcshell") if env == "itdev" { diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index 74d518ad2..5f4e651cb 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -109,7 +109,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( } } else { if agentConfig == nil { - fmt.Println("Auth phase 1") + config.Log.Println("Auth phase 1") trcshConfig.KubeConfig, err = capauth.PenseQuery(config, "kubeconfig") } } @@ -124,7 +124,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( if agentConfig != nil { trcshConfig.VaultAddress, err = agentConfig.PenseFeatherQuery("caddress") } else { - fmt.Println("Auth phase 2") + config.Log.Println("Auth phase 2") trcshConfig.VaultAddress, err = capauth.PenseQuery(config, "caddress") } if err != nil { @@ -157,7 +157,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( if agentConfig != nil { trcshConfig.ConfigRole, err = agentConfig.PenseFeatherQuery("configrole") } else { - fmt.Println("Auth phase 3") + config.Log.Println("Auth phase 3") trcshConfig.ConfigRole, err = capauth.PenseQuery(config, "configrole") } if err != nil { @@ -167,7 +167,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( memprotectopts.MemProtect(nil, trcshConfig.ConfigRole) if agentConfig == nil { - fmt.Println("Auth phase 4") + config.Log.Println("Auth phase 4") trcshConfig.PubRole, err = capauth.PenseQuery(config, "pubrole") if err != nil { return trcshConfig, err @@ -178,7 +178,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( if agentConfig != nil { trcshConfig.CToken, err = agentConfig.PenseFeatherQuery("ctoken") } else { - fmt.Println("Auth phase 5") + config.Log.Println("Auth phase 5") trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") if err != nil { return trcshConfig, err @@ -190,7 +190,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( memprotectopts.MemProtect(nil, trcshConfig.CToken) - fmt.Println("Auth complete.") + config.Log.Println("Auth complete.") return trcshConfig, err } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 860d3c023..4a2bd618f 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -119,6 +119,8 @@ func CommonMain(envPtr *string, if !*agentdeployPtr { fmt.Println("Unsupported plugin type: " + *pluginTypePtr) os.Exit(1) + } else { + fmt.Printf("\nBeginning agent deployment for %s..\n", *pluginNamePtr) } } diff --git a/utils/autoUtil.go b/utils/autoUtil.go index f8d8b6b36..076d103fb 100644 --- a/utils/autoUtil.go +++ b/utils/autoUtil.go @@ -196,7 +196,11 @@ func AutoAuth(config *DriverConfig, return err } } - fmt.Printf("Auth connecting to vault @ %s\n", *addrPtr) + if config.IsShell { + config.Log.Printf("Auth connecting to vault @ %s\n", *addrPtr) + } else { + fmt.Printf("Auth connecting to vault @ %s\n", *addrPtr) + } v, err = sys.NewVault(config.Insecure, *addrPtr, *envPtr, false, ping, false, config.Log) if v != nil { defer v.Close() diff --git a/webapi/server/VaultInit.go b/webapi/server/VaultInit.go index 48c0107a1..a6ea91445 100644 --- a/webapi/server/VaultInit.go +++ b/webapi/server/VaultInit.go @@ -99,7 +99,7 @@ func (s *Server) InitVault(ctx context.Context, req *pb.InitReq) (*pb.InitResp, envStrings := SelectedEnvironment for _, e := range envStrings { mod.Env = e - warn, err = il.UploadTemplateDirectory(mod, templatePath, logger) + warn, err = il.UploadTemplateDirectory(config, mod, templatePath) eUtils.LogErrorObject(config, err, false) eUtils.LogWarningsObject(config, warn, false) } From 0166d3a6c2504e4ab9c80b181a319d219e01cd6a Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 10 Nov 2023 09:37:53 -0800 Subject: [PATCH 12/55] Prepare for release. (#789) * Fixing go.mod --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0714b02fc..f4e070c01 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,7 @@ require ( github.com/jonboulle/clockwork v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lafriks/go-shamir v1.1.0 // indirect @@ -124,7 +124,7 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/oauth2 v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/component-helpers v0.0.0-20230215120307-78f7b9c080f0 // indirect k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a // indirect diff --git a/go.sum b/go.sum index 17f9b58b5..067d1aac7 100644 --- a/go.sum +++ b/go.sum @@ -511,8 +511,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1276,8 +1276,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 h1:AB/lmRny7e2pLhFEYIbl5qkDAUt2h0ZRO4wGPhZf+ik= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= From 540254b3cec39f3a1676937917f6524d0edc8c4e Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:45:52 -0800 Subject: [PATCH 13/55] Prepare for release (#790) * Cleanup config and certify to handle errors more gracefully for better interaction with trcsh. Support multiple deployment configurations on a box. * Certify exit with error code. * Fixing some crashing. * Let this happen async. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfig/trcconfig.go | 8 +- trcconfigbase/trcconfig.go | 163 +++++++++-------- trcconfigbase/utils/configinator.go | 12 ++ trcflow/deploy/process.go | 4 +- trcsh/trcsh.go | 205 ++++++++++++---------- trcvault/trcplgtool/trcplgtool.go | 9 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 86 ++++----- trcvault/util/dbutil.go | 6 +- trcvault/util/properties.go | 5 +- trcvault/util/util.go | 4 +- utils/driverconfig.go | 2 + vaulthelper/kv/Modifier.go | 3 +- 12 files changed, 286 insertions(+), 221 deletions(-) diff --git a/trcconfig/trcconfig.go b/trcconfig/trcconfig.go index 71fbe167d..1446d8f5b 100644 --- a/trcconfig/trcconfig.go +++ b/trcconfig/trcconfig.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" @@ -17,7 +18,7 @@ func main() { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.26") + fmt.Println("Version: " + "1.27") envPtr := flag.String("env", "dev", "Environment to configure") addrPtr := flag.String("addr", "", "API endpoint for the vault") tokenPtr := flag.String("token", "", "Vault access token") @@ -26,5 +27,8 @@ func main() { appRoleIDPtr := flag.String("appRoleID", "", "Public app role ID") tokenNamePtr := flag.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") - trcconfigbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, nil) + err := trcconfigbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, nil) + if err != nil { + os.Exit(1) + } } diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 94236dac7..5d2e1a8b5 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -1,6 +1,7 @@ package trcconfigbase import ( + "errors" "flag" "fmt" "log" @@ -62,16 +63,15 @@ skipSwitch: } func reciever() { - for { - select { - case data := <-resultChannel: - if data != nil && data.inData != nil && data.inPath != "" { - mutex.Lock() - resultMap[data.inPath] = data.inData - mutex.Unlock() - } + + for data := range resultChannel { + if data != nil && data.inData != nil && data.inPath != "" { + mutex.Lock() + resultMap[data.inPath] = data.inData + mutex.Unlock() } } + } func CommonMain(envPtr *string, @@ -82,7 +82,7 @@ func CommonMain(envPtr *string, appRoleIDPtr *string, tokenNamePtr *string, regionPtr *string, - c *eUtils.DriverConfig) { + c *eUtils.DriverConfig) error { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } @@ -102,14 +102,19 @@ func CommonMain(envPtr *string, versionInfoPtr := flag.Bool("versions", false, "Version information about values") insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") noVaultPtr := flag.Bool("novault", false, "Don't pull configuration data from vault.") + isShellSubprocess := false - if c == nil || !c.IsShellSubProcess { + if c != nil { + isShellSubprocess = c.IsShellSubProcess + } + + if c == nil || !isShellSubprocess { args := os.Args[1:] for i := 0; i < len(args); i++ { s := args[i] if s[0] != '-' { fmt.Println("Wrong flag syntax: ", s) - os.Exit(1) + return fmt.Errorf("wrong flag syntax: %s", s) } } flag.Parse() @@ -135,10 +140,11 @@ func CommonMain(envPtr *string, *wantCertsPtr = true } } - - if _, err := os.Stat(*startDirPtr); os.IsNotExist(err) { - fmt.Println("Missing required template folder: " + *startDirPtr) - os.Exit(1) + if !isShellSubprocess { + if _, err := os.Stat(*startDirPtr); os.IsNotExist(err) { + fmt.Println("Missing required template folder: " + *startDirPtr) + return fmt.Errorf("missing required template folder: %s", *startDirPtr) + } } if *zcPtr { @@ -147,7 +153,7 @@ func CommonMain(envPtr *string, if strings.Contains(*envPtr, "*") { fmt.Println("* is not available as an environment suffix.") - os.Exit(1) + return errors.New("* is not available as an environment suffix") } var appRoleConfigPtr *string @@ -170,13 +176,13 @@ func CommonMain(envPtr *string, //Dont allow these combinations of flags if *templateInfoPtr && *diffPtr { fmt.Println("Cannot use -diff flag and -templateInfo flag together") - os.Exit(1) + return errors.New("cannot use -diff flag and -templateInfo flag together") } else if *versionInfoPtr && *diffPtr { fmt.Println("Cannot use -diff flag and -versionInfo flag together") - os.Exit(1) + return errors.New("cannot use -diff flag and -versionInfo flag together") } else if *versionInfoPtr && *templateInfoPtr { fmt.Println("Cannot use -templateInfo flag and -versionInfo flag together") - os.Exit(1) + return errors.New("cannot use -templateInfo flag and -versionInfo flag together") } else if *diffPtr { if strings.ContainsAny(*envPtr, ",") { //Multiple environments *envPtr = strings.ReplaceAll(*envPtr, "latest", "0") @@ -184,12 +190,12 @@ func CommonMain(envPtr *string, envLength = len(envDiffSlice) if len(envDiffSlice) > 4 { fmt.Println("Unsupported number of environments - Maximum: 4") - os.Exit(1) + return errors.New("unsupported number of environments - Maximum: 4") } for i, env := range envDiffSlice { if env == "local" { fmt.Println("Unsupported env: local not available with diff flag") - os.Exit(1) + return errors.New("unsupported env: local not available with diff flag") } if !strings.Contains(env, "_") && env != "filesys" { envDiffSlice[i] = env + "_0" @@ -203,16 +209,16 @@ func CommonMain(envPtr *string, } } else { fmt.Println("Incorrect format for diff: -env=env1,env2,...") - os.Exit(1) + return errors.New("incorrect format for diff: -env=env1,env2") } } else { if strings.ContainsAny(*envPtr, ",") { fmt.Println("-diff flag is required for multiple environments - env: -env=env1,env2,...") - os.Exit(1) + return errors.New("-diff flag is required for multiple environments - env: -env=env1,env2") } if strings.Contains(*envPtr, "filesys") { fmt.Println("Unsupported env: filesys only available with diff flag") - os.Exit(1) + return errors.New("unsupported env: filesys only available with diff flag") } envVersion := strings.Split(*envPtr, "_") //Break apart env+version for token *envPtr = envVersion[0] @@ -221,10 +227,10 @@ func CommonMain(envPtr *string, autoErr := eUtils.AutoAuth(configBase, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, *pingPtr) if autoErr != nil { fmt.Println("Missing auth components.") - os.Exit(1) + return errors.New("missing auth components") } if *pingPtr { - os.Exit(0) + return nil } } else { *tokenPtr = "novault" @@ -234,7 +240,7 @@ func CommonMain(envPtr *string, *envPtr = envVersion[0] + "_" + envVersion[1] if envVersion[1] == "" { fmt.Println("Must declare desired version number after '_' : -env=env1_ver1") - os.Exit(1) + return errors.New("must declare desired version number after '_' : -env=env1_ver1") } } else { *envPtr = envVersion[0] + "_0" @@ -259,16 +265,18 @@ func CommonMain(envPtr *string, removeDuplicateValuesSlice := eUtils.RemoveDuplicateValues(envDiffSlice) if !cmp.Equal(envDiffSlice, removeDuplicateValuesSlice) { fmt.Println("There is a duplicate environment in the -env flag") - os.Exit(1) + return errors.New("there is a duplicate environment in the -env flag") } } - if !*diffPtr { + if !*diffPtr && (c == nil || !c.IsShell) { if len(*envPtr) >= 5 && (*envPtr)[:5] == "local" { var err error *envPtr, err = eUtils.LoginToLocal() fmt.Println(*envPtr) - eUtils.CheckError(configBase, err, true) + if err != nil { + return err + } } } @@ -294,7 +302,7 @@ func CommonMain(envPtr *string, } if len(regions) == 0 { fmt.Println("Unsupported region: " + *regionPtr) - os.Exit(1) + return fmt.Errorf("unsupported region: %s", *regionPtr) } } } @@ -319,10 +327,10 @@ func CommonMain(envPtr *string, autoErr := eUtils.AutoAuth(configBase, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, *pingPtr) if autoErr != nil { fmt.Println("Missing auth components.") - os.Exit(1) + return errors.New("missing auth components") } if *pingPtr { - os.Exit(0) + return nil } } else { *tokenPtr = "novault" @@ -331,7 +339,7 @@ func CommonMain(envPtr *string, *envPtr = envVersion[0] + "_" + envVersion[1] if envVersion[1] == "" { fmt.Println("Must declare desired version number after '_' : -env=env1_ver1,env2_ver2") - os.Exit(1) + return errors.New("must declare desired version number after '_' : -env=env1_ver1,env2_ver2") } } else { *envPtr = envVersion[0] + "_0" @@ -342,27 +350,28 @@ func CommonMain(envPtr *string, } config := eUtils.DriverConfig{ - Insecure: *insecurePtr, - Token: *tokenPtr, - VaultAddress: *addrPtr, - Env: *envPtr, - EnvRaw: eUtils.GetRawEnv(*envPtr), - Regions: regions, - SecretMode: *secretMode, - ServicesWanted: services, - StartDir: append([]string{}, *startDirPtr), - EndDir: *endDirPtr, - WantCerts: *wantCertsPtr, - WantKeystore: *keyStorePtr, - ZeroConfig: *zcPtr, - GenAuth: false, - OutputMemCache: configBase.OutputMemCache, - MemFs: configBase.MemFs, - Log: configBase.Log, - ExitOnFailure: configBase.ExitOnFailure, - Diff: *diffPtr, - Update: messenger, - FileFilter: fileFilterSlice, + IsShellSubProcess: isShellSubprocess, + Insecure: *insecurePtr, + Token: *tokenPtr, + VaultAddress: *addrPtr, + Env: *envPtr, + EnvRaw: eUtils.GetRawEnv(*envPtr), + Regions: regions, + SecretMode: *secretMode, + ServicesWanted: services, + StartDir: append([]string{}, *startDirPtr), + EndDir: *endDirPtr, + WantCerts: *wantCertsPtr, + WantKeystore: *keyStorePtr, + ZeroConfig: *zcPtr, + GenAuth: false, + OutputMemCache: configBase.OutputMemCache, + MemFs: configBase.MemFs, + Log: configBase.Log, + ExitOnFailure: configBase.ExitOnFailure, + Diff: *diffPtr, + Update: messenger, + FileFilter: fileFilterSlice, } configSlice = append(configSlice, config) wg.Add(1) @@ -392,27 +401,28 @@ func CommonMain(envPtr *string, *envPtr = envVersion[0] + "_0" } config := eUtils.DriverConfig{ - Insecure: *insecurePtr, - Token: *tokenPtr, - VaultAddress: *addrPtr, - Env: *envPtr, - EnvRaw: eUtils.GetRawEnv(*envPtr), - Regions: regions, - SecretMode: *secretMode, - ServicesWanted: services, - StartDir: append([]string{}, *startDirPtr), - EndDir: *endDirPtr, - WantCerts: *wantCertsPtr, - WantKeystore: *keyStorePtr, - ZeroConfig: *zcPtr, - GenAuth: false, - OutputMemCache: configBase.OutputMemCache, - MemFs: configBase.MemFs, - ExitOnFailure: configBase.ExitOnFailure, - Log: configBase.Log, - Diff: *diffPtr, - FileFilter: fileFilterSlice, - VersionInfo: eUtils.VersionHelper, + IsShellSubProcess: isShellSubprocess, + Insecure: *insecurePtr, + Token: *tokenPtr, + VaultAddress: *addrPtr, + Env: *envPtr, + EnvRaw: eUtils.GetRawEnv(*envPtr), + Regions: regions, + SecretMode: *secretMode, + ServicesWanted: services, + StartDir: append([]string{}, *startDirPtr), + EndDir: *endDirPtr, + WantCerts: *wantCertsPtr, + WantKeystore: *keyStorePtr, + ZeroConfig: *zcPtr, + GenAuth: false, + OutputMemCache: configBase.OutputMemCache, + MemFs: configBase.MemFs, + ExitOnFailure: configBase.ExitOnFailure, + Log: configBase.Log, + Diff: *diffPtr, + FileFilter: fileFilterSlice, + VersionInfo: eUtils.VersionHelper, } wg.Add(1) go func(c *eUtils.DriverConfig) { @@ -438,4 +448,5 @@ func CommonMain(envPtr *string, }() } wg.Wait() //Wait for diff + return nil } diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index c8f4e95a4..b62bf498f 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -34,7 +34,19 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverCo version := "" if err != nil { eUtils.LogErrorObject(config, err, false) + return nil, err } + if config.IsShellSubProcess && len(config.Deployment) > 0 { + certifyMap, err := modCheck.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", config.Deployment)) + if err != nil { + return nil, err + } + if trcDeployRoot, ok := certifyMap["trcdeployroot"]; ok { + config.StartDir = []string{trcDeployRoot.(string)} + config.EndDir = trcDeployRoot.(string) + } + } + modCheck.VersionFilter = config.VersionFilter //Check if templateInfo is selected for template or values diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index b23d0cf27..5e467e21b 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -330,7 +330,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e factory.PushPluginSha(cConfig, pluginConfig, vaultPluginSignature) eUtils.LogInfo(cConfig, pluginName+": End checkpush sha256") - writeMap, err := cGoMod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + pluginName + "/Certify") + writeMap, err := cGoMod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", pluginName)) if err != nil { eUtils.LogInfo(cConfig, pluginName+": Initializing certification") @@ -348,7 +348,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e if writeMap["trctype"].(string) == "agent" { writeMap["deployed"] = true } - _, err = cGoMod.Write("super-secrets/Index/TrcVault/trcplugin/"+writeMap["trcplugin"].(string)+"/Certify", writeMap, cConfig.Log) + _, err = cGoMod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", writeMap["trcplugin"].(string)), writeMap, cConfig.Log) if err != nil { logger.Printf(fmt.Sprintf("PluginDeployFlow failure: Failed to write plugin state for env: %s and plugin: %s error: %s\n", cConfig.Env, pluginName, err.Error())) } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index bf0a1a1cd..5c8e554d8 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -36,6 +36,90 @@ import ( var gAgentConfig *capauth.AgentConfigs = nil var gTrcshConfig *capauth.TrcShConfig +func ProcessDeployment(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { + go func() { + // Timeout and CtlMessage subscriber + go func() { + for range time.After(120 * time.Second) { + ctlMsg := "Deployment timed out after 120 seconds" + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true) + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + break + } + }() + + for { + perching: + if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_FLAP, deployment+"."+*gAgentConfig.Env, false); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { + // Lookup trcPath from deployment + + // Process the script.... + // This will feed CtlMessages into the Timeout and CtlMessage subscriber + go ProcessDeploy(*gAgentConfig.Env, region, "", "", trcPath, secretId, approleId, false) + + for modeCtl := range gAgentConfig.CtlMessage { + flapMode := cap.MODE_FLAP + "_" + modeCtl + ctlFlapMode := flapMode + var err error = errors.New("init") + + for { + if err == nil && ctlFlapMode == cap.MODE_PERCH { + // Acknowledge perching... + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, deployment+"."+*gAgentConfig.Env, true) + ctlFlapMode = cap.MODE_PERCH + goto perching + } + + if err == nil && flapMode != ctlFlapMode { + // Flap, Gaze, etc... + interruptFun(twoHundredMilliInterruptTicker) + break + } else { + callFlap := flapMode + if err == nil { + interruptFun(twoHundredMilliInterruptTicker) + } else { + if err.Error() != "init" { + interruptFun(multiSecondInterruptTicker) + } + } + ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + callFlap, deployment+"."+*gAgentConfig.Env, true) + } + } + if modeCtl == capauth.TrcCtlComplete { + // Only exit with TrcCtlComplete. + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true) + goto deploycomplete + } + } + } else { + interruptFun(multiSecondInterruptTicker) + } + } + deploycomplete: + }() +} + // This is a controller program that can act as any command line utility. // The Tierceron Shell runs tierceron and kubectl commands in a secure shell. func main() { @@ -43,7 +127,7 @@ func main() { memprotectopts.MemProtectInit(nil) } eUtils.InitHeadless(true) - fmt.Println("trcsh Version: " + "1.20") + fmt.Println("trcsh Version: " + "1.21") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string if runtime.GOOS != "windows" { @@ -58,7 +142,7 @@ func main() { // Running as shell. os.Args[1] = "-c=" + os.Args[1] // Initiate signal handling. - var ic chan os.Signal = make(chan os.Signal) + var ic chan os.Signal = make(chan os.Signal, 3) signal.Notify(ic, os.Interrupt, syscall.SIGTERM) go func() { x := <-ic @@ -84,7 +168,7 @@ func main() { memprotectopts.MemProtect(nil, appRoleIDPtr) //Open deploy script and parse it. - ProcessDeploy(*envPtr, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) + ProcessDeploy(*envPtr, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) } else { gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 1)} deployments := os.Getenv("DEPLOYMENTS") @@ -92,7 +176,6 @@ func main() { agentEnv := os.Getenv("AGENT_ENV") address := os.Getenv("VAULT_ADDR") - envPtr = flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. regionPtr = flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. trcPathPtr = flag.String("c", "", "Optional script to execute.") //If this is blank -> use context otherwise override context. appRoleIDPtr = flag.String("appRoleID", "", "Public app role ID") @@ -126,84 +209,11 @@ func main() { // Preload agent synchronization configs gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) - go func() { - // Timeout and CtlMessage subscriber - go func() { - for range time.After(120 * time.Second) { - ctlMsg := "Deployment timed out after 120 seconds" - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployments+"."+*gAgentConfig.Env, true) - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - break - } - }() - - for { - perching: - if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployments+"."+*gAgentConfig.Env, false); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { - for modeCtl := range gAgentConfig.CtlMessage { - flapMode := cap.MODE_FLAP + "_" + modeCtl - ctlFlapMode := flapMode - var err error = errors.New("init") - - for { - if err == nil && ctlFlapMode == cap.MODE_PERCH { - // Acknowledge perching... - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployments+"."+*gAgentConfig.Env, true) - ctlFlapMode = cap.MODE_PERCH - goto perching - } + deploymentsSlice := strings.Split(deployments, ",") + for _, deployment := range deploymentsSlice { + ProcessDeployment(*gAgentConfig.Env, *regionPtr, deployment, *trcPathPtr, secretIDPtr, appRoleIDPtr, false, deployment) + } - if err == nil && flapMode != ctlFlapMode { - // Flap, Gaze, etc... - interruptFun(twoHundredMilliInterruptTicker) - break - } else { - callFlap := flapMode - if err == nil { - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(multiSecondInterruptTicker) - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, deployments+"."+*gAgentConfig.Env, true) - } - } - if modeCtl == capauth.TrcCtlComplete { - // Only exit with TrcCtlComplete. - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployments+"."+*gAgentConfig.Env, true) - goto deploycomplete - } - } - } else { - interruptFun(multiSecondInterruptTicker) - } - } - deploycomplete: - }() - // Process the script.... - // This will feed CtlMessages into the Timeout and CtlMessage subscriber - ProcessDeploy(*gAgentConfig.Env, *regionPtr, "", *trcPathPtr, secretIDPtr, appRoleIDPtr, false) <-shutdown } } @@ -228,7 +238,7 @@ func interruptFun(tickerInterrupt *time.Ticker) { func featherCtlCb(agentName string) error { if gAgentConfig == nil { - return errors.New("Incorrect agent initialization") + return errors.New("incorrect agent initialization") } else { gAgentConfig.Deployments = &agentName } @@ -281,7 +291,7 @@ func configCmd(env string, token string, argsOrig []string, deployArgLines []string, - configCount *int) { + configCount *int) error { *configCount -= 1 if *configCount != 0 { //This is to keep result channel open - closes on the final config call of the script. config.EndDir = "deploy" @@ -297,7 +307,7 @@ func configCmd(env string, tokenConfig := "" configEnv := env - trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + err := trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -305,6 +315,7 @@ func configCmd(env string, token = "" config.Token = token } + return err } func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, @@ -410,14 +421,14 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, argsOrig []string, deployArgLines []string, configCount *int, - logger *log.Logger) { + logger *log.Logger) error { switch control { case "trcplgtool": config.AppRoleConfig = "" config.EnvRaw = env config.IsShellSubProcess = true - trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) ResetModifier(config) os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -425,8 +436,10 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, token = "" config.Token = token } + return err + case "trcconfig": - configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + err := configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -434,7 +447,9 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, token = "" config.Token = token } + return err } + return errors.New("unsupported control error") } // ProcessDeploy @@ -442,15 +457,17 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, // Parameters: // // - env: Current environment context +// - region: a region // - token: An environment token -// - trcPath: Path to the current executable +// - deployment: name of deployment +// - trcPath: Path to the current deployment script... // - secretId: trcsh secret. // - approleId: trcsh app role. // // Returns: // // Nothing. -func ProcessDeploy(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { +func ProcessDeploy(env string, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { var err error agentToken := false if token != "" { @@ -577,7 +594,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr } if len(os.Args) > 1 || len(trcPath) > 0 { - // Generate .trc code... + // Generate trc code... trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") @@ -688,7 +705,7 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr } } if runtime.GOOS == "windows" { - processWindowsCmds( + err := processWindowsCmds( trcKubeDeploymentConfig, &onceKubeInit, PipeOS, @@ -703,7 +720,11 @@ func ProcessDeploy(env string, region string, token string, trcPath string, secr deployArgLines, &configCount, logger) - gAgentConfig.CtlMessage <- deployLine + if err != nil { + gAgentConfig.CtlMessage <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + } else { + gAgentConfig.CtlMessage <- deployLine + } } else { processPluginCmds( &trcKubeDeploymentConfig, diff --git a/trcvault/trcplgtool/trcplgtool.go b/trcvault/trcplgtool/trcplgtool.go index de529cfe0..0cba48e94 100644 --- a/trcvault/trcplgtool/trcplgtool.go +++ b/trcvault/trcplgtool/trcplgtool.go @@ -3,18 +3,23 @@ package main import ( "flag" "fmt" + "os" plgtbase "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" ) // This executable automates the cerification of a plugin docker image. func main() { - fmt.Println("Version: " + "1.01") + fmt.Println("Version: " + "1.02") envPtr := flag.String("env", "dev", "Environment to configure") addrPtr := flag.String("addr", "", "API endpoint for the vault") tokenPtr := flag.String("token", "", "Vault access token") regionPtr := flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. - plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, nil) + err := plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, nil) + if err != nil { + os.Exit(1) + } + } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 4a2bd618f..e701b5f7f 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -3,6 +3,7 @@ package trcplgtoolbase import ( "bufio" "crypto/sha256" + "errors" "flag" "fmt" "io" @@ -22,7 +23,7 @@ func CommonMain(envPtr *string, addrPtr *string, tokenPtr *string, regionPtr *string, - c *eUtils.DriverConfig) { + c *eUtils.DriverConfig) error { // Main functions are as follows: defineServicePtr := flag.Bool("defineService", false, "Service is defined.") @@ -60,7 +61,7 @@ func CommonMain(envPtr *string, s := args[i] if s[0] != '-' { fmt.Println("Wrong flag syntax: ", s) - os.Exit(1) + return fmt.Errorf("wrong flag syntax: %s", s) } } flag.Parse() @@ -68,7 +69,7 @@ func CommonMain(envPtr *string, // Prints usage if no flags are specified if flag.NFlag() == 0 { flag.Usage() - os.Exit(1) + return errors.New("invalid input parameters") } } else { flag.CommandLine.Parse(os.Args) @@ -77,22 +78,22 @@ func CommonMain(envPtr *string, if *certifyImagePtr && (len(*pluginNamePtr) == 0 || len(*sha256Ptr) == 0) { fmt.Println("Must use -pluginName && -sha256 flags to use -certify flag") - os.Exit(1) + return errors.New("must use -pluginName && -sha256 flags to use -certify flag") } if *checkDeployedPtr && (len(*pluginNamePtr) == 0) { fmt.Println("Must use -pluginName flag to use -checkDeployed flag") - os.Exit(1) + return errors.New("must use -pluginName flag to use -checkDeployed flag") } if *defineServicePtr && (len(*pluginNamePtr) == 0) { fmt.Println("Must use -pluginName flag to use -defineService flag") - os.Exit(1) + return errors.New("must use -pluginName flag to use -defineService flag") } if strings.Contains(*pluginNamePtr, ".") { fmt.Println("-pluginName cannot contain reserved character '.'") - os.Exit(1) + return errors.New("-pluginName cannot contain reserved character '.'") } if *agentdeployPtr || *winservicestopPtr || *winservicestartPtr || *codebundledeployPtr { @@ -105,20 +106,20 @@ func CommonMain(envPtr *string, // TODO: do we want to support Deployment certifications in the pipeline at some point? // If so this is a config check to remove. fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) - os.Exit(-1) + return fmt.Errorf("plugin type %s not supported in trcsh", *pluginTypePtr) } case "agent": // A deployment agent tool. if c != nil { // TODO: do we want to support Deployment certifications in the pipeline at some point? // If so this is a config check to remove. fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) - os.Exit(-1) + return fmt.Errorf("plugin type %s not supported in trcsh", *pluginTypePtr) } case "trcshservice": // A trcshservice managed microservice default: if !*agentdeployPtr { fmt.Println("Unsupported plugin type: " + *pluginTypePtr) - os.Exit(1) + return fmt.Errorf("unsupported plugin type: %s", *pluginTypePtr) } else { fmt.Printf("\nBeginning agent deployment for %s..\n", *pluginNamePtr) } @@ -138,7 +139,7 @@ func CommonMain(envPtr *string, } else { if *agentdeployPtr { fmt.Println("Unsupported agentdeploy outside trcsh") - os.Exit(1) + return errors.New("unsupported agentdeploy outside trcsh") } // If logging production directory does not exist and is selected log to local directory @@ -149,7 +150,9 @@ func CommonMain(envPtr *string, logger = log.New(f, "[INIT]", log.LstdFlags) configBase = &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true, StartDir: []string{*startDirPtr}, SubSectionValue: *pluginNamePtr} - eUtils.CheckError(configBase, err, true) + if err != nil { + return err + } } regions := []string{} @@ -158,7 +161,7 @@ func CommonMain(envPtr *string, pluginConfig = buildopts.ProcessPluginEnvConfig(pluginConfig) //contains logNamespace for InitVaultMod if pluginConfig == nil { fmt.Println("Error: Could not find plugin config") - os.Exit(1) + return errors.New("could not find plugin config") } pluginConfig["env"] = *envPtr pluginConfig["vaddress"] = *addrPtr @@ -174,7 +177,7 @@ func CommonMain(envPtr *string, if err != nil { logger.Println("Error: " + err.Error() + " - 1") logger.Println("Failed to init mod for deploy update") - os.Exit(1) + return err } config.StartDir = []string{*startDirPtr} config.SubSectionValue = *pluginNamePtr @@ -192,7 +195,7 @@ func CommonMain(envPtr *string, } if len(regions) == 0 { fmt.Println("Unsupported region: " + *regionPtr) - os.Exit(1) + return fmt.Errorf("unsupported region: %s", *regionPtr) } } configBase.Regions = regions @@ -202,7 +205,7 @@ func CommonMain(envPtr *string, pluginToolConfig, plcErr := trcvutils.GetPluginToolConfig(configBase, mod, coreopts.ProcessDeployPluginEnvConfig(map[string]interface{}{}), *defineServicePtr) if plcErr != nil { fmt.Println(plcErr.Error()) - os.Exit(1) + return plcErr } if len(*sha256Ptr) > 0 { @@ -211,8 +214,8 @@ func CommonMain(envPtr *string, if fileInfo.Mode().IsRegular() { file, fileOpenErr := os.Open(*sha256Ptr) if fileOpenErr != nil { - fmt.Println(fileOpenErr) - return + fmt.Println(fileOpenErr.Error()) + return fileOpenErr } // Close the file when we're done. @@ -224,7 +227,7 @@ func CommonMain(envPtr *string, pluginImage, imageErr := io.ReadAll(reader) if imageErr != nil { fmt.Println("Failed to read image:" + imageErr.Error()) - return + return imageErr } sha256Bytes := sha256.Sum256(pluginImage) *sha256Ptr = fmt.Sprintf("%x", sha256Bytes) @@ -286,7 +289,7 @@ func CommonMain(envPtr *string, _, err = mod.Write(pluginToolConfig["pluginpath"].(string), writeMap, configBase.Log) if err != nil { fmt.Println(err) - os.Exit(1) + return err } fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { @@ -295,7 +298,7 @@ func CommonMain(envPtr *string, err := cmd.Run() if err != nil { fmt.Println(err) - return + return err } fmt.Printf("Service stopped: %s\n", pluginToolConfig["trcservicename"].(string)) @@ -305,15 +308,16 @@ func CommonMain(envPtr *string, err := cmd.Run() if err != nil { fmt.Println(err) - return + return err } fmt.Printf("Service started: %s\n", pluginToolConfig["trcservicename"].(string)) } else if *codebundledeployPtr { if pluginToolConfig["trcsha256"] != nil && len(pluginToolConfig["trcsha256"].(string)) > 0 { err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { + fmt.Println("Image download failure.") fmt.Println(err.Error()) - os.Exit(1) + return err } } @@ -327,7 +331,8 @@ func CommonMain(envPtr *string, err = os.WriteFile(deployPath, pluginToolConfig["rawImageFile"].([]byte), 0644) if err != nil { fmt.Println(err.Error()) - os.Exit(1) + fmt.Println("Image write failure.") + return err } fmt.Println("Image deployed.") } else { @@ -344,7 +349,7 @@ func CommonMain(envPtr *string, if err != nil { fmt.Println(err.Error()) } - os.Exit(1) + return err } } @@ -366,7 +371,7 @@ func CommonMain(envPtr *string, properties, err := trcvutils.NewProperties(config, vault, mod, mod.Env, "TrcVault", "Certify") if err != nil { fmt.Println("Couldn't create properties for regioned certify:" + err.Error()) - os.Exit(1) + return err } writeMap, replacedFields := properties.GetPluginData(*regionPtr, "Certify", "config", logger) @@ -374,36 +379,36 @@ func CommonMain(envPtr *string, writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginToolConfig["trcplugin"].(string)) if writeErr != nil { fmt.Println(writeErr) - os.Exit(1) + return err } } else { //Non region certify writeMap, readErr := mod.ReadData(pluginToolConfig["pluginpath"].(string)) if readErr != nil { fmt.Println(readErr) - os.Exit(1) + return err } _, err = mod.Write(pluginToolConfig["pluginpath"].(string), WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), configBase.Log) if err != nil { fmt.Println(err) - os.Exit(1) + return err } fmt.Println("Image certified in vault and is ready for release.") } } else { fmt.Println("Invalid or nonexistent image.") - os.Exit(1) + return err } } else if *agentdeployPtr { if config.FeatherCtlCb != nil { err := config.FeatherCtlCb(*pluginNamePtr) if err != nil { fmt.Println("Incorrect installation") - os.Exit(1) + return err } } else { fmt.Println("Incorrect trcplgtool utilization") - os.Exit(1) + return err } } @@ -413,48 +418,49 @@ func CommonMain(envPtr *string, (pluginToolConfig["deployed"] != nil && pluginToolConfig["deployed"].(bool)) && (pluginToolConfig["trcsha256"] != nil && pluginToolConfig["trcsha256"].(string) == *sha256Ptr) { //Compare vault sha with provided sha fmt.Println("Plugin has been copied, deployed & certified.") - os.Exit(0) + return nil } err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println(err.Error()) - os.Exit(1) + return err } if *sha256Ptr == pluginToolConfig["imagesha256"].(string) { //Compare repo image sha with provided sha fmt.Println("Latest plugin image sha matches provided plugin sha. It has been certified.") } else { fmt.Println("Provided plugin sha is not deployable.") - os.Exit(1) + return errors.New("provided plugin sha is not deployable") } fmt.Println("Plugin has not been copied or deployed.") - os.Exit(2) + return nil } if *checkCopiedPtr { if pluginToolConfig["copied"].(bool) && pluginToolConfig["trcsha256"].(string) == *sha256Ptr { //Compare vault sha with provided sha fmt.Println("Plugin has been copied & certified.") - os.Exit(0) + return nil } err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println(err.Error()) - os.Exit(1) + return err } if *sha256Ptr == pluginToolConfig["imagesha256"].(string) { //Compare repo image sha with provided sha fmt.Println("Latest plugin image sha matches provided plugin sha. It has been certified.") } else { fmt.Println("Provided plugin sha is not certified.") - os.Exit(1) + return errors.New("provided plugin sha is not certified") } fmt.Println("Plugin has not been copied or deployed.") - os.Exit(2) + return nil } + return nil } func WriteMapUpdate(writeMap map[string]interface{}, pluginToolConfig map[string]interface{}, defineServicePtr bool, pluginTypePtr string) map[string]interface{} { diff --git a/trcvault/util/dbutil.go b/trcvault/util/dbutil.go index ef802d560..264f554ef 100644 --- a/trcvault/util/dbutil.go +++ b/trcvault/util/dbutil.go @@ -19,19 +19,21 @@ func OpenDirectConnection(config *eUtils.DriverConfig, url string, username stri var conn *sql.DB var err error - if driver == "mysql" || driver == "mariadb" { + switch driver { + case "mysql", "mariadb": if len(port) == 0 { // protocol+transport://user:pass@host/dbname?option1=a&option2=b conn, err = dburl.Open(driver + "://" + username + ":" + password + "@" + server + "/" + dbname + "?tls=skip-verify&parseTime=true") } else { conn, err = dburl.Open(driver + "://" + username + ":" + password + "@" + server + ":" + port + "/" + dbname + "?tls=skip-verify&parseTime=true") } - } else if driver == "sqlserver" { + case "sqlserver": if len(port) == 0 { port = "1433" } conn, err = dburl.Open(driver + "://" + username + ":" + password + "@" + server + ":" + port + "/" + dbname + "?tls=skip-verify") } + if err != nil { if conn != nil { defer conn.Close() diff --git a/trcvault/util/properties.go b/trcvault/util/properties.go index a956ef9b4..779d1f83e 100644 --- a/trcvault/util/properties.go +++ b/trcvault/util/properties.go @@ -1,6 +1,7 @@ package util import ( + "fmt" "log" "strconv" "strings" @@ -165,7 +166,7 @@ func (p *Properties) WritePluginData(pluginData map[string]interface{}, replaced } } - writeMap, readErr := mod.ReadData("super-secrets/Index/TrcVault/trcplugin/" + pluginName + "/Certify") //This read is need to avoid overwritting un-used region data. + writeMap, readErr := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", pluginName)) //This read is need to avoid overwritting un-used region data. if readErr != nil { return readErr } @@ -174,7 +175,7 @@ func (p *Properties) WritePluginData(pluginData map[string]interface{}, replaced writeMap[field] = value } - _, writeErr := mod.Write("super-secrets/Index/TrcVault/trcplugin/"+pluginName+"/Certify", writeMap, log) + _, writeErr := mod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", pluginName), writeMap, log) if writeErr != nil { return writeErr } diff --git a/trcvault/util/util.go b/trcvault/util/util.go index 1e4238f2b..19354420d 100644 --- a/trcvault/util/util.go +++ b/trcvault/util/util.go @@ -104,7 +104,7 @@ func GetJSONFromClientByPost(config *eUtils.DriverConfig, httpClient *http.Clien } if response.StatusCode == http.StatusUnauthorized { - return nil, response.StatusCode, errors.New(fmt.Sprintf("http auth failure: %d", response.StatusCode)) + return nil, response.StatusCode, fmt.Errorf("http auth failure: %d", response.StatusCode) } else if response.StatusCode == http.StatusOK { jsonDataFromHttp, err := io.ReadAll(response.Body) @@ -120,7 +120,7 @@ func GetJSONFromClientByPost(config *eUtils.DriverConfig, httpClient *http.Clien return jsonData, response.StatusCode, nil } - return nil, response.StatusCode, errors.New(fmt.Sprintf("http status failure: %d", response.StatusCode)) + return nil, response.StatusCode, fmt.Errorf("http status failure: %d", response.StatusCode) } func LoadBaseTemplate(config *eUtils.DriverConfig, templateResult *extract.TemplateResultData, goMod *helperkv.Modifier, project string, service string, templatePath string) error { diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 0c7db2062..36bd36e2d 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -78,6 +78,8 @@ type DriverConfig struct { SubSectionValue string ServiceFilter []string // Which tables to use. DynamicPathFilter string // Seeds from a specific path. + + Deployment string // For trcsh to indicate which deployment to work on } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. diff --git a/vaulthelper/kv/Modifier.go b/vaulthelper/kv/Modifier.go index ee13f5c8e..9ecda8af9 100644 --- a/vaulthelper/kv/Modifier.go +++ b/vaulthelper/kv/Modifier.go @@ -105,7 +105,8 @@ func NewModifier(insecure bool, token string, address string, env string, region case checkoutModifier := <-modifierCache[env].modifierChan: return checkoutModifier, nil case <-time.After(time.Millisecond * 200): - if atomic.LoadUint64(&modifierCache[env].modCount) < 20 { + // Someone leaking in trcconfig... this is getting high... + if atomic.LoadUint64(&modifierCache[env].modCount) < 30 { goto modbuild } } From 1669745152d68e2289d36c5117be058e3907d695 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Sat, 11 Nov 2023 17:10:00 -0800 Subject: [PATCH 14/55] Prepare for release (#793) * Add some error handling with early exit. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl | 2 +- trcsh/trcsh.go | 10 ++++++++-- trcvault/README | 5 ++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl b/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl index 350d251aa..956373318 100644 --- a/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl +++ b/trcsh/trc_templates/Tierceron/TrcDeploy/setup.cmd.tmpl @@ -3,5 +3,5 @@ setx /M AGENT_TOKEN "{{.trcDeployAgentToken}}" setx /M DEPLOYMENTS "{{.trcDeployDeployments}}" setx /M VAULT_ADDR "{{.trcDeployVaultAddr}}" -.\nssm.exe install TrcDeploy "{{.trcDeployDirectory}}" +.\nssm.exe install TrcDeploy "{{.trcDeployExecutable}}" .\nssm.exe set TrcDeploy AppDirectory "{{.trcDeployStartDirectory}}" diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 5c8e554d8..c440039c3 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -351,7 +351,10 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, config.Token = token } case "trcconfig": - configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + err := configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + if err != nil { + os.Exit(1) + } case "trcplgtool": config.AppRoleConfig = "" config.EnvRaw = env @@ -364,7 +367,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, } gAgentConfig.Env = &env - trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) config.FeatherCtlCb = nil ResetModifier(config) //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -372,6 +375,9 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, token = "" config.Token = token } + if err != nil { + os.Exit(1) + } case "kubectl": onceKubeInit.Do(func() { diff --git a/trcvault/README b/trcvault/README index 91ea7fad5..a1061dbfc 100644 --- a/trcvault/README +++ b/trcvault/README @@ -13,7 +13,10 @@ trcx -env=QA -token=$VAULT_TOKEN -insecure -indexed=TrcVault -indexFilter=region trcx -env=QA -token=$VAULT_TOKEN -restricted=Identity -serviceFilter=config -indexFilter=config trcx -env=QA -token=$VAULT_TOKEN -restricted=PluginTool -serviceFilter=config -indexFilter=config trcx -env=QA -token=$VAULT_TOKEN -restricted=VaultDatabase -serviceFilter=config -indexFilter=config +trcx -env=dev -token=$VAULT_TOKEN -restricted=TrcDeploy -serviceFilter=setup -indexFilter=setup +trcx -env=dev -token=$VAULT_TOKEN -restricted=TrcshAgent -serviceFilter=config -indexFilter=config # Initialize trcinit -env=QA -token=$VAULT_TOKEN -restricted=VaultDatabase -serviceFilter=config -indexFilter=config -trcinit -env=dev -token=$VAULT_TOKEN_DEV -dynamicPath=Restricted/VaultDatabase/config \ No newline at end of file +trcinit -env=dev -token=$VAULT_TOKEN_DEV -dynamicPath=Restricted/VaultDatabase/config +trcinit -env=dev -token=$VAULT_TOKEN_DEV -dynamicPath=Restricted/TrcshAgent/config From 213f5fa730745010f4fe1404c26f5e58a8c29739 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Sat, 11 Nov 2023 18:20:50 -0800 Subject: [PATCH 15/55] Prepare for release. (#795) * Make deployments re-runnable. * Set timer for each run. Make CtlMessage a buffered channel. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcsh/trcsh.go | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index c440039c3..8eae5979a 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -38,20 +38,6 @@ var gTrcshConfig *capauth.TrcShConfig func ProcessDeployment(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { go func() { - // Timeout and CtlMessage subscriber - go func() { - for range time.After(120 * time.Second) { - ctlMsg := "Deployment timed out after 120 seconds" - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true) - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete - break - } - }() - for { perching: if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, @@ -63,7 +49,23 @@ func ProcessDeployment(env string, region string, token string, trcPath string, // Process the script.... // This will feed CtlMessages into the Timeout and CtlMessage subscriber - go ProcessDeploy(*gAgentConfig.Env, region, "", "", trcPath, secretId, approleId, false) + go func() { + go func() { + // Timeout and CtlMessage subscriber + for range time.After(120 * time.Second) { + ctlMsg := "Deployment timed out after 120 seconds" + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true) + gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + break + } + }() + + ProcessDeploy(*gAgentConfig.Env, region, "", "", trcPath, secretId, approleId, false) + }() for modeCtl := range gAgentConfig.CtlMessage { flapMode := cap.MODE_FLAP + "_" + modeCtl @@ -115,8 +117,8 @@ func ProcessDeployment(env string, region string, token string, trcPath string, } else { interruptFun(multiSecondInterruptTicker) } + deploycomplete: } - deploycomplete: }() } @@ -170,7 +172,7 @@ func main() { //Open deploy script and parse it. ProcessDeploy(*envPtr, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) } else { - gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 1)} + gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 5)} deployments := os.Getenv("DEPLOYMENTS") agentToken := os.Getenv("AGENT_TOKEN") agentEnv := os.Getenv("AGENT_ENV") @@ -362,7 +364,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, config.FeatherCtlCb = featherCtlCb if gAgentConfig == nil { - gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 1)} + gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 5)} gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", "dev") // Feathering always in dev environmnent. } gAgentConfig.Env = &env From 3bec74c09d9665540bab4051b6b0901985a830f4 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Sun, 12 Nov 2023 19:40:19 -0800 Subject: [PATCH 16/55] Prepare for release. (#797) * Cleanup config and certify to handle errors more gracefully for better interaction with trcsh. Support multiple deployment configurations on a box. * Certify exit with error code. * Fixing some crashing. * Let this happen async. * Adding some more commands. * Select more suitable param name. * Add some error handling with early exit. * Make deployments re-runnable. * Set timer for each run. Make CtlMessage a buffered channel. * Couple more fixes for multi-deploy. * Small spelling correction. * Get everything at least running... Output to correct directory not yet functional. * A little more cleanup. * Add support for projectservice. * Add read support for projectservice. * Fix missing import. * Fixing couple bugs and log cleanup. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfigbase/trcconfig.go | 27 ++++-- trcconfigbase/utils/configinator.go | 11 --- trcctl/trcctl.go | 2 +- trcflow/flumen/process.go | 10 +- trcsh/trcsh.go | 95 ++++++++++++++++--- trcsub/trcsub.go | 8 +- trcsubbase/trcsub.go | 72 +++++++++----- .../TrcVault/Certify/config.yml.tmpl | 1 + trcvault/trcplgtoolbase/trcplgtoolbase.go | 10 ++ trcvault/util/util.go | 3 + trcx/xutil/xmanager.go | 2 +- trcxbase/trcxbase.go | 6 +- utils/driverconfig.go | 2 +- 13 files changed, 177 insertions(+), 72 deletions(-) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 5d2e1a8b5..69d1832d1 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -62,7 +62,7 @@ skipSwitch: resultChannel <- &data } -func reciever() { +func receiver() { for data := range resultChannel { if data != nil && data.inData != nil && data.inPath != "" { @@ -168,7 +168,12 @@ func CommonMain(envPtr *string, } else { f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) logger := log.New(f, "["+coreopts.GetFolderPrefix(nil)+"config]", log.LstdFlags) - configBase = &eUtils.DriverConfig{Insecure: true, Log: logger, ExitOnFailure: true} + configBase = &eUtils.DriverConfig{Insecure: true, + StartDir: append([]string{}, *startDirPtr), + EndDir: *endDirPtr, + Log: logger, + ExitOnFailure: true} + appRoleConfigPtr = new(string) eUtils.CheckError(configBase, err, true) } @@ -314,8 +319,8 @@ func CommonMain(envPtr *string, fileFilterSlice[0] = *fileFilterPtr } - //channel reciever - go reciever() + //channel receiver + go receiver() var diffFileCount int if *diffPtr { configSlice := make([]eUtils.DriverConfig, 0, len(envDiffSlice)-1) @@ -359,8 +364,8 @@ func CommonMain(envPtr *string, Regions: regions, SecretMode: *secretMode, ServicesWanted: services, - StartDir: append([]string{}, *startDirPtr), - EndDir: *endDirPtr, + StartDir: configBase.StartDir, + EndDir: configBase.EndDir, WantCerts: *wantCertsPtr, WantKeystore: *keyStorePtr, ZeroConfig: *zcPtr, @@ -410,8 +415,8 @@ func CommonMain(envPtr *string, Regions: regions, SecretMode: *secretMode, ServicesWanted: services, - StartDir: append([]string{}, *startDirPtr), - EndDir: *endDirPtr, + StartDir: configBase.StartDir, + EndDir: configBase.EndDir, WantCerts: *wantCertsPtr, WantKeystore: *keyStorePtr, ZeroConfig: *zcPtr, @@ -424,6 +429,10 @@ func CommonMain(envPtr *string, FileFilter: fileFilterSlice, VersionInfo: eUtils.VersionHelper, } + + if len(configBase.DeploymentConfig) > 0 { + config.DeploymentConfig = configBase.DeploymentConfig + } wg.Add(1) go func(c *eUtils.DriverConfig) { defer wg.Done() @@ -433,7 +442,7 @@ func CommonMain(envPtr *string, wg.Wait() //Wait for templates if c == nil { close(resultChannel) - } else if c.EndDir != "deploy" { + } else if c.EndDir != "deploy" && !c.IsShellSubProcess { close(resultChannel) } if *diffPtr { //Diff if needed diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index b62bf498f..1648384ae 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -36,17 +36,6 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverCo eUtils.LogErrorObject(config, err, false) return nil, err } - if config.IsShellSubProcess && len(config.Deployment) > 0 { - certifyMap, err := modCheck.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", config.Deployment)) - if err != nil { - return nil, err - } - if trcDeployRoot, ok := certifyMap["trcdeployroot"]; ok { - config.StartDir = []string{trcDeployRoot.(string)} - config.EndDir = trcDeployRoot.(string) - } - } - modCheck.VersionFilter = config.VersionFilter //Check if templateInfo is selected for template or values diff --git a/trcctl/trcctl.go b/trcctl/trcctl.go index ab5866dd6..0276b5dd7 100644 --- a/trcctl/trcctl.go +++ b/trcctl/trcctl.go @@ -85,7 +85,7 @@ func main() { case "pub": trcpubbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil) case "sub": - trcsubbase.CommonMain(envPtr, &addrPtr, &envContext) + trcsubbase.CommonMain(envPtr, &addrPtr, &envContext, secretIDPtr, appRoleIDPtr, nil) case "init": trcinitbase.CommonMain(envPtr, &addrPtr, &envContext) case "config": diff --git a/trcflow/flumen/process.go b/trcflow/flumen/process.go index 23cbebdba..a422831cb 100644 --- a/trcflow/flumen/process.go +++ b/trcflow/flumen/process.go @@ -318,11 +318,11 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error }(flowcore.FlowNameType(table), &tfContext, config) controllerInitWG.Wait() //Waiting for remoteDataSource to load up to prevent data race. - if initReciever, ok := tfContext.RemoteDataSource["flowStateInitAlert"].(chan bool); ok { + if initReceiver, ok := tfContext.RemoteDataSource["flowStateInitAlert"].(chan bool); ok { initAlert: //This waits for flow states to be loaded before starting all non-controller flows for { select { - case _, ok := <-initReciever: + case _, ok := <-initReceiver: if ok { break initAlert } @@ -331,9 +331,9 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error } } } else { - initRecieverErr := errors.New("Failed to retrieve channel alert for controller init") - eUtils.LogErrorMessage(config, initRecieverErr.Error(), false) - return initRecieverErr + initReceiverErr := errors.New("Failed to retrieve channel alert for controller init") + eUtils.LogErrorMessage(config, initReceiverErr.Error(), false) + return initReceiverErr } } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 8eae5979a..ae72e3457 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -22,9 +22,11 @@ import ( "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" "github.com/trimble-oss/tierceron/trcconfigbase" + vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" "github.com/trimble-oss/tierceron/trcpubbase" kube "github.com/trimble-oss/tierceron/trcsh/kube/native" "github.com/trimble-oss/tierceron/trcsh/trcshauth" + "github.com/trimble-oss/tierceron/trcsubbase" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" "github.com/trimble-oss/tierceron/trcvault/util" @@ -64,7 +66,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, } }() - ProcessDeploy(*gAgentConfig.Env, region, "", "", trcPath, secretId, approleId, false) + ProcessDeploy(*gAgentConfig.Env, region, "", deployment, trcPath, secretId, approleId, false) }() for modeCtl := range gAgentConfig.CtlMessage { @@ -261,7 +263,7 @@ func featherCtlCb(agentName string) error { if err == nil { if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { ctl := strings.Split(ctlFlapMode, "_") - if len(ctl) > 1 && ctl[1] != "trcctlcomplete" { + if len(ctl) > 1 && ctl[1] != capauth.TrcCtlComplete { fmt.Printf("%s\n", ctl[1]) } callFlap = cap.MODE_GAZE @@ -285,10 +287,11 @@ func featherCtlCb(agentName string) error { } -func configCmd(env string, +func roleBasedRunner(env string, trcshConfig *capauth.TrcShConfig, region string, config *eUtils.DriverConfig, + control string, agentToken bool, token string, argsOrig []string, @@ -303,13 +306,22 @@ func configCmd(env string, config.EnvRaw = env config.WantCerts = false config.IsShellSubProcess = true - + if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { + config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} + config.EndDir = trcDeployRoot.(string) + } configRoleSlice := strings.Split(*trcshConfig.ConfigRole, ":") tokenName := "config_token_" + env tokenConfig := "" configEnv := env + var err error - err := trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + switch control { + case "trcconfig": + err = trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + case "trcsub": + err = trcsubbase.CommonMain(&configEnv, &config.VaultAddress, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], config) + } ResetModifier(config) //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -353,7 +365,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, config.Token = token } case "trcconfig": - err := configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) } @@ -445,9 +457,21 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, config.Token = token } return err + case "trcsub": + // This maybe isn't needed for windows deploys... Since config can + // pull templates directly from vault.. + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } + return err case "trcconfig": - err := configCmd(env, trcshConfig, region, config, agentToken, token, argsOrig, deployArgLines, configCount) + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -526,6 +550,10 @@ func ProcessDeploy(env string, region string, token string, deployment string, t Regions: regions, ExitOnFailure: true} + if len(deployment) > 0 { + config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} + } + if env == "itdev" { config.OutputMemCache = false } @@ -611,6 +639,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t config.EnvRaw = env config.EndDir = "deploy" config.OutputMemCache = true + config.StartDir = []string{"trc_templates"} trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. @@ -636,16 +665,52 @@ func ProcessDeploy(env string, region string, token string, deployment string, t os.Args = []string{os.Args[0]} config.Log.Println("Processing trcshell") } else { - fmt.Println("Processing manual trcshell") - if env == "itdev" { - content, err = os.ReadFile(pwd + "/deploy/buildtest.trc") - if err != nil { - fmt.Println("Error could not find /deploy/buildtest.trc for deployment instructions") + if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { + if deployment, ok := config.DeploymentConfig["trcplugin"]; ok { + // Swapping in project root... + mod, err := helperkv.NewModifier(config.Insecure, *gTrcshConfig.CToken, *gTrcshConfig.VaultAddress, config.EnvRaw, config.Regions, true, config.Log) + if err != nil { + fmt.Println("Unable to obtain resources for deployment") + return + } + mod.Env = config.EnvRaw + deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) + if err != nil { + fmt.Println("Unable to obtain config for deployment") + return + } + config.DeploymentConfig = deploymentConfig + if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { + config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} + config.EndDir = trcDeployRoot.(string) + } + + if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { + trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") + contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, "/deploy/deploy.trc", true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) + if err != nil { + eUtils.LogErrorObject(config, err, false) + return + } + content = []byte(contentArray) + } else { + fmt.Println("Project not configured and ready for deployment. Missing projectservice") + return + } } } else { - content, err = os.ReadFile(pwd + "/deploy/deploy.trc") - if err != nil { - fmt.Println("Error could not find " + pwd + " /deploy/deploy.trc for deployment instructions") + fmt.Println("Processing manual trcshell") + if env == "itdev" { + content, err = os.ReadFile(pwd + "/deploy/buildtest.trc") + if err != nil { + fmt.Println("Error could not find /deploy/buildtest.trc for deployment instructions") + } + } else { + content, err = os.ReadFile(pwd + "/deploy/deploy.trc") + if err != nil { + fmt.Println("Error could not find " + pwd + "/deploy/deploy.trc for deployment instructions") + config.Log.Printf("Error could not find %s/deploy/deploy.trc for deployment instructions", pwd) + } } } } diff --git a/trcsub/trcsub.go b/trcsub/trcsub.go index 15036cf53..fc91afa6c 100644 --- a/trcsub/trcsub.go +++ b/trcsub/trcsub.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/trcsubbase" @@ -22,6 +23,11 @@ func main() { fmt.Println("Version: " + "1.26") envPtr := flag.String("env", "dev", "Environment to configure") addrPtr := flag.String("addr", "", "API endpoint for the vault") + secretIDPtr := flag.String("secretID", "", "Public app role ID") + appRoleIDPtr := flag.String("appRoleID", "", "Secret app role ID") - trcsubbase.CommonMain(envPtr, addrPtr, nil) + err := trcsubbase.CommonMain(envPtr, addrPtr, nil, secretIDPtr, appRoleIDPtr, nil) + if err != nil { + os.Exit(1) + } } diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index 3c6b6d162..33b6c40bf 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -21,15 +21,16 @@ import ( // The file is saved under the data key, and the extension under the ext key // Vault automatically encodes the file into base64 -func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { +func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, + secretIDPtr *string, + appRoleIDPtr *string, + c *eUtils.DriverConfig) error { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.4") + fmt.Println("Version: " + "1.5") dirPtr := flag.String("dir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory containing template files for vault") tokenPtr := flag.String("token", "", "Vault access token") - secretIDPtr := flag.String("secretID", "", "Public app role ID") - appRoleIDPtr := flag.String("appRoleID", "", "Secret app role ID") tokenNamePtr := flag.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") pingPtr := flag.Bool("ping", false, "Ping vault.") insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") @@ -44,51 +45,70 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") os.Exit(1) } + var config *eUtils.DriverConfig + var appRoleConfigPtr *string - // If logging production directory does not exist and is selected log to local directory - if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && *logFilePtr == "/var/log/"+coreopts.GetFolderPrefix(nil)+"sub.log" { - *logFilePtr = "./" + coreopts.GetFolderPrefix(nil) + "sub.log" - } - f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if c != nil { + config = c + appRoleConfigPtr = &c.AppRoleConfig + + } else { + // If logging production directory does not exist and is selected log to local directory + if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && *logFilePtr == "/var/log/"+coreopts.GetFolderPrefix(nil)+"sub.log" { + *logFilePtr = "./" + coreopts.GetFolderPrefix(nil) + "sub.log" + } + f, err := os.OpenFile(*logFilePtr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + fmt.Println("Log init failure") + return err + } - logger := log.New(f, "[INIT]", log.LstdFlags) - config := &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true} - eUtils.CheckError(config, err, true) + logger := log.New(f, "[INIT]", log.LstdFlags) + config = &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true} + appRoleConfigPtr = new(string) + } if len(*envPtr) >= 5 && (*envPtr)[:5] == "local" { var err error *envPtr, err = eUtils.LoginToLocal() fmt.Println(*envPtr) - eUtils.CheckError(config, err, true) + eUtils.CheckError(config, err, false) + return err } fmt.Printf("Connecting to vault @ %s\n", *addrPtr) - autoErr := eUtils.AutoAuth(config, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, "", *pingPtr) + autoErr := eUtils.AutoAuth(config, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, *pingPtr) if autoErr != nil { fmt.Println("Missing auth components.") - os.Exit(1) + return autoErr } if memonly.IsMemonly() { memprotectopts.MemUnprotectAll(nil) memprotectopts.MemProtect(nil, tokenPtr) } - mod, err := helperkv.NewModifier(*insecurePtr, *tokenPtr, *addrPtr, *envPtr, nil, true, logger) + mod, err := helperkv.NewModifier(*insecurePtr, *tokenPtr, *addrPtr, *envPtr, nil, true, config.Log) if mod != nil { defer mod.Release() } - eUtils.CheckError(config, err, true) + if err != nil { + fmt.Println("Failure to init to vault") + config.Log.Println("Failure to init to vault") + return err + } mod.Env = *envPtr if *templatePathsPtr != "" { fmt.Printf("Downloading templates from vault to %s\n", *dirPtr) // The actual download templates goes here. - il.DownloadTemplates(config, mod, *dirPtr, logger, templatePathsPtr) + il.DownloadTemplates(config, mod, *dirPtr, config.Log, templatePathsPtr) } else if *projectInfoPtr { - templateList, err := mod.List("templates/", logger) + templateList, err := mod.List("templates/", config.Log) if err != nil { - eUtils.CheckError(config, err, true) + fmt.Println("Failure read templates") + config.Log.Println("Failure read templates") + return err } fmt.Printf("\nProjects available:\n") for _, templatePath := range templateList.Data { @@ -97,18 +117,20 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string) { fmt.Println(strings.TrimRight(project, "/")) } } - os.Exit(1) + return nil } else { fmt.Printf("Downloading templates from vault to %s\n", *dirPtr) // The actual download templates goes here. - warn, err := il.DownloadTemplateDirectory(config, mod, *dirPtr, logger, filterTemplatePtr) + warn, err := il.DownloadTemplateDirectory(config, mod, *dirPtr, config.Log, filterTemplatePtr) if err != nil { fmt.Println(err) + config.Log.Printf("Failure to download: %s", err.Error()) if strings.Contains(err.Error(), "x509: certificate") { - os.Exit(-1) + return err } } - eUtils.CheckError(config, err, true) - eUtils.CheckWarnings(config, warn, true) + eUtils.CheckError(config, err, false) + eUtils.CheckWarnings(config, warn, false) } + return nil } diff --git a/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl b/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl index 873e43ad3..9913dd36f 100644 --- a/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl +++ b/trcvault/trc_templates/TrcVault/Certify/config.yml.tmpl @@ -5,6 +5,7 @@ copied: {{.copied}} deployed: {{.deployed}} instances: {{.instances}} trctype: {{.trctype}} +trcprojectservice: {{.trcprojectservice}} trcdeployroot: {{.trcdeployroot}} trccodebundle: {{.trccodebundle}} trcservicename: {{.trcservicename}} \ No newline at end of file diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index e701b5f7f..d87bf50f4 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -33,6 +33,7 @@ func CommonMain(envPtr *string, winservicestartPtr := flag.Bool("winservicestart", false, "To start a windows service for a particular plugin.") codebundledeployPtr := flag.Bool("codebundledeploy", false, "To deploy a code bundle.") agentdeployPtr := flag.Bool("agentdeploy", false, "To initiate deployment on agent.") + projectservicePtr := flag.String("projectservice", "", "Provide template root path in form project/service") // Common flags... startDirPtr := flag.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") @@ -239,6 +240,7 @@ func CommonMain(envPtr *string, pluginToolConfig["pluginNamePtr"] = *pluginNamePtr pluginToolConfig["deployrootPtr"] = *deployrootPtr pluginToolConfig["serviceNamePtr"] = *serviceNamePtr + pluginToolConfig["projectservicePtr"] = *projectservicePtr pluginToolConfig["codeBundlePtr"] = *codeBundlePtr if _, ok := pluginToolConfig["trcplugin"].(string); !ok { @@ -267,6 +269,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["codeBundlePtr"].(string); ok { pluginToolConfig["trccodebundle"] = pluginToolConfig["codeBundlePtr"].(string) } + if _, ok := pluginToolConfig["projectServicePtr"].(string); ok { + pluginToolConfig["trcprojectservice"] = pluginToolConfig["projectServicePtr"].(string) + } } } @@ -276,6 +281,7 @@ func CommonMain(envPtr *string, writeMap := make(map[string]interface{}) writeMap["trcplugin"] = *pluginNamePtr writeMap["trctype"] = *pluginTypePtr + writeMap["trcprojectservice"] = *projectservicePtr if _, ok := pluginToolConfig["trcdeployroot"]; ok { writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"] @@ -286,6 +292,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["trccodebundle"]; ok { writeMap["trccodebundle"] = pluginToolConfig["trccodebundle"] } + if _, ok := pluginToolConfig["trcprojectservice"]; ok { + writeMap["trcprojectservice"] = pluginToolConfig["trcprojectservice"] + } _, err = mod.Write(pluginToolConfig["pluginpath"].(string), writeMap, configBase.Log) if err != nil { fmt.Println(err) @@ -475,6 +484,7 @@ func WriteMapUpdate(writeMap map[string]interface{}, pluginToolConfig map[string if defineServicePtr { writeMap["trccodebundle"] = pluginToolConfig["trccodebundle"].(string) writeMap["trcservicename"] = pluginToolConfig["trcservicename"].(string) + writeMap["trcprojectservice"] = pluginToolConfig["trcprojectservice"].(string) writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"].(string) } if _, imgShaOk := pluginToolConfig["imagesha256"].(string); imgShaOk { diff --git a/trcvault/util/util.go b/trcvault/util/util.go index 19354420d..6765fdf93 100644 --- a/trcvault/util/util.go +++ b/trcvault/util/util.go @@ -345,6 +345,9 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl return nil, err } } + if defineService { + pluginEnvConfigClone["pluginpath"] = pluginToolConfig["pluginpath"] + } } config.Log.Println("GetPluginToolConfig end processing plugins.") diff --git a/trcx/xutil/xmanager.go b/trcx/xutil/xmanager.go index af09e91c9..c3c9194ef 100644 --- a/trcx/xutil/xmanager.go +++ b/trcx/xutil/xmanager.go @@ -172,7 +172,7 @@ func GenerateSeedSectionFromVaultRaw(config *eUtils.DriverConfig, templateFromVa } } - //Reciever for configs + //Receiver for configs go func(c *eUtils.DriverConfig) { for { select { diff --git a/trcxbase/trcxbase.go b/trcxbase/trcxbase.go index 1cd35bd94..604d1dfbb 100644 --- a/trcxbase/trcxbase.go +++ b/trcxbase/trcxbase.go @@ -40,7 +40,7 @@ func messenger(inData *string, inPath string) { resultChannel <- &data } -func reciever() { +func receiver() { for { select { case data := <-resultChannel: @@ -373,7 +373,7 @@ skipDiff: var serviceFilterSlice []string if len(*dynamicPathPtr) > 0 { - go reciever() //Channel reciever + go receiver() //Channel receiver dynamicPathParts := strings.Split(*dynamicPathPtr, "/") @@ -645,7 +645,7 @@ skipDiff: } } - go reciever() //Channel reciever + go receiver() //Channel receiver if len(*dynamicPathPtr) == 0 { for _, env := range envSlice { envVersion := eUtils.SplitEnv(env) diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 36bd36e2d..0528f0c66 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -79,7 +79,7 @@ type DriverConfig struct { ServiceFilter []string // Which tables to use. DynamicPathFilter string // Seeds from a specific path. - Deployment string // For trcsh to indicate which deployment to work on + DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. From 73812b5b652f17a22bf8d1ed7f23a8a1388c727f Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 13 Nov 2023 07:42:45 -0800 Subject: [PATCH 17/55] Prepare release (#799) * Revert breaking change. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcsh/trcsh.go | 1 - 1 file changed, 1 deletion(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index ae72e3457..da34474d3 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -639,7 +639,6 @@ func ProcessDeploy(env string, region string, token string, deployment string, t config.EnvRaw = env config.EndDir = "deploy" config.OutputMemCache = true - config.StartDir = []string{"trc_templates"} trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. From 192e2c58b71b5a220a88e6e07aa160ad37778058 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:09:59 -0800 Subject: [PATCH 18/55] Prepare for release. (#801) * Revert breaking change. * Provide appropriate inputs. Handle bad inputs. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfigbase/trcconfig.go | 8 ++++++++ trcsh/trcsh.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 69d1832d1..87b8b6ad1 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -160,6 +160,14 @@ func CommonMain(envPtr *string, var configBase *eUtils.DriverConfig if c != nil { configBase = c + if len(configBase.EndDir) == 0 && len(*endDirPtr) != 0 { + // Bad inputs... use default. + configBase.EndDir = *endDirPtr + } + if len(configBase.StartDir) == 0 || len(configBase.StartDir[0]) == 0 { + // Bad inputs... use default. + configBase.StartDir = append([]string{}, *startDirPtr) + } *insecurePtr = configBase.Insecure appRoleConfigPtr = &(configBase.AppRoleConfig) if configBase.FileFilter != nil { diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index da34474d3..1757ff550 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -639,6 +639,8 @@ func ProcessDeploy(env string, region string, token string, deployment string, t config.EnvRaw = env config.EndDir = "deploy" config.OutputMemCache = true + config.StartDir = []string{"trc_templates"} + config.EndDir = "." trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. From a6db9f4698ad33c3d2a9a909fd1cf5308d5441ed Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:22:59 -0800 Subject: [PATCH 19/55] Fixing release (#803) * Fix hacky logic. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfigbase/trcconfig.go | 2 +- trcsh/trcsh.go | 2 +- utils/driverconfig.go | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 87b8b6ad1..f41ba820d 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -450,7 +450,7 @@ func CommonMain(envPtr *string, wg.Wait() //Wait for templates if c == nil { close(resultChannel) - } else if c.EndDir != "deploy" && !c.IsShellSubProcess { + } else if c.IsShellConfigComplete { close(resultChannel) } if *diffPtr { //Diff if needed diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 1757ff550..857156530 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -299,7 +299,7 @@ func roleBasedRunner(env string, configCount *int) error { *configCount -= 1 if *configCount != 0 { //This is to keep result channel open - closes on the final config call of the script. - config.EndDir = "deploy" + config.IsShellConfigComplete = true } config.AppRoleConfig = "config.yml" config.FileFilter = nil diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 0528f0c66..a4f4b7279 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -22,11 +22,12 @@ type DriverConfig struct { Context ProcessContext // Internal systems... - ExitOnFailure bool // Exit on a failure or try to continue - Insecure bool - IsShell bool // If tool running in shell. - IsShellSubProcess bool // If subshell - Log *log.Logger + ExitOnFailure bool // Exit on a failure or try to continue + Insecure bool + IsShell bool // If tool running in shell. + IsShellSubProcess bool // If subshell + IsShellConfigComplete bool // If all shell trcconfig calls are complete + Log *log.Logger // Vault Configurations... Token string From 0c682da7a0bcb6bcc6ec0754aa9e304cc7b989ee Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:40:25 -0800 Subject: [PATCH 20/55] Prepare release (#805) * Fix so trcsh works with empty directory. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfigbase/trcconfig.go | 14 ++++++++------ trcsh/trcsh.go | 14 +++++++++++--- trcsh/trcshauth/auth.go | 36 ++++++++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index f41ba820d..a4f4bf264 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -102,13 +102,13 @@ func CommonMain(envPtr *string, versionInfoPtr := flag.Bool("versions", false, "Version information about values") insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") noVaultPtr := flag.Bool("novault", false, "Don't pull configuration data from vault.") - isShellSubprocess := false + isShell := false if c != nil { - isShellSubprocess = c.IsShellSubProcess + isShell = c.IsShell } - if c == nil || !isShellSubprocess { + if c == nil || !isShell { args := os.Args[1:] for i := 0; i < len(args); i++ { s := args[i] @@ -140,7 +140,7 @@ func CommonMain(envPtr *string, *wantCertsPtr = true } } - if !isShellSubprocess { + if !isShell { if _, err := os.Stat(*startDirPtr); os.IsNotExist(err) { fmt.Println("Missing required template folder: " + *startDirPtr) return fmt.Errorf("missing required template folder: %s", *startDirPtr) @@ -363,7 +363,8 @@ func CommonMain(envPtr *string, } config := eUtils.DriverConfig{ - IsShellSubProcess: isShellSubprocess, + IsShell: isShell, + IsShellSubProcess: configBase.IsShellSubProcess, Insecure: *insecurePtr, Token: *tokenPtr, VaultAddress: *addrPtr, @@ -414,7 +415,8 @@ func CommonMain(envPtr *string, *envPtr = envVersion[0] + "_0" } config := eUtils.DriverConfig{ - IsShellSubProcess: isShellSubprocess, + IsShell: isShell, + IsShellSubProcess: configBase.IsShellSubProcess, Insecure: *insecurePtr, Token: *tokenPtr, VaultAddress: *addrPtr, diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 05c7bc062..4fb6ebef7 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -634,7 +634,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t fmt.Println("Session Authorized") } - if len(os.Args) > 1 || len(trcPath) > 0 { + if (len(os.Args) > 1 || len(trcPath) > 0) && !strings.Contains(pwd, "TrcDeploy") { // Generate trc code... trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} @@ -646,7 +646,13 @@ func ProcessDeploy(env string, region string, token string, deployment string, t config.StartDir = []string{"trc_templates"} config.EndDir = "." trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. + os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + if !agentToken { + token = "" + config.Token = token + } var memFile billy.File var memFileErr error @@ -692,7 +698,9 @@ func ProcessDeploy(env string, region string, token string, deployment string, t if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") - contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, "/deploy/deploy.trc", true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) + config.ZeroConfig = true + contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) + config.ZeroConfig = false if err != nil { eUtils.LogErrorObject(config, err, false) return diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index 5f4e651cb..85ec3e461 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -8,6 +8,7 @@ import ( "os" "regexp" "strings" + "time" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" @@ -85,8 +86,23 @@ func GetSetEnvAddrContext(env string, envContext string, addrPort string) (strin return env, envContext, addrPort, nil } +func retryingPenseFeatherQuery(agentConfigs *capauth.AgentConfigs, pense string) (*string, error) { + retry := 0 + for retry < 5 { + result, err := agentConfigs.PenseFeatherQuery(pense) + + if err != nil || result == nil || *result == "...." { + time.Sleep(time.Second) + retry = retry + 1 + } else { + return result, err + } + } + return nil, errors.New("unavailable secrets") +} + // Helper function for obtaining auth components. -func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { +func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { trcshConfig := &capauth.TrcShConfig{} var err error @@ -108,7 +124,7 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( return trcshConfig, nil } } else { - if agentConfig == nil { + if agentConfigs == nil { config.Log.Println("Auth phase 1") trcshConfig.KubeConfig, err = capauth.PenseQuery(config, "kubeconfig") } @@ -121,8 +137,8 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( memprotectopts.MemProtect(nil, trcshConfig.KubeConfig) } - if agentConfig != nil { - trcshConfig.VaultAddress, err = agentConfig.PenseFeatherQuery("caddress") + if agentConfigs != nil { + trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(agentConfigs, "caddress") } else { config.Log.Println("Auth phase 2") trcshConfig.VaultAddress, err = capauth.PenseQuery(config, "caddress") @@ -154,8 +170,8 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( config.VaultAddress = *trcshConfig.VaultAddress memprotectopts.MemProtect(nil, &config.VaultAddress) - if agentConfig != nil { - trcshConfig.ConfigRole, err = agentConfig.PenseFeatherQuery("configrole") + if agentConfigs != nil { + trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(agentConfigs, "configrole") } else { config.Log.Println("Auth phase 3") trcshConfig.ConfigRole, err = capauth.PenseQuery(config, "configrole") @@ -166,17 +182,17 @@ func TrcshAuth(agentConfig *capauth.AgentConfigs, config *eUtils.DriverConfig) ( memprotectopts.MemProtect(nil, trcshConfig.ConfigRole) - if agentConfig == nil { + if agentConfigs == nil { config.Log.Println("Auth phase 4") - trcshConfig.PubRole, err = capauth.PenseQuery(config, "pubrole") + trcshConfig.PubRole, err = retryingPenseFeatherQuery(agentConfigs, "pubrole") if err != nil { return trcshConfig, err } memprotectopts.MemProtect(nil, trcshConfig.PubRole) } - if agentConfig != nil { - trcshConfig.CToken, err = agentConfig.PenseFeatherQuery("ctoken") + if agentConfigs != nil { + trcshConfig.CToken, err = retryingPenseFeatherQuery(agentConfigs, "ctoken") } else { config.Log.Println("Auth phase 5") trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") From 9a53ac165c87c44255b1453db7e32a4db294439e Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:15:26 -0800 Subject: [PATCH 21/55] Prepare for release. (#807) * Instrument interruptibility. * Adjust wait times. * Adding window utility function to make life easier. * Fix downloading. * Better error handling of keystore creation. * Honor enddir and startdir when provided. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- capauth/creds.go | 4 +-- trcconfigbase/trcconfig.go | 10 +++--- trcconfigbase/utils/configinator.go | 6 ++-- trcsh/trcsh.go | 40 ++++++++++++++--------- trcvault/trcplgtoolbase/trcplgtoolbase.go | 5 +++ trcxbase/trcxbase.go | 4 +-- utils/coreUtil.go | 7 ++++ utils/diffUtil.go | 15 ++++----- utils/versionUtil.go | 3 +- validator/certs.go | 7 ++-- validator/keystore.go | 5 ++- 11 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 utils/coreUtil.go diff --git a/capauth/creds.go b/capauth/creds.go index da50243b5..3d6934496 100644 --- a/capauth/creds.go +++ b/capauth/creds.go @@ -10,10 +10,10 @@ import ( "math/rand" "net" "os" - "runtime" "strings" "time" + "github.com/trimble-oss/tierceron/utils" "google.golang.org/grpc/credentials" ) @@ -29,7 +29,7 @@ func ReadServerCert() ([]byte, error) { if _, err := os.Stat(ServCert); err == nil { return os.ReadFile(ServCert) } else { - if runtime.GOOS == "windows" { + if utils.IsWindows() { return os.ReadFile(ServCertLocal) } else { return nil, errors.New("file not found") diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index a4f4bf264..61ba17604 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -7,7 +7,6 @@ import ( "log" "os" "path/filepath" - "runtime" "strings" "sync" @@ -15,6 +14,7 @@ import ( "github.com/trimble-oss/tierceron/buildopts/memprotectopts" vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" + "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" "github.com/google/go-cmp/cmp" @@ -160,11 +160,11 @@ func CommonMain(envPtr *string, var configBase *eUtils.DriverConfig if c != nil { configBase = c - if len(configBase.EndDir) == 0 && len(*endDirPtr) != 0 { - // Bad inputs... use default. + if len(configBase.EndDir) == 0 || len(*endDirPtr) > len(configBase.EndDir) { + // Honor inputs if provided... configBase.EndDir = *endDirPtr } - if len(configBase.StartDir) == 0 || len(configBase.StartDir[0]) == 0 { + if len(configBase.StartDir) == 0 || len(configBase.StartDir[0]) == 0 || (len(*startDirPtr) > len(configBase.StartDir[0])) { // Bad inputs... use default. configBase.StartDir = append([]string{}, *startDirPtr) } @@ -266,7 +266,7 @@ func CommonMain(envPtr *string, if len(envVersion) > 1 && envVersion[1] != "" && envVersion[1] != "0" { Yellow := "\033[33m" Reset := "\033[0m" - if runtime.GOOS == "windows" { + if utils.IsWindows() { Reset = "" Yellow = "" } diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index 27fda4443..54b77a8bf 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -8,12 +8,12 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" "sync" "github.com/go-git/go-billy/v5" + "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" "github.com/trimble-oss/tierceron/validator" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -25,7 +25,7 @@ var mutex = &sync.Mutex{} func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverConfig) (interface{}, error) { /*Cyan := "\033[36m" Reset := "\033[0m" - if runtime.GOOS == "windows" { + if utils.IsWindows() { Reset = "" Cyan = "" }*/ @@ -413,7 +413,7 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverCo if config.OutputMemCache { messageBase = "template configured and pre-processed for " } - if runtime.GOOS == "windows" { + if utils.IsWindows() { eUtils.LogInfo(config, messageBase+endPaths[i]) } else { eUtils.LogInfo(config, "\033[0;33m"+messageBase+endPaths[i]+"\033[0m") diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 7f631786d..a1fa650c8 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -9,7 +9,6 @@ import ( "log" "os" "os/signal" - "runtime" "strings" "sync" "syscall" @@ -30,6 +29,7 @@ import ( "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" "github.com/trimble-oss/tierceron/trcvault/util" + "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -46,7 +46,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployment+"."+*gAgentConfig.Env, false); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { + cap.MODE_FLAP, deployment+"."+*gAgentConfig.Env, false, acceptRemote); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { // Lookup trcPath from deployment // Process the script.... @@ -60,7 +60,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true) + cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true, acceptRemote) gAgentConfig.CtlMessage <- capauth.TrcCtlComplete break } @@ -81,7 +81,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployment+"."+*gAgentConfig.Env, true) + cap.MODE_PERCH, deployment+"."+*gAgentConfig.Env, true, acceptRemote) ctlFlapMode = cap.MODE_PERCH goto perching } @@ -96,14 +96,14 @@ func ProcessDeployment(env string, region string, token string, trcPath string, interruptFun(twoHundredMilliInterruptTicker) } else { if err.Error() != "init" { - interruptFun(multiSecondInterruptTicker) + interruptFun(secondInterruptTicker) } } ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - callFlap, deployment+"."+*gAgentConfig.Env, true) + callFlap, deployment+"."+*gAgentConfig.Env, true, acceptRemote) } } if modeCtl == capauth.TrcCtlComplete { @@ -112,12 +112,12 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true) + cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true, acceptRemote) goto deploycomplete } } } else { - interruptFun(multiSecondInterruptTicker) + interruptFun(fiveSecondInterruptTicker) } deploycomplete: } @@ -131,10 +131,10 @@ func main() { memprotectopts.MemProtectInit(nil) } eUtils.InitHeadless(true) - fmt.Println("trcsh Version: " + "1.21") + fmt.Println("trcsh Version: " + "1.22") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string - if runtime.GOOS != "windows" { + if !utils.IsWindows() { if os.Geteuid() == 0 { fmt.Println("Trcsh cannot be run as root.") os.Exit(-1) @@ -224,7 +224,9 @@ func main() { var interruptChan chan os.Signal = make(chan os.Signal) var twoHundredMilliInterruptTicker *time.Ticker = time.NewTicker(200 * time.Millisecond) -var multiSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second) +var secondInterruptTicker *time.Ticker = time.NewTicker(time.Second) +var multiSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 3) +var fiveSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) func interruptFun(tickerInterrupt *time.Ticker) { select { @@ -233,12 +235,20 @@ func interruptFun(tickerInterrupt *time.Ticker) { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true) + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) os.Exit(1) case <-tickerInterrupt.C: } } +// acceptRemote - hook for instrumenting +func acceptRemote(mode int, remote string) bool { + if mode == cap.FEATHER_CTL { + interruptFun(multiSecondInterruptTicker) + } + return true +} + func featherCtlCb(agentName string) error { if gAgentConfig == nil { @@ -281,7 +291,7 @@ func featherCtlCb(agentName string) error { *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env, true) + callFlap, agentName+"."+*gAgentConfig.Env, true, acceptRemote) } } @@ -791,7 +801,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t os.Args = deployArgs } } - if runtime.GOOS == "windows" { + if utils.IsWindows() { err := processWindowsCmds( trcKubeDeploymentConfig, &onceKubeInit, @@ -831,7 +841,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t } } } - if runtime.GOOS == "windows" { + if utils.IsWindows() { gAgentConfig.CtlMessage <- capauth.TrcCtlComplete } //Make the arguments in the script -> os.args. diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index d87bf50f4..2f9124ba4 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -321,6 +321,11 @@ func CommonMain(envPtr *string, } fmt.Printf("Service started: %s\n", pluginToolConfig["trcservicename"].(string)) } else if *codebundledeployPtr { + if pluginToolConfig["trcsha256"] == nil || len(pluginToolConfig["trcsha256"].(string)) == 0 { + if configBase.DeploymentConfig != nil && configBase.DeploymentConfig["trcsha256"] != nil && len(configBase.DeploymentConfig["trcsha256"].(string)) > 0 { + pluginToolConfig["trcsha256"] = configBase.DeploymentConfig["trcsha256"] + } + } if pluginToolConfig["trcsha256"] != nil && len(pluginToolConfig["trcsha256"].(string)) > 0 { err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { diff --git a/trcxbase/trcxbase.go b/trcxbase/trcxbase.go index 604d1dfbb..22fafac50 100644 --- a/trcxbase/trcxbase.go +++ b/trcxbase/trcxbase.go @@ -6,7 +6,6 @@ import ( "fmt" "log" "os" - "runtime" "strings" "sync" "time" @@ -14,6 +13,7 @@ import ( "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" + "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" "github.com/trimble-oss/tierceron/vaulthelper/kv" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -121,7 +121,7 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env Yellow := "\033[33m" Reset := "\033[0m" - if runtime.GOOS == "windows" { + if utils.IsWindows() { Reset = "" Yellow = "" } diff --git a/utils/coreUtil.go b/utils/coreUtil.go new file mode 100644 index 000000000..534dfea94 --- /dev/null +++ b/utils/coreUtil.go @@ -0,0 +1,7 @@ +package utils + +import "runtime" + +func IsWindows() bool { + return runtime.GOOS == "windows" +} diff --git a/utils/diffUtil.go b/utils/diffUtil.go index 27a9a1612..f711c7c6f 100644 --- a/utils/diffUtil.go +++ b/utils/diffUtil.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "net/url" - "runtime" "sort" "strconv" "strings" @@ -36,7 +35,7 @@ func LineByLineDiff(stringA *string, stringB *string, patchData bool, colorSkip var Cyan = "\033[36m" var result string - if runtime.GOOS == "windows" { + if IsWindows() { Reset = "\x1b[0m" Red = "\x1b[31m" Green = "\x1b[32m" @@ -208,13 +207,13 @@ func LineByLineDiff(stringA *string, stringB *string, patchData bool, colorSkip //Diff vs no Diff output if len(strings.TrimSpace(result)) == 0 && patchData { if diffTimeout { - if runtime.GOOS == "windows" { + if IsWindows() { return "@@ Diff Timed Out @@" } return Cyan + "@@ Diff Timed Out @@" + Reset } - if runtime.GOOS == "windows" { + if IsWindows() { return "@@ No Differences @@" } return Cyan + "@@ No Differences @@" + Reset @@ -225,7 +224,7 @@ func LineByLineDiff(stringA *string, stringB *string, patchData bool, colorSkip result = strings.TrimSuffix(result, "\n") } - if runtime.GOOS == "windows" { + if IsWindows() { result = strings.ReplaceAll(result, Reset, "") result = strings.ReplaceAll(result, Green, "") result = strings.ReplaceAll(result, Cyan, "") @@ -239,7 +238,7 @@ func VersionHelper(versionData map[string]interface{}, templateOrValues bool, va Reset := "\033[0m" Cyan := "\033[36m" Red := "\033[31m" - if runtime.GOOS == "windows" { + if IsWindows() { Reset = "" Cyan = "" Red = "" @@ -461,7 +460,7 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri Green := "\033[32m" Yellow := "\033[0;33m" - if runtime.GOOS == "windows" { + if IsWindows() { Reset = "" Red = "" Green = "" @@ -576,7 +575,7 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri } //Seperator - if runtime.GOOS == "windows" { + if IsWindows() { fmt.Printf("======================================================================================\n") } else { fmt.Printf("\033[1;35m======================================================================================\033[0m\n") diff --git a/utils/versionUtil.go b/utils/versionUtil.go index 9fae4e538..27322e53d 100644 --- a/utils/versionUtil.go +++ b/utils/versionUtil.go @@ -2,7 +2,6 @@ package utils import ( "fmt" - "runtime" "sort" "strconv" "strings" @@ -128,7 +127,7 @@ func GetProjectVersions(config *DriverConfig, versionMetadataMap map[string]map[ func BoundCheck(config *DriverConfig, versionNumbers []int, version string) { Cyan := "\033[36m" Reset := "\033[0m" - if runtime.GOOS == "windows" { + if IsWindows() { Reset = "" Cyan = "" } diff --git a/validator/certs.go b/validator/certs.go index 90f112c66..9d7e0f484 100644 --- a/validator/certs.go +++ b/validator/certs.go @@ -8,8 +8,9 @@ import ( "io" "net/http" "os" - "runtime" "time" + + "github.com/trimble-oss/tierceron/utils" ) // Definition here: https://tools.ietf.org/html/rfc5280 @@ -106,7 +107,7 @@ func VerifyCertificate(cert *x509.Certificate, host string) (bool, error) { CurrentTime: time.Now(), } - if runtime.GOOS != "windows" { + if !utils.IsWindows() { rootCAs, err := x509.SystemCertPool() if err != nil { return false, err @@ -116,7 +117,7 @@ func VerifyCertificate(cert *x509.Certificate, host string) (bool, error) { } if _, err := cert.Verify(opts); err != nil { - if runtime.GOOS != "windows" { + if !utils.IsWindows() { if _, ok := err.(x509.UnknownAuthorityError); ok { issuer, issuerErr := getCert("http://r3.i.lencr.org/") if issuerErr != nil { diff --git a/validator/keystore.go b/validator/keystore.go index 88ec8505a..7a4fb0a7c 100644 --- a/validator/keystore.go +++ b/validator/keystore.go @@ -37,7 +37,10 @@ func StoreKeystore(config *eUtils.DriverConfig, trustStorePassword string) ([]by if config.KeyStore == nil { return nil, errors.New("cert bundle not properly named") } - config.KeyStore.Store(keystoreWriter, []byte(trustStorePassword)) + storeErr := config.KeyStore.Store(keystoreWriter, []byte(trustStorePassword)) + if storeErr != nil { + return nil, storeErr + } keystoreWriter.Flush() return buffer.Bytes(), nil From 137442f2b6e24e4c6e80ee128f985ec3e68f5503 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:10:10 -0800 Subject: [PATCH 22/55] Prepare for release. (#809) * Update version for confirmation. * Removing a deadlock. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcconfigbase/trcconfig.go | 13 +++++++++---- trcsh/trcsh.go | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 61ba17604..99643c09d 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -74,6 +74,11 @@ func receiver() { } +var ( + ENDDIR_DEFAULT = "." + STARTDIR_DEFAULT = coreopts.GetFolderPrefix(nil) + "_templates" +) + func CommonMain(envPtr *string, addrPtr *string, tokenPtr *string, @@ -87,8 +92,8 @@ func CommonMain(envPtr *string, memprotectopts.MemProtectInit(nil) } - startDirPtr := flag.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") - endDirPtr := flag.String("endDir", ".", "Directory to put configured templates into") + startDirPtr := flag.String("startDir", STARTDIR_DEFAULT, "Template directory") + endDirPtr := flag.String("endDir", ENDDIR_DEFAULT, "Directory to put configured templates into") secretMode := flag.Bool("secretMode", true, "Only override secret values in templates?") servicesWanted := flag.String("servicesWanted", "", "Services to pull template values for, in the form 'service1,service2' (defaults to all services)") wantCertsPtr := flag.Bool("certs", false, "Pull certificates into directory specified by endDirPtr") @@ -160,11 +165,11 @@ func CommonMain(envPtr *string, var configBase *eUtils.DriverConfig if c != nil { configBase = c - if len(configBase.EndDir) == 0 || len(*endDirPtr) > len(configBase.EndDir) { + if len(configBase.EndDir) == 0 || *endDirPtr != ENDDIR_DEFAULT { // Honor inputs if provided... configBase.EndDir = *endDirPtr } - if len(configBase.StartDir) == 0 || len(configBase.StartDir[0]) == 0 || (len(*startDirPtr) > len(configBase.StartDir[0])) { + if len(configBase.StartDir) == 0 || len(configBase.StartDir[0]) == 0 || *startDirPtr != STARTDIR_DEFAULT { // Bad inputs... use default. configBase.StartDir = append([]string{}, *startDirPtr) } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index a1fa650c8..52d6eda0b 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -42,6 +42,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, go func() { for { perching: + var deployDoneChan chan bool if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, *gAgentConfig.HandshakeHostPort, @@ -54,7 +55,10 @@ func ProcessDeployment(env string, region string, token string, trcPath string, go func() { go func() { // Timeout and CtlMessage subscriber - for range time.After(120 * time.Second) { + select { + case <-deployDoneChan: + break + case <-time.After(120 * time.Second): ctlMsg := "Deployment timed out after 120 seconds" cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, *gAgentConfig.EncryptSalt, @@ -64,6 +68,8 @@ func ProcessDeployment(env string, region string, token string, trcPath string, gAgentConfig.CtlMessage <- capauth.TrcCtlComplete break } + for range time.After(120 * time.Second) { + } }() ProcessDeploy(*gAgentConfig.Env, region, "", deployment, trcPath, secretId, approleId, false) @@ -113,6 +119,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true, acceptRemote) + go func() { deployDoneChan <- true }() goto deploycomplete } } @@ -131,7 +138,7 @@ func main() { memprotectopts.MemProtectInit(nil) } eUtils.InitHeadless(true) - fmt.Println("trcsh Version: " + "1.22") + fmt.Println("trcsh Version: " + "1.23") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string if !utils.IsWindows() { @@ -458,6 +465,7 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, deployArgLines []string, configCount *int, logger *log.Logger) error { + switch control { case "trcplgtool": config.AppRoleConfig = "" @@ -802,6 +810,8 @@ func ProcessDeploy(env string, region string, token string, deployment string, t } } if utils.IsWindows() { + // Log for traceability. + logger.Println(deployLine) err := processWindowsCmds( trcKubeDeploymentConfig, &onceKubeInit, From fdd8da6980a271167bed0091fa97fde5a9773f34 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:07:00 -0800 Subject: [PATCH 23/55] Update to make safe for trcshell. (#811) * Increase timeout. * Adding some retries. * Make re-entrant safe. --- capauth/agentconfig.go | 2 +- trcconfig/trcconfig.go | 18 ++- trcconfigbase/trcconfig.go | 174 ++++++++++++---------- trcconfigbase/utils/configinator.go | 10 +- trcctl/trcctl.go | 19 ++- trcsh/trcsh.go | 55 ++++--- trcsub/trcsub.go | 12 +- trcsubbase/trcsub.go | 33 ++-- trcvault/trcplgtool/trcplgtool.go | 12 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 68 ++++++--- trcx/xutil/xmanager.go | 4 +- trcxbase/trcxbase.go | 130 ++++++++-------- utils/diffUtil.go | 63 ++++---- utils/driverconfig.go | 27 +++- 14 files changed, 347 insertions(+), 280 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index c12f57614..a6d5b1fd6 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -86,7 +86,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error c := cap.NewCapClient(conn) // Contact the server and print out its response. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() r, err := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) diff --git a/trcconfig/trcconfig.go b/trcconfig/trcconfig.go index 1446d8f5b..2d862f0dd 100644 --- a/trcconfig/trcconfig.go +++ b/trcconfig/trcconfig.go @@ -19,15 +19,17 @@ func main() { memprotectopts.MemProtectInit(nil) } fmt.Println("Version: " + "1.27") - envPtr := flag.String("env", "dev", "Environment to configure") - addrPtr := flag.String("addr", "", "API endpoint for the vault") - tokenPtr := flag.String("token", "", "Vault access token") - secretIDPtr := flag.String("secretID", "", "Secret app role ID") - regionPtr := flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. - appRoleIDPtr := flag.String("appRoleID", "", "Public app role ID") - tokenNamePtr := flag.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = flag.Usage + envPtr := flagset.String("env", "dev", "Environment to configure") + addrPtr := flagset.String("addr", "", "API endpoint for the vault") + tokenPtr := flagset.String("token", "", "Vault access token") + secretIDPtr := flagset.String("secretID", "", "Secret app role ID") + regionPtr := flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + appRoleIDPtr := flagset.String("appRoleID", "", "Public app role ID") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") - err := trcconfigbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, nil) + err := trcconfigbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, flagset, os.Args, nil) if err != nil { os.Exit(1) } diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index 99643c09d..a94c24b1b 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" @@ -20,34 +21,21 @@ import ( "github.com/google/go-cmp/cmp" ) -type ResultData struct { - inData *string - inPath string -} - -var resultMap = make(map[string]*string) -var envDiffSlice = make([]string, 0) -var resultChannel = make(chan *ResultData, 5) -var fileSysIndex = -1 -var envLength int -var wg sync.WaitGroup -var mutex = &sync.Mutex{} - -func messenger(inData *string, inPath string) { - var data ResultData - data.inData = inData - data.inPath = inPath +func messenger(configCtx *utils.ConfigContext, inData *string, inPath string) { + var data utils.ResultData + data.InData = inData + data.InPath = inPath inPathSplit := strings.Split(inPath, "||.") - mutex.Lock() - _, present := resultMap["filesys||."+inPathSplit[1]] - mutex.Unlock() + configCtx.Mutex.Lock() + _, present := configCtx.ResultMap["filesys||."+inPathSplit[1]] + configCtx.Mutex.Unlock() //If data is filesys - skip fileSys loop if strings.Contains(inPath, "filesys") { goto skipSwitch } //Read file from filesys once per new file - if fileSysIndex != -1 && !present { + if configCtx.FileSysIndex != -1 && !present { path, err := os.Getwd() fileData, err1 := os.ReadFile(filepath.FromSlash(path + inPathSplit[1])) if err != nil || err1 != nil { @@ -55,20 +43,20 @@ func messenger(inData *string, inPath string) { return } dataStr := string(fileData) - messenger(&dataStr, "filesys||."+inPathSplit[1]) + messenger(configCtx, &dataStr, "filesys||."+inPathSplit[1]) } skipSwitch: - resultChannel <- &data + configCtx.ResultChannel <- &data } -func receiver() { +func receiver(configCtx *utils.ConfigContext) { - for data := range resultChannel { - if data != nil && data.inData != nil && data.inPath != "" { - mutex.Lock() - resultMap[data.inPath] = data.inData - mutex.Unlock() + for data := range configCtx.ResultChannel { + if data != nil && data.InData != nil && data.InPath != "" { + configCtx.Mutex.Lock() + configCtx.ResultMap[data.InPath] = data.InData + configCtx.Mutex.Unlock() } } @@ -87,26 +75,48 @@ func CommonMain(envPtr *string, appRoleIDPtr *string, tokenNamePtr *string, regionPtr *string, + flagset *flag.FlagSet, + argLines []string, c *eUtils.DriverConfig) error { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - startDirPtr := flag.String("startDir", STARTDIR_DEFAULT, "Template directory") - endDirPtr := flag.String("endDir", ENDDIR_DEFAULT, "Directory to put configured templates into") - secretMode := flag.Bool("secretMode", true, "Only override secret values in templates?") - servicesWanted := flag.String("servicesWanted", "", "Services to pull template values for, in the form 'service1,service2' (defaults to all services)") - wantCertsPtr := flag.Bool("certs", false, "Pull certificates into directory specified by endDirPtr") - keyStorePtr := flag.String("keystore", "", "Put certificates into this keystore file.") - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"config.log", "Output path for log file") - pingPtr := flag.Bool("ping", false, "Ping vault.") - zcPtr := flag.Bool("zc", false, "Zero config (no configuration option).") - diffPtr := flag.Bool("diff", false, "Diff files") - fileFilterPtr := flag.String("filter", "", "Filter files for diff") - templateInfoPtr := flag.Bool("templateInfo", false, "Version information about templates") - versionInfoPtr := flag.Bool("versions", false, "Version information about values") - insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") - noVaultPtr := flag.Bool("novault", false, "Don't pull configuration data from vault.") + configCtx := &utils.ConfigContext{ + ResultMap: make(map[string]*string), + EnvSlice: make([]string, 0), + ResultChannel: make(chan *utils.ResultData, 5), + FileSysIndex: -1, + ConfigWg: sync.WaitGroup{}, + Mutex: &sync.Mutex{}, + } + + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) + flagset.Usage = flag.Usage + flagset.String("env", "dev", "Environment to configure") + flagset.String("addr", "", "API endpoint for the vault") + flagset.String("token", "", "Vault access token") + flagset.String("secretID", "", "Secret app role ID") + flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + flagset.String("appRoleID", "", "Public app role ID") + flagset.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") + } + startDirPtr := flagset.String("startDir", STARTDIR_DEFAULT, "Template directory") + endDirPtr := flagset.String("endDir", ENDDIR_DEFAULT, "Directory to put configured templates into") + secretMode := flagset.Bool("secretMode", true, "Only override secret values in templates?") + servicesWanted := flagset.String("servicesWanted", "", "Services to pull template values for, in the form 'service1,service2' (defaults to all services)") + wantCertsPtr := flagset.Bool("certs", false, "Pull certificates into directory specified by endDirPtr") + keyStorePtr := flagset.String("keystore", "", "Put certificates into this keystore file.") + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"config.log", "Output path for log file") + pingPtr := flagset.Bool("ping", false, "Ping vault.") + zcPtr := flagset.Bool("zc", false, "Zero config (no configuration option).") + diffPtr := flagset.Bool("diff", false, "Diff files") + fileFilterPtr := flagset.String("filter", "", "Filter files for diff") + templateInfoPtr := flagset.Bool("templateInfo", false, "Version information about templates") + versionInfoPtr := flagset.Bool("versions", false, "Version information about values") + insecurePtr := flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + noVaultPtr := flagset.Bool("novault", false, "Don't pull configuration data from vault.") isShell := false if c != nil { @@ -114,7 +124,7 @@ func CommonMain(envPtr *string, } if c == nil || !isShell { - args := os.Args[1:] + args := argLines[1:] for i := 0; i < len(args); i++ { s := args[i] if s[0] != '-' { @@ -122,10 +132,10 @@ func CommonMain(envPtr *string, return fmt.Errorf("wrong flag syntax: %s", s) } } - flag.Parse() + flagset.Parse(argLines[1:]) } else { // TODO: rework to support standard arg parsing... - for _, args := range os.Args { + for _, args := range argLines { if args == "-certs" { c.WantCerts = true } else if strings.HasPrefix(args, "-keystore") { @@ -140,7 +150,7 @@ func CommonMain(envPtr *string, } } } - flag.CommandLine.Parse(nil) + flagset.Parse(nil) if c.WantCerts { *wantCertsPtr = true } @@ -204,25 +214,25 @@ func CommonMain(envPtr *string, } else if *diffPtr { if strings.ContainsAny(*envPtr, ",") { //Multiple environments *envPtr = strings.ReplaceAll(*envPtr, "latest", "0") - envDiffSlice = strings.Split(*envPtr, ",") - envLength = len(envDiffSlice) - if len(envDiffSlice) > 4 { + configCtx.EnvSlice = strings.Split(*envPtr, ",") + configCtx.EnvLength = len(configCtx.EnvSlice) + if len(configCtx.EnvSlice) > 4 { fmt.Println("Unsupported number of environments - Maximum: 4") return errors.New("unsupported number of environments - Maximum: 4") } - for i, env := range envDiffSlice { + for i, env := range configCtx.EnvSlice { if env == "local" { fmt.Println("Unsupported env: local not available with diff flag") return errors.New("unsupported env: local not available with diff flag") } if !strings.Contains(env, "_") && env != "filesys" { - envDiffSlice[i] = env + "_0" + configCtx.EnvSlice[i] = env + "_0" } } - for i, env := range envDiffSlice { + for i, env := range configCtx.EnvSlice { if env == "filesys" { - fileSysIndex = i - envDiffSlice = append(envDiffSlice[:i], envDiffSlice[i+1:]...) + configCtx.FileSysIndex = i + configCtx.EnvSlice = append(configCtx.EnvSlice[:i], configCtx.EnvSlice[i+1:]...) } } } else { @@ -279,9 +289,9 @@ func CommonMain(envPtr *string, } } - if len(envDiffSlice) > 1 { - removeDuplicateValuesSlice := eUtils.RemoveDuplicateValues(envDiffSlice) - if !cmp.Equal(envDiffSlice, removeDuplicateValuesSlice) { + if len(configCtx.EnvSlice) > 1 { + removeDuplicateValuesSlice := eUtils.RemoveDuplicateValues(configCtx.EnvSlice) + if !cmp.Equal(configCtx.EnvSlice, removeDuplicateValuesSlice) { fmt.Println("There is a duplicate environment in the -env flag") return errors.New("there is a duplicate environment in the -env flag") } @@ -333,11 +343,11 @@ func CommonMain(envPtr *string, } //channel receiver - go receiver() + go receiver(configCtx) var diffFileCount int if *diffPtr { - configSlice := make([]eUtils.DriverConfig, 0, len(envDiffSlice)-1) - for _, env := range envDiffSlice { + configSlice := make([]eUtils.DriverConfig, 0, len(configCtx.EnvSlice)-1) + for _, env := range configCtx.EnvSlice { envVersion := eUtils.SplitEnv(env) *envPtr = envVersion[0] *tokenPtr = "" @@ -393,10 +403,10 @@ func CommonMain(envPtr *string, FileFilter: fileFilterSlice, } configSlice = append(configSlice, config) - wg.Add(1) + configCtx.ConfigWg.Add(1) go func() { - defer wg.Done() - eUtils.ConfigControl(nil, &configSlice[len(configSlice)-1], vcutils.GenerateConfigsFromVault) + defer configCtx.ConfigWg.Done() + eUtils.ConfigControl(nil, configCtx, &configSlice[len(configSlice)-1], vcutils.GenerateConfigsFromVault) if diffFileCount < configSlice[len(configSlice)-1].DiffCounter { //Without this, resultMap may be missing data when diffing. diffFileCount = configSlice[len(configSlice)-1].DiffCounter //This counter helps the diff wait for results } @@ -448,29 +458,37 @@ func CommonMain(envPtr *string, if len(configBase.DeploymentConfig) > 0 { config.DeploymentConfig = configBase.DeploymentConfig } - wg.Add(1) + configCtx.ConfigWg.Add(1) go func(c *eUtils.DriverConfig) { - defer wg.Done() - eUtils.ConfigControl(nil, c, vcutils.GenerateConfigsFromVault) + defer configCtx.ConfigWg.Done() + eUtils.ConfigControl(nil, configCtx, c, vcutils.GenerateConfigsFromVault) }(&config) } - wg.Wait() //Wait for templates + configCtx.ConfigWg.Wait() //Wait for templates if c == nil { - close(resultChannel) + close(configCtx.ResultChannel) } else if c.IsShellConfigComplete { - close(resultChannel) + select { + case _, ok := <-configCtx.ResultChannel: + if ok { + close(configCtx.ResultChannel) + } + case <-time.NewTicker(200 * time.Millisecond).C: + close(configCtx.ResultChannel) + break + } } if *diffPtr { //Diff if needed - if fileSysIndex != -1 { - envDiffSlice = append(envDiffSlice, "filesys") - envLength = len(envDiffSlice) + if configCtx.FileSysIndex != -1 { + configCtx.EnvSlice = append(configCtx.EnvSlice, "filesys") + configCtx.EnvLength = len(configCtx.EnvSlice) } - wg.Add(1) + configCtx.ConfigWg.Add(1) go func() { - defer wg.Done() - eUtils.DiffHelper(resultMap, envLength, envDiffSlice, fileSysIndex, true, mutex, diffFileCount) + defer configCtx.ConfigWg.Done() + eUtils.DiffHelper(configCtx, true, diffFileCount) }() } - wg.Wait() //Wait for diff + configCtx.ConfigWg.Wait() //Wait for diff return nil } diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index 54b77a8bf..12c5a0ab2 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -22,7 +22,7 @@ import ( var mutex = &sync.Mutex{} // GenerateConfigsFromVault configures the templates in trc_templates and writes them to trcconfig -func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverConfig) (interface{}, error) { +func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.ConfigContext, config *eUtils.DriverConfig) (interface{}, error) { /*Cyan := "\033[36m" Reset := "\033[0m" if utils.IsWindows() { @@ -341,9 +341,9 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverCo } else { if config.Diff { if version != "" { - config.Update(&configuredTemplate, config.Env+"_"+version+"||"+endPaths[i]) + config.Update(configCtx, &configuredTemplate, config.Env+"_"+version+"||"+endPaths[i]) } else { - config.Update(&configuredTemplate, config.Env+"||"+endPaths[i]) + config.Update(configCtx, &configuredTemplate, config.Env+"||"+endPaths[i]) } } else { writeToFile(config, configuredTemplate, endPaths[i]) @@ -397,9 +397,9 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverCo } else { if config.Diff { if version != "" { - config.Update(&configuredTemplate, config.Env+"_"+version+"||"+endPaths[i]) + config.Update(configCtx, &configuredTemplate, config.Env+"_"+version+"||"+endPaths[i]) } else { - config.Update(&configuredTemplate, config.Env+"||"+endPaths[i]) + config.Update(configCtx, &configuredTemplate, config.Env+"||"+endPaths[i]) } } else { writeToFile(config, configuredTemplate, endPaths[i]) diff --git a/trcctl/trcctl.go b/trcctl/trcctl.go index 0276b5dd7..0011316cd 100644 --- a/trcctl/trcctl.go +++ b/trcctl/trcctl.go @@ -28,11 +28,14 @@ func main() { memprotectopts.MemProtectInit(nil) } fmt.Println("Version: " + "1.34") - envPtr := flag.String("env", "", "Environment to be seeded") //If this is blank -> use context otherwise override context. - tokenPtr := flag.String("token", "", "Vault access token") - secretIDPtr := flag.String("secretID", "", "Secret app role ID") - appRoleIDPtr := flag.String("appRoleID", "", "Public app role ID") - tokenNamePtr := flag.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = flag.Usage + + envPtr := flagset.String("env", "", "Environment to be seeded") //If this is blank -> use context otherwise override context. + tokenPtr := flagset.String("token", "", "Vault access token") + secretIDPtr := flagset.String("secretID", "", "Secret app role ID") + appRoleIDPtr := flagset.String("appRoleID", "", "Public app role ID") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") var envContext string var ctl string @@ -48,7 +51,7 @@ func main() { os.Args = os.Args[1:] } } - flag.Parse() + flagset.Parse(os.Args) if ctl != "" { var err error @@ -85,11 +88,11 @@ func main() { case "pub": trcpubbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil) case "sub": - trcsubbase.CommonMain(envPtr, &addrPtr, &envContext, secretIDPtr, appRoleIDPtr, nil) + trcsubbase.CommonMain(envPtr, &addrPtr, &envContext, secretIDPtr, appRoleIDPtr, flagset, os.Args, nil) case "init": trcinitbase.CommonMain(envPtr, &addrPtr, &envContext) case "config": - trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, nil) + trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, flagset, os.Args, nil) case "x": trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil) } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 52d6eda0b..e2e0f1290 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -325,6 +325,9 @@ func roleBasedRunner(env string, config.EnvRaw = env config.WantCerts = false config.IsShellSubProcess = true + if config.VaultAddress == "" { + config.VaultAddress = *gTrcshConfig.VaultAddress + } if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} config.EndDir = trcDeployRoot.(string) @@ -340,13 +343,12 @@ func roleBasedRunner(env string, switch control { case "trcconfig": - err = trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) + err = trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, config) case "trcsub": config.EndDir = config.EndDir + "/trc_templates" - err = trcsubbase.CommonMain(&configEnv, &config.VaultAddress, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], config) + err = trcsubbase.CommonMain(&configEnv, &config.VaultAddress, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], nil, deployArgLines, config) } - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" @@ -404,7 +406,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, } gAgentConfig.Env = &env - err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, nil, deployArgLines, config) config.FeatherCtlCb = nil ResetModifier(config) //Resetting modifier cache to avoid token conflicts. flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. @@ -472,10 +474,8 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, config.EnvRaw = env config.IsShellSubProcess = true - err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, config) + err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, nil, deployArgLines, config) ResetModifier(config) - os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. if !agentToken { token = "" config.Token = token @@ -485,9 +485,6 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, // This maybe isn't needed for windows deploys... Since config can // pull templates directly from vault.. err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. if !agentToken { token = "" config.Token = token @@ -496,9 +493,6 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, case "trcconfig": err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. if !agentToken { token = "" config.Token = token @@ -601,17 +595,21 @@ func ProcessDeploy(env string, region string, token string, deployment string, t // config.VaultAddress = "" // config.Token = "" // Chewbacca: end scrub - if gTrcshConfig == nil { - gTrcshConfig, err = trcshauth.TrcshAuth(gAgentConfig, config) - if err != nil { - fmt.Println("Tierceron bootstrap failure.") - fmt.Println(err.Error()) - logger.Println(err) - os.Exit(-1) + config.Log.Printf("Bootstrap..") + for { + if gTrcshConfig == nil || gTrcshConfig.CToken == nil || gTrcshConfig.ConfigRole == nil || gTrcshConfig.VaultAddress == nil || + *gTrcshConfig.CToken == "" || *gTrcshConfig.ConfigRole == "" || *gTrcshConfig.VaultAddress == "" { + gTrcshConfig, err = trcshauth.TrcshAuth(gAgentConfig, config) + if err != nil { + config.Log.Printf(".") + time.Sleep(time.Second) + continue + } + config.Log.Printf("Auth re-loaded %s\n", env) + } else { + break } - config.Log.Printf("Auth loaded %s\n", env) } - // Chewbacca: Begin dbg comment var auth string mergedVaultAddress := config.VaultAddress @@ -664,10 +662,8 @@ func ProcessDeploy(env string, region string, token string, deployment string, t config.OutputMemCache = true config.StartDir = []string{"trc_templates"} config.EndDir = "." - trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - os.Args = []string{os.Args[0]} //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" config.Token = token @@ -692,7 +688,6 @@ func ProcessDeploy(env string, region string, token string, deployment string, t if env == "itdev" || env == "staging" || env == "prod" { config.OutputMemCache = false } - os.Args = []string{os.Args[0]} config.Log.Println("Processing trcshell") } else { if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { @@ -824,7 +819,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t agentToken, token, argsOrig, - deployArgLines, + strings.Split(deployLine, " "), &configCount, logger) if err != nil { @@ -845,7 +840,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t agentToken, token, argsOrig, - deployArgLines, + strings.Split(deployLine, " "), &configCount, logger) } diff --git a/trcsub/trcsub.go b/trcsub/trcsub.go index fc91afa6c..ba5710823 100644 --- a/trcsub/trcsub.go +++ b/trcsub/trcsub.go @@ -21,12 +21,14 @@ func main() { memprotectopts.MemProtectInit(nil) } fmt.Println("Version: " + "1.26") - envPtr := flag.String("env", "dev", "Environment to configure") - addrPtr := flag.String("addr", "", "API endpoint for the vault") - secretIDPtr := flag.String("secretID", "", "Public app role ID") - appRoleIDPtr := flag.String("appRoleID", "", "Secret app role ID") + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = flag.Usage + envPtr := flagset.String("env", "dev", "Environment to configure") + addrPtr := flagset.String("addr", "", "API endpoint for the vault") + secretIDPtr := flagset.String("secretID", "", "Public app role ID") + appRoleIDPtr := flagset.String("appRoleID", "", "Secret app role ID") - err := trcsubbase.CommonMain(envPtr, addrPtr, nil, secretIDPtr, appRoleIDPtr, nil) + err := trcsubbase.CommonMain(envPtr, addrPtr, nil, secretIDPtr, appRoleIDPtr, flagset, os.Args, nil) if err != nil { os.Exit(1) } diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index ac9d2b9e4..8bb9c19e4 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -24,22 +24,33 @@ import ( func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, secretIDPtr *string, appRoleIDPtr *string, + flagset *flag.FlagSet, + argLines []string, c *eUtils.DriverConfig) error { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.5") - endDirPtr := flag.String("endDir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory to put configured templates into") - tokenPtr := flag.String("token", "", "Vault access token") - tokenNamePtr := flag.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") - pingPtr := flag.Bool("ping", false, "Ping vault.") - insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"sub.log", "Output path for log files") - projectInfoPtr := flag.Bool("projectInfo", false, "Lists all project info") - filterTemplatePtr := flag.String("templateFilter", "", "Specifies which templates to filter") - templatePathsPtr := flag.String("templatePaths", "", "Specifies which specific templates to download.") + fmt.Println("Version: " + "1.6") - flag.Parse() + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) + flagset.Usage = flag.Usage + flagset.String("env", "dev", "Environment to configure") + flagset.String("addr", "", "API endpoint for the vault") + flagset.String("secretID", "", "Public app role ID") + flagset.String("appRoleID", "", "Secret app role ID") + } + endDirPtr := flagset.String("endDir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory to put configured templates into") + tokenPtr := flagset.String("token", "", "Vault access token") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") + pingPtr := flagset.Bool("ping", false, "Ping vault.") + insecurePtr := flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"sub.log", "Output path for log files") + projectInfoPtr := flagset.Bool("projectInfo", false, "Lists all project info") + filterTemplatePtr := flagset.String("templateFilter", "", "Specifies which templates to filter") + templatePathsPtr := flagset.String("templatePaths", "", "Specifies which specific templates to download.") + + flagset.Parse(argLines[1:]) if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") diff --git a/trcvault/trcplgtool/trcplgtool.go b/trcvault/trcplgtool/trcplgtool.go index 0cba48e94..e1229d7aa 100644 --- a/trcvault/trcplgtool/trcplgtool.go +++ b/trcvault/trcplgtool/trcplgtool.go @@ -12,12 +12,14 @@ import ( func main() { fmt.Println("Version: " + "1.02") - envPtr := flag.String("env", "dev", "Environment to configure") - addrPtr := flag.String("addr", "", "API endpoint for the vault") - tokenPtr := flag.String("token", "", "Vault access token") - regionPtr := flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = flag.Usage + envPtr := flagset.String("env", "dev", "Environment to configure") + addrPtr := flagset.String("addr", "", "API endpoint for the vault") + tokenPtr := flagset.String("token", "", "Vault access token") + regionPtr := flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. - err := plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, nil) + err := plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, flagset, os.Args, nil) if err != nil { os.Exit(1) } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 2f9124ba4..287376081 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -23,41 +23,52 @@ func CommonMain(envPtr *string, addrPtr *string, tokenPtr *string, regionPtr *string, + flagset *flag.FlagSet, + argLines []string, c *eUtils.DriverConfig) error { // Main functions are as follows: - defineServicePtr := flag.Bool("defineService", false, "Service is defined.") - certifyImagePtr := flag.Bool("certify", false, "Used to certifies vault plugin.") + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ContinueOnError) + // set and ignore.. + flagset.String("env", "dev", "Environment to configure") + flagset.String("addr", "", "API endpoint for the vault") + flagset.String("token", "", "Vault access token") + flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + flagset.Usage = flag.Usage + } + defineServicePtr := flagset.Bool("defineService", false, "Service is defined.") + certifyImagePtr := flagset.Bool("certify", false, "Used to certifies vault plugin.") // These functions only valid for pluginType trcshservice - winservicestopPtr := flag.Bool("winservicestop", false, "To stop a windows service for a particular plugin.") - winservicestartPtr := flag.Bool("winservicestart", false, "To start a windows service for a particular plugin.") - codebundledeployPtr := flag.Bool("codebundledeploy", false, "To deploy a code bundle.") - agentdeployPtr := flag.Bool("agentdeploy", false, "To initiate deployment on agent.") - projectservicePtr := flag.String("projectservice", "", "Provide template root path in form project/service") + winservicestopPtr := flagset.Bool("winservicestop", false, "To stop a windows service for a particular plugin.") + winservicestartPtr := flagset.Bool("winservicestart", false, "To start a windows service for a particular plugin.") + codebundledeployPtr := flagset.Bool("codebundledeploy", false, "To deploy a code bundle.") + agentdeployPtr := flagset.Bool("agentdeploy", false, "To initiate deployment on agent.") + projectservicePtr := flagset.String("projectservice", "", "Provide template root path in form project/service") // Common flags... - startDirPtr := flag.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") - insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"plgtool.log", "Output path for log files") + startDirPtr := flagset.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") + insecurePtr := flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"plgtool.log", "Output path for log files") // defineService flags... - deployrootPtr := flag.String("deployroot", "", "Optional path for deploying services to.") - serviceNamePtr := flag.String("serviceName", "", "Optional name of service to use in managing service.") - codeBundlePtr := flag.String("codeBundle", "", "Code bundle to deploy.") + deployrootPtr := flagset.String("deployroot", "", "Optional path for deploying services to.") + serviceNamePtr := flagset.String("serviceName", "", "Optional name of service to use in managing service.") + codeBundlePtr := flagset.String("codeBundle", "", "Code bundle to deploy.") // Common plugin flags... - pluginNamePtr := flag.String("pluginName", "", "Used to certify vault plugin") - pluginTypePtr := flag.String("pluginType", "vault", "Used to indicate type of plugin. Default is vault.") + pluginNamePtr := flagset.String("pluginName", "", "Used to certify vault plugin") + pluginTypePtr := flagset.String("pluginType", "vault", "Used to indicate type of plugin. Default is vault.") // Certify flags... - sha256Ptr := flag.String("sha256", "", "Used to certify vault plugin") //This has to match the image that is pulled -> then we write the vault. - checkDeployedPtr := flag.Bool("checkDeployed", false, "Used to check if plugin has been copied, deployed, & certified") - checkCopiedPtr := flag.Bool("checkCopied", false, "Used to check if plugin has been copied & certified") + sha256Ptr := flagset.String("sha256", "", "Used to certify vault plugin") //This has to match the image that is pulled -> then we write the vault. + checkDeployedPtr := flagset.Bool("checkDeployed", false, "Used to check if plugin has been copied, deployed, & certified") + checkCopiedPtr := flagset.Bool("checkCopied", false, "Used to check if plugin has been copied & certified") certifyInit := false if c == nil || !c.IsShellSubProcess { - args := os.Args[1:] + args := argLines[1:] for i := 0; i < len(args); i++ { s := args[i] if s[0] != '-' { @@ -65,16 +76,25 @@ func CommonMain(envPtr *string, return fmt.Errorf("wrong flag syntax: %s", s) } } - flag.Parse() + err := flagset.Parse(argLines[1:]) + if err != nil { + return err + } // Prints usage if no flags are specified - if flag.NFlag() == 0 { - flag.Usage() + if flagset.NFlag() == 0 { + flagset.Usage() return errors.New("invalid input parameters") } } else { - flag.CommandLine.Parse(os.Args) - flag.Parse() + err := flagset.Parse(argLines) + if err != nil { + return err + } + err = flagset.Parse(argLines[1:]) + if err != nil { + return err + } } if *certifyImagePtr && (len(*pluginNamePtr) == 0 || len(*sha256Ptr) == 0) { diff --git a/trcx/xutil/xmanager.go b/trcx/xutil/xmanager.go index c3c9194ef..d4ada8be4 100644 --- a/trcx/xutil/xmanager.go +++ b/trcx/xutil/xmanager.go @@ -596,7 +596,7 @@ func GenerateSeedsFromVaultRaw(config *eUtils.DriverConfig, fromVault bool, temp } // GenerateSeedsFromVault configures the templates in trc_templates and writes them to trcx -func GenerateSeedsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverConfig) (interface{}, error) { +func GenerateSeedsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.ConfigContext, config *eUtils.DriverConfig) (interface{}, error) { if config.Clean { //Clean flag in trcx if strings.HasSuffix(config.Env, "_0") { envVersion := eUtils.SplitEnv(config.Env) @@ -801,7 +801,7 @@ func GenerateSeedsFromVault(ctx eUtils.ProcessContext, config *eUtils.DriverConf if !strings.Contains(config.Env, "_") { config.Env = config.Env + "_0" } - config.Update(&seedData, config.Env+"||"+config.Env+"_seed.yml") + config.Update(configCtx, &seedData, config.Env+"||"+config.Env+"_seed.yml") } else { writeToFile(config, seedData, endPath) // Print that we're done diff --git a/trcxbase/trcxbase.go b/trcxbase/trcxbase.go index 22fafac50..b4382a64d 100644 --- a/trcxbase/trcxbase.go +++ b/trcxbase/trcxbase.go @@ -21,33 +21,21 @@ import ( "github.com/hashicorp/vault/api" ) -type ResultData struct { - inData *string - inPath string +func messenger(configCtx *utils.ConfigContext, inData *string, inPath string) { + var data utils.ResultData + data.InData = inData + data.InPath = inPath + configCtx.ResultChannel <- &data } -var resultMap = make(map[string]*string) -var envSlice = make([]string, 0) -var projectSectionsSlice = make([]string, 0) -var resultChannel = make(chan *ResultData, 5) -var envLength int -var resultMapLock = &sync.Mutex{} - -func messenger(inData *string, inPath string) { - var data ResultData - data.inData = inData - data.inPath = inPath - resultChannel <- &data -} - -func receiver() { +func receiver(configCtx *utils.ConfigContext) { for { select { - case data := <-resultChannel: - if data != nil && data.inData != nil && data.inPath != "" { - resultMapLock.Lock() - resultMap[data.inPath] = data.inData - resultMapLock.Unlock() + case data := <-configCtx.ResultChannel: + if data != nil && data.InData != nil && data.InPath != "" { + configCtx.Mutex.Lock() + configCtx.ResultMap[data.InPath] = data.InData + configCtx.Mutex.Unlock() } } } @@ -104,6 +92,15 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env } flag.Parse() + configCtx := &utils.ConfigContext{ + ResultMap: make(map[string]*string), + EnvSlice: make([]string, 0), + ProjectSectionsSlice: make([]string, 0), + ResultChannel: make(chan *utils.ResultData, 5), + FileSysIndex: -1, + ConfigWg: sync.WaitGroup{}, + Mutex: &sync.Mutex{}, + } config := &eUtils.DriverConfig{ExitOnFailure: true, Insecure: *insecurePtr} @@ -231,7 +228,7 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env fmt.Println(Yellow + "Specified versioning not available, using " + envVersion[0] + " as environment" + Reset) *envPtr = strings.Split(*envPtr, "_")[0] } - envSlice = append(envSlice, *envPtr+"_versionInfo") + configCtx.EnvSlice = append(configCtx.EnvSlice, *envPtr+"_versionInfo") goto skipDiff } @@ -239,19 +236,19 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env if *diffPtr { if strings.ContainsAny(*envPtr, ",") { //Multiple environments *envPtr = strings.ReplaceAll(*envPtr, "latest", "0") - envSlice = strings.Split(*envPtr, ",") - envLength = len(envSlice) - if len(envSlice) > 4 { + configCtx.EnvSlice = strings.Split(*envPtr, ",") + configCtx.EnvLength = len(configCtx.EnvSlice) + if len(configCtx.EnvSlice) > 4 { fmt.Println("Unsupported number of environments - Maximum: 4") os.Exit(1) } - for i, env := range envSlice { + for i, env := range configCtx.EnvSlice { if env == "local" { fmt.Println("Unsupported env: local not available with diff flag") os.Exit(1) } if !strings.Contains(env, "_") { - envSlice[i] = env + "_0" + configCtx.EnvSlice[i] = env + "_0" } } } else { @@ -263,7 +260,7 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env fmt.Println("-diff flag is required for multiple environments - env: -env=env1,env2,...") os.Exit(1) } - envSlice = append(envSlice, (*envPtr)) + configCtx.EnvSlice = append(configCtx.EnvSlice, (*envPtr)) envVersion := strings.Split(*envPtr, "_") //Break apart env+version for token *envPtr = envVersion[0] if !*noVaultPtr { @@ -294,14 +291,14 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env } //Duplicate env check - for _, entry := range envSlice { + for _, entry := range configCtx.EnvSlice { if _, value := keysCheck[entry]; !value { keysCheck[entry] = true listCheck = append(listCheck, entry) } } - if len(listCheck) != len(envSlice) { + if len(listCheck) != len(configCtx.EnvSlice) { fmt.Printf("Cannot diff an environment against itself.\n") os.Exit(1) } @@ -330,7 +327,7 @@ skipDiff: regions := []string{} - if len(envSlice) == 1 && !*noVaultPtr { + if len(configCtx.EnvSlice) == 1 && !*noVaultPtr { if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") { secretIDPtr = nil appRoleIDPtr = nil @@ -345,7 +342,7 @@ skipDiff: } } - if (tokenPtr == nil || *tokenPtr == "") && !*noVaultPtr && len(envSlice) == 1 { + if (tokenPtr == nil || *tokenPtr == "") && !*noVaultPtr && len(configCtx.EnvSlice) == 1 { fmt.Println("Missing required auth token.") os.Exit(1) } @@ -373,11 +370,11 @@ skipDiff: var serviceFilterSlice []string if len(*dynamicPathPtr) > 0 { - go receiver() //Channel receiver + go receiver(configCtx) //Channel receiver dynamicPathParts := strings.Split(*dynamicPathPtr, "/") - for _, env := range envSlice { + for _, env := range configCtx.EnvSlice { envVersion := eUtils.SplitEnv(env) *envPtr = envVersion[0] if secretIDPtr != nil && *secretIDPtr != "" && appRoleIDPtr != nil && *appRoleIDPtr != "" { @@ -387,10 +384,10 @@ skipDiff: var baseEnv string if strings.Contains(*dynamicPathPtr, "%s") { - if strings.Contains(envSlice[0], "_") { - baseEnv = strings.Split(envSlice[0], "_")[0] + if strings.Contains(configCtx.EnvSlice[0], "_") { + baseEnv = strings.Split(configCtx.EnvSlice[0], "_")[0] } else { - baseEnv = envSlice[0] + baseEnv = configCtx.EnvSlice[0] } if !*noVaultPtr && *tokenPtr == "" { //Ask vault for list of dev..* environments, add to envSlice @@ -452,7 +449,7 @@ skipDiff: waitg.Add(1) go func() { defer waitg.Done() - eUtils.ConfigControl(ctx, &config, configDriver) + eUtils.ConfigControl(ctx, configCtx, &config, configDriver) }() return } @@ -514,8 +511,8 @@ skipDiff: // TODO: Deprecated... // 1-800-ROIT - if len(envSlice) == 1 || (len(*eUtils.IndexValueFilterPtr) > 0 && len(*eUtils.IndexedPtr) > 0) { - if strings.Contains(envSlice[0], "*") || len(*eUtils.IndexedPtr) > 0 || len(*eUtils.RestrictedPtr) > 0 || len(*eUtils.ProtectedPtr) > 0 { + if len(configCtx.EnvSlice) == 1 || (len(*eUtils.IndexValueFilterPtr) > 0 && len(*eUtils.IndexedPtr) > 0) { + if strings.Contains(configCtx.EnvSlice[0], "*") || len(*eUtils.IndexedPtr) > 0 || len(*eUtils.RestrictedPtr) > 0 || len(*eUtils.ProtectedPtr) > 0 { if len(*eUtils.IndexedPtr) > 0 { sectionKey = "/Index/" } else if len(*eUtils.RestrictedPtr) > 0 { @@ -527,10 +524,10 @@ skipDiff: newSectionSlice := make([]string, 0) if !*noVaultPtr && !trcxe { var baseEnv string - if strings.Contains(envSlice[0], "_") { - baseEnv = strings.Split(envSlice[0], "_")[0] + if strings.Contains(configCtx.EnvSlice[0], "_") { + baseEnv = strings.Split(configCtx.EnvSlice[0], "_")[0] } else { - baseEnv = envSlice[0] + baseEnv = configCtx.EnvSlice[0] } //Ask vault for list of dev..* environments, add to envSlice authErr := eUtils.AutoAuth(&eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true}, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, &baseEnv, addrPtr, envCtxPtr, "", *pingPtr) @@ -545,20 +542,20 @@ skipDiff: // Only look at index values.... //Checks for indexed projects if len(*eUtils.IndexedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) } if len(*eUtils.RestrictedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.RestrictedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.RestrictedPtr, ",")...) } if len(*eUtils.ProtectedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.ProtectedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.ProtectedPtr, ",")...) } var listValues *api.Secret - if len(projectSectionsSlice) > 0 { //If eid -> look inside Index and grab all environments - subSectionPath := projectSectionsSlice[0] + "/" + if len(configCtx.ProjectSectionsSlice) > 0 { //If eid -> look inside Index and grab all environments + subSectionPath := configCtx.ProjectSectionsSlice[0] + "/" listValues, err = testMod.ListEnv("super-secrets/"+testMod.Env+sectionKey+subSectionPath, config.Log) if err != nil { if strings.Contains(err.Error(), "permission denied") { @@ -607,15 +604,15 @@ skipDiff: } } else { //novault takes this path if len(*eUtils.IndexedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) } if len(*eUtils.RestrictedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.RestrictedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.RestrictedPtr, ",")...) } if len(*eUtils.ProtectedPtr) > 0 { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.ProtectedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.ProtectedPtr, ",")...) } } } @@ -645,9 +642,9 @@ skipDiff: } } - go receiver() //Channel receiver + go receiver(configCtx) //Channel receiver if len(*dynamicPathPtr) == 0 { - for _, env := range envSlice { + for _, env := range configCtx.EnvSlice { envVersion := eUtils.SplitEnv(env) *envPtr = envVersion[0] if secretIDPtr != nil && *secretIDPtr != "" && appRoleIDPtr != nil && *appRoleIDPtr != "" { @@ -675,7 +672,7 @@ skipDiff: var trcxeList []string if trcxe { - projectSectionsSlice = append(projectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) + configCtx.ProjectSectionsSlice = append(configCtx.ProjectSectionsSlice, strings.Split(*eUtils.IndexedPtr, ",")...) trcxeList = append(trcxeList, *fieldsPtr) trcxeList = append(trcxeList, *encryptedPtr) @@ -709,7 +706,7 @@ skipDiff: DynamicPathFilter: *dynamicPathPtr, FileFilter: fileFilter, SubPathFilter: strings.Split(*eUtils.SubPathFilter, ","), - ProjectSections: projectSectionsSlice, + ProjectSections: configCtx.ProjectSectionsSlice, ServiceFilter: serviceFilterSlice, ExitOnFailure: true, Trcxe: trcxeList, @@ -718,31 +715,32 @@ skipDiff: waitg.Add(1) go func() { defer waitg.Done() - eUtils.ConfigControl(ctx, &config, configDriver) + eUtils.ConfigControl(ctx, configCtx, &config, configDriver) }() } } } waitg.Wait() - close(resultChannel) + close(configCtx.ResultChannel) if *diffPtr { //Diff if needed waitg.Add(1) - go func() { + go func(cctx *eUtils.ConfigContext) { defer waitg.Done() retry := 0 for { - resultMapLock.Lock() - if len(resultMap) == len(envSlice)*len(sectionSlice) || retry == 3 { - resultMapLock.Unlock() + cctx.Mutex.Lock() + if len(cctx.ResultMap) == len(cctx.EnvSlice)*len(sectionSlice) || retry == 3 { + cctx.Mutex.Unlock() break } - resultMapLock.Unlock() + cctx.Mutex.Unlock() time.Sleep(time.Duration(time.Second)) retry++ } - eUtils.DiffHelper(resultMap, envLength, envSlice, -1, false, resultMapLock, len(resultMap)/envLength) - }() + configCtx.FileSysIndex = -1 + eUtils.DiffHelper(cctx, false, len(configCtx.ResultMap)/configCtx.EnvLength) + }(configCtx) } waitg.Wait() //Wait for diff diff --git a/utils/diffUtil.go b/utils/diffUtil.go index f711c7c6f..937218deb 100644 --- a/utils/diffUtil.go +++ b/utils/diffUtil.go @@ -8,7 +8,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/sergi/go-diff/diffmatchpatch" @@ -358,27 +357,27 @@ func RemoveDuplicateValues(intSlice []string) []string { return list } -func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []string, fileSysIndex int, config bool, mutex *sync.Mutex, diffFileCount int) { +func DiffHelper(configCtx *ConfigContext, config bool, diffFileCount int) { fileIndex := 0 keys := []string{} - mutex.Lock() - if len(resultMap) == 0 { + configCtx.Mutex.Lock() + if len(configCtx.ResultMap) == 0 { fmt.Println("Couldn't find any data to diff") return } var baseEnv []string diffEnvFound := false - if len(envDiffSlice) > 0 { - baseEnv = SplitEnv(envDiffSlice[0]) + if len(configCtx.EnvSlice) > 0 { + baseEnv = SplitEnv(configCtx.EnvSlice[0]) } //Sort Diff Slice if env are the same - for i, env := range envDiffSlice { //Arranges keys for ordered output + for i, env := range configCtx.EnvSlice { //Arranges keys for ordered output var base []string = SplitEnv(env) if base[1] == "0" { //Special case for latest, so sort adds latest to the back of ordered slice base[1] = "_999999" - envDiffSlice[i] = base[0] + base[1] + configCtx.EnvSlice[i] = base[0] + base[1] } if len(base) > 0 && len(baseEnv) > 0 && baseEnv[0] != base[0] { @@ -387,29 +386,29 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri } if !diffEnvFound { - sort.Strings(envDiffSlice) + sort.Strings(configCtx.EnvSlice) } - for i, env := range envDiffSlice { //Changes latest back - special case + for i, env := range configCtx.EnvSlice { //Changes latest back - special case var base []string = SplitEnv(env) if base[1] == "999999" { base[1] = "_0" - envDiffSlice[i] = base[0] + base[1] + configCtx.EnvSlice[i] = base[0] + base[1] } } fileList := make([]string, diffFileCount) - mutex.Unlock() + configCtx.Mutex.Unlock() sleepCount := 0 - if len(resultMap) != diffFileCount { + if len(configCtx.ResultMap) != diffFileCount { for { time.Sleep(time.Second) sleepCount++ if sleepCount >= 5 { fmt.Println("Timeout: Attempted to wait for remaining configs to come in. Attempting incomplete diff.") break - } else if len(resultMap) == diffFileCount*envLength { + } else if len(configCtx.ResultMap) == diffFileCount*configCtx.EnvLength { break } } @@ -417,7 +416,7 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri if config { //Make fileList - for key := range resultMap { + for key := range configCtx.ResultMap { found := false keySplit := strings.Split(key, "||") @@ -433,7 +432,7 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri } } } else { - for _, env := range envDiffSlice { //Arranges keys for ordered output + for _, env := range configCtx.EnvSlice { //Arranges keys for ordered output keys = append(keys, env+"||"+env+"_seed.yml") } if len(fileList) > 0 { @@ -447,10 +446,10 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri for _, fileName := range fileList { if config { //Arranges keys for ordered output - for _, env := range envDiffSlice { + for _, env := range configCtx.EnvSlice { keys = append(keys, env+"||"+fileName) } - if fileSysIndex == len(envDiffSlice) { + if configCtx.FileSysIndex == len(configCtx.EnvSlice) { keys = append(keys, "filesys||"+fileName) } } @@ -471,20 +470,20 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri keyB := keys[1] keySplitA := strings.Split(keyA, "||") keySplitB := strings.Split(keyB, "||") - mutex.Lock() + configCtx.Mutex.Lock() sortedKeyA := keyA sortedKeyB := keyB - if _, ok := resultMap[sortedKeyA]; !ok { + if _, ok := configCtx.ResultMap[sortedKeyA]; !ok { sortedKeyA = "||" + keySplitA[1] } - if _, ok := resultMap[sortedKeyB]; !ok { + if _, ok := configCtx.ResultMap[sortedKeyB]; !ok { sortedKeyB = "||" + keySplitB[1] } - envFileKeyA := resultMap[sortedKeyA] - envFileKeyB := resultMap[sortedKeyB] - mutex.Unlock() + envFileKeyA := configCtx.ResultMap[sortedKeyA] + envFileKeyB := configCtx.ResultMap[sortedKeyB] + configCtx.Mutex.Unlock() latestVersionACheck := strings.Split(keySplitA[0], "_") if len(latestVersionACheck) > 1 && latestVersionACheck[1] == "0" { @@ -504,16 +503,16 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri fileSplit := strings.Split(keySplitB[1], "_") keySplitB[1] = fileSplit[0] + "_" + fileSplit[len(fileSplit)-1] } - switch envLength { + switch configCtx.EnvLength { case 4: keyC := keys[2] keyD := keys[3] keySplitC := strings.Split(keyC, "||") keySplitD := strings.Split(keyD, "||") - mutex.Lock() - envFileKeyC := resultMap[keyC] - envFileKeyD := resultMap[keyD] - mutex.Unlock() + configCtx.Mutex.Lock() + envFileKeyC := configCtx.ResultMap[keyC] + envFileKeyD := configCtx.ResultMap[keyD] + configCtx.Mutex.Unlock() latestVersionCCheck := strings.Split(keySplitC[0], "_") if len(latestVersionCCheck) > 1 && latestVersionCCheck[1] == "0" { @@ -549,9 +548,9 @@ func DiffHelper(resultMap map[string]*string, envLength int, envDiffSlice []stri case 3: keyC := keys[2] keySplitC := strings.Split(keyC, "||") - mutex.Lock() - envFileKeyC := resultMap[keyC] - mutex.Unlock() + configCtx.Mutex.Lock() + envFileKeyC := configCtx.ResultMap[keyC] + configCtx.Mutex.Unlock() latestVersionCCheck := strings.Split(keySplitC[0], "_") if len(latestVersionCCheck) > 1 && latestVersionCCheck[1] == "0" { diff --git a/utils/driverconfig.go b/utils/driverconfig.go index a4f4b7279..3fec058d1 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -4,6 +4,7 @@ import ( "log" "os" "strings" + "sync" "github.com/go-git/go-billy/v5" "github.com/pavlo-v-chernykh/keystore-go/v4" @@ -12,7 +13,23 @@ import ( type ProcessContext interface{} -type ConfigDriver func(ctx ProcessContext, config *DriverConfig) (interface{}, error) +type ConfigDriver func(ctx ProcessContext, configCtx *ConfigContext, config *DriverConfig) (interface{}, error) + +type ResultData struct { + InData *string + InPath string +} + +type ConfigContext struct { + ResultMap map[string]*string + EnvSlice []string + ProjectSectionsSlice []string + ResultChannel chan *ResultData + FileSysIndex int + EnvLength int + ConfigWg sync.WaitGroup + Mutex *sync.Mutex +} // DriverConfig -- contains many structures necessary for Tierceron tool functionality. type DriverConfig struct { @@ -55,7 +72,7 @@ type DriverConfig struct { Trcxr bool //Used for TRCXR Clean bool - Update func(*string, string) + Update func(*ConfigContext, *string, string) // KeyStore Output tooling KeyStore *keystore.KeyStore @@ -84,7 +101,7 @@ type DriverConfig struct { } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. -func ConfigControl(ctx ProcessContext, config *DriverConfig, drive ConfigDriver) { +func ConfigControl(ctx ProcessContext, configCtx *ConfigContext, config *DriverConfig, drive ConfigDriver) { multiProject := false config.EndDir = strings.Replace(config.EndDir, "\\", "/", -1) @@ -140,7 +157,7 @@ func ConfigControl(ctx ProcessContext, config *DriverConfig, drive ConfigDriver) config.StartDir = startDirs // Drive this set of configurations. - drive(ctx, config) + drive(ctx, configCtx, config) return } @@ -175,5 +192,5 @@ func ConfigControl(ctx ProcessContext, config *DriverConfig, drive ConfigDriver) } // Drive this set of configurations. - drive(ctx, config) + drive(ctx, configCtx, config) } From 5b2fde2ca747cc437b15d99593007f9ebc454fda Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:26:52 -0800 Subject: [PATCH 24/55] Prepare for release. (#812) * Fixing flags and usage. * Changes achieve the following: 1. Make ctoken only available to linux trcshell. 2. Let plugin tool function off approle to perform read only activities. * Update trcx to use flagset as well. * Update xp to provide proper args as well. * Fixing improper references... * Expose diff flag for trcctl. * Fixing some brokenness. * Timeout wasn't issue. Reverting change. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- capauth/agentconfig.go | 7 -- trcconfig/trcconfig.go | 5 +- trcconfigbase/trcconfig.go | 15 +-- trcctl/trcctl.go | 23 +++-- trcinit/init.go | 11 ++- trcinitbase/init.go | 78 ++++++++------- trcinitp/init.go | 3 +- trcpub/trcpub.go | 20 ++-- trcpubbase/trcpub.go | 29 ++++-- trcsh/trcsh.go | 95 ++++++++----------- trcsh/trcshauth/auth.go | 7 +- trcsub/trcsub.go | 5 +- trcsubbase/trcsub.go | 5 +- .../carrierfactory/servercapauth/capauth.go | 4 +- trcvault/trcplgtool/trcplgtool.go | 11 ++- trcvault/trcplgtoolbase/trcplgtoolbase.go | 30 +++++- trcx/trcx.go | 12 ++- trcxbase/trcxbase.go | 77 +++++++++------ trcxp/trcxp.go | 3 +- utils/diffUtil.go | 8 +- utils/driverconfig.go | 13 +++ utils/indexedFlagUtil.go | 34 +++++-- 22 files changed, 307 insertions(+), 188 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index a6d5b1fd6..1fcd83aab 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -152,13 +152,6 @@ func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, } memprotectopts.MemProtect(nil, trcShConfigRole) trcshConfig.ConfigRole = trcShConfigRole - - trcShCToken, penseError := agentconfig.PenseFeatherQuery("ctoken") - if penseError != nil { - return nil, penseError - } - memprotectopts.MemProtect(nil, trcShCToken) - trcshConfig.CToken = trcShCToken } return trcshConfig, nil diff --git a/trcconfig/trcconfig.go b/trcconfig/trcconfig.go index 2d862f0dd..a1b0e0490 100644 --- a/trcconfig/trcconfig.go +++ b/trcconfig/trcconfig.go @@ -20,7 +20,10 @@ func main() { } fmt.Println("Version: " + "1.27") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } envPtr := flagset.String("env", "dev", "Environment to configure") addrPtr := flagset.String("addr", "", "API endpoint for the vault") tokenPtr := flagset.String("token", "", "Vault access token") diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index a94c24b1b..9a1c75ff4 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -93,7 +93,11 @@ func CommonMain(envPtr *string, if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } + flagset.String("env", "dev", "Environment to configure") flagset.String("addr", "", "API endpoint for the vault") flagset.String("token", "", "Vault access token") @@ -321,7 +325,7 @@ func CommonMain(envPtr *string, if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") { supportedRegions := eUtils.GetSupportedProdRegions() - if *regionPtr != "" { + if regionPtr != nil && *regionPtr != "" { for _, supportedRegion := range supportedRegions { if *regionPtr == supportedRegion { regions = append(regions, *regionPtr) @@ -344,7 +348,6 @@ func CommonMain(envPtr *string, //channel receiver go receiver(configCtx) - var diffFileCount int if *diffPtr { configSlice := make([]eUtils.DriverConfig, 0, len(configCtx.EnvSlice)-1) for _, env := range configCtx.EnvSlice { @@ -407,8 +410,8 @@ func CommonMain(envPtr *string, go func() { defer configCtx.ConfigWg.Done() eUtils.ConfigControl(nil, configCtx, &configSlice[len(configSlice)-1], vcutils.GenerateConfigsFromVault) - if diffFileCount < configSlice[len(configSlice)-1].DiffCounter { //Without this, resultMap may be missing data when diffing. - diffFileCount = configSlice[len(configSlice)-1].DiffCounter //This counter helps the diff wait for results + if int(configCtx.GetDiffFileCount()) < configSlice[len(configSlice)-1].DiffCounter { //Without this, resultMap may be missing data when diffing. + configCtx.SetDiffFileCount(configSlice[len(configSlice)-1].DiffCounter) //This counter helps the diff wait for results } }() } @@ -486,7 +489,7 @@ func CommonMain(envPtr *string, configCtx.ConfigWg.Add(1) go func() { defer configCtx.ConfigWg.Done() - eUtils.DiffHelper(configCtx, true, diffFileCount) + eUtils.DiffHelper(configCtx, true) }() } configCtx.ConfigWg.Wait() //Wait for diff diff --git a/trcctl/trcctl.go b/trcctl/trcctl.go index 0011316cd..25c2015ea 100644 --- a/trcctl/trcctl.go +++ b/trcctl/trcctl.go @@ -27,15 +27,18 @@ func main() { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.34") + fmt.Println("Version: " + "1.35") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flagset.Usage = flag.Usage - + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } envPtr := flagset.String("env", "", "Environment to be seeded") //If this is blank -> use context otherwise override context. tokenPtr := flagset.String("token", "", "Vault access token") secretIDPtr := flagset.String("secretID", "", "Secret app role ID") appRoleIDPtr := flagset.String("appRoleID", "", "Public app role ID") tokenNamePtr := flagset.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") + flagset.Bool("diff", false, "Diff files") var envContext string var ctl string @@ -51,7 +54,11 @@ func main() { os.Args = os.Args[1:] } } - flagset.Parse(os.Args) + flagset.Parse(os.Args[1:]) + if flagset.NFlag() == 0 { + flagset.Usage() + os.Exit(0) + } if ctl != "" { var err error @@ -86,15 +93,15 @@ func main() { var addrPtr string switch ctl { case "pub": - trcpubbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil) + trcpubbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, flagset, os.Args, nil) case "sub": trcsubbase.CommonMain(envPtr, &addrPtr, &envContext, secretIDPtr, appRoleIDPtr, flagset, os.Args, nil) case "init": - trcinitbase.CommonMain(envPtr, &addrPtr, &envContext) + trcinitbase.CommonMain(envPtr, &addrPtr, &envContext, flagset, os.Args) case "config": - trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, flagset, os.Args, nil) + trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, nil, os.Args, nil) case "x": - trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil) + trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil, nil, os.Args) } } } diff --git a/trcinit/init.go b/trcinit/init.go index d563390db..edb9ee1a9 100644 --- a/trcinit/init.go +++ b/trcinit/init.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" trcinitbase "github.com/trimble-oss/tierceron/trcinitbase" @@ -17,6 +18,12 @@ func main() { memprotectopts.MemProtectInit(nil) } fmt.Println("Version: " + "1.34") - envPtr := flag.String("env", "dev", "Environment to be seeded") - trcinitbase.CommonMain(envPtr, nil, nil) + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } + + envPtr := flagset.String("env", "dev", "Environment to be seeded") + trcinitbase.CommonMain(envPtr, nil, nil, flagset, os.Args) } diff --git a/trcinitbase/init.go b/trcinitbase/init.go index fed1e0962..d100ee9ee 100644 --- a/trcinitbase/init.go +++ b/trcinitbase/init.go @@ -23,36 +23,50 @@ import ( // This assumes that the vault is completely new, and should only be run for the purpose // of automating setup and initial seeding -func CommonMain(envPtr *string, addrPtrIn *string, envCtxPtr *string) { - devPtr := flag.Bool("dev", false, "Vault server running in dev mode (does not need to be unsealed)") - newPtr := flag.Bool("new", false, "New vault being initialized. Creates engines and requests first-time initialization") - addrPtr := flag.String("addr", "", "API endpoint for the vault") +func CommonMain(envPtr *string, + addrPtrIn *string, + envCtxPtr *string, + flagset *flag.FlagSet, + argLines []string) { + + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } + flagset.String("env", "dev", "Environment to configure") + } + + devPtr := flagset.Bool("dev", false, "Vault server running in dev mode (does not need to be unsealed)") + newPtr := flagset.Bool("new", false, "New vault being initialized. Creates engines and requests first-time initialization") + addrPtr := flagset.String("addr", "", "API endpoint for the vault") if addrPtrIn != nil && *addrPtrIn != "" { addrPtr = addrPtrIn } - seedPtr := flag.String("seeds", coreopts.GetFolderPrefix(nil)+"_seeds", "Directory that contains vault seeds") - tokenPtr := flag.String("token", "", "Vault access token, only use if in dev mode or reseeding") - shardPtr := flag.String("shard", "", "Key shard used to unseal a vault that has been initialized but restarted") - - namespaceVariable := flag.String("namespace", "vault", "name of the namespace") - - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"init.log", "Output path for log files") - servicePtr := flag.String("service", "", "Seeding vault with a single service") - prodPtr := flag.Bool("prod", false, "Prod only seeds vault with staging environment") - uploadCertPtr := flag.Bool("certs", false, "Upload certs if provided") - rotateTokens := flag.Bool("rotateTokens", false, "rotate tokens") - tokenExpiration := flag.Bool("tokenExpiration", false, "Look up Token expiration dates") - pingPtr := flag.Bool("ping", false, "Ping vault.") - updateRole := flag.Bool("updateRole", false, "Update security role") - updatePolicy := flag.Bool("updatePolicy", false, "Update security policy") - initNamespace := flag.Bool("initns", false, "Init namespace (tokens, policy, and role)") - insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") - keyShardPtr := flag.String("totalKeys", "5", "Total number of key shards to make") - unsealShardPtr := flag.String("unsealKeys", "3", "Number of key shards needed to unseal") - tokenFileFilterPtr := flag.String("filter", "", "Filter files for token rotation.") - roleFileFilterPtr := flag.String("approle", "", "Filter files for approle rotation.") - dynamicPathPtr := flag.String("dynamicPath", "", "Seed a specific directory in vault.") - nestPtr := flag.Bool("nest", false, "Seed a specific directory in vault.") + seedPtr := flagset.String("seeds", coreopts.GetFolderPrefix(nil)+"_seeds", "Directory that contains vault seeds") + tokenPtr := flagset.String("token", "", "Vault access token, only use if in dev mode or reseeding") + shardPtr := flagset.String("shard", "", "Key shard used to unseal a vault that has been initialized but restarted") + + namespaceVariable := flagset.String("namespace", "vault", "name of the namespace") + + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"init.log", "Output path for log files") + servicePtr := flagset.String("service", "", "Seeding vault with a single service") + prodPtr := flagset.Bool("prod", false, "Prod only seeds vault with staging environment") + uploadCertPtr := flagset.Bool("certs", false, "Upload certs if provided") + rotateTokens := flagset.Bool("rotateTokens", false, "rotate tokens") + tokenExpiration := flagset.Bool("tokenExpiration", false, "Look up Token expiration dates") + pingPtr := flagset.Bool("ping", false, "Ping vault.") + updateRole := flagset.Bool("updateRole", false, "Update security role") + updatePolicy := flagset.Bool("updatePolicy", false, "Update security policy") + initNamespace := flagset.Bool("initns", false, "Init namespace (tokens, policy, and role)") + insecurePtr := flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + keyShardPtr := flagset.String("totalKeys", "5", "Total number of key shards to make") + unsealShardPtr := flagset.String("unsealKeys", "3", "Number of key shards needed to unseal") + tokenFileFilterPtr := flagset.String("filter", "", "Filter files for token rotation.") + roleFileFilterPtr := flagset.String("approle", "", "Filter files for approle rotation.") + dynamicPathPtr := flagset.String("dynamicPath", "", "Seed a specific directory in vault.") + nestPtr := flagset.Bool("nest", false, "Seed a specific directory in vault.") // indexServiceExtFilterPtr := flag.String("serviceExtFilter", "", "Specifies which nested services (or tables) to filter") //offset or database // indexServiceFilterPtr := flag.String("serviceFilter", "", "Specifies which services (or tables) to filter") // Table names @@ -61,7 +75,7 @@ func CommonMain(envPtr *string, addrPtrIn *string, envCtxPtr *string) { allowNonLocal := false - args := os.Args[1:] + args := argLines[1:] for i := 0; i < len(args); i++ { s := args[i] if s[0] != '-' { @@ -69,16 +83,16 @@ func CommonMain(envPtr *string, addrPtrIn *string, envCtxPtr *string) { os.Exit(1) } } - flag.Parse() - eUtils.CheckInitFlags() + eUtils.CheckInitFlags(flagset) + flagset.Parse(argLines[1:]) if memonly.IsMemonly() { memprotectopts.MemUnprotectAll(nil) memprotectopts.MemProtect(nil, tokenPtr) } // Prints usage if no flags are specified - if flag.NFlag() == 0 { - flag.Usage() + if flagset.NFlag() == 0 { + flagset.Usage() os.Exit(1) } diff --git a/trcinitp/init.go b/trcinitp/init.go index 25f79841b..00ace7ce9 100644 --- a/trcinitp/init.go +++ b/trcinitp/init.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" @@ -19,5 +20,5 @@ func main() { fmt.Println("Version: " + "1.6") env := "local" addr := coreopts.GetVaultHostPort() - trcinitbase.CommonMain(&env, &addr, nil) + trcinitbase.CommonMain(&env, &addr, nil, nil, os.Args) } diff --git a/trcpub/trcpub.go b/trcpub/trcpub.go index 9b3af75af..0ab8bd98e 100644 --- a/trcpub/trcpub.go +++ b/trcpub/trcpub.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" @@ -21,12 +22,17 @@ func main() { memprotectopts.MemProtectInit(nil) } fmt.Println("Version: " + "1.26") - envPtr := flag.String("env", "dev", "Environment to configure") - addrPtr := flag.String("addr", "", "API endpoint for the vault") - tokenPtr := flag.String("token", "", "Vault access token") - secretIDPtr := flag.String("secretID", "", "Public app role ID") - appRoleIDPtr := flag.String("appRoleID", "", "Secret app role ID") - tokenNamePtr := flag.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } + envPtr := flagset.String("env", "dev", "Environment to configure") + addrPtr := flagset.String("addr", "", "API endpoint for the vault") + tokenPtr := flagset.String("token", "", "Vault access token") + secretIDPtr := flagset.String("secretID", "", "Public app role ID") + appRoleIDPtr := flagset.String("appRoleID", "", "Secret app role ID") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") - trcpubbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil) + trcpubbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, flagset, os.Args, nil) } diff --git a/trcpubbase/trcpub.go b/trcpubbase/trcpub.go index 795d2fdea..672776946 100644 --- a/trcpubbase/trcpub.go +++ b/trcpubbase/trcpub.go @@ -28,20 +28,35 @@ func CommonMain(envPtr *string, secretIDPtr *string, appRoleIDPtr *string, tokenNamePtr *string, + flagset *flag.FlagSet, + argLines []string, c *eUtils.DriverConfig) { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - dirPtr := flag.String("dir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory containing template files for vault") - pingPtr := flag.Bool("ping", false, "Ping vault.") - insecurePtr := flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"pub.log", "Output path for log files") - appRolePtr := flag.String("", "config.yml", "Name of auth config file - example.yml") + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } + flagset.String("env", "dev", "Environment to configure") + flagset.String("addr", "", "API endpoint for the vault") + flagset.String("token", "", "Vault access token") + flagset.String("secretID", "", "Public app role ID") + flagset.String("appRoleID", "", "Secret app role ID") + flagset.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"pub to access the vault") + } + dirPtr := flagset.String("dir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory containing template files for vault") + pingPtr := flagset.Bool("ping", false, "Ping vault.") + insecurePtr := flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"pub.log", "Output path for log files") + appRolePtr := flagset.String("approle", "config.yml", "Name of auth config file - example.yml (optional)") if c == nil || !c.IsShellSubProcess { - flag.Parse() + flagset.Parse(argLines[1:]) } else { - flag.CommandLine.Parse(nil) + flagset.Parse(nil) } var configBase *eUtils.DriverConfig diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index e2e0f1290..18776d43b 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -229,11 +229,35 @@ func main() { } } -var interruptChan chan os.Signal = make(chan os.Signal) +var interruptChan chan os.Signal = make(chan os.Signal, 5) var twoHundredMilliInterruptTicker *time.Ticker = time.NewTicker(200 * time.Millisecond) var secondInterruptTicker *time.Ticker = time.NewTicker(time.Second) var multiSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 3) var fiveSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) +var fifteenSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) +var thirtySecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) + +func acceptInterruptFun(tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { + select { + case <-interruptChan: + cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, + *gAgentConfig.EncryptSalt, + *gAgentConfig.HandshakeHostPort, + *gAgentConfig.HandshakeCode, + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) + os.Exit(1) + case <-tickerContinue.C: + // don't break... continue... + return false, nil + case <-tickerBreak.C: + // break and continue + return true, nil + case <-tickerInterrupt.C: + // full stop + return true, errors.New("you shall not pass") + } + return true, errors.New("not possible") +} func interruptFun(tickerInterrupt *time.Ticker) { select { @@ -249,11 +273,11 @@ func interruptFun(tickerInterrupt *time.Ticker) { } // acceptRemote - hook for instrumenting -func acceptRemote(mode int, remote string) bool { +func acceptRemote(mode int, remote string) (bool, error) { if mode == cap.FEATHER_CTL { - interruptFun(multiSecondInterruptTicker) + return acceptInterruptFun(multiSecondInterruptTicker, fifteenSecondInterruptTicker, thirtySecondInterruptTicker) } - return true + return true, nil } func featherCtlCb(agentName string) error { @@ -342,6 +366,9 @@ func roleBasedRunner(env string, var err error switch control { + case "trcplgtool": + tokenConfig := token + err = trcplgtoolbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, config) case "trcconfig": err = trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, config) case "trcsub": @@ -382,9 +409,8 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, tokenPub := "" pubEnv := env - trcpubbase.CommonMain(&pubEnv, &config.VaultAddress, &tokenPub, &trcshConfig.EnvContext, &pubRoleSlice[1], &pubRoleSlice[0], &tokenName, config) - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. + trcpubbase.CommonMain(&pubEnv, &config.VaultAddress, &tokenPub, &trcshConfig.EnvContext, &pubRoleSlice[1], &pubRoleSlice[0], &tokenName, nil, deployArgLines, config) + ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" config.Token = token @@ -395,25 +421,8 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, os.Exit(1) } case "trcplgtool": - config.AppRoleConfig = "" - config.EnvRaw = env - config.IsShellSubProcess = true - - config.FeatherCtlCb = featherCtlCb - if gAgentConfig == nil { - gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 5)} - gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", "dev") // Feathering always in dev environmnent. - } - gAgentConfig.Env = &env - - err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, nil, deployArgLines, config) - config.FeatherCtlCb = nil - ResetModifier(config) //Resetting modifier cache to avoid token conflicts. - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) //Reset flag parse to allow more toolset calls. - if !agentToken { - token = "" - config.Token = token - } + // Utilize elevated CToken to perform certifications if asked. + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) } @@ -468,38 +477,8 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, configCount *int, logger *log.Logger) error { - switch control { - case "trcplgtool": - config.AppRoleConfig = "" - config.EnvRaw = env - config.IsShellSubProcess = true - - err := trcplgtoolbase.CommonMain(&env, &config.VaultAddress, trcshConfig.CToken, ®ion, nil, deployArgLines, config) - ResetModifier(config) - if !agentToken { - token = "" - config.Token = token - } - return err - case "trcsub": - // This maybe isn't needed for windows deploys... Since config can - // pull templates directly from vault.. - err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) - if !agentToken { - token = "" - config.Token = token - } - return err - - case "trcconfig": - err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) - if !agentToken { - token = "" - config.Token = token - } - return err - } - return errors.New("unsupported control error") + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) + return err } // ProcessDeploy diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index 38272b40b..e5bd8d543 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -191,21 +191,18 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) memprotectopts.MemProtect(nil, trcshConfig.PubRole) } - if agentConfigs != nil { - trcshConfig.CToken, err = retryingPenseFeatherQuery(agentConfigs, "ctoken") - } else { + if agentConfigs == nil { config.Log.Println("Auth phase 5") trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") if err != nil { return trcshConfig, err } + memprotectopts.MemProtect(nil, trcshConfig.CToken) } if err != nil { return trcshConfig, err } - memprotectopts.MemProtect(nil, trcshConfig.CToken) - config.Log.Println("Auth complete.") return trcshConfig, err diff --git a/trcsub/trcsub.go b/trcsub/trcsub.go index ba5710823..c229e70ff 100644 --- a/trcsub/trcsub.go +++ b/trcsub/trcsub.go @@ -22,7 +22,10 @@ func main() { } fmt.Println("Version: " + "1.26") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } envPtr := flagset.String("env", "dev", "Environment to configure") addrPtr := flagset.String("addr", "", "API endpoint for the vault") secretIDPtr := flagset.String("secretID", "", "Public app role ID") diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index 8bb9c19e4..7f7cbea97 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -34,7 +34,10 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } flagset.String("env", "dev", "Environment to configure") flagset.String("addr", "", "API endpoint for the vault") flagset.String("secretID", "", "Public app role ID") diff --git a/trcvault/carrierfactory/servercapauth/capauth.go b/trcvault/carrierfactory/servercapauth/capauth.go index cd2320556..e89c5d364 100644 --- a/trcvault/carrierfactory/servercapauth/capauth.go +++ b/trcvault/carrierfactory/servercapauth/capauth.go @@ -115,10 +115,10 @@ func Memorize(memorizeFields map[string]interface{}, logger *log.Logger) { // Insecure things can be remembered here... logger.Println("EyeRemember: " + key) tap.TapEyeRemember(key, value.(string)) - case "vaddress", "caddress", "ctoken", "configrole": + case "vaddress", "caddress", "configrole": cap.TapFeather(key, value.(string)) fallthrough - case "pubrole", "kubeconfig": + case "pubrole", "ctoken", "kubeconfig": logger.Println("Memorizing: " + key) cap.TapMemorize(key, value.(string)) default: diff --git a/trcvault/trcplgtool/trcplgtool.go b/trcvault/trcplgtool/trcplgtool.go index e1229d7aa..6623c322b 100644 --- a/trcvault/trcplgtool/trcplgtool.go +++ b/trcvault/trcplgtool/trcplgtool.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/trimble-oss/tierceron/buildopts/coreopts" plgtbase "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" ) @@ -13,13 +14,19 @@ func main() { fmt.Println("Version: " + "1.02") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } envPtr := flagset.String("env", "dev", "Environment to configure") addrPtr := flagset.String("addr", "", "API endpoint for the vault") tokenPtr := flagset.String("token", "", "Vault access token") regionPtr := flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. + secretIDPtr := flagset.String("secretID", "", "Secret app role ID") + appRoleIDPtr := flagset.String("appRoleID", "", "Public app role ID") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this"+coreopts.GetFolderPrefix(nil)+"config to access the vault") - err := plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, regionPtr, flagset, os.Args, nil) + err := plgtbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, flagset, os.Args, nil) if err != nil { os.Exit(1) } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 287376081..b02118b77 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -22,6 +22,10 @@ import ( func CommonMain(envPtr *string, addrPtr *string, tokenPtr *string, + envCtxPtr *string, + secretIDPtr *string, + appRoleIDPtr *string, + tokenNamePtr *string, regionPtr *string, flagset *flag.FlagSet, argLines []string, @@ -35,7 +39,10 @@ func CommonMain(envPtr *string, flagset.String("addr", "", "API endpoint for the vault") flagset.String("token", "", "Vault access token") flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. - flagset.Usage = flag.Usage + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } } defineServicePtr := flagset.Bool("defineService", false, "Service is defined.") certifyImagePtr := flagset.Bool("certify", false, "Used to certifies vault plugin.") @@ -129,6 +136,11 @@ func CommonMain(envPtr *string, fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) return fmt.Errorf("plugin type %s not supported in trcsh", *pluginTypePtr) } + if *codebundledeployPtr { + fmt.Printf("codebundledeploy not supported for plugin type %s in trcsh\n", *pluginTypePtr) + return fmt.Errorf("codebundledeploy not supported for plugin type %s in trcsh", *pluginTypePtr) + } + case "agent": // A deployment agent tool. if c != nil { // TODO: do we want to support Deployment certifications in the pipeline at some point? @@ -136,6 +148,10 @@ func CommonMain(envPtr *string, fmt.Printf("Plugin type %s not supported in trcsh.\n", *pluginTypePtr) return fmt.Errorf("plugin type %s not supported in trcsh", *pluginTypePtr) } + if *codebundledeployPtr { + fmt.Printf("codebundledeploy not supported for plugin type %s in trcsh\n", *pluginTypePtr) + return fmt.Errorf("codebundledeploy not supported for plugin type %s in trcsh", *pluginTypePtr) + } case "trcshservice": // A trcshservice managed microservice default: if !*agentdeployPtr { @@ -150,12 +166,14 @@ func CommonMain(envPtr *string, *regionPtr = "" } + var appRoleConfigPtr *string var configBase *eUtils.DriverConfig var logger *log.Logger if c != nil { configBase = c logger = c.Log configBase.SubSectionValue = *pluginNamePtr + appRoleConfigPtr = &(configBase.AppRoleConfig) *insecurePtr = configBase.Insecure } else { if *agentdeployPtr { @@ -171,11 +189,21 @@ func CommonMain(envPtr *string, logger = log.New(f, "[INIT]", log.LstdFlags) configBase = &eUtils.DriverConfig{Insecure: *insecurePtr, Log: logger, ExitOnFailure: true, StartDir: []string{*startDirPtr}, SubSectionValue: *pluginNamePtr} + appRoleConfigPtr = new(string) if err != nil { return err } } + // + if tokenNamePtr == nil || *tokenNamePtr == "" { + autoErr := eUtils.AutoAuth(configBase, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, false) + if autoErr != nil { + eUtils.LogErrorMessage(configBase, "Auth failure: "+autoErr.Error(), false) + return errors.New("auth failure") + } + } + regions := []string{} pluginConfig := map[string]interface{}{} diff --git a/trcx/trcx.go b/trcx/trcx.go index 6014d6a45..ef99aa21b 100644 --- a/trcx/trcx.go +++ b/trcx/trcx.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/trcvault/opts/memonly" @@ -16,8 +17,13 @@ func main() { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.25") - envPtr := flag.String("env", "dev", "Environment to get seed data for.") + fmt.Println("Version: " + "1.26") + flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) + flagset.PrintDefaults() + } + envPtr := flagset.String("env", "dev", "Environment to get seed data for.") - trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, nil, nil, nil) + trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, nil, nil, nil, flagset, os.Args) } diff --git a/trcxbase/trcxbase.go b/trcxbase/trcxbase.go index b4382a64d..46076c7fb 100644 --- a/trcxbase/trcxbase.go +++ b/trcxbase/trcxbase.go @@ -43,46 +43,64 @@ func receiver(configCtx *utils.ConfigContext) { // CommonMain This executable automates the creation of seed files from template file(s). // New seed files are written (or overwrite current seed files) to the specified directory. -func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, envPtr *string, addrPtrIn *string, envCtxPtr *string, insecurePtrIn *bool) { +func CommonMain(ctx eUtils.ProcessContext, + configDriver eUtils.ConfigDriver, + envPtr *string, + addrPtrIn *string, + envCtxPtr *string, + insecurePtrIn *bool, + flagset *flag.FlagSet, + argLines []string) { // Executable input arguments(flags) addrPtr := flag.String("addr", "", "API endpoint for the vault") if addrPtrIn != nil && *addrPtrIn != "" { addrPtr = addrPtrIn } - startDirPtr := flag.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Pull templates from this directory") - endDirPtr := flag.String("endDir", "./"+coreopts.GetFolderPrefix(nil)+"_seeds/", "Write generated seed files to this directory") - logFilePtr := flag.String("log", "./"+coreopts.GetFolderPrefix(nil)+"x.log", "Output path for log file") - helpPtr := flag.Bool("h", false, "Provide options for "+coreopts.GetFolderPrefix(nil)+"x") - tokenPtr := flag.String("token", "", "Vault access token") - secretMode := flag.Bool("secretMode", true, "Only override secret values in templates?") - genAuth := flag.Bool("genAuth", false, "Generate auth section of seed data?") - cleanPtr := flag.Bool("clean", false, "Cleans seed files locally") - secretIDPtr := flag.String("secretID", "", "Secret app role ID") - appRoleIDPtr := flag.String("appRoleID", "", "Public app role ID") - tokenNamePtr := flag.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"x to access the vault") - noVaultPtr := flag.Bool("novault", false, "Don't pull configuration data from vault.") - pingPtr := flag.Bool("ping", false, "Ping vault.") - - fileAddrPtr := flag.String("seedpath", "", "Path for seed file") - fieldsPtr := flag.String("fields", "", "Fields to enter") - encryptedPtr := flag.String("encrypted", "", "Fields to encrypt") - readOnlyPtr := flag.Bool("readonly", false, "Fields to encrypt") - dynamicPathPtr := flag.String("dynamicPath", "", "Generate seeds for a dynamic path in vault.") + if flagset == nil { + flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) + flagset.Usage = func() { + fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) + flagset.PrintDefaults() + } + flagset.String("env", "dev", "Environment to configure") + } + + startDirPtr := flagset.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Pull templates from this directory") + endDirPtr := flagset.String("endDir", "./"+coreopts.GetFolderPrefix(nil)+"_seeds/", "Write generated seed files to this directory") + logFilePtr := flagset.String("log", "./"+coreopts.GetFolderPrefix(nil)+"x.log", "Output path for log file") + helpPtr := flagset.Bool("h", false, "Provide options for "+coreopts.GetFolderPrefix(nil)+"x") + tokenPtr := flagset.String("token", "", "Vault access token") + secretMode := flagset.Bool("secretMode", true, "Only override secret values in templates?") + genAuth := flagset.Bool("genAuth", false, "Generate auth section of seed data?") + cleanPtr := flagset.Bool("clean", false, "Cleans seed files locally") + secretIDPtr := flagset.String("secretID", "", "Secret app role ID") + appRoleIDPtr := flagset.String("appRoleID", "", "Public app role ID") + tokenNamePtr := flagset.String("tokenName", "", "Token name used by this "+coreopts.GetFolderPrefix(nil)+"x to access the vault") + noVaultPtr := flagset.Bool("novault", false, "Don't pull configuration data from vault.") + pingPtr := flagset.Bool("ping", false, "Ping vault.") + + fileAddrPtr := flagset.String("seedpath", "", "Path for seed file") + fieldsPtr := flagset.String("fields", "", "Fields to enter") + encryptedPtr := flagset.String("encrypted", "", "Fields to encrypt") + readOnlyPtr := flagset.Bool("readonly", false, "Fields to encrypt") + dynamicPathPtr := flagset.String("dynamicPath", "", "Generate seeds for a dynamic path in vault.") var insecurePtr *bool if insecurePtrIn == nil { - insecurePtr = flag.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") + insecurePtr = flagset.Bool("insecure", false, "By default, every ssl connection is secure. Allows to continue with server connections considered insecure.") } else { insecurePtr = insecurePtrIn } - diffPtr := flag.Bool("diff", false, "Diff files") - versionPtr := flag.Bool("versions", false, "Gets version metadata information") - wantCertsPtr := flag.Bool("certs", false, "Pull certificates into directory specified by endDirPtr") - filterTemplatePtr := flag.String("templateFilter", "", "Specifies which templates to filter") // -templateFilter=config.yml + diffPtr := flagset.Bool("diff", false, "Diff files") + versionPtr := flagset.Bool("versions", false, "Gets version metadata information") + wantCertsPtr := flagset.Bool("certs", false, "Pull certificates into directory specified by endDirPtr") + filterTemplatePtr := flagset.String("templateFilter", "", "Specifies which templates to filter") // -templateFilter=config.yml + + eUtils.CheckInitFlags(flagset) // Checks for proper flag input - args := os.Args[1:] + args := argLines[1:] for i := 0; i < len(args); i++ { s := args[i] if s[0] != '-' { @@ -91,7 +109,7 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env } } - flag.Parse() + flagset.Parse(argLines[1:]) configCtx := &utils.ConfigContext{ ResultMap: make(map[string]*string), EnvSlice: make([]string, 0), @@ -306,7 +324,7 @@ func CommonMain(ctx eUtils.ProcessContext, configDriver eUtils.ConfigDriver, env skipDiff: // Prints usage if no flags are specified if *helpPtr { - flag.Usage() + flagset.Usage() os.Exit(1) } if ctx == nil { @@ -739,7 +757,8 @@ skipDiff: retry++ } configCtx.FileSysIndex = -1 - eUtils.DiffHelper(cctx, false, len(configCtx.ResultMap)/configCtx.EnvLength) + cctx.SetDiffFileCount(len(configCtx.ResultMap) / configCtx.EnvLength) + eUtils.DiffHelper(cctx, false) }(configCtx) } waitg.Wait() //Wait for diff diff --git a/trcxp/trcxp.go b/trcxp/trcxp.go index 529c86888..bc8a2e3b6 100644 --- a/trcxp/trcxp.go +++ b/trcxp/trcxp.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" @@ -19,5 +20,5 @@ func main() { fmt.Println("Version: " + "1.5") env := "local" addr := coreopts.GetVaultHostPort() - trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, &env, &addr, nil, nil) + trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, &env, &addr, nil, nil, nil, os.Args) } diff --git a/utils/diffUtil.go b/utils/diffUtil.go index 937218deb..8347bf6d7 100644 --- a/utils/diffUtil.go +++ b/utils/diffUtil.go @@ -357,7 +357,7 @@ func RemoveDuplicateValues(intSlice []string) []string { return list } -func DiffHelper(configCtx *ConfigContext, config bool, diffFileCount int) { +func DiffHelper(configCtx *ConfigContext, config bool) { fileIndex := 0 keys := []string{} configCtx.Mutex.Lock() @@ -397,18 +397,18 @@ func DiffHelper(configCtx *ConfigContext, config bool, diffFileCount int) { } } - fileList := make([]string, diffFileCount) + fileList := make([]string, configCtx.DiffFileCount) configCtx.Mutex.Unlock() sleepCount := 0 - if len(configCtx.ResultMap) != diffFileCount { + if len(configCtx.ResultMap) != int(configCtx.DiffFileCount) { for { time.Sleep(time.Second) sleepCount++ if sleepCount >= 5 { fmt.Println("Timeout: Attempted to wait for remaining configs to come in. Attempting incomplete diff.") break - } else if len(configCtx.ResultMap) == diffFileCount*configCtx.EnvLength { + } else if len(configCtx.ResultMap) == int(configCtx.DiffFileCount)*configCtx.EnvLength { break } } diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 3fec058d1..dd7bb1e70 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -2,9 +2,11 @@ package utils import ( "log" + "math" "os" "strings" "sync" + "sync/atomic" "github.com/go-git/go-billy/v5" "github.com/pavlo-v-chernykh/keystore-go/v4" @@ -26,11 +28,22 @@ type ConfigContext struct { ProjectSectionsSlice []string ResultChannel chan *ResultData FileSysIndex int + DiffFileCount int32 EnvLength int ConfigWg sync.WaitGroup Mutex *sync.Mutex } +func (cfgContext *ConfigContext) SetDiffFileCount(cnt int) { + if cnt < math.MaxInt32 { + atomic.StoreInt32(&cfgContext.DiffFileCount, int32(cnt)) + } +} + +func (cfgContext *ConfigContext) GetDiffFileCount() int32 { + return cfgContext.DiffFileCount +} + // DriverConfig -- contains many structures necessary for Tierceron tool functionality. type DriverConfig struct { // Process context used by new tool implementations in callbacks. diff --git a/utils/indexedFlagUtil.go b/utils/indexedFlagUtil.go index e73b6ea6e..2377ee66c 100644 --- a/utils/indexedFlagUtil.go +++ b/utils/indexedFlagUtil.go @@ -7,17 +7,29 @@ import ( "strings" ) -var SubPathFilter = flag.String("subPathFilter", "", "Specifies subpath templates to filter") // a sub path filter. -var ServiceNameFilterPtr = flag.String("serviceExtFilter", "", "Specifies which nested services (or tables) to filter") //offset or database -var ServiceFilterPtr = flag.String("serviceFilter", "", "Specifies which services (or tables) to filter") // Table names -var IndexNameFilterPtr = flag.String("indexFilter", "", "Specifies which index names to filter") // column index, table to filter. -var IndexValueFilterPtr = flag.String("indexValueFilter", "", "Specifies which index values to filter") // column index value to filter on. -var IndexedPtr = flag.String("indexed", "", "Specifies which projects are indexed") // Indicates indexed projects... -var RestrictedPtr = flag.String("restricted", "", "Specifies which projects have restricted access.") -var ProtectedPtr = flag.String("protected", "", "Specifies which projects have protected access.") -var BasePtr = flag.Bool("base", false, "Specifies whether the base env seed file will be seeded") +var SubPathFilter *string +var ServiceNameFilterPtr *string +var ServiceFilterPtr *string +var IndexNameFilterPtr *string +var IndexValueFilterPtr *string +var IndexedPtr *string +var RestrictedPtr *string +var ProtectedPtr *string +var BasePtr *bool var OnlyBasePtr = false +func initglobals(flagset *flag.FlagSet) { + SubPathFilter = flagset.String("subPathFilter", "", "Specifies subpath templates to filter") // a sub path filter. + ServiceNameFilterPtr = flagset.String("serviceExtFilter", "", "Specifies which nested services (or tables) to filter") //offset or database + ServiceFilterPtr = flagset.String("serviceFilter", "", "Specifies which services (or tables) to filter") // Table names + IndexNameFilterPtr = flagset.String("indexFilter", "", "Specifies which index names to filter") // column index, table to filter. + IndexValueFilterPtr = flagset.String("indexValueFilter", "", "Specifies which index values to filter") // column index value to filter on. + IndexedPtr = flagset.String("indexed", "", "Specifies which projects are indexed") // Indicates indexed projects... + RestrictedPtr = flagset.String("restricted", "", "Specifies which projects have restricted access.") + ProtectedPtr = flagset.String("protected", "", "Specifies which projects have protected access.") + BasePtr = flagset.Bool("base", false, "Specifies whether the base env seed file will be seeded") +} + func checkInitFlagHelper() { if len(*IndexValueFilterPtr) > 0 { if len(*IndexNameFilterPtr) > 0 { @@ -34,8 +46,10 @@ func checkInitFlagHelper() { os.Exit(1) } } -func CheckInitFlags() { + +func CheckInitFlags(flagset *flag.FlagSet) { filtered := false + initglobals(flagset) //Cannot specify a pathed indexed/restricted seed file while specifying a restricted/indexed section. if len(*IndexNameFilterPtr) > 0 || len(*ServiceNameFilterPtr) > 0 || len(*IndexValueFilterPtr) > 0 || len(*ServiceNameFilterPtr) > 0 { filtered = true From f4fe338f05caf6e95f9b4ff194d325ba9e332581 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:01:53 -0800 Subject: [PATCH 25/55] Make concurrent safe ctlmessage (#814) * Concurrency fix: 1. Move CtlMessage into concurrent safe config. 2. Move config init into init routine for use by both linux and windows logic branches. --------- --- capauth/agentconfig.go | 1 - trcsh/trcsh.go | 182 ++++++++++++++++++++++------------------- utils/driverconfig.go | 3 +- 3 files changed, 100 insertions(+), 86 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index 1fcd83aab..096446a73 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -33,7 +33,6 @@ type AgentConfigs struct { EncryptSalt *string Deployments *string Env *string - CtlMessage chan string } var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 18776d43b..da71cb4da 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -38,8 +38,63 @@ import ( var gAgentConfig *capauth.AgentConfigs = nil var gTrcshConfig *capauth.TrcShConfig +func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.DriverConfig, error) { + if len(env) == 0 { + env = os.Getenv("TRC_ENV") + } + if len(region) == 0 { + region = os.Getenv("TRC_REGION") + } + + regions := []string{} + if strings.HasPrefix(env, "staging") || strings.HasPrefix(env, "prod") || strings.HasPrefix(env, "dev") { + supportedRegions := eUtils.GetSupportedProdRegions() + if region != "" { + for _, supportedRegion := range supportedRegions { + if region == supportedRegion { + regions = append(regions, region) + break + } + } + if len(regions) == 0 { + fmt.Println("Unsupported region: " + region) + return nil, errors.New("Unsupported region: " + region) + } + } + } + + fmt.Println("trcsh env: " + env) + fmt.Printf("trcsh regions: %s\n", strings.Join(regions, ", ")) + + logFile := "./" + coreopts.GetFolderPrefix(nil) + "deploy.log" + if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && logFile == "/var/log/"+coreopts.GetFolderPrefix(nil)+"deploy.log" { + logFile = "./" + coreopts.GetFolderPrefix(nil) + "deploy.log" + } + f, errOpenFile := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) + if errOpenFile != nil { + return nil, errOpenFile + } + logger := log.New(f, "[DEPLOY]", log.LstdFlags) + config := &eUtils.DriverConfig{Insecure: true, + Env: env, + EnvRaw: env, + Log: logger, + IsShell: true, + IsShellSubProcess: false, + OutputMemCache: outputMemCache, + MemFs: memfs.New(), + Regions: regions, + ExitOnFailure: true} + return config, nil +} + func ProcessDeployment(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { - go func() { + config, err := TrcshInitConfig(env, region, outputMemCache) + if err != nil { + fmt.Printf("Initialization setup error: %s\n", err.Error()) + } + + go func(c0 *eUtils.DriverConfig) { for { perching: var deployDoneChan chan bool @@ -52,8 +107,8 @@ func ProcessDeployment(env string, region string, token string, trcPath string, // Process the script.... // This will feed CtlMessages into the Timeout and CtlMessage subscriber - go func() { - go func() { + go func(c1 *eUtils.DriverConfig) { + go func(c2 *eUtils.DriverConfig) { // Timeout and CtlMessage subscriber select { case <-deployDoneChan: @@ -65,17 +120,17 @@ func ProcessDeployment(env string, region string, token string, trcPath string, *gAgentConfig.HandshakeHostPort, *gAgentConfig.HandshakeCode, cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + c2.DeploymentCtlMessage <- capauth.TrcCtlComplete break } for range time.After(120 * time.Second) { } - }() + }(c1) - ProcessDeploy(*gAgentConfig.Env, region, "", deployment, trcPath, secretId, approleId, false) - }() + ProcessDeploy(c1, region, "", deployment, trcPath, secretId, approleId, false) + }(c0) - for modeCtl := range gAgentConfig.CtlMessage { + for modeCtl := range c0.DeploymentCtlMessage { flapMode := cap.MODE_FLAP + "_" + modeCtl ctlFlapMode := flapMode var err error = errors.New("init") @@ -128,7 +183,7 @@ func ProcessDeployment(env string, region string, token string, trcPath string, } deploycomplete: } - }() + }(config) } // This is a controller program that can act as any command line utility. @@ -178,10 +233,16 @@ func main() { memprotectopts.MemProtect(nil, secretIDPtr) memprotectopts.MemProtect(nil, appRoleIDPtr) + config, err := TrcshInitConfig(*envPtr, *regionPtr, true) + if err != nil { + fmt.Printf("trcsh config setup failure: %s\n", err.Error()) + os.Exit(-1) + } + //Open deploy script and parse it. - ProcessDeploy(*envPtr, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) + ProcessDeploy(config, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) } else { - gAgentConfig = &capauth.AgentConfigs{CtlMessage: make(chan string, 5)} + gAgentConfig = &capauth.AgentConfigs{} deployments := os.Getenv("DEPLOYMENTS") agentToken := os.Getenv("AGENT_TOKEN") agentEnv := os.Getenv("AGENT_ENV") @@ -396,8 +457,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, token string, argsOrig []string, deployArgLines []string, - configCount *int, - logger *log.Logger) { + configCount *int) { switch control { case "trcpub": config.AppRoleConfig = "configpub.yml" @@ -443,19 +503,19 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, kubectlErrChan := make(chan error, 1) - go func() { - config.Log.Println("Executing kubectl") - kubectlErrChan <- kube.KubeCtl(*trcKubeDeploymentConfig, config) - }() + go func(c *eUtils.DriverConfig) { + c.Log.Println("Executing kubectl") + kubectlErrChan <- kube.KubeCtl(*trcKubeDeploymentConfig, c) + }(config) select { case <-time.After(15 * time.Second): fmt.Println("Agent is not yet ready..") - logger.Println("Timed out waiting for KubeCtl.") + config.Log.Println("Timed out waiting for KubeCtl.") os.Exit(-1) case kubeErr := <-kubectlErrChan: if kubeErr != nil { - logger.Println(kubeErr) + config.Log.Println(kubeErr) os.Exit(-1) } } @@ -474,8 +534,7 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, token string, argsOrig []string, deployArgLines []string, - configCount *int, - logger *log.Logger) error { + configCount *int) error { err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, token, argsOrig, deployArgLines, configCount) return err @@ -496,66 +555,24 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, // Returns: // // Nothing. -func ProcessDeploy(env string, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { - var err error +func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { agentToken := false if token != "" { agentToken = true } pwd, _ := os.Getwd() var content []byte - if len(env) == 0 { - env = os.Getenv("TRC_ENV") - } - if len(region) == 0 { - region = os.Getenv("TRC_REGION") - } - - regions := []string{} - if strings.HasPrefix(env, "staging") || strings.HasPrefix(env, "prod") || strings.HasPrefix(env, "dev") { - supportedRegions := eUtils.GetSupportedProdRegions() - if region != "" { - for _, supportedRegion := range supportedRegions { - if region == supportedRegion { - regions = append(regions, region) - break - } - } - if len(regions) == 0 { - fmt.Println("Unsupported region: " + region) - os.Exit(1) - } - } - } - - fmt.Println("trcsh env: " + env) - fmt.Printf("trcsh regions: %s\n", strings.Join(regions, ", ")) - - logFile := "./" + coreopts.GetFolderPrefix(nil) + "deploy.log" - if _, err := os.Stat("/var/log/"); os.IsNotExist(err) && logFile == "/var/log/"+coreopts.GetFolderPrefix(nil)+"deploy.log" { - logFile = "./" + coreopts.GetFolderPrefix(nil) + "deploy.log" - } - f, _ := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) - logger := log.New(f, "[DEPLOY]", log.LstdFlags) - config := &eUtils.DriverConfig{Insecure: true, - EnvRaw: env, - Log: logger, - IsShell: true, - IsShellSubProcess: false, - OutputMemCache: outputMemCache, - MemFs: memfs.New(), - Regions: regions, - ExitOnFailure: true} if len(deployment) > 0 { config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} + config.DeploymentCtlMessage = make(chan string, 5) } - if env == "itdev" { + if config.EnvRaw == "itdev" { config.OutputMemCache = false } fmt.Println("Logging initialized.") - logger.Printf("Logging initialized for env:%s\n", env) + config.Log.Printf("Logging initialized for env:%s\n", config.EnvRaw) // Chewbacca: scrub before checkin // This data is generated by TrcshAuth @@ -574,6 +591,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t // config.VaultAddress = "" // config.Token = "" // Chewbacca: end scrub + var err error config.Log.Printf("Bootstrap..") for { if gTrcshConfig == nil || gTrcshConfig.CToken == nil || gTrcshConfig.ConfigRole == nil || gTrcshConfig.VaultAddress == nil || @@ -584,7 +602,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t time.Sleep(time.Second) continue } - config.Log.Printf("Auth re-loaded %s\n", env) + config.Log.Printf("Auth re-loaded %s\n", config.EnvRaw) } else { break } @@ -635,13 +653,11 @@ func ProcessDeploy(env string, region string, token string, deployment string, t trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") - tokenName := "config_token_" + env - configEnv := env - config.EnvRaw = env + tokenName := "config_token_" + config.EnvRaw config.OutputMemCache = true config.StartDir = []string{"trc_templates"} config.EndDir = "." - trcconfigbase.CommonMain(&configEnv, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" @@ -664,7 +680,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t token = "" config.Token = token } - if env == "itdev" || env == "staging" || env == "prod" { + if config.EnvRaw == "itdev" || config.EnvRaw == "staging" || config.EnvRaw == "prod" { config.OutputMemCache = false } config.Log.Println("Processing trcshell") @@ -706,7 +722,7 @@ func ProcessDeploy(env string, region string, token string, deployment string, t } } else { fmt.Println("Processing manual trcshell") - if env == "itdev" { + if config.EnvRaw == "itdev" { content, err = os.ReadFile(pwd + "/deploy/buildtest.trc") if err != nil { fmt.Println("Error could not find /deploy/buildtest.trc for deployment instructions") @@ -785,12 +801,12 @@ func ProcessDeploy(env string, region string, token string, deployment string, t } if utils.IsWindows() { // Log for traceability. - logger.Println(deployLine) + config.Log.Println(deployLine) err := processWindowsCmds( trcKubeDeploymentConfig, &onceKubeInit, PipeOS, - env, + config.EnvRaw, gTrcshConfig, region, config, @@ -799,19 +815,18 @@ func ProcessDeploy(env string, region string, token string, deployment string, t token, argsOrig, strings.Split(deployLine, " "), - &configCount, - logger) + &configCount) if err != nil { - gAgentConfig.CtlMessage <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + config.DeploymentCtlMessage <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) } else { - gAgentConfig.CtlMessage <- deployLine + config.DeploymentCtlMessage <- deployLine } } else { processPluginCmds( &trcKubeDeploymentConfig, &onceKubeInit, PipeOS, - env, + config.EnvRaw, gTrcshConfig, region, config, @@ -820,13 +835,12 @@ func ProcessDeploy(env string, region string, token string, deployment string, t token, argsOrig, strings.Split(deployLine, " "), - &configCount, - logger) + &configCount) } } } if utils.IsWindows() { - gAgentConfig.CtlMessage <- capauth.TrcCtlComplete + config.DeploymentCtlMessage <- capauth.TrcCtlComplete } //Make the arguments in the script -> os.args. diff --git a/utils/driverconfig.go b/utils/driverconfig.go index dd7bb1e70..2cf072e24 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -110,7 +110,8 @@ type DriverConfig struct { ServiceFilter []string // Which tables to use. DynamicPathFilter string // Seeds from a specific path. - DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on + DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on + DeploymentCtlMessage chan string } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. From 2f25b89ec507c0b84c79062921909110e71a5a21 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:47:25 -0800 Subject: [PATCH 26/55] Prepare for release. (#817) * Update wording for any. * Feathering only in lower environments. * A little cleanup and a little error handling. * Fix env selection problem in trcsh. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcflow/deploy/process.go | 34 ++++++++++++++------------ trcsh/trcsh.go | 15 ++++++++---- trcvault/deploy/deploy.sh | 4 +++ trcvault/deploy/refreshcarriertoken.sh | 8 ++++++ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index 5e467e21b..5ab94a64b 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -67,23 +67,27 @@ func PluginDeployEnvFlow(pluginConfig map[string]interface{}, logger *log.Logger if ok, err := servercapauth.ValidatePathSha(goMod, pluginConfig, logger); true || ok { onceAuth.Do(func() { - logger.Printf("Cap auth init for env: %s\n", pluginConfig["env"].(string)) - var featherAuth *servercapauth.FeatherAuth - featherAuth, err = servercapauth.Init(goMod, pluginConfig, logger) - if err != nil { - eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) - return - } - gCapInitted = true - - pluginConfig["trcHatSecretsPort"] = featherAuth.SecretsPort + if pluginConfig["env"].(string) == "dev" || pluginConfig["env"].(string) == "staging" { + // Ensure only dev is the cap auth... + logger.Printf("Cap auth init for env: %s\n", pluginConfig["env"].(string)) + var featherAuth *servercapauth.FeatherAuth = nil + if pluginConfig["env"].(string) == "dev" { + featherAuth, err = servercapauth.Init(goMod, pluginConfig, logger) + if err != nil { + eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) + return + } + pluginConfig["trcHatSecretsPort"] = featherAuth.SecretsPort + } - servercapauth.Memorize(pluginConfig, logger) + servercapauth.Memorize(pluginConfig, logger) - // TODO: Support variables for different environments... - // Not really clear how cap auth would do this... - go servercapauth.Start(featherAuth, pluginConfig["env"].(string), logger) - logger.Printf("Cap auth init complete for env: %s\n", pluginConfig["env"].(string)) + // TODO: Support variables for different environments... + // Not really clear how cap auth would do this... + go servercapauth.Start(featherAuth, pluginConfig["env"].(string), logger) + logger.Printf("Cap auth init complete for env: %s\n", pluginConfig["env"].(string)) + gCapInitted = true + } }) } else { eUtils.LogErrorMessage(config, fmt.Sprintf("Mismatched sha256 cap auth for env: %s. Skipping.", pluginConfig["env"].(string)), false) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index ebb3c33d2..9b20d3c5c 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -286,8 +286,12 @@ func main() { memprotectopts.MemProtect(nil, &address) shutdown := make(chan bool) - // Preload agent synchronization configs - gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) + // Preload agent synchronization configs... + _, errAgentLoad := gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) + if errAgentLoad != nil { + fmt.Println("trcsh agent bootstrap failure.") + os.Exit(-1) + } deploymentsSlice := strings.Split(deployments, ",") for _, deployment := range deploymentsSlice { @@ -353,8 +357,6 @@ func featherCtlCb(agentName string) error { if gAgentConfig == nil { return errors.New("incorrect agent initialization") - } else { - gAgentConfig.Deployments = &agentName } flapMode := cap.MODE_GAZE ctlFlapMode := flapMode @@ -491,9 +493,12 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, // Utilize elevated CToken to perform certifications if asked. config.FeatherCtlCb = featherCtlCb if gAgentConfig == nil { + // Prepare the configuration triggering mechanism. + // Bootstrap deployment is replaced during callback with the agent name. gAgentConfig = &capauth.AgentConfigs{} - gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", "dev") // Feathering always in dev environmnent. + gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", config.Env) // Feathering always in dev environmnent. } + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) diff --git a/trcvault/deploy/deploy.sh b/trcvault/deploy/deploy.sh index ef31a6c1b..2ad5631ce 100755 --- a/trcvault/deploy/deploy.sh +++ b/trcvault/deploy/deploy.sh @@ -41,7 +41,11 @@ read SECRET_VAULT_ENV_TOKEN fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi read SECRET_VAULT_PLUGIN_TOKEN fi diff --git a/trcvault/deploy/refreshcarriertoken.sh b/trcvault/deploy/refreshcarriertoken.sh index 5e9e6d9f6..3860bfb68 100755 --- a/trcvault/deploy/refreshcarriertoken.sh +++ b/trcvault/deploy/refreshcarriertoken.sh @@ -15,7 +15,11 @@ read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi read SECRET_VAULT_PLUGIN_TOKEN fi @@ -25,7 +29,11 @@ read VAULT_TOKEN fi if [[ -z "${VAULT_ENV_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then echo "Enter agent vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " +else +echo "Enter agent vault *plugin* environment token with tightly confined write permissions(config_token_pluginany): " +fi read VAULT_ENV_TOKEN fi From a3d5c9dda615d3b4af3574120118f7b0eba0c204 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:47:01 -0800 Subject: [PATCH 27/55] Prepare for trcdeploy release. (#819) * Fix cert validation for rqa and auto. * Add support for subpath deployments on codedeploy. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- trcinit/initlib/vault-seed.go | 4 +++- trcvault/trcplgtoolbase/trcplgtoolbase.go | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index 26d8441a3..a5719c153 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -513,8 +513,10 @@ func seedVaultWithCertsFromEntry(config *eUtils.DriverConfig, mod *helperkv.Modi certHost = strings.Replace(certHost, "*", "develop", 1) case "QA": certHost = strings.Replace(certHost, "*", "qa", 1) + case "RQA": + certHost = strings.Replace(certHost, "*", "qa", 1) case "auto": - certHost = strings.Replace(certHost, "*", "auto", 1) + certHost = strings.Replace(certHost, "*", "qa", 1) case "performance": certHost = strings.Replace(certHost, "*", "performance", 1) } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 3cbb52163..190616059 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -10,6 +10,7 @@ import ( "log" "os" "os/exec" + "path/filepath" "strings" "github.com/trimble-oss/tierceron/buildopts" @@ -52,6 +53,7 @@ func CommonMain(envPtr *string, codebundledeployPtr := flagset.Bool("codebundledeploy", false, "To deploy a code bundle.") agentdeployPtr := flagset.Bool("agentdeploy", false, "To initiate deployment on agent.") projectservicePtr := flagset.String("projectservice", "", "Provide template root path in form project/service") + deploysubpathPtr := flagset.String("deploysubpath", "", "Subpath under root to deliver code bundles.") // Common flags... startDirPtr := flagset.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") @@ -286,9 +288,10 @@ func CommonMain(envPtr *string, pluginToolConfig["trcsha256"] = *sha256Ptr pluginToolConfig["pluginNamePtr"] = *pluginNamePtr - pluginToolConfig["deployrootPtr"] = *deployrootPtr pluginToolConfig["serviceNamePtr"] = *serviceNamePtr pluginToolConfig["projectservicePtr"] = *projectservicePtr + pluginToolConfig["deployrootPtr"] = *deployrootPtr + pluginToolConfig["deploysubpathPtr"] = *deploysubpathPtr pluginToolConfig["codeBundlePtr"] = *codeBundlePtr if _, ok := pluginToolConfig["trcplugin"].(string); !ok { @@ -310,7 +313,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["deployrootPtr"].(string); ok { pluginToolConfig["trcdeployroot"] = pluginToolConfig["deployrootPtr"].(string) } - + if _, ok := pluginToolConfig["deploysubpathPtr"]; ok { + pluginToolConfig["trcdeploysubpath"] = pluginToolConfig["deploysubpathPtr"] + } if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) } @@ -330,10 +335,12 @@ func CommonMain(envPtr *string, writeMap["trcplugin"] = *pluginNamePtr writeMap["trctype"] = *pluginTypePtr writeMap["trcprojectservice"] = *projectservicePtr - if _, ok := pluginToolConfig["trcdeployroot"]; ok { writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"] } + if _, ok := pluginToolConfig["trcdeploysubpath"]; ok { + writeMap["trcdeploysubpath"] = pluginToolConfig["trcdeploysubpath"] + } if _, ok := pluginToolConfig["trcservicename"]; ok { writeMap["trcservicename"] = pluginToolConfig["trcservicename"] } @@ -387,7 +394,12 @@ func CommonMain(envPtr *string, pluginToolConfig["imagesha256"] != nil && pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { // Write the image to the destination... - deployPath := fmt.Sprintf("%s\\%s", pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) + var deployPath string + if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok { + deployPath = filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string), pluginToolConfig["trccodebundle"].(string)) + } else { + deployPath = filepath.Join(pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) + } fmt.Printf("Deploying image to: %s\n", deployPath) err = os.WriteFile(deployPath, pluginToolConfig["rawImageFile"].([]byte), 0644) From 0b0d79ce690b1e6904c73935d8f532059dc7b16f Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:20:01 -0800 Subject: [PATCH 28/55] Prepare for release. (#821) * This should return an error, not exit. * Only exit on failure for command driven. * Better handling of acceptable errors. --------- --- trcsubbase/trcsub.go | 7 +++++-- trcvault/trcplgtoolbase/trcplgtoolbase.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index 7f7cbea97..9387a29b5 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -1,6 +1,7 @@ package trcsubbase import ( + "errors" "flag" "fmt" "log" @@ -32,6 +33,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, } fmt.Println("Version: " + "1.6") + exitOnFailure := false if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) flagset.Usage = func() { @@ -42,6 +44,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, flagset.String("addr", "", "API endpoint for the vault") flagset.String("secretID", "", "Public app role ID") flagset.String("appRoleID", "", "Secret app role ID") + exitOnFailure = true } endDirPtr := flagset.String("endDir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory to put configured templates into") tokenPtr := flagset.String("token", "", "Vault access token") @@ -57,7 +60,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") - os.Exit(1) + return errors.New("must specify either -projectInfo or -templateFilter flag") } var configBase *eUtils.DriverConfig var appRoleConfigPtr *string @@ -85,7 +88,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, configBase = &eUtils.DriverConfig{Insecure: *insecurePtr, EndDir: *endDirPtr, Log: logger, - ExitOnFailure: true} + ExitOnFailure: exitOnFailure} appRoleConfigPtr = new(string) } diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 190616059..b7fee5664 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -360,7 +360,7 @@ func CommonMain(envPtr *string, fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) cmd := exec.Command("sc", "stop", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil { + if err != nil && !strings.Contains(err.Error(), "1062") && !strings.Contains(err.Error(), "1052") { fmt.Println(err) return err } @@ -370,7 +370,7 @@ func CommonMain(envPtr *string, fmt.Printf("Starting service %s\n", pluginToolConfig["trcservicename"].(string)) cmd := exec.Command("sc", "start", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil { + if err != nil && !strings.Contains(err.Error(), "1056") { fmt.Println(err) return err } From 1985cc76f0744e058ffc6c946a2437dc16deb63d Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:37:44 -0800 Subject: [PATCH 29/55] Prepare for release. (#824) * Concurrency improvements. (#823) * Checking in for safekeeping. * Remove fatal exits. * Clarifying cleanup. * Only deploy lines when running state is running or started. --- capauth/agentconfig.go | 95 ++++----- trcsh/trcsh.go | 243 ++++++++-------------- trcsh/trcshauth/auth.go | 11 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 2 +- utils/driverconfig.go | 8 +- 5 files changed, 136 insertions(+), 223 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index a77f81231..7a3345f24 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "log" "math/rand" "net" "os" @@ -15,6 +14,7 @@ import ( "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron-hat/cap/tap" + captiplib "github.com/trimble-oss/tierceron-hat/captip/captiplib" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" eUtils "github.com/trimble-oss/tierceron/utils" @@ -25,15 +25,12 @@ import ( var TrcCtlComplete string = "trcctlcomplete" type AgentConfigs struct { - AgentToken *string - HandshakeHostPort *string - FeatherHostPort *string - HandshakeCode *string - DeployRoleID *string - EncryptPass *string - EncryptSalt *string - Deployments *string - Env *string + cap.FeatherContext + AgentToken *string + FeatherHostPort *string + DeployRoleID *string + Deployments *string + Env *string } var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -58,16 +55,12 @@ func ValidateVhost(host string) error { return errors.New("Bad host: " + host) } -func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error) { +func (agentconfig *AgentConfigs) PenseFeatherQuery(featherCtx *cap.FeatherContext, pense string) (*string, error) { penseCode := randomString(7 + rand.Intn(7)) penseArray := sha256.Sum256([]byte(penseCode)) penseSum := hex.EncodeToString(penseArray[:]) - _, featherErr := cap.FeatherWriter(*agentconfig.EncryptPass, - *agentconfig.EncryptSalt, - *agentconfig.HandshakeHostPort, - *agentconfig.HandshakeCode, - penseSum) + _, featherErr := cap.FeatherWriter(featherCtx, penseSum) if featherErr != nil { return nil, featherErr } @@ -80,7 +73,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error conn, err := grpc.Dial(*agentconfig.FeatherHostPort, grpc.WithTransportCredentials(creds)) if err != nil { - log.Fatalf("did not connect: %v", err) + return nil, err } defer conn.Close() c := cap.NewCapClient(conn) @@ -91,7 +84,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error r, err := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) if err != nil { - log.Fatalf("did not connect: %v", err) + return nil, err } var penseProtect *string rPense := r.GetPense() @@ -101,9 +94,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error return penseProtect, nil } -func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, deployments string, env string) (*TrcShConfig, error) { - var trcshConfig *TrcShConfig - +func NewAgentConfig(address string, agentToken string, deployments string, env string) (*AgentConfigs, *TrcShConfig, error) { mod, modErr := helperkv.NewModifier(false, agentToken, address, env, nil, true, nil) if modErr != nil { fmt.Println("trcsh Failed to bootstrap") @@ -114,47 +105,45 @@ func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, data, readErr := mod.ReadData("super-secrets/Restricted/TrcshAgent/config") if readErr != nil { - return nil, readErr + return nil, nil, readErr } else { - trcHatEncryptPass := data["trcHatEncryptPass"].(string) - memprotectopts.MemProtect(nil, &trcHatEncryptPass) - trcHatEncryptSalt := data["trcHatEncryptSalt"].(string) - memprotectopts.MemProtect(nil, &trcHatEncryptSalt) + featherCtx := captiplib.FeatherCtlInit(nil, + "", + data["trcHatEncryptPass"].(string), + data["trcHatEncryptSalt"].(string), + fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatHandshakePort"].(string)), + data["trcHatHandshakeCode"].(string), + "sessionIdDynamicFill", captiplib.AcceptRemote, nil) + + featherHostPort := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatSecretsPort"].(string)) trcHatEnv := data["trcHatEnv"].(string) - memprotectopts.MemProtect(nil, &trcHatEnv) - trcHatHandshakeCode := data["trcHatHandshakeCode"].(string) - memprotectopts.MemProtect(nil, &trcHatHandshakeCode) - trcHatHandshakePort := data["trcHatHandshakePort"].(string) - memprotectopts.MemProtect(nil, &trcHatHandshakePort) - trcHatHost := data["trcHatHost"].(string) - memprotectopts.MemProtect(nil, &trcHatHost) - trcHatSecretsPort := data["trcHatSecretsPort"].(string) - memprotectopts.MemProtect(nil, &trcHatSecretsPort) - trcHandshakeHostPort := trcHatHost + ":" + trcHatHandshakePort - memprotectopts.MemProtect(nil, &trcHandshakeHostPort) - trcFeatherHostPort := trcHatHost + ":" + trcHatSecretsPort - memprotectopts.MemProtect(nil, &trcFeatherHostPort) - - agentconfig.HandshakeHostPort = &trcHandshakeHostPort - agentconfig.FeatherHostPort = &trcFeatherHostPort - agentconfig.HandshakeCode = &trcHatHandshakeCode - agentconfig.EncryptPass = &trcHatEncryptPass - agentconfig.EncryptSalt = &trcHatEncryptSalt - agentconfig.Deployments = &deployments - agentconfig.Env = &trcHatEnv - - trcshConfig = &TrcShConfig{Env: trcHatEnv, + // TODO: Figure out.... + // memprotectopts.MemProtect(nil, &featherCtx.EncryptPass) + // memprotectopts.MemProtect(nil, &featherCtx.EncryptSalt) + // memprotectopts.MemProtect(nil, &featherCtx.HostAddr) + // memprotectopts.MemProtect(nil, &featherCtx.HandshakeCode) + + agentconfig := &AgentConfigs{ + *featherCtx, + &agentToken, + &featherHostPort, + new(string), + &deployments, + &trcHatEnv, + } + trcshConfig := &TrcShConfig{Env: trcHatEnv, EnvContext: trcHatEnv, } - trcShConfigRole, penseError := agentconfig.PenseFeatherQuery("configrole") + + trcShConfigRole, penseError := agentconfig.PenseFeatherQuery(featherCtx, "configrole") if penseError != nil { - return nil, penseError + return nil, nil, penseError } memprotectopts.MemProtect(nil, trcShConfigRole) trcshConfig.ConfigRole = trcShConfigRole - } - return trcshConfig, nil + return agentconfig, trcshConfig, nil + } } func PenseQuery(config *eUtils.DriverConfig, pense string) (*string, error) { diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 9b20d3c5c..f9c4568db 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -11,12 +11,14 @@ import ( "os/signal" "strings" "sync" + "sync/atomic" "syscall" "time" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/trimble-oss/tierceron-hat/cap" + captiplib "github.com/trimble-oss/tierceron-hat/captip/captiplib" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" @@ -88,110 +90,49 @@ func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.Dr return config, nil } -func ProcessDeployment(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { +func emote(msg string) { + // TODO: emote msg somewhere somehow? +} + +// deployCtl -- is the deployment controller or manager if you will. +func deployCtlInterrupted(featherCtx *cap.FeatherContext) error { + os.Exit(-1) + return nil +} + +// deployer -- does the work of deploying.. +func deployerInterrupted(featherCtx *cap.FeatherContext) error { + cap.FeatherCtlEmit(featherCtx, cap.MODE_PERCH, featherCtx.SessionIdentifier, true) + return nil +} + +// EnableDeploy - initializes and starts running deployer for provided deployment and environment. +func EnableDeployer(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { config, err := TrcshInitConfig(env, region, outputMemCache) if err != nil { fmt.Printf("Initialization setup error: %s\n", err.Error()) } if len(deployment) > 0 { config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} - config.DeploymentCtlMessage = make(chan string, 5) + config.DeploymentCtlMessageChan = make(chan string, 5) } - go func() { - for { - perching: - endTimerChan := make(chan bool, 5) // Anticipate a few of these. - deployDoneChan := make(chan bool, 5) // Could be a few... - if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployment+"."+*gAgentConfig.Env, false, acceptRemote); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { - // Lookup trcPath from deployment - - // Process the script.... - // This will feed CtlMessages into the Timeout and CtlMessage subscriber - go func() { - go func() { - // Timeout and CtlMessage subscriber - select { - case <-endTimerChan: - break - case <-time.After(120 * time.Second): - ctlMsg := "Deployment timed out after 120 seconds" - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - go func() { config.DeploymentCtlMessage <- capauth.TrcCtlComplete }() - break - } - }() - - ProcessDeploy(config, region, "", deployment, trcPath, secretId, approleId, false) - }() - - for { - select { - case modeCtl := <-config.DeploymentCtlMessage: - flapMode := cap.MODE_FLAP + "_" + modeCtl - ctlFlapMode := flapMode - var err error = errors.New("init") - - for { - if err == nil && ctlFlapMode == cap.MODE_PERCH { - // Acknowledge perching... - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - ctlFlapMode = cap.MODE_PERCH - deployDoneChan <- true - break - } - - if err == nil && flapMode != ctlFlapMode && ctlFlapMode != " " { - // Flap, Gaze, etc... - interruptFun(twoHundredMilliInterruptTicker) - break - } else { - callFlap := strings.ReplaceAll(flapMode, ":", "#") - if err == nil { - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(secondInterruptTicker) - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - } - } - if modeCtl == capauth.TrcCtlComplete { - // Only exit with TrcCtlComplete. - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - deployDoneChan <- true - } - case <-deployDoneChan: - go func() { endTimerChan <- true }() - goto perching - } - } - } else { - interruptFun(fiveSecondInterruptTicker) - } - } - }() + // + // Each deployer needs it's own context. + // + config.FeatherCtx = captiplib.FeatherCtlInit(interruptChan, + "", + gAgentConfig.EncryptPass, + gAgentConfig.EncryptSalt, + gAgentConfig.HostAddr, + gAgentConfig.HandshakeCode, + deployment+"."+*gAgentConfig.Env, /*Session identifier */ + captiplib.AcceptRemote, + deployerInterrupted) + + go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, emote, nil) + + go ProcessDeploy(config.FeatherCtx, config, region, "", deployment, trcPath, secretId, approleId, false) } // This is a controller program that can act as any command line utility. @@ -248,7 +189,7 @@ func main() { } //Open deploy script and parse it. - ProcessDeploy(config, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) + ProcessDeploy(nil, config, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) } else { deployments := os.Getenv("DEPLOYMENTS") agentToken := os.Getenv("AGENT_TOKEN") @@ -280,14 +221,16 @@ func main() { fmt.Println("trcsh on windows requires VAULT_ADDR address.") os.Exit(-1) } - gAgentConfig = &capauth.AgentConfigs{AgentToken: &agentToken} - + if err := capauth.ValidateVhost(address); err != nil { + fmt.Printf("trcsh on windows requires supported VAULT_ADDR address: %s\n", err.Error()) + os.Exit(-1) + } memprotectopts.MemProtect(nil, &agentToken) memprotectopts.MemProtect(nil, &address) shutdown := make(chan bool) // Preload agent synchronization configs... - _, errAgentLoad := gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) + gAgentConfig, _, errAgentLoad := capauth.NewAgentConfig(address, agentToken, deployments, agentEnv) if errAgentLoad != nil { fmt.Println("trcsh agent bootstrap failure.") os.Exit(-1) @@ -295,7 +238,7 @@ func main() { deploymentsSlice := strings.Split(deployments, ",") for _, deployment := range deploymentsSlice { - ProcessDeployment(*gAgentConfig.Env, *regionPtr, deployment, *trcPathPtr, secretIDPtr, appRoleIDPtr, false, deployment) + EnableDeployer(*gAgentConfig.Env, *regionPtr, deployment, *trcPathPtr, secretIDPtr, appRoleIDPtr, false, deployment) } <-shutdown @@ -310,14 +253,11 @@ var fiveSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) var fifteenSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) var thirtySecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) -func acceptInterruptFun(tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { +func acceptInterruptFun(featherCtx *cap.FeatherContext, tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { select { case <-interruptChan: - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) + cap.FeatherCtlEmit(featherCtx, + cap.MODE_PERCH, featherCtx.SessionIdentifier, true) os.Exit(1) case <-tickerContinue.C: // don't break... continue... @@ -332,71 +272,33 @@ func acceptInterruptFun(tickerContinue *time.Ticker, tickerBreak *time.Ticker, t return true, errors.New("not possible") } -func interruptFun(tickerInterrupt *time.Ticker) { +func interruptFun(featherCtx *cap.FeatherContext, tickerInterrupt *time.Ticker) { select { case <-interruptChan: - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) + cap.FeatherCtlEmit(featherCtx, + cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true) os.Exit(1) case <-tickerInterrupt.C: } } // acceptRemote - hook for instrumenting -func acceptRemote(mode int, remote string) (bool, error) { +func acceptRemote(featherCtx *cap.FeatherContext, mode int, remote string) (bool, error) { if mode == cap.FEATHER_CTL { - return acceptInterruptFun(multiSecondInterruptTicker, fifteenSecondInterruptTicker, thirtySecondInterruptTicker) + return acceptInterruptFun(featherCtx, multiSecondInterruptTicker, fifteenSecondInterruptTicker, thirtySecondInterruptTicker) } return true, nil } -func featherCtlCb(agentName string) error { +func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { if gAgentConfig == nil { return errors.New("incorrect agent initialization") } - flapMode := cap.MODE_GAZE - ctlFlapMode := flapMode - var err error = errors.New("init") - - for { - if err == nil && ctlFlapMode == cap.MODE_GLIDE { - if err != nil { - fmt.Printf("\nDeployment error.\n") - } else { - fmt.Printf("\nDeployment complete.\n") - } - os.Exit(0) - } else { - callFlap := flapMode - if err == nil { - if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { - ctl := strings.Split(ctlFlapMode, "_") - if len(ctl) > 1 && ctl[1] != capauth.TrcCtlComplete { - fmt.Printf("%s\n", ctl[1]) - } - callFlap = cap.MODE_GAZE - } else { - callFlap = cap.MODE_GAZE - } - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(multiSecondInterruptTicker) - callFlap = cap.MODE_GAZE - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env, true, acceptRemote) - } - } + featherCtx.SessionIdentifier = agentName + "." + *gAgentConfig.Env + captiplib.FeatherCtl(featherCtx, featherCtx.SessionIdentifier) + return nil } func roleBasedRunner(env string, @@ -495,8 +397,12 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, if gAgentConfig == nil { // Prepare the configuration triggering mechanism. // Bootstrap deployment is replaced during callback with the agent name. - gAgentConfig = &capauth.AgentConfigs{} - gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", config.Env) // Feathering always in dev environmnent. + gAgentConfig, _, errAgentLoad := capauth.NewAgentConfig(config.VaultAddress, *trcshConfig.CToken, "bootstrap", config.Env) + gAgentConfig.InterruptHandlerFunc = deployCtlInterrupted + if errAgentLoad != nil { + os.Exit(1) + } + config.FeatherCtx = &gAgentConfig.FeatherContext } err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) @@ -572,7 +478,7 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, // Returns: // // Nothing. -func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { +func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { agentToken := false if token != "" { agentToken = true @@ -608,7 +514,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep for { if gTrcshConfig == nil || !gTrcshConfig.IsValid(gAgentConfig) { // Loop until we have something usable... - gTrcshConfig, err = trcshauth.TrcshAuth(gAgentConfig, config) + gTrcshConfig, err = trcshauth.TrcshAuth(featherCtx, gAgentConfig, config) if err != nil { config.Log.Printf(".") time.Sleep(time.Second) @@ -769,6 +675,15 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep var onceKubeInit sync.Once var PipeOS billy.File + atomic.StoreInt64(&featherCtx.RunState, cap.RUN_STARTED) +rerun: + for { + if atomic.LoadInt64(&featherCtx.RunState) == cap.RUN_STARTED || atomic.LoadInt64(&featherCtx.RunState) == cap.RUNNING { + break + } else { + time.Sleep(time.Second * 3) + } + } for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") if strings.HasPrefix(deployPipeline, "#") || deployPipeline == "" { @@ -840,11 +755,17 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep strings.Split(deployLine, " "), &configCount) if err != nil { - config.DeploymentCtlMessage <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + config.DeploymentCtlMessageChan <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete + goto rerun } else { - config.DeploymentCtlMessage <- deployLine + config.DeploymentCtlMessageChan <- deployLine + } + if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { + goto rerun } } else { + config.FeatherCtx = featherCtx processPluginCmds( &trcKubeDeploymentConfig, &onceKubeInit, @@ -863,7 +784,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep } } if utils.IsWindows() { - config.DeploymentCtlMessage <- capauth.TrcCtlComplete + config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete } //Make the arguments in the script -> os.args. diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index e5bd8d543..637a36876 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" eUtils "github.com/trimble-oss/tierceron/utils" @@ -86,10 +87,10 @@ func GetSetEnvAddrContext(env string, envContext string, addrPort string) (strin return env, envContext, addrPort, nil } -func retryingPenseFeatherQuery(agentConfigs *capauth.AgentConfigs, pense string) (*string, error) { +func retryingPenseFeatherQuery(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfigs, pense string) (*string, error) { retry := 0 for retry < 5 { - result, err := agentConfigs.PenseFeatherQuery(pense) + result, err := agentConfigs.PenseFeatherQuery(featherCtx, pense) if err != nil || result == nil || *result == "...." { time.Sleep(time.Second) @@ -102,7 +103,7 @@ func retryingPenseFeatherQuery(agentConfigs *capauth.AgentConfigs, pense string) } // Helper function for obtaining auth components. -func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { +func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { trcshConfig := &capauth.TrcShConfig{} var err error @@ -138,7 +139,7 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) } if agentConfigs != nil { - trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(agentConfigs, "caddress") + trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "caddress") } else { config.Log.Println("Auth phase 2") trcshConfig.VaultAddress, err = capauth.PenseQuery(config, "caddress") @@ -171,7 +172,7 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) memprotectopts.MemProtect(nil, &config.VaultAddress) if agentConfigs != nil { - trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(agentConfigs, "configrole") + trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "configrole") } else { config.Log.Println("Auth phase 3") trcshConfig.ConfigRole, err = capauth.PenseQuery(config, "configrole") diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index b7fee5664..b5c20ca78 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -475,7 +475,7 @@ func CommonMain(envPtr *string, } } else if *agentdeployPtr { if config.FeatherCtlCb != nil { - err := config.FeatherCtlCb(*pluginNamePtr) + err := config.FeatherCtlCb(config.FeatherCtx, *pluginNamePtr) if err != nil { fmt.Println("Incorrect installation") return err diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 7f594b571..cee507925 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -10,6 +10,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/pavlo-v-chernykh/keystore-go/v4" + "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/coreopts" ) @@ -74,7 +75,8 @@ type DriverConfig struct { EndDir string OutputMemCache bool MemFs billy.Filesystem - FeatherCtlCb func(string) error + FeatherCtx *cap.FeatherContext + FeatherCtlCb func(*cap.FeatherContext, string) error // Config modes.... WantCerts bool @@ -110,8 +112,8 @@ type DriverConfig struct { ServiceFilter []string // Which tables to use. DynamicPathFilter string // Seeds from a specific path. - DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on - DeploymentCtlMessage chan string + DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on + DeploymentCtlMessageChan chan string } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. From f1ef570b0ebf9a73604edc48f8e5888dc70f95c3 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 27 Nov 2023 09:17:29 -0800 Subject: [PATCH 30/55] Small library upgrades. (#827) * More security updates. * Update to latest. * Using more concise session identifier. --------- --- capauth/agentconfig.go | 35 ++++++++++++++++++++++++---------- go.mod | 14 +++++++------- go.sum | 12 ++++++++++++ trcchatproxy/go.mod | 10 +++++----- trcchatproxy/go.sum | 10 ++++++++++ trcsh/trcsh.go | 42 +++++++++++++++++++++++++++-------------- trcsh/trcshauth/auth.go | 10 +++++----- 7 files changed, 92 insertions(+), 41 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index 7a3345f24..20400a93d 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -46,7 +46,10 @@ func randomString(n int) string { } func ValidateVhost(host string) error { - protocolHost := "https://" + host + protocolHost := host + if !strings.HasPrefix(host, "https://") { + protocolHost = fmt.Sprintf("https://%s", host) + } for _, endpoint := range coreopts.GetSupportedEndpoints() { if strings.HasPrefix(endpoint, protocolHost) { return nil @@ -107,15 +110,27 @@ func NewAgentConfig(address string, agentToken string, deployments string, env s if readErr != nil { return nil, nil, readErr } else { + trcHatHostLocal := new(string) + trcHatEncryptPass := data["trcHatEncryptPass"].(string) + memprotectopts.MemProtect(nil, &trcHatEncryptPass) + trcHatEncryptSalt := data["trcHatEncryptSalt"].(string) + memprotectopts.MemProtect(nil, &trcHatEncryptSalt) + hatHandshakeHostAddr := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatHandshakePort"].(string)) + memprotectopts.MemProtect(nil, &hatHandshakeHostAddr) + trcHatHandshakeCode := data["trcHatHandshakeCode"].(string) + memprotectopts.MemProtect(nil, &trcHatHandshakeCode) + sessionIdentifier := "sessionIdDynamicFill" + featherCtx := captiplib.FeatherCtlInit(nil, - "", - data["trcHatEncryptPass"].(string), - data["trcHatEncryptSalt"].(string), - fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatHandshakePort"].(string)), - data["trcHatHandshakeCode"].(string), - "sessionIdDynamicFill", captiplib.AcceptRemote, nil) - - featherHostPort := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatSecretsPort"].(string)) + trcHatHostLocal, + &trcHatEncryptPass, + &trcHatEncryptSalt, + &hatHandshakeHostAddr, + &trcHatHandshakeCode, + &sessionIdentifier, captiplib.AcceptRemote, nil) + + hatFeatherHostAddr := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatSecretsPort"].(string)) + memprotectopts.MemProtect(nil, &hatFeatherHostAddr) trcHatEnv := data["trcHatEnv"].(string) // TODO: Figure out.... // memprotectopts.MemProtect(nil, &featherCtx.EncryptPass) @@ -126,7 +141,7 @@ func NewAgentConfig(address string, agentToken string, deployments string, env s agentconfig := &AgentConfigs{ *featherCtx, &agentToken, - &featherHostPort, + &hatFeatherHostAddr, new(string), &deployments, &trcHatEnv, diff --git a/go.mod b/go.mod index f4e070c01..3ef9d7120 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,8 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/twitchtv/twirp v5.12.1+incompatible github.com/xo/dburl v0.9.0 - golang.org/x/crypto v0.14.0 - golang.org/x/sys v0.13.0 + golang.org/x/crypto v0.15.0 + golang.org/x/sys v0.14.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -176,7 +176,7 @@ require ( github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9 // indirect github.com/stretchr/testify v1.8.1 // indirect github.com/yuin/goldmark v1.4.13 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.14.0 // indirect golang.org/x/mobile v0.0.0-20220307220422-55113b94f09c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2 // indirect @@ -226,13 +226,13 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/term v0.14.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/grpc v1.58.3 + google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 067d1aac7..fd9dfebed 100644 --- a/go.sum +++ b/go.sum @@ -893,6 +893,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -913,6 +914,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap golang.org/x/image v0.0.0-20220601225756-64ec528b34cd/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -998,6 +1001,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1109,10 +1114,13 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1126,6 +1134,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1308,6 +1318,8 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod index ed147ef01..d30af4270 100644 --- a/trcchatproxy/go.mod +++ b/trcchatproxy/go.mod @@ -14,14 +14,14 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect - google.golang.org/grpc v1.58.2 // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/trcchatproxy/go.sum b/trcchatproxy/go.sum index 9c1066b54..9f0ad2421 100644 --- a/trcchatproxy/go.sum +++ b/trcchatproxy/go.sum @@ -63,6 +63,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -76,6 +78,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= @@ -90,11 +94,15 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -123,6 +131,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index f9c4568db..50a8819c5 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -40,6 +40,10 @@ import ( var gAgentConfig *capauth.AgentConfigs = nil var gTrcshConfig *capauth.TrcShConfig +var ( + MODE_PERCH_STR string = string([]byte{cap.MODE_PERCH}) +) + func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.DriverConfig, error) { if len(env) == 0 { env = os.Getenv("TRC_ENV") @@ -90,8 +94,14 @@ func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.Dr return config, nil } -func emote(msg string) { - // TODO: emote msg somewhere somehow? +func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg string) { + if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { + fmt.Printf(msg) + } +} + +func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { + featherCtx.Log.Printf(msg) } // deployCtl -- is the deployment controller or manager if you will. @@ -102,7 +112,7 @@ func deployCtlInterrupted(featherCtx *cap.FeatherContext) error { // deployer -- does the work of deploying.. func deployerInterrupted(featherCtx *cap.FeatherContext) error { - cap.FeatherCtlEmit(featherCtx, cap.MODE_PERCH, featherCtx.SessionIdentifier, true) + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) return nil } @@ -120,17 +130,20 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec // // Each deployer needs it's own context. // + localHostAddr := "" + sessionIdentifier := deployment + "." + *gAgentConfig.Env config.FeatherCtx = captiplib.FeatherCtlInit(interruptChan, - "", + &localHostAddr, gAgentConfig.EncryptPass, gAgentConfig.EncryptSalt, gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, - deployment+"."+*gAgentConfig.Env, /*Session identifier */ + &sessionIdentifier, /*Session identifier */ captiplib.AcceptRemote, deployerInterrupted) + config.FeatherCtx.Log = *config.Log - go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, emote, nil) + go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, deployerEmote, nil) go ProcessDeploy(config.FeatherCtx, config, region, "", deployment, trcPath, secretId, approleId, false) } @@ -230,7 +243,8 @@ func main() { shutdown := make(chan bool) // Preload agent synchronization configs... - gAgentConfig, _, errAgentLoad := capauth.NewAgentConfig(address, agentToken, deployments, agentEnv) + var errAgentLoad error + gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(address, agentToken, deployments, agentEnv) if errAgentLoad != nil { fmt.Println("trcsh agent bootstrap failure.") os.Exit(-1) @@ -256,8 +270,7 @@ var thirtySecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) func acceptInterruptFun(featherCtx *cap.FeatherContext, tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { select { case <-interruptChan: - cap.FeatherCtlEmit(featherCtx, - cap.MODE_PERCH, featherCtx.SessionIdentifier, true) + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(1) case <-tickerContinue.C: // don't break... continue... @@ -275,8 +288,7 @@ func acceptInterruptFun(featherCtx *cap.FeatherContext, tickerContinue *time.Tic func interruptFun(featherCtx *cap.FeatherContext, tickerInterrupt *time.Ticker) { select { case <-interruptChan: - cap.FeatherCtlEmit(featherCtx, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true) + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(1) case <-tickerInterrupt.C: } @@ -296,8 +308,9 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { return errors.New("incorrect agent initialization") } - featherCtx.SessionIdentifier = agentName + "." + *gAgentConfig.Env - captiplib.FeatherCtl(featherCtx, featherCtx.SessionIdentifier) + sessionIdentifier := agentName + "." + *gAgentConfig.Env + featherCtx.SessionIdentifier = &sessionIdentifier + captiplib.FeatherCtl(featherCtx, *featherCtx.SessionIdentifier, deployerCtlEmote) return nil } @@ -755,13 +768,14 @@ rerun: strings.Split(deployLine, " "), &configCount) if err != nil { - config.DeploymentCtlMessageChan <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + config.DeploymentCtlMessageChan <- fmt.Sprintf("%s\nEncountered errors--%s\n", deployLine, err.Error()) config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete goto rerun } else { config.DeploymentCtlMessageChan <- deployLine } if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { + config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete goto rerun } } else { diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index 637a36876..3cb55bb19 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -125,7 +125,7 @@ func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfig return trcshConfig, nil } } else { - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 1") trcshConfig.KubeConfig, err = capauth.PenseQuery(config, "kubeconfig") } @@ -138,7 +138,7 @@ func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfig memprotectopts.MemProtect(nil, trcshConfig.KubeConfig) } - if agentConfigs != nil { + if featherCtx != nil { trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "caddress") } else { config.Log.Println("Auth phase 2") @@ -171,7 +171,7 @@ func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfig config.VaultAddress = *trcshConfig.VaultAddress memprotectopts.MemProtect(nil, &config.VaultAddress) - if agentConfigs != nil { + if featherCtx != nil { trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "configrole") } else { config.Log.Println("Auth phase 3") @@ -183,7 +183,7 @@ func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfig memprotectopts.MemProtect(nil, trcshConfig.ConfigRole) - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 4") trcshConfig.PubRole, err = capauth.PenseQuery(config, "pubrole") if err != nil { @@ -192,7 +192,7 @@ func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfig memprotectopts.MemProtect(nil, trcshConfig.PubRole) } - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 5") trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") if err != nil { From 8237174951d6a89e5fd2d9d95bfb3162102bdd57 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:15:22 -0800 Subject: [PATCH 31/55] Prepare for release. (#829) * Log errors when it makes sense to. * Small changes to provide better error handling. --------- --- trcsh/trcsh.go | 32 ++++++++++++++++------- trcvault/trcplgtoolbase/trcplgtoolbase.go | 6 ++++- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 1ee3e4e96..defd7ec1c 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -31,7 +31,6 @@ import ( "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" "github.com/trimble-oss/tierceron/trcvault/util" - "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -95,9 +94,14 @@ func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.Dr } func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg string) { + if ctlFlapMode == "p_trcctlcomplete" { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(0) + } if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { - fmt.Printf(msg) + fmt.Printf("%s", msg) } + featherCtx.Log.Printf("ctl: %s msg: %s\n", ctlFlapMode, strings.Trim(msg, "\n")) } func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { @@ -158,7 +162,7 @@ func main() { fmt.Println("trcsh Version: " + "1.23") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string - if !utils.IsWindows() { + if !eUtils.IsWindows() { if os.Geteuid() == 0 { fmt.Println("Trcsh cannot be run as root.") os.Exit(-1) @@ -314,7 +318,7 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { sessionIdentifier := agentName + "." + *gAgentConfig.Env featherCtx.SessionIdentifier = &sessionIdentifier - captiplib.FeatherCtl(featherCtx, *featherCtx.SessionIdentifier, deployerCtlEmote) + captiplib.FeatherCtl(featherCtx, deployerCtlEmote) return nil } @@ -431,7 +435,10 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, gAgentConfig.EncryptSalt, gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, - new(string), gAgentConfig.AcceptRemoteFunc, nil) + new(string), gAgentConfig.AcceptRemoteFunc, gAgentConfig.InterruptHandlerFunc) + if config.Log != nil { + config.FeatherCtx.Log = *config.Log + } err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) if err != nil { @@ -525,9 +532,9 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, // cToken := "" // configRole := "" // pubRole := "" - // fileBytes, err := ioutil.ReadFile("") + // fileBytes, _ := os.ReadFile("") // kc := base64.StdEncoding.EncodeToString(fileBytes) - // gTrcshConfig = &trcshauth.TrcShConfig{Env: "dev", + // gTrcshConfig = &capauth.TrcShConfig{Env: "dev", // EnvContext: "dev", // CToken: &cToken, // ConfigRole: &configRole, @@ -535,6 +542,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, // KubeConfig: &kc, // } // config.VaultAddress = "" + // gTrcshConfig.VaultAddress = &config.VaultAddress // config.Token = "" // Chewbacca: end scrub var err error @@ -771,7 +779,7 @@ collaboratorReRun: os.Args = deployArgs } } - if utils.IsWindows() { + if eUtils.IsWindows() { // Log for traceability. config.Log.Println(deployLine) err := processWindowsCmds( @@ -789,7 +797,11 @@ collaboratorReRun: strings.Split(deployLine, " "), &configCount) if err != nil { - config.DeploymentCtlMessageChan <- fmt.Sprintf("%s\nEncountered errors--%s\n", deployLine, err.Error()) + if strings.Contains(err.Error(), "Forbidden") { + // Critical agent setup error. + os.Exit(-1) + } + config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors\n", deployLine) config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete goto collaboratorReRun } else { @@ -818,7 +830,7 @@ collaboratorReRun: } } } - if utils.IsWindows() { + if eUtils.IsWindows() { config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete } //Make the arguments in the script -> os.args. diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 2aa81f3ca..7adeb6b29 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -387,7 +387,11 @@ func CommonMain(envPtr *string, err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println("Image download failure.") - fmt.Println(err.Error()) + if configBase.FeatherCtx != nil { + configBase.FeatherCtx.Log.Printf("%s", err.Error()) + } else { + fmt.Println(err.Error()) + } return err } } From 5f252dadceac10376795bca915776eef2cb73db2 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:26:56 -0800 Subject: [PATCH 32/55] Prepare for release. (#831) * Fix for trcsh state processing and windows. --------- --- trcsh/trcsh.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index defd7ec1c..b6b1c107e 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -99,7 +99,7 @@ func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg st os.Exit(0) } if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { - fmt.Printf("%s", msg) + fmt.Printf("%s\n", msg) } featherCtx.Log.Printf("ctl: %s msg: %s\n", ctlFlapMode, strings.Trim(msg, "\n")) } @@ -718,13 +718,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, collaboratorReRun: if featherCtx != nil { - for { - if atomic.LoadInt64(&featherCtx.RunState) == cap.RUN_STARTED || atomic.LoadInt64(&featherCtx.RunState) == cap.RUNNING { - break - } else { - time.Sleep(time.Second * 3) - } - } + atomic.StoreInt64(&featherCtx.RunState, cap.RUN_STARTED) } for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") @@ -802,13 +796,11 @@ collaboratorReRun: os.Exit(-1) } config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors\n", deployLine) - config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete goto collaboratorReRun } else { config.DeploymentCtlMessageChan <- deployLine } if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { - config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete goto collaboratorReRun } } else { @@ -831,7 +823,15 @@ collaboratorReRun: } } if eUtils.IsWindows() { - config.DeploymentCtlMessageChan <- capauth.TrcCtlComplete + config.DeploymentCtlMessageChan <- cap.CTL_COMPLETE + for { + if atomic.LoadInt64(&featherCtx.RunState) == cap.RUNNING { + time.Sleep(time.Second) + } else { + break + } + } + goto collaboratorReRun } //Make the arguments in the script -> os.args. From f8a69c8c638146b292696cb5b322330934b2fc8a Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:24:33 -0800 Subject: [PATCH 33/55] Prepare for release. (#833) * Fix for trcsh state processing and windows. * A little cleanup. --------- --- capauth/agentconfig.go | 2 -- trcsh/trcsh.go | 27 ++++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index 2480a5162..707113e62 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -22,8 +22,6 @@ import ( "google.golang.org/grpc" ) -var TrcCtlComplete string = "trcctlcomplete" - type AgentConfigs struct { *cap.FeatherContext AgentToken *string diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index b6b1c107e..b9e8793a8 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -93,17 +93,24 @@ func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.Dr return config, nil } +// Logging of deployer controller activities.. func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg string) { - if ctlFlapMode == "p_trcctlcomplete" { + if strings.HasSuffix(ctlFlapMode, cap.CTL_COMPLETE) { cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(0) } + if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { fmt.Printf("%s\n", msg) } featherCtx.Log.Printf("ctl: %s msg: %s\n", ctlFlapMode, strings.Trim(msg, "\n")) + if strings.Contains(msg, "encountered errors") { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(0) + } } +// Logging of deployer activities.. func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { featherCtx.Log.Printf(msg) } @@ -146,6 +153,8 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec captiplib.AcceptRemote, deployerInterrupted) config.FeatherCtx.Log = *config.Log + // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) + atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, deployerEmote, nil) @@ -711,14 +720,17 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, var onceKubeInit sync.Once var PipeOS billy.File +collaboratorReRun: if featherCtx != nil { // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) - atomic.StoreInt64(&featherCtx.RunState, cap.RUN_STARTED) - } + for { + if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { + break + } else { + time.Sleep(time.Second * 3) + } + } -collaboratorReRun: - if featherCtx != nil { - atomic.StoreInt64(&featherCtx.RunState, cap.RUN_STARTED) } for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") @@ -800,9 +812,6 @@ collaboratorReRun: } else { config.DeploymentCtlMessageChan <- deployLine } - if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { - goto collaboratorReRun - } } else { config.FeatherCtx = featherCtx processPluginCmds( From 3f898d257cfa2ad90990f8cd8b1b86832681d86c Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 30 Nov 2023 09:36:01 -0800 Subject: [PATCH 34/55] Prepare for release. (#835) * Adding error messaging for incorrect project setup. * Reduce logging. * Tweak wording. --------- --- .../Vault/Tierceron/deploy/deploy.sh.tmpl | 99 +++++++++++++------ .../Tierceron/deploy/refreshtoken.sh.tmpl | 44 +++++++-- trcsh/trcsh.go | 14 ++- trcvault/deploy/deploy.sh | 41 ++++++-- trcvault/deploy/refreshtoken.sh | 4 +- 5 files changed, 152 insertions(+), 50 deletions(-) diff --git a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl index 208971d24..1cd6df620 100755 --- a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl +++ b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl @@ -22,8 +22,13 @@ fi FILESHAVAL=$(cat $FILESHA) +echo "Enter agent environment: " +read VAULT_ENV + +if [[ -z "${VAULT_ADDR}" ]]; then echo "Enter agent vault host base url: " read VAULT_ADDR +fi if [[ -z "${SECRET_VAULT_ADDR}" ]]; then echo "Enter organization vault host base url including port: " @@ -31,14 +36,22 @@ read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_ENV_TOKEN}" ]]; then -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read SECRET_VAULT_ENV_TOKEN fi + +if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi +read SECRET_VAULT_PLUGIN_TOKEN +fi + echo "Enter agent root token: " read VAULT_TOKEN -echo "Enter agent environment: " -read VAULT_ENV echo "Is this plugin an agent deployment tool (Y or N): " read VAULT_AGENT @@ -46,13 +59,16 @@ read VAULT_AGENT if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then PRE_CERTIFY="Y" else - echo "Enter trc plugin runtime environment token with write permissions unrestricted: " - read VAULT_ENV_TOKEN + if [[ -z "${VAULT_ENV_TOKEN}" ]]; then + echo "Enter trc plugin runtime environment token with write permissions unrestricted(config_token_"$VAULT_ENV"_unrestricted): " + read VAULT_ENV_TOKEN + fi if [ "$VAULT_PLUGIN_DIR" ] then echo "Deploying using local vault strategy." - PRE_CERTIFY="N" + echo "Precertify plugin (Y or N): " + read PRE_CERTIFY else echo "Precertify plugin (Y or N): " read PRE_CERTIFY @@ -61,19 +77,27 @@ fi if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = "y" ]; then - if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then - echo "Deploying agent deploy tool" + if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then + echo "Certifying agent deployment tool plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent certifystatus=$? - if [ $certifystatus -eq 0 ]; then - echo "No problems encountered." + if [ $certifystatus -eq 0 ]; then + echo "No certification problems encountered." exit $certifystatus else - echo "Unexpected certifyication errorerror." + echo "Unexpected certification error." exit $certifystatus fi else + echo "Certifying vault type plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) + certifystatus=$? + if [ $certifystatus -eq 0 ]; then + echo "No certification problems encountered." + else + echo "Unexpected certification error." + exit $certifystatus + fi fi fi @@ -83,23 +107,44 @@ fi if [ "$VAULT_PLUGIN_DIR" ] then - echo "Local plugin registration skipping certification." + echo "Local plugin registration skipping certified check." else echo "Checking plugin deploy status." - echo "Certifying plugin for env $VAULT_ENV." + echo "Checking deployment status on plugin for env $VAULT_ENV." trcplgtool -env=$VAULT_ENV -checkDeployed -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) status=$? - echo "Plugin certified with result $status." + echo "Plugin deployment had status result $status." - if [ $status -eq 0 ]; then - echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." - vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR - exit $status + if [ $status -eq 0 ]; then + echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." + + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi + + vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + vaultUpdateStatus=$? + if [ $vaultUpdateStatus -eq 0 ]; then + exit $status + else + echo "But it's not deployed to vault for $VAULT_ENV. Continuing..." + fi elif [ $status -eq 1 ]; then echo "Existing plugin does not match repository plugin - cannot continue." exit $status elif [ $status -eq 2 ]; then - echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi else echo "Unexpected error response." exit $status @@ -154,16 +199,10 @@ if [ "$SHAVAL" != "$FILESHAVAL" ]; then exit -1 fi -PROD_EXT="" -for x in "staging" "prod"; do - if [ $x = $VAULT_ENV ]; then - PROD_EXT="-prod" - fi -done echo "Registering new plugin." vault plugin register \ - -command=$TRC_PLUGIN_NAME$PROD_EXT \ + -command=$TRC_PLUGIN_NAME \ -sha256=$(echo $SHAVAL) \ -args=`backendUUID=789` \ $TRC_PLUGIN_NAME @@ -177,6 +216,10 @@ vault secrets enable \ #Activates/starts the deployed plugin. # Note: plugin should update deployed flag for itself. -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR -vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME \ No newline at end of file +echo "Activating plugin." +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + +echo "Notifying carrier." +vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME +echo "Deployment complete." diff --git a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl index 91904d82a..85ebab5be 100755 --- a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl +++ b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl @@ -8,21 +8,49 @@ if [ "$TRC_PLUGIN_NAME" = 'trc-vault-carrier-plugin' ] ; then exit 1 fi -echo "Enter organization vault host base url including port: " +echo "Enter environment: " +read VAULT_ENV + +if [[ -z "${SECRET_VAULT_ADDR}" ]]; then +echo "Enter organization vault host base url including port (hit enter if just refreshing org tokens): " +read SECRET_VAULT_ADDR +fi + +if [[ -z "${SECRET_ENV_TOKEN}" ]]; then +echo "Enter organization vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " +read SECRET_ENV_TOKEN +fi + +if [[ -z "${VAULT_ADDR}" ]]; then +echo "Enter agent vault host base url including port: " read VAULT_ADDR +fi -echo "Enter organization vault root token: " +if [[ -z "${VAULT_TOKEN}" ]]; then +echo "Enter agent vault root token: " read VAULT_TOKEN +fi -echo "Enter environment: " -read VAULT_ENV - -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read VAULT_ENV_TOKEN -VAULT_API_ADDR=VAULT_ADDR +VAULT_API_ADDR=$VAULT_ADDR +VAULT_API_TOKEN=$VAULT_TOKEN export VAULT_ADDR export VAULT_TOKEN export VAULT_API_ADDR +export VAULT_API_TOKEN + +echo "Secret vault: " $SECRET_VAULT_ADDR +echo "Agent vault: " $VAULT_ADDR + +echo "Agent and secrets stored in secrets vault only? (Y or N): " +read SECRETS_ONLY + +if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_API_ADDR=$SECRET_VAULT_ADDR + SECRET_ENV_TOKEN=$VAULT_ENV_TOKEN +fi + +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index b9e8793a8..1e3f835ed 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -112,7 +112,9 @@ func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg st // Logging of deployer activities.. func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { - featherCtx.Log.Printf(msg) + if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH { + featherCtx.Log.Printf(msg) + } } // deployCtl -- is the deployment controller or manager if you will. @@ -136,6 +138,7 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec if len(deployment) > 0 { config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} config.DeploymentCtlMessageChan = make(chan string, 5) + config.Log.Printf("Starting deployer: %s\n", deployment) } // @@ -615,6 +618,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, if (len(os.Args) > 1 || len(trcPath) > 0) && !strings.Contains(pwd, "TrcDeploy") { // Generate trc code... + config.Log.Println("Preload setup") trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") @@ -622,7 +626,13 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, config.OutputMemCache = true config.StartDir = []string{"trc_templates"} config.EndDir = "." - trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + config.Log.Println("Preloading") + configErr := trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + if configErr != nil { + fmt.Println("Preload failed. Couldn't find required resource.") + config.Log.Printf("Preload Error %s\n", configErr.Error()) + os.Exit(-1) + } ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" diff --git a/trcvault/deploy/deploy.sh b/trcvault/deploy/deploy.sh index 2ad5631ce..1cd6df620 100755 --- a/trcvault/deploy/deploy.sh +++ b/trcvault/deploy/deploy.sh @@ -42,9 +42,9 @@ fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then -echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " else -echo "Enter organization vault plugin token for certification(config_token_pluginany): " +echo "Enter organization vault plugin token for certification(config_token_pluginany): " fi read SECRET_VAULT_PLUGIN_TOKEN fi @@ -77,11 +77,11 @@ fi if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = "y" ]; then - if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then + if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then echo "Certifying agent deployment tool plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent certifystatus=$? - if [ $certifystatus -eq 0 ]; then + if [ $certifystatus -eq 0 ]; then echo "No certification problems encountered." exit $certifystatus else @@ -92,7 +92,7 @@ if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = echo "Certifying vault type plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) certifystatus=$? - if [ $certifystatus -eq 0 ]; then + if [ $certifystatus -eq 0 ]; then echo "No certification problems encountered." else echo "Unexpected certification error." @@ -115,15 +115,36 @@ else status=$? echo "Plugin deployment had status result $status." - if [ $status -eq 0 ]; then - echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." - vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR - exit $status + if [ $status -eq 0 ]; then + echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." + + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi + + vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + vaultUpdateStatus=$? + if [ $vaultUpdateStatus -eq 0 ]; then + exit $status + else + echo "But it's not deployed to vault for $VAULT_ENV. Continuing..." + fi elif [ $status -eq 1 ]; then echo "Existing plugin does not match repository plugin - cannot continue." exit $status elif [ $status -eq 2 ]; then - echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi else echo "Unexpected error response." exit $status diff --git a/trcvault/deploy/refreshtoken.sh b/trcvault/deploy/refreshtoken.sh index 091426635..85ebab5be 100755 --- a/trcvault/deploy/refreshtoken.sh +++ b/trcvault/deploy/refreshtoken.sh @@ -49,8 +49,8 @@ read SECRETS_ONLY if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then VAULT_API_ADDR=$SECRET_VAULT_ADDR - VAULT_API_TOKEN=$SECRET_ENV_TOKEN + SECRET_ENV_TOKEN=$VAULT_ENV_TOKEN fi -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_API_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME From 0a2c8550f9e9c4233516589918c3830ceecb0c0f Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:48:43 -0800 Subject: [PATCH 35/55] Improve feedback. (#837) * Funnel errors upstream better. --------- --- trcsh/trcsh.go | 4 +++- trcvault/trcplgtoolbase/trcplgtoolbase.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 1e3f835ed..a1fd87546 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -817,7 +817,9 @@ collaboratorReRun: // Critical agent setup error. os.Exit(-1) } - config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors\n", deployLine) + errMessage := err.Error() + errMessageFiltered := strings.ReplaceAll(errMessage, ":", "-") + config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors - %s\n", deployLine, errMessageFiltered) goto collaboratorReRun } else { config.DeploymentCtlMessageChan <- deployLine diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 7adeb6b29..1b7693b4a 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -388,7 +388,7 @@ func CommonMain(envPtr *string, if err != nil { fmt.Println("Image download failure.") if configBase.FeatherCtx != nil { - configBase.FeatherCtx.Log.Printf("%s", err.Error()) + configBase.FeatherCtx.Log.Printf("Image download failure: %s", err.Error()) } else { fmt.Println(err.Error()) } @@ -416,7 +416,13 @@ func CommonMain(envPtr *string, } fmt.Printf("Image deployed to: %s\n", deployPath) } else { - fmt.Printf("Image not certified. Cannot deploy image for %s\n", pluginToolConfig["trcplugin"]) + errMessage := fmt.Sprintf("image not certified. cannot deploy image for %s", pluginToolConfig["trcplugin"]) + if configBase.FeatherCtx != nil { + configBase.FeatherCtx.Log.Printf(errMessage) + } else { + fmt.Printf("%s\n", errMessage) + } + return errors.New(errMessage) } } else if *certifyImagePtr { //Certify Image From 66323c18f96a3397c1b08a5c0dc8a7204377b7f9 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 1 Dec 2023 07:47:52 -0800 Subject: [PATCH 36/55] Better support for plugin aliasing. (#839) * Make pluginName come from deployer. * More assurance that service goes down before copying. * Fix problem with certifying. * Update to latest. * Adding some assurances. * Add clarifying comment. --------- --- trcflow/core/dataFlowStatistics.go | 1 + trcflow/deploy/process.go | 2 +- trcsh/trcsh.go | 5 +-- trcvault/trcplgtoolbase/trcplgtoolbase.go | 28 +++++++++++++--- trcvault/util/properties.go | 3 ++ vaulthelper/kv/Modifier.go | 40 +++++++++++++++-------- 6 files changed, 58 insertions(+), 21 deletions(-) diff --git a/trcflow/core/dataFlowStatistics.go b/trcflow/core/dataFlowStatistics.go index 5df24dc24..8f8ecea88 100644 --- a/trcflow/core/dataFlowStatistics.go +++ b/trcflow/core/dataFlowStatistics.go @@ -551,6 +551,7 @@ func (dfs *TTDINode) FinishStatistic(tfmContext *TrcFlowMachineContext, tfContex mod.SectionPath = "" if vaultWriteBack { + mod.SectionPath = "" _, writeErr := mod.Write("super-secrets/PublicIndex/"+indexPath+"/"+idName+"/"+id+"/DataFlowStatistics/DataFlowGroup/"+decodedStatData["FlowGroup"].(string)+"/dataFlowName/"+decodedStatData["FlowName"].(string)+"/"+decodedStatData["StateCode"].(string), statMap, logger) if writeErr != nil && decodedData["LogFunc"] != nil { logFunc := decodedData["LogFunc"].(func(string, error)) diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index 5ab94a64b..f1eeab27e 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -352,6 +352,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e if writeMap["trctype"].(string) == "agent" { writeMap["deployed"] = true } + cGoMod.SectionPath = "" _, err = cGoMod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", writeMap["trcplugin"].(string)), writeMap, cConfig.Log) if err != nil { logger.Printf(fmt.Sprintf("PluginDeployFlow failure: Failed to write plugin state for env: %s and plugin: %s error: %s\n", cConfig.Env, pluginName, err.Error())) @@ -447,7 +448,6 @@ func PluginDeployedUpdate(config *eUtils.DriverConfig, mod *helperkv.Modifier, v if hostRegion != "" { pluginData["deployed"] = true //Update deploy status if region exist otherwise this will block regionless deploys if set for regionless status } - statusUpdateErr := properties.WritePluginData(pluginData, replacedFields, mod, config.Log, hostRegion, pluginName) if err != nil { return statusUpdateErr diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index a1fd87546..6b01644f0 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -155,7 +155,7 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec &sessionIdentifier, /*Session identifier */ captiplib.AcceptRemote, deployerInterrupted) - config.FeatherCtx.Log = *config.Log + config.FeatherCtx.Log = config.Log // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) @@ -449,7 +449,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, gAgentConfig.HandshakeCode, new(string), gAgentConfig.AcceptRemoteFunc, gAgentConfig.InterruptHandlerFunc) if config.Log != nil { - config.FeatherCtx.Log = *config.Log + config.FeatherCtx.Log = config.Log } err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) @@ -683,6 +683,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, fmt.Println("Unable to obtain config for deployment") return } + deploymentConfig["trcpluginalias"] = deployment config.DeploymentConfig = deploymentConfig if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 1b7693b4a..3ad6d24ed 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -67,6 +67,7 @@ func CommonMain(envPtr *string, // Common plugin flags... pluginNamePtr := flagset.String("pluginName", "", "Used to certify vault plugin") + pluginNameAliasPtr := flagset.String("pluginNameAlias", "", "Name used to define an alias for a plugin") pluginTypePtr := flagset.String("pluginType", "vault", "Used to indicate type of plugin. Default is vault.") // Certify flags... @@ -106,6 +107,11 @@ func CommonMain(envPtr *string, } } + if c != nil && c.DeploymentConfig["trcpluginalias"] != nil { + // Prefer internal definition of alias + *pluginNameAliasPtr = c.DeploymentConfig["trcpluginalias"].(string) + } + if *certifyImagePtr && (len(*pluginNamePtr) == 0 || len(*sha256Ptr) == 0) { fmt.Println("Must use -pluginName && -sha256 flags to use -certify flag") return errors.New("must use -pluginName && -sha256 flags to use -certify flag") @@ -174,7 +180,11 @@ func CommonMain(envPtr *string, if c != nil { configBase = c logger = c.Log - configBase.SubSectionValue = *pluginNamePtr + if *pluginNameAliasPtr != "" { + configBase.SubSectionValue = *pluginNameAliasPtr + } else { + configBase.SubSectionValue = *pluginNamePtr + } appRoleConfigPtr = &(configBase.AppRoleConfig) *insecurePtr = configBase.Insecure } else { @@ -233,7 +243,11 @@ func CommonMain(envPtr *string, return err } config.StartDir = []string{*startDirPtr} - config.SubSectionValue = *pluginNamePtr + if *pluginNameAliasPtr != "" { + configBase.SubSectionValue = *pluginNameAliasPtr + } else { + configBase.SubSectionValue = *pluginNamePtr + } mod.Env = *envPtr eUtils.CheckError(config, err, true) @@ -360,9 +374,9 @@ func CommonMain(envPtr *string, fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("sc", "stop", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) err := cmd.Run() - if err != nil && !strings.Contains(err.Error(), "1062") && !strings.Contains(err.Error(), "1052") { + if err != nil && !strings.Contains(err.Error(), "1") && !strings.Contains(err.Error(), "5") { fmt.Println(err) return err } @@ -462,7 +476,11 @@ func CommonMain(envPtr *string, writeMap, replacedFields := properties.GetPluginData(*regionPtr, "Certify", "config", logger) - writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginToolConfig["trcplugin"].(string)) + pluginTarget := pluginToolConfig["trcplugin"].(string) + if strings.HasPrefix(*pluginNamePtr, pluginTarget) { + pluginTarget = *pluginNamePtr + } + writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginTarget) if writeErr != nil { fmt.Println(writeErr) return err diff --git a/trcvault/util/properties.go b/trcvault/util/properties.go index 779d1f83e..3e0023337 100644 --- a/trcvault/util/properties.go +++ b/trcvault/util/properties.go @@ -154,6 +154,8 @@ func (p *Properties) GetPluginData(region string, service string, config string, func (p *Properties) WritePluginData(pluginData map[string]interface{}, replacedFields map[string]interface{}, mod *helperkv.Modifier, log *log.Logger, hostRegion string, pluginName string) error { //writeMap := make(map[string]interface{}) + // If SectionPath is set like + mod.SectionPath = "" regionSuffix := "" if hostRegion != "" { regionSuffix = "~" + hostRegion @@ -175,6 +177,7 @@ func (p *Properties) WritePluginData(pluginData map[string]interface{}, replaced writeMap[field] = value } + mod.SectionPath = "" _, writeErr := mod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", pluginName), writeMap, log) if writeErr != nil { return writeErr diff --git a/vaulthelper/kv/Modifier.go b/vaulthelper/kv/Modifier.go index cb4311b96..95acbec5b 100644 --- a/vaulthelper/kv/Modifier.go +++ b/vaulthelper/kv/Modifier.go @@ -39,19 +39,20 @@ type Modifier struct { httpClient *http.Client // Handle to http client. client *api.Client // Client connected to vault logical *api.Logical // Logical used for read/write options - Env string // Environment (local/dev/QA; Initialized to secrets) - Regions []string // Supported regions - SecretDictionary *api.Secret // Current Secret Dictionary Cache. - Version string // Version for data - VersionFilter []string // Used to filter vault paths - RawEnv string - TemplatePath string // Path to template we are processing. - ProjectIndex []string // Which projects are indexed. - SectionKey string // The section key: Index or Restricted. - SectionName string // The name of the actual section. - SubSectionName string // The name of the actual subsection. - SubSectionValue string // The actual value for the sub section. - SectionPath string // The path to the Index (both seed and vault) + SecretDictionary *api.Secret // Current Secret Dictionary Cache -- populated by mod.List("templates" + + Env string // Environment (local/dev/QA; Initialized to secrets) + RawEnv string + Regions []string // Supported regions + Version string // Version for data + VersionFilter []string // Used to filter vault paths + TemplatePath string // Path to template we are processing. + ProjectIndex []string // Which projects are indexed. + SectionKey string // The section key: Index or Restricted. + SectionName string // The name of the actual section. + SubSectionName string // The name of the actual subsection. + SubSectionValue string // The actual value for the sub section. + SectionPath string // The path to the Index (both seed and vault) } type modCache struct { @@ -97,6 +98,19 @@ func NewModifier(insecure bool, token string, address string, env string, region PruneCache(env, 10) checkoutModifier, err := cachedModifierHelper(env) if err == nil && checkoutModifier != nil { + checkoutModifier.Insecure = insecure + checkoutModifier.RawEnv = env + checkoutModifier.Regions = regions + checkoutModifier.Version = "" // Version for data + checkoutModifier.VersionFilter = []string{} // Used to filter vault paths + checkoutModifier.TemplatePath = "" // Path to template we are processing. + checkoutModifier.ProjectIndex = []string{} // Which projects are indexed. + checkoutModifier.SectionKey = "" // The section key: Index or Restricted. + checkoutModifier.SectionName = "" // The name of the actual section. + checkoutModifier.SubSectionName = "" // The name of the actual subsection. + checkoutModifier.SubSectionValue = "" // The actual value for the sub section. + checkoutModifier.SectionPath = "" // The path to the Index (both seed and vault) + return checkoutModifier, nil } } From 29d621e3e0688e1e112408923314a5aa91170d99 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:34:42 -0800 Subject: [PATCH 37/55] Update tools. (#840) * Prepare for release. (#841) * Better support for plugin aliasing. (#839) * Make pluginName come from deployer. * More assurance that service goes down before copying. * Fix problem with certifying. * Update to latest. * Adding some assurances. * Add clarifying comment. --------- Signed-off-by: dependabot[bot] --- azure-pipelines-plugin.yml | 2 +- azure-pipelines.yml | 2 +- capauth/agentconfig.go | 98 +++-- go.mod | 14 +- go.sum | 36 +- trcchatproxy/go.mod | 10 +- trcchatproxy/go.sum | 10 + trcconfigbase/utils/configinator.go | 6 +- trcflow/core/context.go | 10 +- trcflow/core/dataFlowStatistics.go | 1 + trcflow/deploy/process.go | 36 +- trcflow/flumen/process.go | 8 +- trcinit/initlib/vault-seed.go | 4 +- .../Vault/Tierceron/deploy/deploy.sh.tmpl | 99 +++-- .../Tierceron/deploy/refreshtoken.sh.tmpl | 44 ++- trcinitbase/init.go | 6 +- trcsh/trcsh.go | 337 +++++++++--------- trcsh/trcshauth/auth.go | 21 +- trcsubbase/trcsub.go | 7 +- trcvault/deploy/deploy.sh | 43 ++- trcvault/deploy/refreshcarriertoken.sh | 8 + trcvault/deploy/refreshtoken.sh | 4 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 72 +++- trcvault/util/dbutil.go | 7 +- trcvault/util/properties.go | 3 + utils/driverconfig.go | 8 +- validator/db.go | 14 +- vaulthelper/kv/Modifier.go | 50 ++- 28 files changed, 573 insertions(+), 387 deletions(-) diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index dc738e255..fe12f8278 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -128,7 +128,7 @@ steps: fi echo "Compile Windows" - GOBIN= CGO_ENABLED=0 GOOS=windows GOARCH=amd64 GOCACHE=$GOCACHE go install -tags "tc memonly azrcr windows" -ldflags '-w' -tags "tc" github.com/trimble-oss/tierceron/trcsh + GOBIN= CGO_ENABLED=0 GOOS=windows GOARCH=amd64 GOCACHE=$GOCACHE go install -tags "tc memonly azrcr azure windows" -ldflags '-w' github.com/trimble-oss/tierceron/trcsh if [[ ! -f $GOBIN/windows_amd64/trcsh.exe ]] ; then echo File trcsh.exe was not built, aborting. exit 1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 471bb6a22..90562dcb8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,7 +129,7 @@ steps: done echo "Compile Windows" - GOBIN= CGO_ENABLED=0 GOOS=windows GOARCH=amd64 GOCACHE=$GOCACHE go install -tags "tc memonly windows vaultname" -ldflags '-w' -tags "tc" github.com/trimble-oss/tierceron/{trcconfig,trcpub,trcinit,trcinitp,trcx,trcxp,trcsub} + GOBIN= CGO_ENABLED=0 GOOS=windows GOARCH=amd64 GOCACHE=$GOCACHE go install -tags "tc memonly windows vaultname" -ldflags '-w' github.com/trimble-oss/tierceron/{trcconfig,trcpub,trcinit,trcinitp,trcx,trcxp,trcsub} for p in trcconfig trcpub trcinit trcinitp trcx trcxp trcsub; do echo Compiling $p if [[ ! -f $GOBIN/windows_amd64/$p.exe ]] ; then diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index a77f81231..707113e62 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "log" "math/rand" "net" "os" @@ -15,6 +14,7 @@ import ( "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron-hat/cap/tap" + captiplib "github.com/trimble-oss/tierceron-hat/captip/captiplib" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" eUtils "github.com/trimble-oss/tierceron/utils" @@ -22,18 +22,13 @@ import ( "google.golang.org/grpc" ) -var TrcCtlComplete string = "trcctlcomplete" - type AgentConfigs struct { - AgentToken *string - HandshakeHostPort *string - FeatherHostPort *string - HandshakeCode *string - DeployRoleID *string - EncryptPass *string - EncryptSalt *string - Deployments *string - Env *string + *cap.FeatherContext + AgentToken *string + FeatherHostPort *string + DeployRoleID *string + Deployments *string + Env *string } var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -49,7 +44,10 @@ func randomString(n int) string { } func ValidateVhost(host string) error { - protocolHost := "https://" + host + protocolHost := host + if !strings.HasPrefix(host, "https://") { + protocolHost = fmt.Sprintf("https://%s", host) + } for _, endpoint := range coreopts.GetSupportedEndpoints() { if strings.HasPrefix(endpoint, protocolHost) { return nil @@ -58,16 +56,12 @@ func ValidateVhost(host string) error { return errors.New("Bad host: " + host) } -func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error) { +func (agentconfig *AgentConfigs) PenseFeatherQuery(featherCtx *cap.FeatherContext, pense string) (*string, error) { penseCode := randomString(7 + rand.Intn(7)) penseArray := sha256.Sum256([]byte(penseCode)) penseSum := hex.EncodeToString(penseArray[:]) - _, featherErr := cap.FeatherWriter(*agentconfig.EncryptPass, - *agentconfig.EncryptSalt, - *agentconfig.HandshakeHostPort, - *agentconfig.HandshakeCode, - penseSum) + _, featherErr := cap.FeatherWriter(featherCtx, penseSum) if featherErr != nil { return nil, featherErr } @@ -80,7 +74,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error conn, err := grpc.Dial(*agentconfig.FeatherHostPort, grpc.WithTransportCredentials(creds)) if err != nil { - log.Fatalf("did not connect: %v", err) + return nil, err } defer conn.Close() c := cap.NewCapClient(conn) @@ -91,7 +85,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error r, err := c.Pense(ctx, &cap.PenseRequest{Pense: penseCode, PenseIndex: pense}) if err != nil { - log.Fatalf("did not connect: %v", err) + return nil, err } var penseProtect *string rPense := r.GetPense() @@ -101,9 +95,7 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(pense string) (*string, error return penseProtect, nil } -func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, deployments string, env string) (*TrcShConfig, error) { - var trcshConfig *TrcShConfig - +func NewAgentConfig(address string, agentToken string, deployments string, env string) (*AgentConfigs, *TrcShConfig, error) { mod, modErr := helperkv.NewModifier(false, agentToken, address, env, nil, true, nil) if modErr != nil { fmt.Println("trcsh Failed to bootstrap") @@ -114,47 +106,51 @@ func (agentconfig *AgentConfigs) LoadConfigs(address string, agentToken string, data, readErr := mod.ReadData("super-secrets/Restricted/TrcshAgent/config") if readErr != nil { - return nil, readErr + return nil, nil, readErr } else { + trcHatHostLocal := new(string) trcHatEncryptPass := data["trcHatEncryptPass"].(string) memprotectopts.MemProtect(nil, &trcHatEncryptPass) trcHatEncryptSalt := data["trcHatEncryptSalt"].(string) memprotectopts.MemProtect(nil, &trcHatEncryptSalt) - trcHatEnv := data["trcHatEnv"].(string) - memprotectopts.MemProtect(nil, &trcHatEnv) + hatHandshakeHostAddr := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatHandshakePort"].(string)) + memprotectopts.MemProtect(nil, &hatHandshakeHostAddr) trcHatHandshakeCode := data["trcHatHandshakeCode"].(string) memprotectopts.MemProtect(nil, &trcHatHandshakeCode) - trcHatHandshakePort := data["trcHatHandshakePort"].(string) - memprotectopts.MemProtect(nil, &trcHatHandshakePort) - trcHatHost := data["trcHatHost"].(string) - memprotectopts.MemProtect(nil, &trcHatHost) - trcHatSecretsPort := data["trcHatSecretsPort"].(string) - memprotectopts.MemProtect(nil, &trcHatSecretsPort) - trcHandshakeHostPort := trcHatHost + ":" + trcHatHandshakePort - memprotectopts.MemProtect(nil, &trcHandshakeHostPort) - trcFeatherHostPort := trcHatHost + ":" + trcHatSecretsPort - memprotectopts.MemProtect(nil, &trcFeatherHostPort) - - agentconfig.HandshakeHostPort = &trcHandshakeHostPort - agentconfig.FeatherHostPort = &trcFeatherHostPort - agentconfig.HandshakeCode = &trcHatHandshakeCode - agentconfig.EncryptPass = &trcHatEncryptPass - agentconfig.EncryptSalt = &trcHatEncryptSalt - agentconfig.Deployments = &deployments - agentconfig.Env = &trcHatEnv - - trcshConfig = &TrcShConfig{Env: trcHatEnv, + sessionIdentifier := "sessionIdDynamicFill" + + hatFeatherHostAddr := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatSecretsPort"].(string)) + memprotectopts.MemProtect(nil, &hatFeatherHostAddr) + trcHatEnv := data["trcHatEnv"].(string) + + agentconfig := &AgentConfigs{ + captiplib.FeatherCtlInit(nil, + trcHatHostLocal, + &trcHatEncryptPass, + &trcHatEncryptSalt, + &hatHandshakeHostAddr, + &trcHatHandshakeCode, + &sessionIdentifier, captiplib.AcceptRemote, nil), + &agentToken, + &hatFeatherHostAddr, + new(string), + &deployments, + &trcHatEnv, + } + + trcshConfig := &TrcShConfig{Env: trcHatEnv, EnvContext: trcHatEnv, } - trcShConfigRole, penseError := agentconfig.PenseFeatherQuery("configrole") + + trcShConfigRole, penseError := agentconfig.PenseFeatherQuery(agentconfig.FeatherContext, "configrole") if penseError != nil { - return nil, penseError + return nil, nil, penseError } memprotectopts.MemProtect(nil, trcShConfigRole) trcshConfig.ConfigRole = trcShConfigRole - } - return trcshConfig, nil + return agentconfig, trcshConfig, nil + } } func PenseQuery(config *eUtils.DriverConfig, pense string) (*string, error) { diff --git a/go.mod b/go.mod index f4e070c01..3ef9d7120 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,8 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/twitchtv/twirp v5.12.1+incompatible github.com/xo/dburl v0.9.0 - golang.org/x/crypto v0.14.0 - golang.org/x/sys v0.13.0 + golang.org/x/crypto v0.15.0 + golang.org/x/sys v0.14.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -176,7 +176,7 @@ require ( github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9 // indirect github.com/stretchr/testify v1.8.1 // indirect github.com/yuin/goldmark v1.4.13 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.14.0 // indirect golang.org/x/mobile v0.0.0-20220307220422-55113b94f09c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2 // indirect @@ -226,13 +226,13 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/term v0.14.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/grpc v1.58.3 + google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 067d1aac7..4097e6fe1 100644 --- a/go.sum +++ b/go.sum @@ -890,9 +890,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -911,8 +910,8 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20210504121937-7319ad40d33e/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20220601225756-64ec528b34cd/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -939,7 +938,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -995,9 +993,8 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1025,7 +1022,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1101,18 +1097,16 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1123,9 +1117,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1192,7 +1185,6 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1306,8 +1298,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod index ed147ef01..d30af4270 100644 --- a/trcchatproxy/go.mod +++ b/trcchatproxy/go.mod @@ -14,14 +14,14 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect - google.golang.org/grpc v1.58.2 // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/trcchatproxy/go.sum b/trcchatproxy/go.sum index 9c1066b54..9f0ad2421 100644 --- a/trcchatproxy/go.sum +++ b/trcchatproxy/go.sum @@ -63,6 +63,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -76,6 +78,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= @@ -90,11 +94,15 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -123,6 +131,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index 12c5a0ab2..f4077e8f4 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -55,7 +55,7 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi } versionData := make(map[string]interface{}) if config.Token != "novault" { - if valid, errValidateEnvironment := modCheck.ValidateEnvironment(modCheck.RawEnv, false, "", config.Log); errValidateEnvironment != nil || !valid { + if valid, baseDesiredPolicy, errValidateEnvironment := modCheck.ValidateEnvironment(modCheck.RawEnv, false, "", config.Log); errValidateEnvironment != nil || !valid { if errValidateEnvironment != nil { if urlErr, urlErrOk := errValidateEnvironment.(*url.Error); urlErrOk { if _, sErrOk := urlErr.Err.(*tls.CertificateVerificationError); sErrOk { @@ -64,8 +64,8 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi } } - if unrestrictedValid, errValidateUnrestrictedEnvironment := modCheck.ValidateEnvironment(modCheck.RawEnv, false, "_unrestricted", config.Log); errValidateUnrestrictedEnvironment != nil || !unrestrictedValid { - return nil, eUtils.LogAndSafeExit(config, "Mismatched token for requested environment: "+config.Env, 1) + if unrestrictedValid, desiredPolicy, errValidateUnrestrictedEnvironment := modCheck.ValidateEnvironment(modCheck.RawEnv, false, "_unrestricted", config.Log); errValidateUnrestrictedEnvironment != nil || !unrestrictedValid { + return nil, eUtils.LogAndSafeExit(config, fmt.Sprintf("Mismatched token for requested environment: %s base policy: %s policy: %s", config.Env, baseDesiredPolicy, desiredPolicy), 1) } } } diff --git a/trcflow/core/context.go b/trcflow/core/context.go index 8da6bc35f..e93319ef5 100644 --- a/trcflow/core/context.go +++ b/trcflow/core/context.go @@ -318,7 +318,7 @@ func (tfmContext *TrcFlowMachineContext) AddTableSchema(tableSchema sqle.Primary } else { tfContext.FlowLock.Unlock() } - case <-time.After(7 * time.Second): + case <-time.After(15 * time.Second): { tfContext.FlowState = flowcorehelper.CurrentFlowState{State: 0, SyncMode: "nosync", SyncFilter: ""} tfmContext.Log("Flow ready for use (but inactive due to invalid setup): "+tfContext.Flow.TableName(), nil) @@ -976,9 +976,11 @@ func (tfmContext *TrcFlowMachineContext) ProcessFlow( retryConnectionAccess: dbsourceConn, err := trcvutils.OpenDirectConnection(config, sourceDatabaseConnectionMap["dbsourceurl"].(string), sourceDatabaseConnectionMap["dbsourceuser"].(string), sourceDatabaseConnectionMap["dbsourcepassword"].(string)) - if retryCount < 3 && err != nil && dbsourceConn == nil { - retryCount = retryCount + 1 - goto retryConnectionAccess + if err.Error() != "incorrect URL format" { + if retryCount < 3 && err != nil && dbsourceConn == nil { + retryCount = retryCount + 1 + goto retryConnectionAccess + } } if err != nil { diff --git a/trcflow/core/dataFlowStatistics.go b/trcflow/core/dataFlowStatistics.go index 5df24dc24..8f8ecea88 100644 --- a/trcflow/core/dataFlowStatistics.go +++ b/trcflow/core/dataFlowStatistics.go @@ -551,6 +551,7 @@ func (dfs *TTDINode) FinishStatistic(tfmContext *TrcFlowMachineContext, tfContex mod.SectionPath = "" if vaultWriteBack { + mod.SectionPath = "" _, writeErr := mod.Write("super-secrets/PublicIndex/"+indexPath+"/"+idName+"/"+id+"/DataFlowStatistics/DataFlowGroup/"+decodedStatData["FlowGroup"].(string)+"/dataFlowName/"+decodedStatData["FlowName"].(string)+"/"+decodedStatData["StateCode"].(string), statMap, logger) if writeErr != nil && decodedData["LogFunc"] != nil { logFunc := decodedData["LogFunc"].(func(string, error)) diff --git a/trcflow/deploy/process.go b/trcflow/deploy/process.go index 5e467e21b..f1eeab27e 100644 --- a/trcflow/deploy/process.go +++ b/trcflow/deploy/process.go @@ -67,23 +67,27 @@ func PluginDeployEnvFlow(pluginConfig map[string]interface{}, logger *log.Logger if ok, err := servercapauth.ValidatePathSha(goMod, pluginConfig, logger); true || ok { onceAuth.Do(func() { - logger.Printf("Cap auth init for env: %s\n", pluginConfig["env"].(string)) - var featherAuth *servercapauth.FeatherAuth - featherAuth, err = servercapauth.Init(goMod, pluginConfig, logger) - if err != nil { - eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) - return - } - gCapInitted = true - - pluginConfig["trcHatSecretsPort"] = featherAuth.SecretsPort + if pluginConfig["env"].(string) == "dev" || pluginConfig["env"].(string) == "staging" { + // Ensure only dev is the cap auth... + logger.Printf("Cap auth init for env: %s\n", pluginConfig["env"].(string)) + var featherAuth *servercapauth.FeatherAuth = nil + if pluginConfig["env"].(string) == "dev" { + featherAuth, err = servercapauth.Init(goMod, pluginConfig, logger) + if err != nil { + eUtils.LogErrorMessage(config, "Skipping cap auth init.", false) + return + } + pluginConfig["trcHatSecretsPort"] = featherAuth.SecretsPort + } - servercapauth.Memorize(pluginConfig, logger) + servercapauth.Memorize(pluginConfig, logger) - // TODO: Support variables for different environments... - // Not really clear how cap auth would do this... - go servercapauth.Start(featherAuth, pluginConfig["env"].(string), logger) - logger.Printf("Cap auth init complete for env: %s\n", pluginConfig["env"].(string)) + // TODO: Support variables for different environments... + // Not really clear how cap auth would do this... + go servercapauth.Start(featherAuth, pluginConfig["env"].(string), logger) + logger.Printf("Cap auth init complete for env: %s\n", pluginConfig["env"].(string)) + gCapInitted = true + } }) } else { eUtils.LogErrorMessage(config, fmt.Sprintf("Mismatched sha256 cap auth for env: %s. Skipping.", pluginConfig["env"].(string)), false) @@ -348,6 +352,7 @@ func PluginDeployFlow(pluginConfig map[string]interface{}, logger *log.Logger) e if writeMap["trctype"].(string) == "agent" { writeMap["deployed"] = true } + cGoMod.SectionPath = "" _, err = cGoMod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", writeMap["trcplugin"].(string)), writeMap, cConfig.Log) if err != nil { logger.Printf(fmt.Sprintf("PluginDeployFlow failure: Failed to write plugin state for env: %s and plugin: %s error: %s\n", cConfig.Env, pluginName, err.Error())) @@ -443,7 +448,6 @@ func PluginDeployedUpdate(config *eUtils.DriverConfig, mod *helperkv.Modifier, v if hostRegion != "" { pluginData["deployed"] = true //Update deploy status if region exist otherwise this will block regionless deploys if set for regionless status } - statusUpdateErr := properties.WritePluginData(pluginData, replacedFields, mod, config.Log, hostRegion, pluginName) if err != nil { return statusUpdateErr diff --git a/trcflow/flumen/process.go b/trcflow/flumen/process.go index a422831cb..0a53ad6fe 100644 --- a/trcflow/flumen/process.go +++ b/trcflow/flumen/process.go @@ -49,8 +49,12 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error if pluginNameList, ok := pluginConfig["pluginNameList"].([]string); ok || true { tempAddr := pluginConfig["vaddress"] tempToken := pluginConfig["token"] - pluginConfig["vaddress"] = pluginConfig["caddress"] - pluginConfig["token"] = pluginConfig["ctoken"] + if caddress, cOk := pluginConfig["caddress"]; cOk { + pluginConfig["vaddress"] = caddress + } + if cToken, cOk := pluginConfig["ctoken"]; cOk { + pluginConfig["token"] = cToken + } pluginConfig["exitOnFailure"] = true cConfig, cGoMod, cVault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index 26d8441a3..a5719c153 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -513,8 +513,10 @@ func seedVaultWithCertsFromEntry(config *eUtils.DriverConfig, mod *helperkv.Modi certHost = strings.Replace(certHost, "*", "develop", 1) case "QA": certHost = strings.Replace(certHost, "*", "qa", 1) + case "RQA": + certHost = strings.Replace(certHost, "*", "qa", 1) case "auto": - certHost = strings.Replace(certHost, "*", "auto", 1) + certHost = strings.Replace(certHost, "*", "qa", 1) case "performance": certHost = strings.Replace(certHost, "*", "performance", 1) } diff --git a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl index 208971d24..1cd6df620 100755 --- a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl +++ b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/deploy.sh.tmpl @@ -22,8 +22,13 @@ fi FILESHAVAL=$(cat $FILESHA) +echo "Enter agent environment: " +read VAULT_ENV + +if [[ -z "${VAULT_ADDR}" ]]; then echo "Enter agent vault host base url: " read VAULT_ADDR +fi if [[ -z "${SECRET_VAULT_ADDR}" ]]; then echo "Enter organization vault host base url including port: " @@ -31,14 +36,22 @@ read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_ENV_TOKEN}" ]]; then -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read SECRET_VAULT_ENV_TOKEN fi + +if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi +read SECRET_VAULT_PLUGIN_TOKEN +fi + echo "Enter agent root token: " read VAULT_TOKEN -echo "Enter agent environment: " -read VAULT_ENV echo "Is this plugin an agent deployment tool (Y or N): " read VAULT_AGENT @@ -46,13 +59,16 @@ read VAULT_AGENT if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then PRE_CERTIFY="Y" else - echo "Enter trc plugin runtime environment token with write permissions unrestricted: " - read VAULT_ENV_TOKEN + if [[ -z "${VAULT_ENV_TOKEN}" ]]; then + echo "Enter trc plugin runtime environment token with write permissions unrestricted(config_token_"$VAULT_ENV"_unrestricted): " + read VAULT_ENV_TOKEN + fi if [ "$VAULT_PLUGIN_DIR" ] then echo "Deploying using local vault strategy." - PRE_CERTIFY="N" + echo "Precertify plugin (Y or N): " + read PRE_CERTIFY else echo "Precertify plugin (Y or N): " read PRE_CERTIFY @@ -61,19 +77,27 @@ fi if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = "y" ]; then - if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then - echo "Deploying agent deploy tool" + if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then + echo "Certifying agent deployment tool plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent certifystatus=$? - if [ $certifystatus -eq 0 ]; then - echo "No problems encountered." + if [ $certifystatus -eq 0 ]; then + echo "No certification problems encountered." exit $certifystatus else - echo "Unexpected certifyication errorerror." + echo "Unexpected certification error." exit $certifystatus fi else + echo "Certifying vault type plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) + certifystatus=$? + if [ $certifystatus -eq 0 ]; then + echo "No certification problems encountered." + else + echo "Unexpected certification error." + exit $certifystatus + fi fi fi @@ -83,23 +107,44 @@ fi if [ "$VAULT_PLUGIN_DIR" ] then - echo "Local plugin registration skipping certification." + echo "Local plugin registration skipping certified check." else echo "Checking plugin deploy status." - echo "Certifying plugin for env $VAULT_ENV." + echo "Checking deployment status on plugin for env $VAULT_ENV." trcplgtool -env=$VAULT_ENV -checkDeployed -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) status=$? - echo "Plugin certified with result $status." + echo "Plugin deployment had status result $status." - if [ $status -eq 0 ]; then - echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." - vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR - exit $status + if [ $status -eq 0 ]; then + echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." + + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi + + vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + vaultUpdateStatus=$? + if [ $vaultUpdateStatus -eq 0 ]; then + exit $status + else + echo "But it's not deployed to vault for $VAULT_ENV. Continuing..." + fi elif [ $status -eq 1 ]; then echo "Existing plugin does not match repository plugin - cannot continue." exit $status elif [ $status -eq 2 ]; then - echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi else echo "Unexpected error response." exit $status @@ -154,16 +199,10 @@ if [ "$SHAVAL" != "$FILESHAVAL" ]; then exit -1 fi -PROD_EXT="" -for x in "staging" "prod"; do - if [ $x = $VAULT_ENV ]; then - PROD_EXT="-prod" - fi -done echo "Registering new plugin." vault plugin register \ - -command=$TRC_PLUGIN_NAME$PROD_EXT \ + -command=$TRC_PLUGIN_NAME \ -sha256=$(echo $SHAVAL) \ -args=`backendUUID=789` \ $TRC_PLUGIN_NAME @@ -177,6 +216,10 @@ vault secrets enable \ #Activates/starts the deployed plugin. # Note: plugin should update deployed flag for itself. -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR -vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME \ No newline at end of file +echo "Activating plugin." +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + +echo "Notifying carrier." +vault write vaultcarrier/$VAULT_ENV plugin=$TRC_PLUGIN_NAME +echo "Deployment complete." diff --git a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl index 91904d82a..85ebab5be 100755 --- a/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl +++ b/trcinit/vaultfiles/azure/trcagent/trc_templates/Vault/Tierceron/deploy/refreshtoken.sh.tmpl @@ -8,21 +8,49 @@ if [ "$TRC_PLUGIN_NAME" = 'trc-vault-carrier-plugin' ] ; then exit 1 fi -echo "Enter organization vault host base url including port: " +echo "Enter environment: " +read VAULT_ENV + +if [[ -z "${SECRET_VAULT_ADDR}" ]]; then +echo "Enter organization vault host base url including port (hit enter if just refreshing org tokens): " +read SECRET_VAULT_ADDR +fi + +if [[ -z "${SECRET_ENV_TOKEN}" ]]; then +echo "Enter organization vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " +read SECRET_ENV_TOKEN +fi + +if [[ -z "${VAULT_ADDR}" ]]; then +echo "Enter agent vault host base url including port: " read VAULT_ADDR +fi -echo "Enter organization vault root token: " +if [[ -z "${VAULT_TOKEN}" ]]; then +echo "Enter agent vault root token: " read VAULT_TOKEN +fi -echo "Enter environment: " -read VAULT_ENV - -echo "Enter organization vault unrestricted environment token with write permissions: " +echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read VAULT_ENV_TOKEN -VAULT_API_ADDR=VAULT_ADDR +VAULT_API_ADDR=$VAULT_ADDR +VAULT_API_TOKEN=$VAULT_TOKEN export VAULT_ADDR export VAULT_TOKEN export VAULT_API_ADDR +export VAULT_API_TOKEN + +echo "Secret vault: " $SECRET_VAULT_ADDR +echo "Agent vault: " $VAULT_ADDR + +echo "Agent and secrets stored in secrets vault only? (Y or N): " +read SECRETS_ONLY + +if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_API_ADDR=$SECRET_VAULT_ADDR + SECRET_ENV_TOKEN=$VAULT_ENV_TOKEN +fi + +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR diff --git a/trcinitbase/init.go b/trcinitbase/init.go index d100ee9ee..2e4b3b6b8 100644 --- a/trcinitbase/init.go +++ b/trcinitbase/init.go @@ -693,9 +693,9 @@ func CommonMain(envPtr *string, } mod.Env = *envPtr mod.RawEnv = eUtils.GetRawEnv(*envPtr) - if valid, errValidateEnvironment := mod.ValidateEnvironment(mod.RawEnv, *uploadCertPtr, "", config.Log); errValidateEnvironment != nil || !valid { - if unrestrictedValid, errValidateUnrestrictedEnvironment := mod.ValidateEnvironment(mod.RawEnv, false, "_unrestricted", config.Log); errValidateUnrestrictedEnvironment != nil || !unrestrictedValid { - eUtils.LogAndSafeExit(config, "Mismatched token for requested environment: "+mod.Env, 1) + if valid, baseDesiredPolicy, errValidateEnvironment := mod.ValidateEnvironment(mod.RawEnv, *uploadCertPtr, "", config.Log); errValidateEnvironment != nil || !valid { + if unrestrictedValid, desiredPolicy, errValidateUnrestrictedEnvironment := mod.ValidateEnvironment(mod.RawEnv, false, "_unrestricted", config.Log); errValidateUnrestrictedEnvironment != nil || !unrestrictedValid { + eUtils.LogAndSafeExit(config, fmt.Sprintf("Mismatched token for requested environment: %s base policy: %s policy: %s", mod.Env, baseDesiredPolicy, desiredPolicy), 1) return } } diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 84682e6d1..6b01644f0 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -11,12 +11,14 @@ import ( "os/signal" "strings" "sync" + "sync/atomic" "syscall" "time" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/trimble-oss/tierceron-hat/cap" + captiplib "github.com/trimble-oss/tierceron-hat/captip/captiplib" "github.com/trimble-oss/tierceron/buildopts/coreopts" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" @@ -29,7 +31,6 @@ import ( "github.com/trimble-oss/tierceron/trcvault/opts/memonly" "github.com/trimble-oss/tierceron/trcvault/trcplgtoolbase" "github.com/trimble-oss/tierceron/trcvault/util" - "github.com/trimble-oss/tierceron/utils" eUtils "github.com/trimble-oss/tierceron/utils" helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" @@ -38,6 +39,10 @@ import ( var gAgentConfig *capauth.AgentConfigs = nil var gTrcshConfig *capauth.TrcShConfig +var ( + MODE_PERCH_STR string = string([]byte{cap.MODE_PERCH}) +) + func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.DriverConfig, error) { if len(env) == 0 { env = os.Getenv("TRC_ENV") @@ -88,110 +93,75 @@ func TrcshInitConfig(env string, region string, outputMemCache bool) (*eUtils.Dr return config, nil } -func ProcessDeployment(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { +// Logging of deployer controller activities.. +func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg string) { + if strings.HasSuffix(ctlFlapMode, cap.CTL_COMPLETE) { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(0) + } + + if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { + fmt.Printf("%s\n", msg) + } + featherCtx.Log.Printf("ctl: %s msg: %s\n", ctlFlapMode, strings.Trim(msg, "\n")) + if strings.Contains(msg, "encountered errors") { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(0) + } +} + +// Logging of deployer activities.. +func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { + if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH { + featherCtx.Log.Printf(msg) + } +} + +// deployCtl -- is the deployment controller or manager if you will. +func deployCtlInterrupted(featherCtx *cap.FeatherContext) error { + os.Exit(-1) + return nil +} + +// deployer -- does the work of deploying.. +func deployerInterrupted(featherCtx *cap.FeatherContext) error { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + return nil +} + +// EnableDeploy - initializes and starts running deployer for provided deployment and environment. +func EnableDeployer(env string, region string, token string, trcPath string, secretId *string, approleId *string, outputMemCache bool, deployment string) { config, err := TrcshInitConfig(env, region, outputMemCache) if err != nil { fmt.Printf("Initialization setup error: %s\n", err.Error()) } if len(deployment) > 0 { config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} - config.DeploymentCtlMessage = make(chan string, 5) + config.DeploymentCtlMessageChan = make(chan string, 5) + config.Log.Printf("Starting deployer: %s\n", deployment) } - go func() { - for { - perching: - endTimerChan := make(chan bool, 5) // Anticipate a few of these. - deployDoneChan := make(chan bool, 5) // Could be a few... - if featherMode, featherErr := cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_FLAP, deployment+"."+*gAgentConfig.Env, false, acceptRemote); featherErr == nil && strings.HasPrefix(featherMode, cap.MODE_GAZE) { - // Lookup trcPath from deployment - - // Process the script.... - // This will feed CtlMessages into the Timeout and CtlMessage subscriber - go func() { - go func() { - // Timeout and CtlMessage subscriber - select { - case <-endTimerChan: - break - case <-time.After(120 * time.Second): - ctlMsg := "Deployment timed out after 120 seconds" - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH+"_"+ctlMsg, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - go func() { config.DeploymentCtlMessage <- capauth.TrcCtlComplete }() - break - } - }() - - ProcessDeploy(config, region, "", deployment, trcPath, secretId, approleId, false) - }() - - for { - select { - case modeCtl := <-config.DeploymentCtlMessage: - flapMode := cap.MODE_FLAP + "_" + modeCtl - ctlFlapMode := flapMode - var err error = errors.New("init") - - for { - if err == nil && ctlFlapMode == cap.MODE_PERCH { - // Acknowledge perching... - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - ctlFlapMode = cap.MODE_PERCH - deployDoneChan <- true - break - } - - if err == nil && flapMode != ctlFlapMode && ctlFlapMode != " " { - // Flap, Gaze, etc... - interruptFun(twoHundredMilliInterruptTicker) - break - } else { - callFlap := strings.ReplaceAll(flapMode, ":", "#") - if err == nil { - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(secondInterruptTicker) - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - } - } - if modeCtl == capauth.TrcCtlComplete { - // Only exit with TrcCtlComplete. - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_GLIDE, deployment+"."+*gAgentConfig.Env, true, acceptRemote) - deployDoneChan <- true - } - case <-deployDoneChan: - go func() { endTimerChan <- true }() - goto perching - } - } - } else { - interruptFun(fiveSecondInterruptTicker) - } - } - }() + // + // Each deployer needs it's own context. + // + localHostAddr := "" + sessionIdentifier := deployment + "." + *gAgentConfig.Env + config.FeatherCtx = captiplib.FeatherCtlInit(interruptChan, + &localHostAddr, + gAgentConfig.EncryptPass, + gAgentConfig.EncryptSalt, + gAgentConfig.HostAddr, + gAgentConfig.HandshakeCode, + &sessionIdentifier, /*Session identifier */ + captiplib.AcceptRemote, + deployerInterrupted) + config.FeatherCtx.Log = config.Log + // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) + atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) + + go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, deployerEmote, nil) + + go ProcessDeploy(config.FeatherCtx, config, region, "", deployment, trcPath, secretId, approleId, false) } // This is a controller program that can act as any command line utility. @@ -204,7 +174,7 @@ func main() { fmt.Println("trcsh Version: " + "1.23") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string - if !utils.IsWindows() { + if !eUtils.IsWindows() { if os.Geteuid() == 0 { fmt.Println("Trcsh cannot be run as root.") os.Exit(-1) @@ -248,7 +218,7 @@ func main() { } //Open deploy script and parse it. - ProcessDeploy(config, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) + ProcessDeploy(nil, config, *regionPtr, "", "", *trcPathPtr, secretIDPtr, appRoleIDPtr, true) } else { deployments := os.Getenv("DEPLOYMENTS") agentToken := os.Getenv("AGENT_TOKEN") @@ -280,18 +250,25 @@ func main() { fmt.Println("trcsh on windows requires VAULT_ADDR address.") os.Exit(-1) } - gAgentConfig = &capauth.AgentConfigs{AgentToken: &agentToken} - + if err := capauth.ValidateVhost(address); err != nil { + fmt.Printf("trcsh on windows requires supported VAULT_ADDR address: %s\n", err.Error()) + os.Exit(-1) + } memprotectopts.MemProtect(nil, &agentToken) memprotectopts.MemProtect(nil, &address) shutdown := make(chan bool) - // Preload agent synchronization configs - gAgentConfig.LoadConfigs(address, agentToken, deployments, agentEnv) + // Preload agent synchronization configs... + var errAgentLoad error + gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(address, agentToken, deployments, agentEnv) + if errAgentLoad != nil { + fmt.Println("trcsh agent bootstrap failure.") + os.Exit(-1) + } deploymentsSlice := strings.Split(deployments, ",") for _, deployment := range deploymentsSlice { - ProcessDeployment(*gAgentConfig.Env, *regionPtr, deployment, *trcPathPtr, secretIDPtr, appRoleIDPtr, false, deployment) + EnableDeployer(*gAgentConfig.Env, *regionPtr, deployment, *trcPathPtr, secretIDPtr, appRoleIDPtr, false, deployment) } <-shutdown @@ -306,14 +283,10 @@ var fiveSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) var fifteenSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) var thirtySecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) -func acceptInterruptFun(tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { +func acceptInterruptFun(featherCtx *cap.FeatherContext, tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { select { case <-interruptChan: - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(1) case <-tickerContinue.C: // don't break... continue... @@ -328,73 +301,37 @@ func acceptInterruptFun(tickerContinue *time.Ticker, tickerBreak *time.Ticker, t return true, errors.New("not possible") } -func interruptFun(tickerInterrupt *time.Ticker) { +func interruptFun(featherCtx *cap.FeatherContext, tickerInterrupt *time.Ticker) { select { case <-interruptChan: - cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - cap.MODE_PERCH, *gAgentConfig.Deployments+"."+*gAgentConfig.Env, true, nil) + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(1) case <-tickerInterrupt.C: } } // acceptRemote - hook for instrumenting -func acceptRemote(mode int, remote string) (bool, error) { +func acceptRemote(featherCtx *cap.FeatherContext, mode int, remote string) (bool, error) { if mode == cap.FEATHER_CTL { - return acceptInterruptFun(multiSecondInterruptTicker, fifteenSecondInterruptTicker, thirtySecondInterruptTicker) + return acceptInterruptFun(featherCtx, multiSecondInterruptTicker, fifteenSecondInterruptTicker, thirtySecondInterruptTicker) } return true, nil } -func featherCtlCb(agentName string) error { +func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { if gAgentConfig == nil { return errors.New("incorrect agent initialization") - } else { - gAgentConfig.Deployments = &agentName } - flapMode := cap.MODE_GAZE - ctlFlapMode := flapMode - var err error = errors.New("init") - for { - if err == nil && ctlFlapMode == cap.MODE_GLIDE { - if err != nil { - fmt.Printf("\nDeployment error.\n") - } else { - fmt.Printf("\nDeployment complete.\n") - } - os.Exit(0) - } else { - callFlap := flapMode - if err == nil { - if strings.HasPrefix(ctlFlapMode, cap.MODE_FLAP) { - ctl := strings.Split(ctlFlapMode, "_") - if len(ctl) > 1 && ctl[1] != capauth.TrcCtlComplete { - fmt.Printf("%s\n", ctl[1]) - } - callFlap = cap.MODE_GAZE - } else { - callFlap = cap.MODE_GAZE - } - interruptFun(twoHundredMilliInterruptTicker) - } else { - if err.Error() != "init" { - interruptFun(multiSecondInterruptTicker) - callFlap = cap.MODE_GAZE - } - } - ctlFlapMode, err = cap.FeatherCtlEmit(*gAgentConfig.EncryptPass, - *gAgentConfig.EncryptSalt, - *gAgentConfig.HandshakeHostPort, - *gAgentConfig.HandshakeCode, - callFlap, agentName+"."+*gAgentConfig.Env, true, acceptRemote) - } + if featherCtx == nil { + return errors.New("incorrect feathering") } + sessionIdentifier := agentName + "." + *gAgentConfig.Env + featherCtx.SessionIdentifier = &sessionIdentifier + captiplib.FeatherCtl(featherCtx, deployerCtlEmote) + return nil } func roleBasedRunner(env string, @@ -434,6 +371,10 @@ func roleBasedRunner(env string, tokenConfig := token err = trcplgtoolbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, config) case "trcconfig": + if config.EnvRaw == "itdev" || config.EnvRaw == "staging" || config.EnvRaw == "prod" || + config.Env == "itdev" || config.Env == "staging" || config.Env == "prod" { + config.OutputMemCache = false + } err = trcconfigbase.CommonMain(&configEnv, &config.VaultAddress, &tokenConfig, &trcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, config) case "trcsub": config.EndDir = config.EndDir + "/trc_templates" @@ -487,9 +428,30 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, // Utilize elevated CToken to perform certifications if asked. config.FeatherCtlCb = featherCtlCb if gAgentConfig == nil { - gAgentConfig = &capauth.AgentConfigs{} - gAgentConfig.LoadConfigs(config.VaultAddress, *trcshConfig.CToken, "bootstrap", "dev") // Feathering always in dev environmnent. + var errAgentLoad error + // Prepare the configuration triggering mechanism. + // Bootstrap deployment is replaced during callback with the agent name. + gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(config.VaultAddress, *trcshConfig.CToken, "bootstrap", config.Env) + if errAgentLoad != nil { + fmt.Printf("Permissions failure. Incorrect deployment") + os.Exit(1) + } + if gAgentConfig.FeatherContext == nil { + fmt.Printf("Warning! Permissions failure. Incorrect feathering") + } + gAgentConfig.InterruptHandlerFunc = deployCtlInterrupted + } + config.FeatherCtx = captiplib.FeatherCtlInit(nil, + gAgentConfig.LocalHostAddr, + gAgentConfig.EncryptPass, + gAgentConfig.EncryptSalt, + gAgentConfig.HostAddr, + gAgentConfig.HandshakeCode, + new(string), gAgentConfig.AcceptRemoteFunc, gAgentConfig.InterruptHandlerFunc) + if config.Log != nil { + config.FeatherCtx.Log = config.Log } + err := roleBasedRunner(env, trcshConfig, region, config, control, agentToken, *trcshConfig.CToken, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) @@ -563,7 +525,7 @@ func processWindowsCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, // Returns: // // Nothing. -func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { +func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, region string, token string, deployment string, trcPath string, secretId *string, approleId *string, outputMemCache bool) { agentToken := false if token != "" { agentToken = true @@ -582,9 +544,9 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep // cToken := "" // configRole := "" // pubRole := "" - // fileBytes, err := ioutil.ReadFile("") + // fileBytes, _ := os.ReadFile("") // kc := base64.StdEncoding.EncodeToString(fileBytes) - // gTrcshConfig = &trcshauth.TrcShConfig{Env: "dev", + // gTrcshConfig = &capauth.TrcShConfig{Env: "dev", // EnvContext: "dev", // CToken: &cToken, // ConfigRole: &configRole, @@ -592,6 +554,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep // KubeConfig: &kc, // } // config.VaultAddress = "" + // gTrcshConfig.VaultAddress = &config.VaultAddress // config.Token = "" // Chewbacca: end scrub var err error @@ -599,7 +562,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep for { if gTrcshConfig == nil || !gTrcshConfig.IsValid(gAgentConfig) { // Loop until we have something usable... - gTrcshConfig, err = trcshauth.TrcshAuth(gAgentConfig, config) + gTrcshConfig, err = trcshauth.TrcshAuth(featherCtx, gAgentConfig, config) if err != nil { config.Log.Printf(".") time.Sleep(time.Second) @@ -655,6 +618,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep if (len(os.Args) > 1 || len(trcPath) > 0) && !strings.Contains(pwd, "TrcDeploy") { // Generate trc code... + config.Log.Println("Preload setup") trcPathParts := strings.Split(trcPath, "/") config.FileFilter = []string{trcPathParts[len(trcPathParts)-1]} configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") @@ -662,7 +626,13 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep config.OutputMemCache = true config.StartDir = []string{"trc_templates"} config.EndDir = "." - trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + config.Log.Println("Preloading") + configErr := trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) + if configErr != nil { + fmt.Println("Preload failed. Couldn't find required resource.") + config.Log.Printf("Preload Error %s\n", configErr.Error()) + os.Exit(-1) + } ResetModifier(config) //Resetting modifier cache to avoid token conflicts. if !agentToken { token = "" @@ -713,6 +683,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep fmt.Println("Unable to obtain config for deployment") return } + deploymentConfig["trcpluginalias"] = deployment config.DeploymentConfig = deploymentConfig if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} @@ -760,6 +731,18 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep var onceKubeInit sync.Once var PipeOS billy.File +collaboratorReRun: + if featherCtx != nil { + // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) + for { + if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { + break + } else { + time.Sleep(time.Second * 3) + } + } + + } for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") if strings.HasPrefix(deployPipeline, "#") || deployPipeline == "" { @@ -813,7 +796,7 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep os.Args = deployArgs } } - if utils.IsWindows() { + if eUtils.IsWindows() { // Log for traceability. config.Log.Println(deployLine) err := processWindowsCmds( @@ -831,11 +814,19 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep strings.Split(deployLine, " "), &configCount) if err != nil { - config.DeploymentCtlMessage <- fmt.Sprintf("%s\nEncountered errors: %s\n", deployLine, err.Error()) + if strings.Contains(err.Error(), "Forbidden") { + // Critical agent setup error. + os.Exit(-1) + } + errMessage := err.Error() + errMessageFiltered := strings.ReplaceAll(errMessage, ":", "-") + config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors - %s\n", deployLine, errMessageFiltered) + goto collaboratorReRun } else { - config.DeploymentCtlMessage <- deployLine + config.DeploymentCtlMessageChan <- deployLine } } else { + config.FeatherCtx = featherCtx processPluginCmds( &trcKubeDeploymentConfig, &onceKubeInit, @@ -853,8 +844,16 @@ func ProcessDeploy(config *eUtils.DriverConfig, region string, token string, dep } } } - if utils.IsWindows() { - config.DeploymentCtlMessage <- capauth.TrcCtlComplete + if eUtils.IsWindows() { + config.DeploymentCtlMessageChan <- cap.CTL_COMPLETE + for { + if atomic.LoadInt64(&featherCtx.RunState) == cap.RUNNING { + time.Sleep(time.Second) + } else { + break + } + } + goto collaboratorReRun } //Make the arguments in the script -> os.args. diff --git a/trcsh/trcshauth/auth.go b/trcsh/trcshauth/auth.go index e5bd8d543..3cb55bb19 100755 --- a/trcsh/trcshauth/auth.go +++ b/trcsh/trcshauth/auth.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" eUtils "github.com/trimble-oss/tierceron/utils" @@ -86,10 +87,10 @@ func GetSetEnvAddrContext(env string, envContext string, addrPort string) (strin return env, envContext, addrPort, nil } -func retryingPenseFeatherQuery(agentConfigs *capauth.AgentConfigs, pense string) (*string, error) { +func retryingPenseFeatherQuery(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfigs, pense string) (*string, error) { retry := 0 for retry < 5 { - result, err := agentConfigs.PenseFeatherQuery(pense) + result, err := agentConfigs.PenseFeatherQuery(featherCtx, pense) if err != nil || result == nil || *result == "...." { time.Sleep(time.Second) @@ -102,7 +103,7 @@ func retryingPenseFeatherQuery(agentConfigs *capauth.AgentConfigs, pense string) } // Helper function for obtaining auth components. -func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { +func TrcshAuth(featherCtx *cap.FeatherContext, agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) (*capauth.TrcShConfig, error) { trcshConfig := &capauth.TrcShConfig{} var err error @@ -124,7 +125,7 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) return trcshConfig, nil } } else { - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 1") trcshConfig.KubeConfig, err = capauth.PenseQuery(config, "kubeconfig") } @@ -137,8 +138,8 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) memprotectopts.MemProtect(nil, trcshConfig.KubeConfig) } - if agentConfigs != nil { - trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(agentConfigs, "caddress") + if featherCtx != nil { + trcshConfig.VaultAddress, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "caddress") } else { config.Log.Println("Auth phase 2") trcshConfig.VaultAddress, err = capauth.PenseQuery(config, "caddress") @@ -170,8 +171,8 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) config.VaultAddress = *trcshConfig.VaultAddress memprotectopts.MemProtect(nil, &config.VaultAddress) - if agentConfigs != nil { - trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(agentConfigs, "configrole") + if featherCtx != nil { + trcshConfig.ConfigRole, err = retryingPenseFeatherQuery(featherCtx, agentConfigs, "configrole") } else { config.Log.Println("Auth phase 3") trcshConfig.ConfigRole, err = capauth.PenseQuery(config, "configrole") @@ -182,7 +183,7 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) memprotectopts.MemProtect(nil, trcshConfig.ConfigRole) - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 4") trcshConfig.PubRole, err = capauth.PenseQuery(config, "pubrole") if err != nil { @@ -191,7 +192,7 @@ func TrcshAuth(agentConfigs *capauth.AgentConfigs, config *eUtils.DriverConfig) memprotectopts.MemProtect(nil, trcshConfig.PubRole) } - if agentConfigs == nil { + if featherCtx == nil { config.Log.Println("Auth phase 5") trcshConfig.CToken, err = capauth.PenseQuery(config, "ctoken") if err != nil { diff --git a/trcsubbase/trcsub.go b/trcsubbase/trcsub.go index 7f7cbea97..9387a29b5 100644 --- a/trcsubbase/trcsub.go +++ b/trcsubbase/trcsub.go @@ -1,6 +1,7 @@ package trcsubbase import ( + "errors" "flag" "fmt" "log" @@ -32,6 +33,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, } fmt.Println("Version: " + "1.6") + exitOnFailure := false if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) flagset.Usage = func() { @@ -42,6 +44,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, flagset.String("addr", "", "API endpoint for the vault") flagset.String("secretID", "", "Public app role ID") flagset.String("appRoleID", "", "Secret app role ID") + exitOnFailure = true } endDirPtr := flagset.String("endDir", coreopts.GetFolderPrefix(nil)+"_templates", "Directory to put configured templates into") tokenPtr := flagset.String("token", "", "Vault access token") @@ -57,7 +60,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") - os.Exit(1) + return errors.New("must specify either -projectInfo or -templateFilter flag") } var configBase *eUtils.DriverConfig var appRoleConfigPtr *string @@ -85,7 +88,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, configBase = &eUtils.DriverConfig{Insecure: *insecurePtr, EndDir: *endDirPtr, Log: logger, - ExitOnFailure: true} + ExitOnFailure: exitOnFailure} appRoleConfigPtr = new(string) } diff --git a/trcvault/deploy/deploy.sh b/trcvault/deploy/deploy.sh index ef31a6c1b..1cd6df620 100755 --- a/trcvault/deploy/deploy.sh +++ b/trcvault/deploy/deploy.sh @@ -41,7 +41,11 @@ read SECRET_VAULT_ENV_TOKEN fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then -echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then +echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi read SECRET_VAULT_PLUGIN_TOKEN fi @@ -73,11 +77,11 @@ fi if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = "y" ]; then - if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then + if [ "$VAULT_AGENT" = 'Y' ] || [ "$VAULT_AGENT" = 'y' ]; then echo "Certifying agent deployment tool plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) -pluginType=agent certifystatus=$? - if [ $certifystatus -eq 0 ]; then + if [ $certifystatus -eq 0 ]; then echo "No certification problems encountered." exit $certifystatus else @@ -88,7 +92,7 @@ if [ "$PRE_CERTIFY" = "Y" ] || [ "$PRE_CERTIFY" = "yes" ] || [ "$PRE_CERTIFY" = echo "Certifying vault type plugin..." trcplgtool -env=$VAULT_ENV -certify -addr=$SECRET_VAULT_ADDR -token=$SECRET_VAULT_ENV_TOKEN -pluginName=$TRC_PLUGIN_NAME -sha256=$(cat target/$TRC_PLUGIN_NAME.sha256) certifystatus=$? - if [ $certifystatus -eq 0 ]; then + if [ $certifystatus -eq 0 ]; then echo "No certification problems encountered." else echo "Unexpected certification error." @@ -111,15 +115,36 @@ else status=$? echo "Plugin deployment had status result $status." - if [ $status -eq 0 ]; then - echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." - vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR - exit $status + if [ $status -eq 0 ]; then + echo "This version of the plugin has already been deployed - enabling for environment $VAULT_ENV." + + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi + + vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_VAULT_PLUGIN_TOKEN + vaultUpdateStatus=$? + if [ $vaultUpdateStatus -eq 0 ]; then + exit $status + else + echo "But it's not deployed to vault for $VAULT_ENV. Continuing..." + fi elif [ $status -eq 1 ]; then echo "Existing plugin does not match repository plugin - cannot continue." exit $status elif [ $status -eq 2 ]; then - echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "This version of the plugin has not been deployed for environment $VAULT_ENV. Beginning installation process." + echo "Agent and secrets stored in secrets vault only? (Y or N): " + read SECRETS_ONLY + + if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then + VAULT_ADDR=$SECRET_VAULT_ADDR + SECRET_VAULT_PLUGIN_TOKEN=$VAULT_ENV_TOKEN + fi else echo "Unexpected error response." exit $status diff --git a/trcvault/deploy/refreshcarriertoken.sh b/trcvault/deploy/refreshcarriertoken.sh index 5e9e6d9f6..3860bfb68 100755 --- a/trcvault/deploy/refreshcarriertoken.sh +++ b/trcvault/deploy/refreshcarriertoken.sh @@ -15,7 +15,11 @@ read SECRET_VAULT_ADDR fi if [[ -z "${SECRET_VAULT_PLUGIN_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then echo "Enter organization vault plugin token for certification(config_token_plugin$VAULT_ENV): " +else +echo "Enter organization vault plugin token for certification(config_token_pluginany): " +fi read SECRET_VAULT_PLUGIN_TOKEN fi @@ -25,7 +29,11 @@ read VAULT_TOKEN fi if [[ -z "${VAULT_ENV_TOKEN}" ]]; then +if [ "$VAULT_ENV" = "staging" ] || [ "$VAULT_ENV" = "prod" ]; then echo "Enter agent vault *plugin* environment token with tightly confined write permissions(config_token_plugin$VAULT_ENV): " +else +echo "Enter agent vault *plugin* environment token with tightly confined write permissions(config_token_pluginany): " +fi read VAULT_ENV_TOKEN fi diff --git a/trcvault/deploy/refreshtoken.sh b/trcvault/deploy/refreshtoken.sh index 091426635..85ebab5be 100755 --- a/trcvault/deploy/refreshtoken.sh +++ b/trcvault/deploy/refreshtoken.sh @@ -49,8 +49,8 @@ read SECRETS_ONLY if [ "$SECRETS_ONLY" = "Y" ] || [ "$SECRETS_ONLY" = "yes" ] || [ "$SECRETS_ONLY" = "y" ]; then VAULT_API_ADDR=$SECRET_VAULT_ADDR - VAULT_API_TOKEN=$SECRET_ENV_TOKEN + SECRET_ENV_TOKEN=$VAULT_ENV_TOKEN fi -vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_API_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME +vault write $TRC_PLUGIN_NAME/$VAULT_ENV token=$VAULT_ENV_TOKEN vaddress=$VAULT_API_ADDR caddress=$SECRET_VAULT_ADDR ctoken=$SECRET_ENV_TOKEN plugin=$TRC_PLUGIN_NAME diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 3cbb52163..3ad6d24ed 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -10,6 +10,7 @@ import ( "log" "os" "os/exec" + "path/filepath" "strings" "github.com/trimble-oss/tierceron/buildopts" @@ -52,6 +53,7 @@ func CommonMain(envPtr *string, codebundledeployPtr := flagset.Bool("codebundledeploy", false, "To deploy a code bundle.") agentdeployPtr := flagset.Bool("agentdeploy", false, "To initiate deployment on agent.") projectservicePtr := flagset.String("projectservice", "", "Provide template root path in form project/service") + deploysubpathPtr := flagset.String("deploysubpath", "", "Subpath under root to deliver code bundles.") // Common flags... startDirPtr := flagset.String("startDir", coreopts.GetFolderPrefix(nil)+"_templates", "Template directory") @@ -65,6 +67,7 @@ func CommonMain(envPtr *string, // Common plugin flags... pluginNamePtr := flagset.String("pluginName", "", "Used to certify vault plugin") + pluginNameAliasPtr := flagset.String("pluginNameAlias", "", "Name used to define an alias for a plugin") pluginTypePtr := flagset.String("pluginType", "vault", "Used to indicate type of plugin. Default is vault.") // Certify flags... @@ -104,6 +107,11 @@ func CommonMain(envPtr *string, } } + if c != nil && c.DeploymentConfig["trcpluginalias"] != nil { + // Prefer internal definition of alias + *pluginNameAliasPtr = c.DeploymentConfig["trcpluginalias"].(string) + } + if *certifyImagePtr && (len(*pluginNamePtr) == 0 || len(*sha256Ptr) == 0) { fmt.Println("Must use -pluginName && -sha256 flags to use -certify flag") return errors.New("must use -pluginName && -sha256 flags to use -certify flag") @@ -172,7 +180,11 @@ func CommonMain(envPtr *string, if c != nil { configBase = c logger = c.Log - configBase.SubSectionValue = *pluginNamePtr + if *pluginNameAliasPtr != "" { + configBase.SubSectionValue = *pluginNameAliasPtr + } else { + configBase.SubSectionValue = *pluginNamePtr + } appRoleConfigPtr = &(configBase.AppRoleConfig) *insecurePtr = configBase.Insecure } else { @@ -223,13 +235,19 @@ func CommonMain(envPtr *string, } config, mod, vault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) config.FeatherCtlCb = configBase.FeatherCtlCb + config.FeatherCtx = configBase.FeatherCtx + if err != nil { logger.Println("Error: " + err.Error() + " - 1") logger.Println("Failed to init mod for deploy update") return err } config.StartDir = []string{*startDirPtr} - config.SubSectionValue = *pluginNamePtr + if *pluginNameAliasPtr != "" { + configBase.SubSectionValue = *pluginNameAliasPtr + } else { + configBase.SubSectionValue = *pluginNamePtr + } mod.Env = *envPtr eUtils.CheckError(config, err, true) @@ -286,9 +304,10 @@ func CommonMain(envPtr *string, pluginToolConfig["trcsha256"] = *sha256Ptr pluginToolConfig["pluginNamePtr"] = *pluginNamePtr - pluginToolConfig["deployrootPtr"] = *deployrootPtr pluginToolConfig["serviceNamePtr"] = *serviceNamePtr pluginToolConfig["projectservicePtr"] = *projectservicePtr + pluginToolConfig["deployrootPtr"] = *deployrootPtr + pluginToolConfig["deploysubpathPtr"] = *deploysubpathPtr pluginToolConfig["codeBundlePtr"] = *codeBundlePtr if _, ok := pluginToolConfig["trcplugin"].(string); !ok { @@ -310,7 +329,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["deployrootPtr"].(string); ok { pluginToolConfig["trcdeployroot"] = pluginToolConfig["deployrootPtr"].(string) } - + if _, ok := pluginToolConfig["deploysubpathPtr"]; ok { + pluginToolConfig["trcdeploysubpath"] = pluginToolConfig["deploysubpathPtr"] + } if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) } @@ -330,10 +351,12 @@ func CommonMain(envPtr *string, writeMap["trcplugin"] = *pluginNamePtr writeMap["trctype"] = *pluginTypePtr writeMap["trcprojectservice"] = *projectservicePtr - if _, ok := pluginToolConfig["trcdeployroot"]; ok { writeMap["trcdeployroot"] = pluginToolConfig["trcdeployroot"] } + if _, ok := pluginToolConfig["trcdeploysubpath"]; ok { + writeMap["trcdeploysubpath"] = pluginToolConfig["trcdeploysubpath"] + } if _, ok := pluginToolConfig["trcservicename"]; ok { writeMap["trcservicename"] = pluginToolConfig["trcservicename"] } @@ -351,9 +374,9 @@ func CommonMain(envPtr *string, fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("sc", "stop", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) err := cmd.Run() - if err != nil { + if err != nil && !strings.Contains(err.Error(), "1") && !strings.Contains(err.Error(), "5") { fmt.Println(err) return err } @@ -363,7 +386,7 @@ func CommonMain(envPtr *string, fmt.Printf("Starting service %s\n", pluginToolConfig["trcservicename"].(string)) cmd := exec.Command("sc", "start", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil { + if err != nil && !strings.Contains(err.Error(), "1056") { fmt.Println(err) return err } @@ -378,7 +401,11 @@ func CommonMain(envPtr *string, err := repository.GetImageAndShaFromDownload(configBase, pluginToolConfig) if err != nil { fmt.Println("Image download failure.") - fmt.Println(err.Error()) + if configBase.FeatherCtx != nil { + configBase.FeatherCtx.Log.Printf("Image download failure: %s", err.Error()) + } else { + fmt.Println(err.Error()) + } return err } } @@ -387,7 +414,12 @@ func CommonMain(envPtr *string, pluginToolConfig["imagesha256"] != nil && pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) { // Write the image to the destination... - deployPath := fmt.Sprintf("%s\\%s", pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) + var deployPath string + if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok { + deployPath = filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string), pluginToolConfig["trccodebundle"].(string)) + } else { + deployPath = filepath.Join(pluginToolConfig["trcdeployroot"].(string), pluginToolConfig["trccodebundle"].(string)) + } fmt.Printf("Deploying image to: %s\n", deployPath) err = os.WriteFile(deployPath, pluginToolConfig["rawImageFile"].([]byte), 0644) @@ -396,9 +428,15 @@ func CommonMain(envPtr *string, fmt.Println("Image write failure.") return err } - fmt.Println("Image deployed.") + fmt.Printf("Image deployed to: %s\n", deployPath) } else { - fmt.Printf("Image not certified. Cannot deploy image for %s\n", pluginToolConfig["trcplugin"]) + errMessage := fmt.Sprintf("image not certified. cannot deploy image for %s", pluginToolConfig["trcplugin"]) + if configBase.FeatherCtx != nil { + configBase.FeatherCtx.Log.Printf(errMessage) + } else { + fmt.Printf("%s\n", errMessage) + } + return errors.New(errMessage) } } else if *certifyImagePtr { //Certify Image @@ -438,7 +476,11 @@ func CommonMain(envPtr *string, writeMap, replacedFields := properties.GetPluginData(*regionPtr, "Certify", "config", logger) - writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginToolConfig["trcplugin"].(string)) + pluginTarget := pluginToolConfig["trcplugin"].(string) + if strings.HasPrefix(*pluginNamePtr, pluginTarget) { + pluginTarget = *pluginNamePtr + } + writeErr := properties.WritePluginData(WriteMapUpdate(writeMap, pluginToolConfig, *defineServicePtr, *pluginTypePtr), replacedFields, mod, config.Log, *regionPtr, pluginTarget) if writeErr != nil { fmt.Println(writeErr) return err @@ -463,9 +505,9 @@ func CommonMain(envPtr *string, } } else if *agentdeployPtr { if config.FeatherCtlCb != nil { - err := config.FeatherCtlCb(*pluginNamePtr) + err := config.FeatherCtlCb(config.FeatherCtx, *pluginNamePtr) if err != nil { - fmt.Println("Incorrect installation") + fmt.Printf("Incorrect installation: %s\n", err.Error()) return err } } else { diff --git a/trcvault/util/dbutil.go b/trcvault/util/dbutil.go index 264f554ef..79f5f4047 100644 --- a/trcvault/util/dbutil.go +++ b/trcvault/util/dbutil.go @@ -14,10 +14,13 @@ import ( // OpenDirectConnection opens connection to a database using various sql urls used by Spectrum. func OpenDirectConnection(config *eUtils.DriverConfig, url string, username string, password string) (*sql.DB, error) { - driver, server, port, dbname := validator.ParseURL(config, url) + driver, server, port, dbname, err := validator.ParseURL(config, url) + + if err != nil { + return nil, err + } var conn *sql.DB - var err error switch driver { case "mysql", "mariadb": diff --git a/trcvault/util/properties.go b/trcvault/util/properties.go index 779d1f83e..3e0023337 100644 --- a/trcvault/util/properties.go +++ b/trcvault/util/properties.go @@ -154,6 +154,8 @@ func (p *Properties) GetPluginData(region string, service string, config string, func (p *Properties) WritePluginData(pluginData map[string]interface{}, replacedFields map[string]interface{}, mod *helperkv.Modifier, log *log.Logger, hostRegion string, pluginName string) error { //writeMap := make(map[string]interface{}) + // If SectionPath is set like + mod.SectionPath = "" regionSuffix := "" if hostRegion != "" { regionSuffix = "~" + hostRegion @@ -175,6 +177,7 @@ func (p *Properties) WritePluginData(pluginData map[string]interface{}, replaced writeMap[field] = value } + mod.SectionPath = "" _, writeErr := mod.Write(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", pluginName), writeMap, log) if writeErr != nil { return writeErr diff --git a/utils/driverconfig.go b/utils/driverconfig.go index 7f594b571..cee507925 100644 --- a/utils/driverconfig.go +++ b/utils/driverconfig.go @@ -10,6 +10,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/pavlo-v-chernykh/keystore-go/v4" + "github.com/trimble-oss/tierceron-hat/cap" "github.com/trimble-oss/tierceron/buildopts/coreopts" ) @@ -74,7 +75,8 @@ type DriverConfig struct { EndDir string OutputMemCache bool MemFs billy.Filesystem - FeatherCtlCb func(string) error + FeatherCtx *cap.FeatherContext + FeatherCtlCb func(*cap.FeatherContext, string) error // Config modes.... WantCerts bool @@ -110,8 +112,8 @@ type DriverConfig struct { ServiceFilter []string // Which tables to use. DynamicPathFilter string // Seeds from a specific path. - DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on - DeploymentCtlMessage chan string + DeploymentConfig map[string]interface{} // For trcsh to indicate which deployment to work on + DeploymentCtlMessageChan chan string } // ConfigControl Setup initializes the directory structures in preparation for parsing templates. diff --git a/validator/db.go b/validator/db.go index e2afa2b63..f1570347c 100644 --- a/validator/db.go +++ b/validator/db.go @@ -19,8 +19,10 @@ import ( // Heartbeat validates the database connection func Heartbeat(config *eUtils.DriverConfig, url string, username string, password string) (bool, error) { //extract driver, server, port and dbname with regex - driver, server, port, dbname := ParseURL(config, url) - var err error + driver, server, port, dbname, err := ParseURL(config, url) + if err != nil { + return false, err + } var conn *sql.DB switch driver { case "mysql": @@ -53,12 +55,14 @@ func Heartbeat(config *eUtils.DriverConfig, url string, username string, passwor } return true, nil } -func ParseURL(config *eUtils.DriverConfig, url string) (string, string, string, string) { +func ParseURL(config *eUtils.DriverConfig, url string) (string, string, string, string, error) { //only works with jdbc:mysql or jdbc:sqlserver. regex := regexp.MustCompile(`(?i)(?:jdbc:(mysql|sqlserver|mariadb))://([\w\-\.]+)(?::(\d{0,5}))?(?:/|.*;DatabaseName=)(\w+).*`) m := regex.FindStringSubmatch(url) if m == nil { - eUtils.LogErrorObject(config, errors.New("incorrect URL format"), false) + err := errors.New("incorrect URL format") + eUtils.LogErrorObject(config, err, false) + return "", "", "", "", err } - return m[1], m[2], m[3], m[4] + return m[1], m[2], m[3], m[4], nil } diff --git a/vaulthelper/kv/Modifier.go b/vaulthelper/kv/Modifier.go index 1eb0db581..95acbec5b 100644 --- a/vaulthelper/kv/Modifier.go +++ b/vaulthelper/kv/Modifier.go @@ -39,19 +39,20 @@ type Modifier struct { httpClient *http.Client // Handle to http client. client *api.Client // Client connected to vault logical *api.Logical // Logical used for read/write options - Env string // Environment (local/dev/QA; Initialized to secrets) - Regions []string // Supported regions - SecretDictionary *api.Secret // Current Secret Dictionary Cache. - Version string // Version for data - VersionFilter []string // Used to filter vault paths - RawEnv string - TemplatePath string // Path to template we are processing. - ProjectIndex []string // Which projects are indexed. - SectionKey string // The section key: Index or Restricted. - SectionName string // The name of the actual section. - SubSectionName string // The name of the actual subsection. - SubSectionValue string // The actual value for the sub section. - SectionPath string // The path to the Index (both seed and vault) + SecretDictionary *api.Secret // Current Secret Dictionary Cache -- populated by mod.List("templates" + + Env string // Environment (local/dev/QA; Initialized to secrets) + RawEnv string + Regions []string // Supported regions + Version string // Version for data + VersionFilter []string // Used to filter vault paths + TemplatePath string // Path to template we are processing. + ProjectIndex []string // Which projects are indexed. + SectionKey string // The section key: Index or Restricted. + SectionName string // The name of the actual section. + SubSectionName string // The name of the actual subsection. + SubSectionValue string // The actual value for the sub section. + SectionPath string // The path to the Index (both seed and vault) } type modCache struct { @@ -97,6 +98,19 @@ func NewModifier(insecure bool, token string, address string, env string, region PruneCache(env, 10) checkoutModifier, err := cachedModifierHelper(env) if err == nil && checkoutModifier != nil { + checkoutModifier.Insecure = insecure + checkoutModifier.RawEnv = env + checkoutModifier.Regions = regions + checkoutModifier.Version = "" // Version for data + checkoutModifier.VersionFilter = []string{} // Used to filter vault paths + checkoutModifier.TemplatePath = "" // Path to template we are processing. + checkoutModifier.ProjectIndex = []string{} // Which projects are indexed. + checkoutModifier.SectionKey = "" // The section key: Index or Restricted. + checkoutModifier.SectionName = "" // The name of the actual section. + checkoutModifier.SubSectionName = "" // The name of the actual subsection. + checkoutModifier.SubSectionValue = "" // The actual value for the sub section. + checkoutModifier.SectionPath = "" // The path to the Index (both seed and vault) + return checkoutModifier, nil } } @@ -214,12 +228,12 @@ func (m *Modifier) CleanCache(limit uint64) { } // ValidateEnvironment Ensures token has access to requested data. -func (m *Modifier) ValidateEnvironment(environment string, init bool, policySuffix string, logger *log.Logger) (bool, error) { +func (m *Modifier) ValidateEnvironment(environment string, init bool, policySuffix string, logger *log.Logger) (bool, string, error) { env, sub, _, envErr := PreCheckEnvironment(environment) if envErr != nil { logger.Printf("Environment format error: %v\n", envErr) - return false, envErr + return false, "", envErr } else { if sub != "" { environment = env @@ -241,10 +255,10 @@ func (m *Modifier) ValidateEnvironment(environment string, init bool, policySuff logger.Printf("LookupSelf Auth failure: %v\n", err) if urlErr, urlErrOk := err.(*url.Error); urlErrOk { if _, sErrOk := urlErr.Err.(*tls.CertificateVerificationError); sErrOk { - return false, err + return false, desiredPolicy, err } } else if strings.Contains(err.Error(), "x509: certificate") { - return false, err + return false, desiredPolicy, err } } @@ -265,7 +279,7 @@ func (m *Modifier) ValidateEnvironment(environment string, init bool, policySuff } - return valid, nil + return valid, desiredPolicy, nil } // Writes the key,value pairs in data to the vault From 642ff006f5856cf28006d879433faf1ee4890f86 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Sun, 3 Dec 2023 16:54:38 -0800 Subject: [PATCH 38/55] Prepare for release (#842) * Fixing break on windows. * Add more robust support for servicesWanted and filter. * Deployments should match a deployment run. Always load deployment during a run. --------- --- buildopts/coreopts/options_common.go | 3 +- buildopts/coreopts/options_tc.go | 3 +- trcconfig/trcconfig.go | 2 +- trcconfigbase/trcconfig.go | 5 +- trcconfigbase/utils/configinator.go | 29 +-------- trcconfigbase/utils/filter.go | 53 ++++++++++++++++ trcsh/deployutil/deployutil.go | 66 +++++++++++++++++++ trcsh/trcsh.go | 95 ++++++++-------------------- 8 files changed, 155 insertions(+), 101 deletions(-) create mode 100644 trcconfigbase/utils/filter.go create mode 100644 trcsh/deployutil/deployutil.go diff --git a/buildopts/coreopts/options_common.go b/buildopts/coreopts/options_common.go index 71b7a52ea..09124bb36 100644 --- a/buildopts/coreopts/options_common.go +++ b/buildopts/coreopts/options_common.go @@ -6,6 +6,7 @@ package coreopts import ( "database/sql" "errors" + "os" "strings" ) @@ -17,7 +18,7 @@ func GetFolderPrefix(custom []string) string { endTi = 0 for endTi = ti; endTi > 0; endTi-- { - if custom[0][endTi] == '/' { + if custom[0][endTi] == '/' || custom[0][endTi] == os.PathSeparator { endTi = endTi + 1 break } diff --git a/buildopts/coreopts/options_tc.go b/buildopts/coreopts/options_tc.go index 76406af2d..6cd4d42ef 100644 --- a/buildopts/coreopts/options_tc.go +++ b/buildopts/coreopts/options_tc.go @@ -4,6 +4,7 @@ package coreopts import ( + "os" "strings" bcore "VaultConfig.Bootstrap/configcore" @@ -21,7 +22,7 @@ func GetFolderPrefix(custom []string) string { endTi = 0 for endTi = ti; endTi > 0; endTi-- { - if custom[0][endTi] == '/' { + if custom[0][endTi] == '/' || custom[0][endTi] == os.PathSeparator { endTi = endTi + 1 break } diff --git a/trcconfig/trcconfig.go b/trcconfig/trcconfig.go index a1b0e0490..2e56f95be 100644 --- a/trcconfig/trcconfig.go +++ b/trcconfig/trcconfig.go @@ -18,7 +18,7 @@ func main() { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.27") + fmt.Println("Version: " + "1.28") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) flagset.Usage = func() { fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index c9dcb3f73..24efcb542 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -320,10 +320,6 @@ func CommonMain(envPtr *string, services = strings.Split(*servicesWanted, ",") } - // TODO: This wasn't doing anything useful... possibly remove? - // for _, service := range services { - // service = strings.TrimSpace(service) - // } regions := []string{} if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") { @@ -408,6 +404,7 @@ func CommonMain(envPtr *string, Update: messenger, FileFilter: fileFilterSlice, } + configSlice = append(configSlice, config) configCtx.ConfigWg.Add(1) go func() { diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index f4077e8f4..d77747f14 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -107,34 +107,9 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi } //File filter - fileFound := true - fileFilterIndex := make([]int, len(config.FileFilter)) - fileFilterCounter := 0 - if len(config.FileFilter) != 0 && config.FileFilter[0] != "" { - for _, FileFilter := range config.FileFilter { - for i, templatePath := range templatePaths { - if strings.Contains(templatePath, FileFilter) { - fileFilterIndex[fileFilterCounter] = i - fileFilterCounter++ - fileFound = true - break - } - } - } - if !fileFound { - return nil, eUtils.LogAndSafeExit(config, "Could not find specified file in templates", 1) - } + templatePaths, endPaths = FilterPaths(templatePaths, endPaths, config.FileFilter, false) - fileTemplatePaths := []string{} - fileEndPaths := []string{} - for _, index := range fileFilterIndex { - fileTemplatePaths = append(fileTemplatePaths, templatePaths[index]) - fileEndPaths = append(fileEndPaths, endPaths[index]) - } - - templatePaths = fileTemplatePaths - endPaths = fileEndPaths - } + templatePaths, endPaths = FilterPaths(templatePaths, endPaths, config.ServicesWanted, false) for _, templatePath := range templatePaths { if !config.WantCerts && strings.Contains(templatePath, "Common") { diff --git a/trcconfigbase/utils/filter.go b/trcconfigbase/utils/filter.go new file mode 100644 index 000000000..8ed954a67 --- /dev/null +++ b/trcconfigbase/utils/filter.go @@ -0,0 +1,53 @@ +package utils + +import ( + "strings" +) + +// FilterPaths -- filters based on provided fileFilter +func FilterPaths(templatePaths []string, endPaths []string, fileFilter []string, prefix bool) ([]string, []string) { + fileFilterIndex := make([]int, len(templatePaths)) + fileFilterCounter := 0 + + if len(fileFilter) != 0 && fileFilter[0] != "" { + for _, FileFilter := range fileFilter { + for i, templatePath := range templatePaths { + if !prefix { + ii := -1 + if strings.Contains(templatePath, FileFilter) { + ii = i + } + if len(fileFilterIndex) > fileFilterCounter { + fileFilterIndex[fileFilterCounter] = ii + fileFilterCounter++ + } else { + break + } + } else { + if strings.HasPrefix(templatePath, FileFilter) { + if len(fileFilterIndex) > fileFilterCounter { + fileFilterIndex[fileFilterCounter] = i + fileFilterCounter++ + } else { + break + } + } + + } + } + } + + fileTemplatePaths := []string{} + fileEndPaths := []string{} + for _, index := range fileFilterIndex { + if index > 0 { + fileTemplatePaths = append(fileTemplatePaths, templatePaths[index]) + fileEndPaths = append(fileEndPaths, endPaths[index]) + } + } + + templatePaths = fileTemplatePaths + endPaths = fileEndPaths + } + return templatePaths, endPaths +} diff --git a/trcsh/deployutil/deployutil.go b/trcsh/deployutil/deployutil.go new file mode 100644 index 000000000..90e1671ae --- /dev/null +++ b/trcsh/deployutil/deployutil.go @@ -0,0 +1,66 @@ +package deployutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/trimble-oss/tierceron/capauth" + vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" + eUtils "github.com/trimble-oss/tierceron/utils" + helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" +) + +// Loads a plugin's deploy.trc script directly from vault. +func LoadPluginDeploymentScript(config *eUtils.DriverConfig, trcshConfig *capauth.TrcShConfig, pwd string) ([]byte, error) { + if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { + if deployment, ok := config.DeploymentConfig["trcplugin"]; ok { + mergedEnvRaw := config.EnvRaw + // Swapping in project root... + configRoleSlice := strings.Split(*trcshConfig.ConfigRole, ":") + tokenName := "config_token_" + config.EnvRaw + readToken := "" + autoErr := eUtils.AutoAuth(config, &configRoleSlice[1], &configRoleSlice[0], &readToken, &tokenName, &config.Env, &config.VaultAddress, &mergedEnvRaw, "config.yml", false) + if autoErr != nil { + fmt.Println("Missing auth components.") + return nil, autoErr + } + + mod, err := helperkv.NewModifier(config.Insecure, readToken, *trcshConfig.VaultAddress, config.EnvRaw, config.Regions, true, config.Log) + if err != nil { + fmt.Println("Unable to obtain resources for deployment") + return nil, err + } + mod.Env = config.EnvRaw + deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) + if err != nil { + fmt.Println("Unable to obtain config for deployment") + return nil, err + } + deploymentConfig["trcpluginalias"] = deployment + config.DeploymentConfig = deploymentConfig + if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { + config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} + config.EndDir = trcDeployRoot.(string) + } + + if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { + var content []byte + trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") + config.ZeroConfig = true + contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) + config.ZeroConfig = false + if err != nil { + eUtils.LogErrorObject(config, err, false) + return nil, err + } + content = []byte(contentArray) + return content, nil + } else { + fmt.Println("Project not configured and ready for deployment. Missing projectservice") + return nil, errors.New("project not configured and ready for deployment. missing projectservice") + } + } + } + return nil, errors.New("not deployer") +} diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 87861627c..03f3ace9b 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -24,8 +24,8 @@ import ( "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" "github.com/trimble-oss/tierceron/trcconfigbase" - vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" "github.com/trimble-oss/tierceron/trcpubbase" + "github.com/trimble-oss/tierceron/trcsh/deployutil" kube "github.com/trimble-oss/tierceron/trcsh/kube/native" "github.com/trimble-oss/tierceron/trcsh/trcshauth" "github.com/trimble-oss/tierceron/trcsubbase" @@ -114,15 +114,9 @@ func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg st // Logging of deployer activities.. func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { - if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH { + if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH && msg != captiplib.MSG_PERCH_AND_GAZE { featherCtx.Log.Printf(msg) } - deployerId, _ := deployers.GetDecodedDeployerId(*featherCtx.SessionIdentifier) - - if *featherCtx.SessionIdentifier == "spectrumformssub.dev" { - fmt.Printf("deployer: %s %s", deployerId, msg) - featherCtx.Log.Printf("%s %s", *featherCtx.SessionIdentifier, msg) - } } func deployCtlAcceptRemote(featherCtx *cap.FeatherContext, x int, y string) (bool, error) { @@ -152,6 +146,8 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec fmt.Printf("Initialization setup error: %s\n", err.Error()) } if len(deployment) > 0 { + // Set the name of the plugin to deploy in "trcplugin" + // Used later by codedeploy config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} config.DeploymentCtlMessageChan = make(chan string, 5) fmt.Printf("Starting deployer: %s\n", deployment) @@ -182,8 +178,6 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) - config.Log.Printf("Starting deployer session: %s\n", sessionIdentifier) - go captiplib.FeatherCtlEmitter(config.FeatherCtx, config.DeploymentCtlMessageChan, deployerEmote, nil) go ProcessDeploy(config.FeatherCtx, config, region, "", deployment, trcPath, secretId, approleId, false) @@ -501,7 +495,8 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, gAgentConfig.EncryptSalt, gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, - new(string), gAgentConfig.AcceptRemoteFunc, gAgentConfig.InterruptHandlerFunc) + new(string), deployCtlAcceptRemote, + deployCtlInterrupted) if config.Log != nil { config.FeatherCtx.Log = config.Log } @@ -714,52 +709,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, } config.Log.Println("Processing trcshell") } else { - if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { - if deployment, ok := config.DeploymentConfig["trcplugin"]; ok { - // Swapping in project root... - configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") - tokenName := "config_token_" + config.EnvRaw - readToken := "" - autoErr := eUtils.AutoAuth(config, &configRoleSlice[1], &configRoleSlice[0], &readToken, &tokenName, &config.Env, &config.VaultAddress, &mergedEnvRaw, "config.yml", false) - if autoErr != nil { - fmt.Println("Missing auth components.") - return - } - - mod, err := helperkv.NewModifier(config.Insecure, readToken, *gTrcshConfig.VaultAddress, config.EnvRaw, config.Regions, true, config.Log) - if err != nil { - fmt.Println("Unable to obtain resources for deployment") - return - } - mod.Env = config.EnvRaw - deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) - if err != nil { - fmt.Println("Unable to obtain config for deployment") - return - } - deploymentConfig["trcpluginalias"] = deployment - config.DeploymentConfig = deploymentConfig - if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { - config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} - config.EndDir = trcDeployRoot.(string) - } - - if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { - trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") - config.ZeroConfig = true - contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) - config.ZeroConfig = false - if err != nil { - eUtils.LogErrorObject(config, err, false) - return - } - content = []byte(contentArray) - } else { - fmt.Println("Project not configured and ready for deployment. Missing projectservice") - return - } - } - } else { + if !strings.Contains(pwd, "TrcDeploy") || len(config.DeploymentConfig) == 0 { fmt.Println("Processing manual trcshell") if config.EnvRaw == "itdev" { content, err = os.ReadFile(pwd + "/deploy/buildtest.trc") @@ -776,15 +726,6 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, } } - deployArgLines := strings.Split(string(content), "\n") - configCount := strings.Count(string(content), "trcconfig") //Uses this to close result channel on last run. - - argsOrig := os.Args - - var trcKubeDeploymentConfig *kube.TrcKubeConfig - var onceKubeInit sync.Once - var PipeOS billy.File - collaboratorReRun: if featherCtx != nil { // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) @@ -792,11 +733,29 @@ collaboratorReRun: if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { break } else { - time.Sleep(time.Second * 3) + acceptRemote(featherCtx, cap.FEATHER_CTL, "") } } + if content == nil { + content, err = deployutil.LoadPluginDeploymentScript(config, gTrcshConfig, pwd) + if err != nil { + config.Log.Printf("Failure to load deployment: %s\n", config.DeploymentConfig["trcplugin"]) + time.Sleep(time.Minute) + goto collaboratorReRun + } + } } + + deployArgLines := strings.Split(string(content), "\n") + configCount := strings.Count(string(content), "trcconfig") //Uses this to close result channel on last run. + + argsOrig := os.Args + + var trcKubeDeploymentConfig *kube.TrcKubeConfig + var onceKubeInit sync.Once + var PipeOS billy.File + for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") if strings.HasPrefix(deployPipeline, "#") || deployPipeline == "" { @@ -881,6 +840,7 @@ collaboratorReRun: }(deliverableMsg) atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) + content = nil goto collaboratorReRun } else { config.DeploymentCtlMessageChan <- deployLine @@ -919,6 +879,7 @@ collaboratorReRun: break } } + content = nil goto collaboratorReRun } //Make the arguments in the script -> os.args. From bd20face193d47f0e17ce1e849a50443716407d0 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:06:47 -0800 Subject: [PATCH 39/55] Prepare for release. (#844) * Prepare for release (#842) * Fixing break on windows. * Add more robust support for servicesWanted and filter. * Deployments should match a deployment run. Always load deployment during a run. * Add ability for cmd override to support multiple deployers for multiple environments on a single box. --------- Signed-off-by: dependabot[bot] --- azure-pipelines-legacy.yml | 7 + azure-pipelines-linux.yml | 7 + azure-pipelines-plugin-develop.yml | 7 + azure-pipelines-plugin.yml | 7 + azure-pipelines.yml | 7 + buildopts/coreopts/options_common.go | 3 +- buildopts/coreopts/options_tc.go | 3 +- capauth/agentconfig.go | 7 +- go.mod | 8 +- trcconfig/trcconfig.go | 2 +- trcconfigbase/trcconfig.go | 5 +- trcconfigbase/utils/configinator.go | 29 +--- trcconfigbase/utils/filter.go | 53 +++++++ trcsh/deployutil/deployutil.go | 66 +++++++++ trcsh/trcsh.go | 206 ++++++++++++++++----------- 15 files changed, 292 insertions(+), 125 deletions(-) create mode 100644 trcconfigbase/utils/filter.go create mode 100644 trcsh/deployutil/deployutil.go diff --git a/azure-pipelines-legacy.yml b/azure-pipelines-legacy.yml index 01f7bdf5d..87e233151 100644 --- a/azure-pipelines-legacy.yml +++ b/azure-pipelines-legacy.yml @@ -44,6 +44,11 @@ resources: endpoint: trimble-oss name: trimble-oss/tierceron-hat ref: main + - repository: tierceron-succinctly + type: github + endpoint: trimble-oss + name: trimble-oss/tierceron-succinctly + ref: main - repository: tierceron-nute type: github endpoint: trimble-oss @@ -93,6 +98,8 @@ steps: clean: false - checkout: tierceron-hat clean: false +- checkout: tierceron-succinctly + clean: false - checkout: tierceron-nute clean: false - checkout: client-go diff --git a/azure-pipelines-linux.yml b/azure-pipelines-linux.yml index db38e7a48..36c08154c 100644 --- a/azure-pipelines-linux.yml +++ b/azure-pipelines-linux.yml @@ -39,6 +39,11 @@ resources: endpoint: trimble-oss name: trimble-oss/tierceron-hat ref: main + - repository: tierceron-succinctly + type: github + endpoint: trimble-oss + name: trimble-oss/tierceron-succinctly + ref: main - repository: tierceron-nute type: github endpoint: trimble-oss @@ -89,6 +94,8 @@ steps: clean: false - checkout: tierceron-hat clean: false +- checkout: tierceron-succinctly + clean: false - checkout: tierceron-nute clean: false - checkout: client-go diff --git a/azure-pipelines-plugin-develop.yml b/azure-pipelines-plugin-develop.yml index 9f1f4a34f..544932f4b 100644 --- a/azure-pipelines-plugin-develop.yml +++ b/azure-pipelines-plugin-develop.yml @@ -42,6 +42,11 @@ resources: endpoint: trimble-oss name: trimble-oss/tierceron-hat ref: main + - repository: tierceron-succinctly + type: github + endpoint: trimble-oss + name: trimble-oss/tierceron-succinctly + ref: main - repository: tierceron-nute type: github endpoint: trimble-oss @@ -91,6 +96,8 @@ steps: clean: false - checkout: tierceron-hat clean: false +- checkout: tierceron-succinctly + clean: false - checkout: tierceron-nute clean: false - checkout: client-go diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index fe12f8278..38b403d03 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -47,6 +47,11 @@ resources: endpoint: trimble-oss name: trimble-oss/tierceron-hat ref: main + - repository: tierceron-succinctly + type: github + endpoint: trimble-oss + name: trimble-oss/tierceron-succinctly + ref: main - repository: tierceron-nute type: github endpoint: trimble-oss @@ -96,6 +101,8 @@ steps: clean: false - checkout: tierceron-hat clean: false +- checkout: tierceron-succinctly + clean: false - checkout: tierceron-nute clean: false - checkout: client-go diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 90562dcb8..7101e0e8d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -45,6 +45,11 @@ resources: endpoint: trimble-oss name: trimble-oss/tierceron-hat ref: main + - repository: tierceron-succinctly + type: github + endpoint: trimble-oss + name: trimble-oss/tierceron-succinctly + ref: main - repository: tierceron-nute type: github endpoint: trimble-oss @@ -94,6 +99,8 @@ steps: clean: false - checkout: tierceron-hat clean: false +- checkout: tierceron-succinctly + clean: false - checkout: tierceron-nute clean: false - checkout: client-go diff --git a/buildopts/coreopts/options_common.go b/buildopts/coreopts/options_common.go index 71b7a52ea..09124bb36 100644 --- a/buildopts/coreopts/options_common.go +++ b/buildopts/coreopts/options_common.go @@ -6,6 +6,7 @@ package coreopts import ( "database/sql" "errors" + "os" "strings" ) @@ -17,7 +18,7 @@ func GetFolderPrefix(custom []string) string { endTi = 0 for endTi = ti; endTi > 0; endTi-- { - if custom[0][endTi] == '/' { + if custom[0][endTi] == '/' || custom[0][endTi] == os.PathSeparator { endTi = endTi + 1 break } diff --git a/buildopts/coreopts/options_tc.go b/buildopts/coreopts/options_tc.go index 76406af2d..6cd4d42ef 100644 --- a/buildopts/coreopts/options_tc.go +++ b/buildopts/coreopts/options_tc.go @@ -4,6 +4,7 @@ package coreopts import ( + "os" "strings" bcore "VaultConfig.Bootstrap/configcore" @@ -21,7 +22,7 @@ func GetFolderPrefix(custom []string) string { endTi = 0 for endTi = ti; endTi > 0; endTi-- { - if custom[0][endTi] == '/' { + if custom[0][endTi] == '/' || custom[0][endTi] == os.PathSeparator { endTi = endTi + 1 break } diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index 707113e62..d2401fd23 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -95,7 +95,12 @@ func (agentconfig *AgentConfigs) PenseFeatherQuery(featherCtx *cap.FeatherContex return penseProtect, nil } -func NewAgentConfig(address string, agentToken string, deployments string, env string) (*AgentConfigs, *TrcShConfig, error) { +func NewAgentConfig(address string, + agentToken string, + deployments string, + env string, + acceptRemoteFunc func(*cap.FeatherContext, int, string) (bool, error), + interruptedFunc func(*cap.FeatherContext) error) (*AgentConfigs, *TrcShConfig, error) { mod, modErr := helperkv.NewModifier(false, agentToken, address, env, nil, true, nil) if modErr != nil { fmt.Println("trcsh Failed to bootstrap") diff --git a/go.mod b/go.mod index 3ef9d7120..31f8fff94 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/trimble-oss/tierceron -go 1.21 - -toolchain go1.21.4 +go 1.21.4 require ( VaultConfig.Bootstrap v0.0.0-00010101000000-000000000000 @@ -39,6 +37,8 @@ require ( require github.com/trimble-oss/tierceron-nute v0.0.0-20230128181737-65043c9e434b +require github.com/trimble-oss/tierceron-succinctly v0.0.0-00010101000000-000000000000 // indirect + require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 @@ -246,6 +246,8 @@ replace VaultConfig.Test => ../VaultConfig.Test replace github.com/trimble-oss/tierceron-hat => ../tierceron-hat +replace github.com/trimble-oss/tierceron-succinctly => ../tierceron-succinctly + replace github.com/dolthub/go-mysql-server => github.com/trimble-oss/go-mysql-server v0.12.0-1.6 replace github.com/trimble-oss/tierceron-nute => ../tierceron-nute diff --git a/trcconfig/trcconfig.go b/trcconfig/trcconfig.go index a1b0e0490..2e56f95be 100644 --- a/trcconfig/trcconfig.go +++ b/trcconfig/trcconfig.go @@ -18,7 +18,7 @@ func main() { if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } - fmt.Println("Version: " + "1.27") + fmt.Println("Version: " + "1.28") flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError) flagset.Usage = func() { fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) diff --git a/trcconfigbase/trcconfig.go b/trcconfigbase/trcconfig.go index c9dcb3f73..24efcb542 100644 --- a/trcconfigbase/trcconfig.go +++ b/trcconfigbase/trcconfig.go @@ -320,10 +320,6 @@ func CommonMain(envPtr *string, services = strings.Split(*servicesWanted, ",") } - // TODO: This wasn't doing anything useful... possibly remove? - // for _, service := range services { - // service = strings.TrimSpace(service) - // } regions := []string{} if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") { @@ -408,6 +404,7 @@ func CommonMain(envPtr *string, Update: messenger, FileFilter: fileFilterSlice, } + configSlice = append(configSlice, config) configCtx.ConfigWg.Add(1) go func() { diff --git a/trcconfigbase/utils/configinator.go b/trcconfigbase/utils/configinator.go index f4077e8f4..d77747f14 100644 --- a/trcconfigbase/utils/configinator.go +++ b/trcconfigbase/utils/configinator.go @@ -107,34 +107,9 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi } //File filter - fileFound := true - fileFilterIndex := make([]int, len(config.FileFilter)) - fileFilterCounter := 0 - if len(config.FileFilter) != 0 && config.FileFilter[0] != "" { - for _, FileFilter := range config.FileFilter { - for i, templatePath := range templatePaths { - if strings.Contains(templatePath, FileFilter) { - fileFilterIndex[fileFilterCounter] = i - fileFilterCounter++ - fileFound = true - break - } - } - } - if !fileFound { - return nil, eUtils.LogAndSafeExit(config, "Could not find specified file in templates", 1) - } + templatePaths, endPaths = FilterPaths(templatePaths, endPaths, config.FileFilter, false) - fileTemplatePaths := []string{} - fileEndPaths := []string{} - for _, index := range fileFilterIndex { - fileTemplatePaths = append(fileTemplatePaths, templatePaths[index]) - fileEndPaths = append(fileEndPaths, endPaths[index]) - } - - templatePaths = fileTemplatePaths - endPaths = fileEndPaths - } + templatePaths, endPaths = FilterPaths(templatePaths, endPaths, config.ServicesWanted, false) for _, templatePath := range templatePaths { if !config.WantCerts && strings.Contains(templatePath, "Common") { diff --git a/trcconfigbase/utils/filter.go b/trcconfigbase/utils/filter.go new file mode 100644 index 000000000..8ed954a67 --- /dev/null +++ b/trcconfigbase/utils/filter.go @@ -0,0 +1,53 @@ +package utils + +import ( + "strings" +) + +// FilterPaths -- filters based on provided fileFilter +func FilterPaths(templatePaths []string, endPaths []string, fileFilter []string, prefix bool) ([]string, []string) { + fileFilterIndex := make([]int, len(templatePaths)) + fileFilterCounter := 0 + + if len(fileFilter) != 0 && fileFilter[0] != "" { + for _, FileFilter := range fileFilter { + for i, templatePath := range templatePaths { + if !prefix { + ii := -1 + if strings.Contains(templatePath, FileFilter) { + ii = i + } + if len(fileFilterIndex) > fileFilterCounter { + fileFilterIndex[fileFilterCounter] = ii + fileFilterCounter++ + } else { + break + } + } else { + if strings.HasPrefix(templatePath, FileFilter) { + if len(fileFilterIndex) > fileFilterCounter { + fileFilterIndex[fileFilterCounter] = i + fileFilterCounter++ + } else { + break + } + } + + } + } + } + + fileTemplatePaths := []string{} + fileEndPaths := []string{} + for _, index := range fileFilterIndex { + if index > 0 { + fileTemplatePaths = append(fileTemplatePaths, templatePaths[index]) + fileEndPaths = append(fileEndPaths, endPaths[index]) + } + } + + templatePaths = fileTemplatePaths + endPaths = fileEndPaths + } + return templatePaths, endPaths +} diff --git a/trcsh/deployutil/deployutil.go b/trcsh/deployutil/deployutil.go new file mode 100644 index 000000000..90e1671ae --- /dev/null +++ b/trcsh/deployutil/deployutil.go @@ -0,0 +1,66 @@ +package deployutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/trimble-oss/tierceron/capauth" + vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" + eUtils "github.com/trimble-oss/tierceron/utils" + helperkv "github.com/trimble-oss/tierceron/vaulthelper/kv" +) + +// Loads a plugin's deploy.trc script directly from vault. +func LoadPluginDeploymentScript(config *eUtils.DriverConfig, trcshConfig *capauth.TrcShConfig, pwd string) ([]byte, error) { + if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { + if deployment, ok := config.DeploymentConfig["trcplugin"]; ok { + mergedEnvRaw := config.EnvRaw + // Swapping in project root... + configRoleSlice := strings.Split(*trcshConfig.ConfigRole, ":") + tokenName := "config_token_" + config.EnvRaw + readToken := "" + autoErr := eUtils.AutoAuth(config, &configRoleSlice[1], &configRoleSlice[0], &readToken, &tokenName, &config.Env, &config.VaultAddress, &mergedEnvRaw, "config.yml", false) + if autoErr != nil { + fmt.Println("Missing auth components.") + return nil, autoErr + } + + mod, err := helperkv.NewModifier(config.Insecure, readToken, *trcshConfig.VaultAddress, config.EnvRaw, config.Regions, true, config.Log) + if err != nil { + fmt.Println("Unable to obtain resources for deployment") + return nil, err + } + mod.Env = config.EnvRaw + deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) + if err != nil { + fmt.Println("Unable to obtain config for deployment") + return nil, err + } + deploymentConfig["trcpluginalias"] = deployment + config.DeploymentConfig = deploymentConfig + if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { + config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} + config.EndDir = trcDeployRoot.(string) + } + + if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { + var content []byte + trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") + config.ZeroConfig = true + contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) + config.ZeroConfig = false + if err != nil { + eUtils.LogErrorObject(config, err, false) + return nil, err + } + content = []byte(contentArray) + return content, nil + } else { + fmt.Println("Project not configured and ready for deployment. Missing projectservice") + return nil, errors.New("project not configured and ready for deployment. missing projectservice") + } + } + } + return nil, errors.New("not deployer") +} diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index 6b01644f0..d98c8e8cb 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "VaultConfig.TenantConfig/util/buildopts/deployers" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/trimble-oss/tierceron-hat/cap" @@ -23,8 +24,8 @@ import ( "github.com/trimble-oss/tierceron/buildopts/memprotectopts" "github.com/trimble-oss/tierceron/capauth" "github.com/trimble-oss/tierceron/trcconfigbase" - vcutils "github.com/trimble-oss/tierceron/trcconfigbase/utils" "github.com/trimble-oss/tierceron/trcpubbase" + "github.com/trimble-oss/tierceron/trcsh/deployutil" kube "github.com/trimble-oss/tierceron/trcsh/kube/native" "github.com/trimble-oss/tierceron/trcsh/trcshauth" "github.com/trimble-oss/tierceron/trcsubbase" @@ -103,7 +104,8 @@ func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg st if len(ctlFlapMode) > 0 && ctlFlapMode[0] == cap.MODE_FLAP { fmt.Printf("%s\n", msg) } - featherCtx.Log.Printf("ctl: %s msg: %s\n", ctlFlapMode, strings.Trim(msg, "\n")) + deployerId, _ := deployers.GetDecodedDeployerId(*featherCtx.SessionIdentifier) + featherCtx.Log.Printf("deployer: %s ctl: %s msg: %s\n", deployerId, ctlFlapMode, strings.Trim(msg, "\n")) if strings.Contains(msg, "encountered errors") { cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) os.Exit(0) @@ -112,17 +114,25 @@ func deployerCtlEmote(featherCtx *cap.FeatherContext, ctlFlapMode string, msg st // Logging of deployer activities.. func deployerEmote(featherCtx *cap.FeatherContext, ctlFlapMode []byte, msg string) { - if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH { + if len(ctlFlapMode) > 0 && ctlFlapMode[0] != cap.MODE_PERCH && msg != captiplib.MSG_PERCH_AND_GAZE { featherCtx.Log.Printf(msg) } } +func deployCtlAcceptRemote(featherCtx *cap.FeatherContext, x int, y string) (bool, error) { + return acceptInterruptFun(featherCtx, featherCtx.MultiSecondInterruptTicker, featherCtx.FifteenSecondInterruptTicker, featherCtx.ThirtySecondInterruptTicker) +} + // deployCtl -- is the deployment controller or manager if you will. func deployCtlInterrupted(featherCtx *cap.FeatherContext) error { os.Exit(-1) return nil } +func deployerAcceptRemote(featherCtx *cap.FeatherContext, x int, y string) (bool, error) { + return acceptInterruptFun(featherCtx, featherCtx.MultiSecondInterruptTicker, featherCtx.FifteenSecondInterruptTicker, featherCtx.ThirtySecondInterruptTicker) +} + // deployer -- does the work of deploying.. func deployerInterrupted(featherCtx *cap.FeatherContext) error { cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) @@ -136,8 +146,11 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec fmt.Printf("Initialization setup error: %s\n", err.Error()) } if len(deployment) > 0 { + // Set the name of the plugin to deploy in "trcplugin" + // Used later by codedeploy config.DeploymentConfig = map[string]interface{}{"trcplugin": deployment} config.DeploymentCtlMessageChan = make(chan string, 5) + fmt.Printf("Starting deployer: %s\n", deployment) config.Log.Printf("Starting deployer: %s\n", deployment) } @@ -145,7 +158,13 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec // Each deployer needs it's own context. // localHostAddr := "" - sessionIdentifier := deployment + "." + *gAgentConfig.Env + var sessionIdentifier string + if sessionId, ok := deployers.GetEncodedDeployerId(deployment, *gAgentConfig.Env); ok { + sessionIdentifier = sessionId + } else { + fmt.Printf("Unsupported deployer: %s\n", deployment) + os.Exit(-1) + } config.FeatherCtx = captiplib.FeatherCtlInit(interruptChan, &localHostAddr, gAgentConfig.EncryptPass, @@ -153,7 +172,7 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, &sessionIdentifier, /*Session identifier */ - captiplib.AcceptRemote, + deployerAcceptRemote, deployerInterrupted) config.FeatherCtx.Log = config.Log // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) @@ -173,6 +192,21 @@ func main() { eUtils.InitHeadless(true) fmt.Println("trcsh Version: " + "1.23") var envPtr, regionPtr, trcPathPtr, appRoleIDPtr, secretIDPtr *string + // Initiate signal handling. + var ic chan os.Signal = make(chan os.Signal, 5) + + if !eUtils.IsWindows() { + signal.Notify(ic, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGABRT) + } else { + signal.Notify(ic, os.Interrupt) + } + go func() { + x := <-ic + interruptChan <- x + }() + + // Initialize the supported + deployers.InitSupportedDeployers() if !eUtils.IsWindows() { if os.Geteuid() == 0 { @@ -185,13 +219,6 @@ func main() { if strings.Contains(os.Args[1], "trc") && !strings.Contains(os.Args[1], "-c") { // Running as shell. os.Args[1] = "-c=" + os.Args[1] - // Initiate signal handling. - var ic chan os.Signal = make(chan os.Signal, 3) - signal.Notify(ic, os.Interrupt, syscall.SIGTERM) - go func() { - x := <-ic - interruptChan <- x - }() } } envPtr = flag.String("env", "", "Environment to be processed") //If this is blank -> use context otherwise override context. @@ -225,6 +252,7 @@ func main() { agentEnv := os.Getenv("AGENT_ENV") address := os.Getenv("VAULT_ADDR") + envPtr := flag.String("env", "", "Environment to configure") regionPtr = flag.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. trcPathPtr = flag.String("c", "", "Optional script to execute.") //If this is blank -> use context otherwise override context. appRoleIDPtr = flag.String("appRoleID", "", "Public app role ID") @@ -246,6 +274,10 @@ func main() { os.Exit(-1) } + if len(*envPtr) > 0 { + agentEnv = *envPtr + } + if len(address) == 0 { fmt.Println("trcsh on windows requires VAULT_ADDR address.") os.Exit(-1) @@ -260,7 +292,10 @@ func main() { // Preload agent synchronization configs... var errAgentLoad error - gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(address, agentToken, deployments, agentEnv) + gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(address, + agentToken, + deployments, + agentEnv, nil, nil) if errAgentLoad != nil { fmt.Println("trcsh agent bootstrap failure.") os.Exit(-1) @@ -284,29 +319,36 @@ var fifteenSecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) var thirtySecondInterruptTicker *time.Ticker = time.NewTicker(time.Second * 5) func acceptInterruptFun(featherCtx *cap.FeatherContext, tickerContinue *time.Ticker, tickerBreak *time.Ticker, tickerInterrupt *time.Ticker) (bool, error) { + result := false + var resultError error = nil select { - case <-interruptChan: - cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) - os.Exit(1) case <-tickerContinue.C: // don't break... continue... - return false, nil + result = false + resultError = nil case <-tickerBreak.C: // break and continue - return true, nil + result = true + resultError = nil case <-tickerInterrupt.C: // full stop - return true, errors.New("you shall not pass") + result = true + resultError = errors.New("you shall not pass") } - return true, errors.New("not possible") + if len(featherCtx.InterruptChan) > 0 { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(1) + } + return result, resultError } func interruptFun(featherCtx *cap.FeatherContext, tickerInterrupt *time.Ticker) { select { - case <-interruptChan: - cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) - os.Exit(1) case <-tickerInterrupt.C: + if len(featherCtx.InterruptChan) > 0 { + cap.FeatherCtlEmit(featherCtx, MODE_PERCH_STR, *featherCtx.SessionIdentifier, true) + os.Exit(1) + } } } @@ -328,9 +370,15 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { return errors.New("incorrect feathering") } - sessionIdentifier := agentName + "." + *gAgentConfig.Env - featherCtx.SessionIdentifier = &sessionIdentifier - captiplib.FeatherCtl(featherCtx, deployerCtlEmote) + if sessionIdentifier, ok := deployers.GetEncodedDeployerId(agentName, *gAgentConfig.Env); ok { + featherCtx.SessionIdentifier = &sessionIdentifier + featherCtx.Log.Printf("Starting deploy ctl session: %s\n", sessionIdentifier) + captiplib.FeatherCtl(featherCtx, deployerCtlEmote) + } else { + fmt.Println(fmt.Sprintf("Unsupported agent: %s", agentName)) + os.Exit(-1) + } + return nil } @@ -431,7 +479,12 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, var errAgentLoad error // Prepare the configuration triggering mechanism. // Bootstrap deployment is replaced during callback with the agent name. - gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(config.VaultAddress, *trcshConfig.CToken, "bootstrap", config.Env) + gAgentConfig, _, errAgentLoad = capauth.NewAgentConfig(config.VaultAddress, + *trcshConfig.CToken, + "bootstrap", + config.Env, + deployCtlAcceptRemote, + deployCtlInterrupted) if errAgentLoad != nil { fmt.Printf("Permissions failure. Incorrect deployment") os.Exit(1) @@ -441,13 +494,14 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, } gAgentConfig.InterruptHandlerFunc = deployCtlInterrupted } - config.FeatherCtx = captiplib.FeatherCtlInit(nil, + config.FeatherCtx = captiplib.FeatherCtlInit(interruptChan, gAgentConfig.LocalHostAddr, gAgentConfig.EncryptPass, gAgentConfig.EncryptSalt, gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, - new(string), gAgentConfig.AcceptRemoteFunc, gAgentConfig.InterruptHandlerFunc) + new(string), deployCtlAcceptRemote, + deployCtlInterrupted) if config.Log != nil { config.FeatherCtx.Log = config.Log } @@ -660,52 +714,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, } config.Log.Println("Processing trcshell") } else { - if strings.Contains(pwd, "TrcDeploy") && len(config.DeploymentConfig) > 0 { - if deployment, ok := config.DeploymentConfig["trcplugin"]; ok { - // Swapping in project root... - configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") - tokenName := "config_token_" + config.EnvRaw - readToken := "" - autoErr := eUtils.AutoAuth(config, &configRoleSlice[1], &configRoleSlice[0], &readToken, &tokenName, &config.Env, &config.VaultAddress, &mergedEnvRaw, "config.yml", false) - if autoErr != nil { - fmt.Println("Missing auth components.") - return - } - - mod, err := helperkv.NewModifier(config.Insecure, readToken, *gTrcshConfig.VaultAddress, config.EnvRaw, config.Regions, true, config.Log) - if err != nil { - fmt.Println("Unable to obtain resources for deployment") - return - } - mod.Env = config.EnvRaw - deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) - if err != nil { - fmt.Println("Unable to obtain config for deployment") - return - } - deploymentConfig["trcpluginalias"] = deployment - config.DeploymentConfig = deploymentConfig - if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { - config.StartDir = []string{fmt.Sprintf("%s/trc_templates", trcDeployRoot.(string))} - config.EndDir = trcDeployRoot.(string) - } - - if trcProjectService, ok := config.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { - trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") - config.ZeroConfig = true - contentArray, _, _, err := vcutils.ConfigTemplate(config, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) - config.ZeroConfig = false - if err != nil { - eUtils.LogErrorObject(config, err, false) - return - } - content = []byte(contentArray) - } else { - fmt.Println("Project not configured and ready for deployment. Missing projectservice") - return - } - } - } else { + if !strings.Contains(pwd, "TrcDeploy") || len(config.DeploymentConfig) == 0 { fmt.Println("Processing manual trcshell") if config.EnvRaw == "itdev" { content, err = os.ReadFile(pwd + "/deploy/buildtest.trc") @@ -722,15 +731,6 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, } } - deployArgLines := strings.Split(string(content), "\n") - configCount := strings.Count(string(content), "trcconfig") //Uses this to close result channel on last run. - - argsOrig := os.Args - - var trcKubeDeploymentConfig *kube.TrcKubeConfig - var onceKubeInit sync.Once - var PipeOS billy.File - collaboratorReRun: if featherCtx != nil { // featherCtx initialization is delayed for the self contained deployments (kubernetes, etc...) @@ -738,11 +738,29 @@ collaboratorReRun: if atomic.LoadInt64(&featherCtx.RunState) == cap.RESETTING { break } else { - time.Sleep(time.Second * 3) + acceptRemote(featherCtx, cap.FEATHER_CTL, "") } } + if content == nil { + content, err = deployutil.LoadPluginDeploymentScript(config, gTrcshConfig, pwd) + if err != nil { + config.Log.Printf("Failure to load deployment: %s\n", config.DeploymentConfig["trcplugin"]) + time.Sleep(time.Minute) + goto collaboratorReRun + } + } } + + deployArgLines := strings.Split(string(content), "\n") + configCount := strings.Count(string(content), "trcconfig") //Uses this to close result channel on last run. + + argsOrig := os.Args + + var trcKubeDeploymentConfig *kube.TrcKubeConfig + var onceKubeInit sync.Once + var PipeOS billy.File + for _, deployPipeline := range deployArgLines { deployPipeline = strings.TrimLeft(deployPipeline, " ") if strings.HasPrefix(deployPipeline, "#") || deployPipeline == "" { @@ -820,7 +838,14 @@ collaboratorReRun: } errMessage := err.Error() errMessageFiltered := strings.ReplaceAll(errMessage, ":", "-") - config.DeploymentCtlMessageChan <- fmt.Sprintf("%s encountered errors - %s\n", deployLine, errMessageFiltered) + deliverableMsg := fmt.Sprintf("%s encountered errors - %s\n", deployLine, errMessageFiltered) + go func(dMesg string) { + config.DeploymentCtlMessageChan <- dMesg + config.DeploymentCtlMessageChan <- cap.CTL_COMPLETE + }(deliverableMsg) + + atomic.StoreInt64(&config.FeatherCtx.RunState, cap.RUN_STARTED) + content = nil goto collaboratorReRun } else { config.DeploymentCtlMessageChan <- deployLine @@ -845,14 +870,21 @@ collaboratorReRun: } } if eUtils.IsWindows() { - config.DeploymentCtlMessageChan <- cap.CTL_COMPLETE for { + completeOnce := false if atomic.LoadInt64(&featherCtx.RunState) == cap.RUNNING { + if !completeOnce { + go func() { + config.DeploymentCtlMessageChan <- cap.CTL_COMPLETE + }() + completeOnce = true + } time.Sleep(time.Second) } else { break } } + content = nil goto collaboratorReRun } //Make the arguments in the script -> os.args. From 310a57c686c7c2427bed344825b9fa45f4beb062 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 5 Dec 2023 07:12:42 -0800 Subject: [PATCH 40/55] Prepare for release. (#845) * Provide service name always if defined. --------- --- trcvault/trcplgtoolbase/trcplgtoolbase.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 3ad6d24ed..cb6bb5e40 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -312,6 +312,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["trcplugin"].(string); !ok { pluginToolConfig["trcplugin"] = pluginToolConfig["pluginNamePtr"].(string) + if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { + pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) + } if *certifyImagePtr { certifyInit = true } @@ -332,9 +335,6 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["deploysubpathPtr"]; ok { pluginToolConfig["trcdeploysubpath"] = pluginToolConfig["deploysubpathPtr"] } - if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { - pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) - } if _, ok := pluginToolConfig["codeBundlePtr"].(string); ok { pluginToolConfig["trccodebundle"] = pluginToolConfig["codeBundlePtr"].(string) } From 18ac74b46cfd874b11ac511a884d37edd3870c5a Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 5 Dec 2023 13:40:43 -0800 Subject: [PATCH 41/55] Prepare for release (#847) * Switch to something compatible with nssm --------- --- trcvault/trcplgtoolbase/trcplgtoolbase.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index cb6bb5e40..a62552d97 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -374,9 +374,12 @@ func CommonMain(envPtr *string, fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) + //cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) + cmd := exec.Command("net", "stop", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil && !strings.Contains(err.Error(), "1") && !strings.Contains(err.Error(), "5") { + if err != nil && !strings.Contains(err.Error(), "3521") { + // 3521 means service isn't running... + // Everything else treated as error. fmt.Println(err) return err } @@ -384,7 +387,7 @@ func CommonMain(envPtr *string, } else if *winservicestartPtr { fmt.Printf("Starting service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("sc", "start", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("net", "start", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() if err != nil && !strings.Contains(err.Error(), "1056") { fmt.Println(err) From fe7386a2a0815edbb37613e64692ad29e105e032 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:36:18 -0800 Subject: [PATCH 42/55] Prepare for release (#849) * Switch to something compatible with nssm * Go to combination of net stop, taskkill, and sc start. --------- --- trcvault/trcplgtoolbase/trcplgtoolbase.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index a62552d97..2568c2e29 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -374,20 +374,20 @@ func CommonMain(envPtr *string, fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) - //cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) cmd := exec.Command("net", "stop", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil && !strings.Contains(err.Error(), "3521") { - // 3521 means service isn't running... - // Everything else treated as error. + if err != nil && strings.Contains(err.Error(), "2185") { + // Only break if service isn't defined... fmt.Println(err) return err } + cmdKill := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) + cmdKill.Run() fmt.Printf("Service stopped: %s\n", pluginToolConfig["trcservicename"].(string)) } else if *winservicestartPtr { fmt.Printf("Starting service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("net", "start", pluginToolConfig["trcservicename"].(string)) + cmd := exec.Command("sc", "start", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() if err != nil && !strings.Contains(err.Error(), "1056") { fmt.Println(err) From 309d6085bc351559ef563607f3b2305e3f893a23 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:59:22 -0800 Subject: [PATCH 43/55] Prepare for release (#851) * Some mod cleanup. * Updating to go 1.21.5 --------- --- azure-pipelines-legacy.yml | 2 +- azure-pipelines-linux.yml | 2 +- azure-pipelines-plugin-develop.yml | 2 +- azure-pipelines-plugin.yml | 2 +- azure-pipelines.yml | 2 +- go.mod | 5 ++--- trcchatproxy/go.mod | 2 +- 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/azure-pipelines-legacy.yml b/azure-pipelines-legacy.yml index 87e233151..cb1e97eb0 100644 --- a/azure-pipelines-legacy.yml +++ b/azure-pipelines-legacy.yml @@ -80,7 +80,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-linux.yml b/azure-pipelines-linux.yml index 36c08154c..cc4b9bfdc 100644 --- a/azure-pipelines-linux.yml +++ b/azure-pipelines-linux.yml @@ -76,7 +76,7 @@ steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin-develop.yml b/azure-pipelines-plugin-develop.yml index 544932f4b..ba27c7637 100644 --- a/azure-pipelines-plugin-develop.yml +++ b/azure-pipelines-plugin-develop.yml @@ -78,7 +78,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index 38b403d03..b19cac96f 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -83,7 +83,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7101e0e8d..cbb217335 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,7 +81,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/go.mod b/go.mod index 31f8fff94..9fbd81408 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/trimble-oss/tierceron -go 1.21.4 +go 1.21.5 require ( VaultConfig.Bootstrap v0.0.0-00010101000000-000000000000 - VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 github.com/denisenkom/go-mssqldb v0.12.0 github.com/dolthub/go-mysql-server v0.12.0 github.com/go-sql-driver/mysql v1.6.0 @@ -24,7 +23,6 @@ require ( ) require ( - VaultConfig.Test v0.0.0-00010101000000-000000000000 github.com/g3n/engine v0.2.0 github.com/golang-jwt/jwt v3.2.2+incompatible //github.com/trimble-oss/tierceron-nute v0.0.0-20221220150022-370bea61eb44 @@ -56,6 +54,7 @@ require ( ) require ( + VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod index d30af4270..21c12c366 100644 --- a/trcchatproxy/go.mod +++ b/trcchatproxy/go.mod @@ -1,6 +1,6 @@ module github.com/trimble-oss/tierceron/trcchatproxy -go 1.21.4 +go 1.21.5 require google.golang.org/api v0.145.0 From 507c98f7d22804705be8c027cdd733dd3a7c3c49 Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:58:51 -0800 Subject: [PATCH 44/55] Merging Develop into Main (#854) --- trcflow/core/context.go | 1 - trcflow/core/context_helper.go | 4 ---- trcinit/initlib/vault-seed.go | 8 +------- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/trcflow/core/context.go b/trcflow/core/context.go index 89522c357..fb898f79e 100644 --- a/trcflow/core/context.go +++ b/trcflow/core/context.go @@ -488,7 +488,6 @@ func (tfmContext *TrcFlowMachineContext) seedVaultCycle(tfContext *TrcFlowContex } } } - tfmContext.Config.Log.Println("Signal recieved for writeback for" + tfContext.Flow.TableName()) tfmContext.vaultPersistPushRemoteChanges( tfContext, identityColumnName, diff --git a/trcflow/core/context_helper.go b/trcflow/core/context_helper.go index 10227cfb0..bfa38da5e 100644 --- a/trcflow/core/context_helper.go +++ b/trcflow/core/context_helper.go @@ -221,10 +221,6 @@ func (tfmContext *TrcFlowMachineContext) vaultPersistPushRemoteChanges( } } - if matrixChangedEntries != nil { - tfmContext.Config.Log.Println("Changes found for writeback for" + tfContext.Flow.TableName()) - } - for _, changedEntry := range matrixChangedEntries { var changedTableQuery string var changedId interface{} diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index 05dd5bc9e..96e7fd5e5 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -791,13 +791,7 @@ func WriteData(config *eUtils.DriverConfig, path string, data map[string]interfa return } } - - config.Log.Println("Writeback for " + path + " initiated.") - warn, err := mod.Write(path, data, config.Log) - if err != nil { - config.Log.Println("Encountered error while writing to vault...") - } else { - config.Log.Println("Writeback for " + path + " occured.") + warn, err := mod.Write(path, data, config.Log) } eUtils.LogWarningsObject(config, warn, false) eUtils.LogErrorObject(config, err, false) From 1bf2f36992472192f37d3292128c18bde649ca9c Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:24:32 -0800 Subject: [PATCH 45/55] Merging Develop into Main (#855) --- trcinit/initlib/vault-seed.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index 96e7fd5e5..30852d957 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -791,8 +791,8 @@ func WriteData(config *eUtils.DriverConfig, path string, data map[string]interfa return } } - warn, err := mod.Write(path, data, config.Log) - } + warn, err := mod.Write(path, data, config.Log) + eUtils.LogWarningsObject(config, warn, false) eUtils.LogErrorObject(config, err, false) // Update value metrics to reflect credential use From 03d62f0d7b972627091379ac213bab3308a2d583 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:28:31 -0800 Subject: [PATCH 46/55] Prepare for release. (#853) * Prepare for release. (#852) * Some mod cleanup. * Updating to go 1.21.5 --------- * Merging Develop into Main (#854) * Merging Develop into Main (#855) --------- Signed-off-by: dependabot[bot] Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- azure-pipelines-legacy.yml | 2 +- azure-pipelines-linux.yml | 2 +- azure-pipelines-plugin-develop.yml | 2 +- azure-pipelines-plugin.yml | 2 +- azure-pipelines.yml | 2 +- go.mod | 5 ++--- trcchatproxy/go.mod | 2 +- trcflow/core/context.go | 1 - trcinit/initlib/vault-seed.go | 1 - trcvault/trcplgtoolbase/trcplgtoolbase.go | 13 ++++++++----- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines-legacy.yml b/azure-pipelines-legacy.yml index 87e233151..cb1e97eb0 100644 --- a/azure-pipelines-legacy.yml +++ b/azure-pipelines-legacy.yml @@ -80,7 +80,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-linux.yml b/azure-pipelines-linux.yml index 36c08154c..cc4b9bfdc 100644 --- a/azure-pipelines-linux.yml +++ b/azure-pipelines-linux.yml @@ -76,7 +76,7 @@ steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin-develop.yml b/azure-pipelines-plugin-develop.yml index 544932f4b..ba27c7637 100644 --- a/azure-pipelines-plugin-develop.yml +++ b/azure-pipelines-plugin-develop.yml @@ -78,7 +78,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines-plugin.yml b/azure-pipelines-plugin.yml index 38b403d03..b19cac96f 100644 --- a/azure-pipelines-plugin.yml +++ b/azure-pipelines-plugin.yml @@ -83,7 +83,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7101e0e8d..cbb217335 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,7 +81,7 @@ variables: steps: - task: GoTool@0 inputs: - version: '1.21.4' + version: '1.21.5' - script: | mkdir -p '$(GOBIN)' mkdir -p '$(GOPATH)/pkg' diff --git a/go.mod b/go.mod index 31f8fff94..9fbd81408 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/trimble-oss/tierceron -go 1.21.4 +go 1.21.5 require ( VaultConfig.Bootstrap v0.0.0-00010101000000-000000000000 - VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 github.com/denisenkom/go-mssqldb v0.12.0 github.com/dolthub/go-mysql-server v0.12.0 github.com/go-sql-driver/mysql v1.6.0 @@ -24,7 +23,6 @@ require ( ) require ( - VaultConfig.Test v0.0.0-00010101000000-000000000000 github.com/g3n/engine v0.2.0 github.com/golang-jwt/jwt v3.2.2+incompatible //github.com/trimble-oss/tierceron-nute v0.0.0-20221220150022-370bea61eb44 @@ -56,6 +54,7 @@ require ( ) require ( + VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect diff --git a/trcchatproxy/go.mod b/trcchatproxy/go.mod index d30af4270..21c12c366 100644 --- a/trcchatproxy/go.mod +++ b/trcchatproxy/go.mod @@ -1,6 +1,6 @@ module github.com/trimble-oss/tierceron/trcchatproxy -go 1.21.4 +go 1.21.5 require google.golang.org/api v0.145.0 diff --git a/trcflow/core/context.go b/trcflow/core/context.go index e93319ef5..fb898f79e 100644 --- a/trcflow/core/context.go +++ b/trcflow/core/context.go @@ -975,7 +975,6 @@ func (tfmContext *TrcFlowMachineContext) ProcessFlow( eUtils.LogInfo(config, "Obtaining resource connections for : "+flow.ServiceName()) retryConnectionAccess: dbsourceConn, err := trcvutils.OpenDirectConnection(config, sourceDatabaseConnectionMap["dbsourceurl"].(string), sourceDatabaseConnectionMap["dbsourceuser"].(string), sourceDatabaseConnectionMap["dbsourcepassword"].(string)) - if err.Error() != "incorrect URL format" { if retryCount < 3 && err != nil && dbsourceConn == nil { retryCount = retryCount + 1 diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index a5719c153..30852d957 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -791,7 +791,6 @@ func WriteData(config *eUtils.DriverConfig, path string, data map[string]interfa return } } - warn, err := mod.Write(path, data, config.Log) eUtils.LogWarningsObject(config, warn, false) diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 3ad6d24ed..2568c2e29 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -312,6 +312,9 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["trcplugin"].(string); !ok { pluginToolConfig["trcplugin"] = pluginToolConfig["pluginNamePtr"].(string) + if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { + pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) + } if *certifyImagePtr { certifyInit = true } @@ -332,9 +335,6 @@ func CommonMain(envPtr *string, if _, ok := pluginToolConfig["deploysubpathPtr"]; ok { pluginToolConfig["trcdeploysubpath"] = pluginToolConfig["deploysubpathPtr"] } - if _, ok := pluginToolConfig["serviceNamePtr"].(string); ok { - pluginToolConfig["trcservicename"] = pluginToolConfig["serviceNamePtr"].(string) - } if _, ok := pluginToolConfig["codeBundlePtr"].(string); ok { pluginToolConfig["trccodebundle"] = pluginToolConfig["codeBundlePtr"].(string) } @@ -374,12 +374,15 @@ func CommonMain(envPtr *string, fmt.Println("Deployment definition applied to vault and is ready for deployments.") } else if *winservicestopPtr { fmt.Printf("Stopping service %s\n", pluginToolConfig["trcservicename"].(string)) - cmd := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) + cmd := exec.Command("net", "stop", pluginToolConfig["trcservicename"].(string)) err := cmd.Run() - if err != nil && !strings.Contains(err.Error(), "1") && !strings.Contains(err.Error(), "5") { + if err != nil && strings.Contains(err.Error(), "2185") { + // Only break if service isn't defined... fmt.Println(err) return err } + cmdKill := exec.Command("taskkill", "/F", "/T", "/FI", fmt.Sprintf("\"SERVICES eq %s\"", pluginToolConfig["trcservicename"].(string))) + cmdKill.Run() fmt.Printf("Service stopped: %s\n", pluginToolConfig["trcservicename"].(string)) } else if *winservicestartPtr { From 10377480af2af1b5b9d95c788591028657cbbe6c Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:47:30 -0800 Subject: [PATCH 47/55] Prepare for release. (#856) * Adding support in trcshell for environments like dev-1. * Provide env overrides. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- capauth/agentconfig.go | 15 ++++++++++++--- trcsh/trcsh.go | 11 +++++++---- trcvault/deploy/refreshtoken.sh | 2 +- trcvault/trcplgtoolbase/trcplgtoolbase.go | 7 ++++++- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/capauth/agentconfig.go b/capauth/agentconfig.go index d2401fd23..d97c3517a 100644 --- a/capauth/agentconfig.go +++ b/capauth/agentconfig.go @@ -107,7 +107,9 @@ func NewAgentConfig(address string, os.Exit(-1) } mod.Direct = true - mod.Env = env + envParts := strings.Split(env, "-") + + mod.Env = envParts[0] data, readErr := mod.ReadData("super-secrets/Restricted/TrcshAgent/config") if readErr != nil { @@ -126,7 +128,12 @@ func NewAgentConfig(address string, hatFeatherHostAddr := fmt.Sprintf("%s:%s", data["trcHatHost"].(string), data["trcHatSecretsPort"].(string)) memprotectopts.MemProtect(nil, &hatFeatherHostAddr) - trcHatEnv := data["trcHatEnv"].(string) + var trcHatEnv string + if strings.HasPrefix(env, data["trcHatEnv"].(string)) { + trcHatEnv = env + } else { + trcHatEnv = data["trcHatEnv"].(string) + } agentconfig := &AgentConfigs{ captiplib.FeatherCtlInit(nil, @@ -135,7 +142,9 @@ func NewAgentConfig(address string, &trcHatEncryptSalt, &hatHandshakeHostAddr, &trcHatHandshakeCode, - &sessionIdentifier, captiplib.AcceptRemote, nil), + &sessionIdentifier, + &env, + captiplib.AcceptRemote, nil), &agentToken, &hatFeatherHostAddr, new(string), diff --git a/trcsh/trcsh.go b/trcsh/trcsh.go index d98c8e8cb..eb874775d 100644 --- a/trcsh/trcsh.go +++ b/trcsh/trcsh.go @@ -172,6 +172,7 @@ func EnableDeployer(env string, region string, token string, trcPath string, sec gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, &sessionIdentifier, /*Session identifier */ + &env, deployerAcceptRemote, deployerInterrupted) config.FeatherCtx.Log = config.Log @@ -370,7 +371,7 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { return errors.New("incorrect feathering") } - if sessionIdentifier, ok := deployers.GetEncodedDeployerId(agentName, *gAgentConfig.Env); ok { + if sessionIdentifier, ok := deployers.GetEncodedDeployerId(agentName, *featherCtx.Env); ok { featherCtx.SessionIdentifier = &sessionIdentifier featherCtx.Log.Printf("Starting deploy ctl session: %s\n", sessionIdentifier) captiplib.FeatherCtl(featherCtx, deployerCtlEmote) @@ -500,7 +501,9 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, gAgentConfig.EncryptSalt, gAgentConfig.HostAddr, gAgentConfig.HandshakeCode, - new(string), deployCtlAcceptRemote, + new(string), + &env, + deployCtlAcceptRemote, deployCtlInterrupted) if config.Log != nil { config.FeatherCtx.Log = config.Log @@ -670,7 +673,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, fmt.Println("Session Authorized") } - if (len(os.Args) > 1 || len(trcPath) > 0) && !strings.Contains(pwd, "TrcDeploy") { + if (len(os.Args) > 1 && len(trcPath) > 0) && !strings.Contains(pwd, "TrcDeploy") { // Generate trc code... config.Log.Println("Preload setup") trcPathParts := strings.Split(trcPath, "/") @@ -680,7 +683,7 @@ func ProcessDeploy(featherCtx *cap.FeatherContext, config *eUtils.DriverConfig, config.OutputMemCache = true config.StartDir = []string{"trc_templates"} config.EndDir = "." - config.Log.Println("Preloading") + config.Log.Printf("Preloading path %s env %s\n", trcPath, config.EnvRaw) configErr := trcconfigbase.CommonMain(&config.EnvRaw, &mergedVaultAddress, &token, &mergedEnvRaw, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, []string{"trcsh"}, config) if configErr != nil { fmt.Println("Preload failed. Couldn't find required resource.") diff --git a/trcvault/deploy/refreshtoken.sh b/trcvault/deploy/refreshtoken.sh index 85ebab5be..2c6bd7499 100755 --- a/trcvault/deploy/refreshtoken.sh +++ b/trcvault/deploy/refreshtoken.sh @@ -31,7 +31,7 @@ echo "Enter agent vault root token: " read VAULT_TOKEN fi -echo "Enter organization vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " +echo "Enter agent vault unrestricted environment token with write permissions(config_token_"$VAULT_ENV"_unrestricted): " read VAULT_ENV_TOKEN VAULT_API_ADDR=$VAULT_ADDR diff --git a/trcvault/trcplgtoolbase/trcplgtoolbase.go b/trcvault/trcplgtoolbase/trcplgtoolbase.go index 2568c2e29..a729cb6c3 100644 --- a/trcvault/trcplgtoolbase/trcplgtoolbase.go +++ b/trcvault/trcplgtoolbase/trcplgtoolbase.go @@ -32,11 +32,12 @@ func CommonMain(envPtr *string, argLines []string, c *eUtils.DriverConfig) error { + var flagEnvPtr *string // Main functions are as follows: if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ContinueOnError) // set and ignore.. - flagset.String("env", "dev", "Environment to configure") + flagEnvPtr = flagset.String("env", "dev", "Environment to configure") flagset.String("addr", "", "API endpoint for the vault") flagset.String("token", "", "Vault access token") flagset.String("region", "", "Region to be processed") //If this is blank -> use context otherwise override context. @@ -236,6 +237,10 @@ func CommonMain(envPtr *string, config, mod, vault, err := eUtils.InitVaultModForPlugin(pluginConfig, logger) config.FeatherCtlCb = configBase.FeatherCtlCb config.FeatherCtx = configBase.FeatherCtx + if config.FeatherCtx != nil && flagEnvPtr != nil && strings.HasPrefix(*flagEnvPtr, *config.FeatherCtx.Env) { + // take on the environment context of the provided flag... like dev-1 + config.FeatherCtx.Env = flagEnvPtr + } if err != nil { logger.Println("Error: " + err.Error() + " - 1") From 50c86f585bd60bedaec1ecc21d5747c8d54ccfb6 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 8 Dec 2023 06:33:49 -0800 Subject: [PATCH 48/55] Prepare for release. (#858) * Adding support in trcshell for environments like dev-1. * Provide env overrides. * Support plugin config loading even when utilizing custom environments. --------- --- trcsh/deployutil/deployutil.go | 6 +++++- trcvault/util/util.go | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/trcsh/deployutil/deployutil.go b/trcsh/deployutil/deployutil.go index 90e1671ae..c02132a97 100644 --- a/trcsh/deployutil/deployutil.go +++ b/trcsh/deployutil/deployutil.go @@ -31,12 +31,16 @@ func LoadPluginDeploymentScript(config *eUtils.DriverConfig, trcshConfig *capaut fmt.Println("Unable to obtain resources for deployment") return nil, err } - mod.Env = config.EnvRaw + tempEnv := config.EnvRaw + envParts := strings.Split(config.EnvRaw, "-") + mod.Env = envParts[0] + fmt.Printf("Loading deployment details for %s and env %s", deployment, mod.Env) deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) if err != nil { fmt.Println("Unable to obtain config for deployment") return nil, err } + mod.Env = tempEnv deploymentConfig["trcpluginalias"] = deployment config.DeploymentConfig = deploymentConfig if trcDeployRoot, ok := config.DeploymentConfig["trcdeployroot"]; ok { diff --git a/trcvault/util/util.go b/trcvault/util/util.go index 6765fdf93..4477939fc 100644 --- a/trcvault/util/util.go +++ b/trcvault/util/util.go @@ -256,6 +256,9 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl templatePaths := pluginConfig["templatePath"].([]string) config.Log.Println("GetPluginToolConfig reading base configurations.") + tempEnv := mod.Env + envParts := strings.Split(mod.Env, "-") + mod.Env = envParts[0] pluginToolConfig, err := mod.ReadData("super-secrets/Restricted/PluginTool/config") if err != nil { @@ -304,6 +307,7 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl mod.SectionPath = "super-secrets/Index/" + project + "/trcplugin/" + config.SubSectionValue + "/" + service } ptc1, err = mod.ReadData(mod.SectionPath) + pluginToolConfig["pluginpath"] = mod.SectionPath if err != nil || ptc1 == nil { config.Log.Println("No data found for project: " + project + " plugin: " + config.SubSectionValue + " service: " + service) @@ -326,6 +330,7 @@ func GetPluginToolConfig(config *eUtils.DriverConfig, mod *helperkv.Modifier, pl } mod.SectionPath = "" config.Log.Println("GetPluginToolConfig plugin data load process complete.") + mod.Env = tempEnv if pluginEnvConfigClone == nil { config.Log.Println("No data found for plugin.") From 841074a4cacca359dc0c74e308bf6dd446fdf90d Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:30:15 -0800 Subject: [PATCH 49/55] Merging Develop into Main (#860) --- LICENSE | 21 +++++++++++ LICENSE.txt | 3 -- SECURITY.md | 9 +++++ trcflow/core/context.go | 77 ++++++++++++++++------------------------- 4 files changed, 60 insertions(+), 50 deletions(-) create mode 100644 LICENSE delete mode 100644 LICENSE.txt create mode 100644 SECURITY.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..b4e568dd8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Trimble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 45d10b38d..000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,3 +0,0 @@ - THIS VERSION OF THE TOOLS AND DOCUMENTATION IS FOR INTERNAL TRIMBLE USE ONLY. - NOT APPROVED FOR EXTERNAL USE BY, OR DISTRIBUTION TO, THIRD PARTIES. - Copyright © 2021 Trimble, Inc. All Rights Reserved. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..57a78115c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security Policy + +## Reporting a Vulnerability + +Security issues and bugs should be reported privately, via email, to [cybersecurity@trimble.com](mailto:cybersecurity@trimble.com). + +If you do not receive a response within 24 hours, please follow up via email to ensure we received your original message. + +Please do not open issues for anything you think might have a security implication. diff --git a/trcflow/core/context.go b/trcflow/core/context.go index d1ad8e0e6..37d0b9a73 100644 --- a/trcflow/core/context.go +++ b/trcflow/core/context.go @@ -51,6 +51,7 @@ var AskFlumeFlow FlowNameType = "AskFlumeFlow" var signalChannel chan os.Signal var sourceDatabaseConnectionsMap map[string]map[string]interface{} var tfmContextMap = make(map[string]*TrcFlowMachineContext, 5) +var cleanerInit = false const ( TableSyncFlow FlowType = iota @@ -89,16 +90,6 @@ func (fnt FlowNameType) ServiceName() string { func TriggerChangeChannel(table string) { for _, tfmContext := range tfmContextMap { if notificationFlowChannel, notificationChannelOk := tfmContext.ChannelMap[FlowNameType(table)]; notificationChannelOk { - if len(notificationFlowChannel) == 5 { - emptied: - for i := 0; i < 3; i++ { - select { - case <-notificationFlowChannel: - default: - break emptied - } - } - } go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -108,8 +99,30 @@ func TriggerChangeChannel(table string) { } func TriggerAllChangeChannel(table string, changeIds map[string]string) { - for _, tfmContext := range tfmContextMap { + if !cleanerInit { //Kicks off cleaner goroutine if not already active. + cleanerInit = true + for _, tfmContext := range tfmContextMap { + if notificationFlowChannel, notificationChannelOk := tfmContext.ChannelMap[FlowNameType(table)]; notificationChannelOk { + go func(nFC chan bool) { + for { + if len(nFC) >= 8 { + emptied: + for i := 0; i < 4; i++ { + select { + case <-nFC: + default: + nFC <- true + break emptied + } + } + } + } + }(notificationFlowChannel) + } + } + } + for _, tfmContext := range tfmContextMap { // If changIds identified, manually trigger a change. if table != "" { for changeIdKey, changeIdValue := range changeIds { @@ -125,16 +138,6 @@ func TriggerAllChangeChannel(table string, changeIds map[string]string) { } } if notificationFlowChannel, notificationChannelOk := tfmContext.ChannelMap[FlowNameType(table)]; notificationChannelOk { - if len(notificationFlowChannel) == 5 { - emptied: - for i := 0; i < 3; i++ { - select { - case <-notificationFlowChannel: - default: - break emptied - } - } - } go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -143,17 +146,7 @@ func TriggerAllChangeChannel(table string, changeIds map[string]string) { } for _, notificationFlowChannel := range tfmContext.ChannelMap { - if len(notificationFlowChannel) == 5 { - emptied1: - for i := 0; i < 3; i++ { - select { - case <-notificationFlowChannel: - default: - break emptied1 - } - } - } - if len(notificationFlowChannel) < 3 { + if len(notificationFlowChannel) < 10 { go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -277,7 +270,7 @@ func (tfmContext *TrcFlowMachineContext) Init( tfmContext.ChannelMap = make(map[FlowNameType]chan bool) for _, table := range tableNames { - tfmContext.ChannelMap[FlowNameType(table)] = make(chan bool, 5) + tfmContext.ChannelMap[FlowNameType(table)] = make(chan bool, 10) } for _, f := range additionalFlowNames { @@ -478,16 +471,6 @@ func (tfmContext *TrcFlowMachineContext) seedVaultCycle(tfContext *TrcFlowContex eUtils.LogErrorMessage(tfmContext.Config, "Receiving shutdown presumably from vault.", true) os.Exit(0) case <-flowChangedChannel: - if len(flowChangedChannel) == 5 { - emptied: - for i := 0; i < 3; i++ { - select { - case <-flowChangedChannel: - default: - break emptied - } - } - } tfmContext.vaultPersistPushRemoteChanges( tfContext, identityColumnName, @@ -782,7 +765,7 @@ func (tfmContext *TrcFlowMachineContext) CallDBQuery(tfContext *TrcFlowContext, // look up channels and notify them too. for _, flowNotification := range flowNotifications { if notificationFlowChannel, ok := tfmContext.ChannelMap[flowNotification]; ok { - if len(notificationFlowChannel) < 5 { + if len(notificationFlowChannel) < 10 { go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -795,7 +778,7 @@ func (tfmContext *TrcFlowMachineContext) CallDBQuery(tfContext *TrcFlowContext, additionalTestFlows := tfmContext.GetAdditionalFlowsByState(flowtestState) for _, flowNotification := range additionalTestFlows { if notificationFlowChannel, ok := tfmContext.ChannelMap[flowNotification]; ok { - if len(notificationFlowChannel) < 5 { + if len(notificationFlowChannel) < 10 { go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -876,7 +859,7 @@ func (tfmContext *TrcFlowMachineContext) CallDBQuery(tfContext *TrcFlowContext, // look up channels and notify them too. for _, flowNotification := range flowNotifications { if notificationFlowChannel, ok := tfmContext.ChannelMap[flowNotification]; ok { - if len(notificationFlowChannel) < 5 { + if len(notificationFlowChannel) < 10 { go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) @@ -889,7 +872,7 @@ func (tfmContext *TrcFlowMachineContext) CallDBQuery(tfContext *TrcFlowContext, additionalTestFlows := tfmContext.GetAdditionalFlowsByState(flowtestState) for _, flowNotification := range additionalTestFlows { if notificationFlowChannel, ok := tfmContext.ChannelMap[flowNotification]; ok { - if len(notificationFlowChannel) < 5 { + if len(notificationFlowChannel) < 10 { go func(nfc chan bool) { nfc <- true }(notificationFlowChannel) From c4011a66e4bec47668ab4f71ad29d8c9dcffea12 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Tue, 12 Dec 2023 09:30:26 -0800 Subject: [PATCH 50/55] Prepare for release. (#862) * Fix locking issue. * Roll back since not needed anymore. * Update to more concurrent safe go-mysql * Update to latest go-mysql-server build. --------- Co-authored-by: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Co-authored-by: Karnveer Gill --- go.mod | 5 +++-- go.sum | 29 ++++++++--------------------- trcflow/core/context_helper.go | 1 + 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index d8fa61d3b..8b3f6fbf3 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,8 @@ require github.com/trimble-oss/tierceron-nute v0.0.0-20230128181737-65043c9e434b require github.com/trimble-oss/tierceron-succinctly v0.0.0-00010101000000-000000000000 // indirect require ( + VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 + VaultConfig.Test v0.0.0-00010101000000-000000000000 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/containers/azcontainerregistry v0.2.0 @@ -54,7 +56,6 @@ require ( ) require ( - VaultConfig.TenantConfig v0.0.0-00010101000000-000000000000 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect @@ -247,7 +248,7 @@ replace github.com/trimble-oss/tierceron-hat => ../tierceron-hat replace github.com/trimble-oss/tierceron-succinctly => ../tierceron-succinctly -replace github.com/dolthub/go-mysql-server => github.com/trimble-oss/go-mysql-server v0.12.0-1.7 +replace github.com/dolthub/go-mysql-server => github.com/trimble-oss/go-mysql-server v0.12.0-1.10 replace github.com/trimble-oss/tierceron-nute => ../tierceron-nute diff --git a/go.sum b/go.sum index 0fd858766..25aee0e5e 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -166,8 +168,6 @@ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5O github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dolthub/vitess v0.0.0-20220930181015-759b1acd9188 h1:TJVpwZ8XY/fNK1ZgV9QMw+DE6WKJRi28Ruilk/qTmOU= -github.com/dolthub/vitess v0.0.0-20220930181015-759b1acd9188/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs= github.com/dolthub/vitess v0.0.0-20221128235342-e80b37ceed86 h1:ZZpN3gM/lKUnZ/jxuPrdVqd8pCtkbz2iEda+ssMaXHg= github.com/dolthub/vitess v0.0.0-20221128235342-e80b37ceed86/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -408,7 +408,6 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -422,8 +421,6 @@ github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= -github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -467,8 +464,6 @@ github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267 h1:e1ok06zG github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267/go.mod h1:WX57W2PwkrOPQ6rVQk+dy5/htHIaB4aBM70EwKThu10= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d h1:W+SIwDdl3+jXWeidYySAgzytE3piq6GumXeBjFBG67c= -github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -483,9 +478,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackmordaunt/icns/v2 v2.2.1/go.mod h1:6aYIB9eSzyfHHMKqDf17Xrs1zetQPReAkiUSHzdw4cI= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= -github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -772,8 +766,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -815,8 +807,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/templexxx/cpu v0.1.0 h1:wVM+WIJP2nYaxVxqgHPD4wGA2aJ9rvrQRV8CvFzNb40= @@ -828,10 +820,10 @@ github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K0 github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/trimble-oss/go-mysql-server v0.12.0-1.6 h1:MSaLKPO+FyLkcFGK12Qx03ouU/BLmf1lYFstH4tdz/0= -github.com/trimble-oss/go-mysql-server v0.12.0-1.6/go.mod h1:qQReFVsJ0CPJLxX3lAuZxYWi6T4twzErI//D2bSSttY= -github.com/trimble-oss/go-mysql-server v0.12.0-1.7 h1:yqusp2FPi09XLyxFajiHd5mR2rcyZUWpW9mROAcbqI4= -github.com/trimble-oss/go-mysql-server v0.12.0-1.7/go.mod h1:gKm3pHgnop43wPmdX4LuB90Yiru1osP1K8XBCTE90dw= +github.com/trimble-oss/go-mysql-server v0.12.0-1.8 h1:+jzvlTmCVLXGK5uD4VSs7axLJYRynHRqIDGZbz86G78= +github.com/trimble-oss/go-mysql-server v0.12.0-1.8/go.mod h1:Fw74iKSS3oIwtxB3ZJmnC7chclpYJIQjHIdUIvs2JDk= +github.com/trimble-oss/go-mysql-server v0.12.0-1.9 h1:9EHUCvZZ7NAEujmZVsYNTIfdfslMtfqQB4Ay4Ul6pDk= +github.com/trimble-oss/go-mysql-server v0.12.0-1.9/go.mod h1:BWcyVjTqYp/wL9LYTRJ341vcOaJDaDRPldmyay0CbmU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchtv/twirp v5.12.1+incompatible h1:UnrJ4Z8llkdjnQbLqJBWRBwaDGojBsU5lft3DrD/SvY= github.com/twitchtv/twirp v5.12.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= @@ -952,7 +944,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1034,8 +1025,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1237,7 +1226,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1284,7 +1272,6 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 h1:AB/lmRny7e2pLhFEYIbl5qkDAUt2h0ZRO4wGPhZf+ik= google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE= -google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/trcflow/core/context_helper.go b/trcflow/core/context_helper.go index bfa38da5e..e27a03612 100644 --- a/trcflow/core/context_helper.go +++ b/trcflow/core/context_helper.go @@ -501,6 +501,7 @@ func (tfmContext *TrcFlowMachineContext) seedTrcDbFromVault( } } tfContext.Inserter.Close(tfmContext.TierceronEngine.Context) + tfContext.Inserter = nil return nil From f44a85dd1efcc23a02d608cb51b257c2bb232e27 Mon Sep 17 00:00:00 2001 From: Joel Rieke Date: Tue, 12 Dec 2023 09:48:30 -0800 Subject: [PATCH 51/55] Update go.sum --- go.sum | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index 25aee0e5e..8b2ce0d1e 100644 --- a/go.sum +++ b/go.sum @@ -820,10 +820,8 @@ github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K0 github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/trimble-oss/go-mysql-server v0.12.0-1.8 h1:+jzvlTmCVLXGK5uD4VSs7axLJYRynHRqIDGZbz86G78= -github.com/trimble-oss/go-mysql-server v0.12.0-1.8/go.mod h1:Fw74iKSS3oIwtxB3ZJmnC7chclpYJIQjHIdUIvs2JDk= -github.com/trimble-oss/go-mysql-server v0.12.0-1.9 h1:9EHUCvZZ7NAEujmZVsYNTIfdfslMtfqQB4Ay4Ul6pDk= -github.com/trimble-oss/go-mysql-server v0.12.0-1.9/go.mod h1:BWcyVjTqYp/wL9LYTRJ341vcOaJDaDRPldmyay0CbmU= +github.com/trimble-oss/go-mysql-server v0.12.0-1.10 h1:sGq+oXobGEuN2oaX0+47T6d86FGQ62aWQaCXlOUw3Kc= +github.com/trimble-oss/go-mysql-server v0.12.0-1.10/go.mod h1:BWcyVjTqYp/wL9LYTRJ341vcOaJDaDRPldmyay0CbmU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchtv/twirp v5.12.1+incompatible h1:UnrJ4Z8llkdjnQbLqJBWRBwaDGojBsU5lft3DrD/SvY= github.com/twitchtv/twirp v5.12.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= From c96d0b82b8ddd7791f598f678f43c3ceb1f743c3 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:05:09 -0800 Subject: [PATCH 52/55] Various codeql fixes. (#865) * Adding bounds check. * Add actual bounds checking. * Bounds check totalKeyShard * Disallow complex path navigation. * Add log sanitizing. * More sanitization. * Directly applying recommendation from dependabot --------- --- go.mod | 3 ++- go.sum | 6 ++++-- trcinit/initlib/vault-seed.go | 6 ++++++ trcinit/initlib/verify.go | 2 +- trcinitbase/init.go | 13 ++++++++----- utils/diffUtil.go | 10 ++++++++-- utils/errorUtils.go | 15 +++++++++++---- webapi/apiRouter/router.go | 10 +++++++--- 8 files changed, 47 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 8b3f6fbf3..5b1515277 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/daviddengcn/go-colortext v1.0.0 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect @@ -106,6 +106,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect diff --git a/go.sum b/go.sum index 8b2ce0d1e..a321d57f5 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfz github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dolthub/vitess v0.0.0-20221128235342-e80b37ceed86 h1:ZZpN3gM/lKUnZ/jxuPrdVqd8pCtkbz2iEda+ssMaXHg= github.com/dolthub/vitess v0.0.0-20221128235342-e80b37ceed86/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs= @@ -658,6 +658,8 @@ github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdM github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/trcinit/initlib/vault-seed.go b/trcinit/initlib/vault-seed.go index 30852d957..2c5a740d5 100644 --- a/trcinit/initlib/vault-seed.go +++ b/trcinit/initlib/vault-seed.go @@ -471,6 +471,12 @@ func seedVaultWithCertsFromEntry(config *eUtils.DriverConfig, mod *helperkv.Modi certPath = strings.Replace(certPath, "ENV", config.EnvRaw, 1) } } + if strings.Contains(certPath, "..") { + errMsg := eUtils.SanitizeForLogging("Invalid cert path: " + certPath + " Certs not allowed to contain complex path navigation.") + fmt.Println(errMsg) + config.Log.Println(errMsg) + return + } certPath = coreopts.GetFolderPrefix(nil) + "_seeds/" + certPath eUtils.LogInfo(config, fmt.Sprintf("Inspecting certificate: "+certPath+".")) cert, err := os.ReadFile(certPath) diff --git a/trcinit/initlib/verify.go b/trcinit/initlib/verify.go index b0f0a0a18..040907c0c 100644 --- a/trcinit/initlib/verify.go +++ b/trcinit/initlib/verify.go @@ -32,7 +32,7 @@ func verify(config *eUtils.DriverConfig, mod *helperkv.Modifier, v map[interface if err != nil { return nil, err } - config.Log.Printf("Verifying %s as type %s\n", service, vType) + config.Log.Print(eUtils.SanitizeForLogging(fmt.Sprintf("Verifying %s as type %s\n", service, vType))) switch vType { case "db": if url, ok := serviceData["url"].(string); ok { diff --git a/trcinitbase/init.go b/trcinitbase/init.go index 2e4b3b6b8..94a7ea82a 100644 --- a/trcinitbase/init.go +++ b/trcinitbase/init.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "log" + "math" "os" "path/filepath" "strconv" @@ -270,14 +271,16 @@ func CommonMain(envPtr *string, v.SetToken(*tokenPtr) } else { // Unseal and grab keys/root token totalKeyShard, err := strconv.ParseUint(*keyShardPtr, 10, 32) - if err != nil { + if err != nil || totalKeyShard > math.MaxInt { fmt.Println("Unable to parse totalKeyShard into int") + os.Exit(-1) } - unsealShardPtr, err := strconv.ParseUint(*unsealShardPtr, 10, 32) - if err != nil { - fmt.Println("Unable to parse unsealShardPtr into int") + keyThreshold, err := strconv.ParseUint(*unsealShardPtr, 10, 32) + if err != nil || keyThreshold > math.MaxInt { + fmt.Println("Unable to parse keyThreshold into int") + os.Exit(-1) } - keyToken, err := v.InitVault(int(totalKeyShard), int(unsealShardPtr)) + keyToken, err := v.InitVault(int(totalKeyShard), int(keyThreshold)) eUtils.LogErrorObject(config, err, true) v.SetToken(keyToken.Token) v.SetShards(keyToken.Keys) diff --git a/utils/diffUtil.go b/utils/diffUtil.go index 8347bf6d7..1c03688dd 100644 --- a/utils/diffUtil.go +++ b/utils/diffUtil.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "log" + "math" "net/url" "sort" "strconv" @@ -317,8 +318,13 @@ func VersionHelper(versionData map[string]interface{}, templateOrValues bool, va keys := make([]int, 0, len(versionData)) for versionNumber := range versionData { - versionNo, _ := strconv.ParseInt(versionNumber, 10, 64) - keys = append(keys, int(versionNo)) + versionNo, err := strconv.ParseInt(versionNumber, 10, 64) + if err == nil && versionNo <= math.MaxInt { + keys = append(keys, int(versionNo)) + } else { + fmt.Printf("Version limit exceeded: %s\n", versionNumber) + return + } } sort.Ints(keys) for _, key := range keys { diff --git a/utils/errorUtils.go b/utils/errorUtils.go index b3b6cf99e..eb0de1ef6 100644 --- a/utils/errorUtils.go +++ b/utils/errorUtils.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "strings" ) var headlessService bool @@ -167,9 +168,9 @@ func LogErrorObject(config *DriverConfig, err error, exit bool) { if !headlessService { fmt.Printf("Errors encountered, exiting and writing to log file: %v\n", err) } - config.Log.Fatal(err) + config.Log.Fatal(SanitizeForLogging(err.Error())) } else { - config.Log.Println(err) + config.Log.Println(SanitizeForLogging(err.Error())) config.Log.SetPrefix(_prefix) } } @@ -178,12 +179,12 @@ func LogErrorObject(config *DriverConfig, err error, exit bool) { // LogErrorObject writes errors to the passed logger object and exits func LogInfo(config *DriverConfig, msg string) { if !headlessService { - fmt.Println(msg) + fmt.Println(SanitizeForLogging(msg)) } if config != nil && config.Log != nil { _prefix := config.Log.Prefix() config.Log.SetPrefix("[INFO]") - config.Log.Println(msg) + config.Log.Println(SanitizeForLogging(msg)) config.Log.SetPrefix(_prefix) } } @@ -232,3 +233,9 @@ func LogErrorAndSafeExit(config *DriverConfig, err error, code int) error { return err } + +func SanitizeForLogging(errMsg string) string { + errMsgSanitized := strings.ReplaceAll(errMsg, "\n", "") + errMsgSanitized = strings.ReplaceAll(errMsgSanitized, "\r", "") + return errMsgSanitized +} diff --git a/webapi/apiRouter/router.go b/webapi/apiRouter/router.go index 7bc2969c4..b733bd21b 100644 --- a/webapi/apiRouter/router.go +++ b/webapi/apiRouter/router.go @@ -39,7 +39,8 @@ func authrouter(restHandler http.Handler, isAuth bool) *rtr.Router { // Simply route noauth := func(w http.ResponseWriter, r *http.Request, ps rtr.Params) { s.Log.SetPrefix("[INFO]") - s.Log.Printf("Incoming request %s %s\n \tfrom %s\n", r.Method, r.URL.String(), r.RemoteAddr) + errMsg := eUtils.SanitizeForLogging(fmt.Sprintf("Incoming request %s %s From %s", r.Method, r.URL.String(), r.RemoteAddr)) + s.Log.Print(errMsg) s.Log.Println("Handling with no auth") restHandler.ServeHTTP(w, r) } @@ -53,7 +54,8 @@ func authrouter(restHandler http.Handler, isAuth bool) *rtr.Router { var errMsg string s.Log.SetPrefix("[INFO]") - s.Log.Printf("Incoming request %s %s\n \tFrom %s\n", r.Method, r.URL.String(), r.RemoteAddr) + errMsg = eUtils.SanitizeForLogging(fmt.Sprintf("Incoming request %s %s From %s", r.Method, r.URL.String(), r.RemoteAddr)) + s.Log.Print(errMsg) s.Log.SetPrefix("[ERROR]") authString := r.Header.Get("Authorization") @@ -98,6 +100,8 @@ func authrouter(restHandler http.Handler, isAuth bool) *rtr.Router { // Token claims not in json format errMsg = "Format error with auth token claims" http.Error(w, errMsg, 401) + + errMsg = eUtils.SanitizeForLogging(errMsg) s.Log.Printf("%d: %s", 401, errMsg) return } @@ -110,7 +114,7 @@ func authrouter(restHandler http.Handler, isAuth bool) *rtr.Router { // Auth method passed but is not a bearer token errMsg = "Invalid auth method " + splitAuth[0] http.Error(w, errMsg, 401) - s.Log.Printf("%d: %s", 401, errMsg) + s.Log.Print(eUtils.SanitizeForLogging(fmt.Sprintf("%d: %s", 401, errMsg))) return } // No token to authenticate against From 0b2b5ecade88fcb4d44d64b8cbf7bf5e4d821d35 Mon Sep 17 00:00:00 2001 From: Karnveer Gill <101755160+KarnveerSGill@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:05:00 -0800 Subject: [PATCH 53/55] Merging Develop into Main (#868) --- trcflow/core/context.go | 4 ++-- trctest/maintest.go | 2 ++ vaulthelper/kv/Modifier.go | 46 +++++++++++++++++++------------------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/trcflow/core/context.go b/trcflow/core/context.go index 1c31b83be..1a5c57e67 100644 --- a/trcflow/core/context.go +++ b/trcflow/core/context.go @@ -650,8 +650,8 @@ func (tfmContext *TrcFlowMachineContext) SyncTableCycle(tfContext *TrcFlowContex // Second row here // Not sure if necessary to copy entire ReportStatistics method - //tenantIndexPath, tenantDFSIdPath := utilcore.GetDFSPathName() - //df.FinishStatistic(tfmContext, tfContext, tfContext.GoMod, "flume", tenantIndexPath, tenantDFSIdPath, tfmContext.Config.Log, false) + tenantIndexPath, tenantDFSIdPath := utilcore.GetDFSPathName() + df.FinishStatistic(tfmContext, tfContext, tfContext.GoMod, "flume", tenantIndexPath, tenantDFSIdPath, tfmContext.Config.Log, false) //df.FinishStatistic(tfmContext, tfContext, tfContext.GoMod, ...) tfmContext.FlowControllerLock.Lock() diff --git a/trctest/maintest.go b/trctest/maintest.go index fb41dd57c..b31339c80 100644 --- a/trctest/maintest.go +++ b/trctest/maintest.go @@ -29,7 +29,9 @@ func main() { pluginConfig := testopts.GetTestConfig(*tokenPtr, false) pluginConfig["address"] = "" pluginConfig["vaddress"] = "" + pluginConfig["caddress"] = "" pluginConfig["token"] = "" + pluginConfig["ctoken"] = "" pluginConfig["env"] = "dev" pluginConfig["insecure"] = true diff --git a/vaulthelper/kv/Modifier.go b/vaulthelper/kv/Modifier.go index 95acbec5b..822975af3 100644 --- a/vaulthelper/kv/Modifier.go +++ b/vaulthelper/kv/Modifier.go @@ -95,8 +95,8 @@ func PreCheckEnvironment(environment string) (string, string, bool, error) { // Any errors generated in creating the client func NewModifier(insecure bool, token string, address string, env string, regions []string, useCache bool, logger *log.Logger) (*Modifier, error) { if useCache { - PruneCache(env, 10) - checkoutModifier, err := cachedModifierHelper(env) + PruneCache(env, address, 10) + checkoutModifier, err := cachedModifierHelper(env, address) if err == nil && checkoutModifier != nil { checkoutModifier.Insecure = insecure checkoutModifier.RawEnv = env @@ -142,20 +142,20 @@ func NewModifier(insecure bool, token string, address string, env string, region return newModifier, nil } -func checkInitModCache(env string) { - if _, ok := modifierCache[env]; !ok { +func checkInitModCache(env string, addr string) { + if _, ok := modifierCache[fmt.Sprintf("%s+%s", env, addr)]; !ok { modifierCachLock.Lock() - modifierCache[env] = &modCache{modCount: 0, modifierChan: make(chan *Modifier, 20)} + modifierCache[fmt.Sprintf("%s+%s", env, addr)] = &modCache{modCount: 0, modifierChan: make(chan *Modifier, 20)} modifierCachLock.Unlock() } } -func cachedModifierHelper(env string) (*Modifier, error) { - checkInitModCache(env) +func cachedModifierHelper(env string, addr string) (*Modifier, error) { + checkInitModCache(env, addr) for { select { - case checkoutModifier := <-modifierCache[env].modifierChan: + case checkoutModifier := <-modifierCache[fmt.Sprintf("%s+%s", env, addr)].modifierChan: atomic.AddUint64(&modifierCache[env].modCount, ^uint64(0)) return checkoutModifier, nil case <-time.After(time.Millisecond * 200): @@ -174,32 +174,32 @@ func (m *Modifier) Release() { } func (m *Modifier) releaseHelper(env string) { - checkInitModCache(env) + checkInitModCache(env, m.client.Address()) // Since modifiers are re-used now, this may not be necessary or even desired for that // matter. // m.httpClient.CloseIdleConnections() - if modifierCache[env].modCount > 10 { + if modifierCache[fmt.Sprintf("%s+%s", env, m.client.Address())].modCount > 10 { m.CleanCache(10) } - atomic.AddUint64(&modifierCache[env].modCount, 1) - modifierCache[env].modifierChan <- m + atomic.AddUint64(&modifierCache[fmt.Sprintf("%s+%s", env, m.client.Address())].modCount, 1) + modifierCache[fmt.Sprintf("%s+%s", env, m.client.Address())].modifierChan <- m } func (m *Modifier) RemoveFromCache() { m.CleanCache(20) } -func cleanCacheHelper(env string, limit uint64) { +func cleanCacheHelper(env string, addr string, limit uint64) { modifierCachLock.Lock() - if modifierCache[env].modCount > 1 { + if modifierCache[fmt.Sprintf("%s+%s", env, addr)].modCount > 1 { emptied: for i := uint64(0); i < limit; i++ { select { - case mod := <-modifierCache[env].modifierChan: + case mod := <-modifierCache[fmt.Sprintf("%s+%s", env, addr)].modifierChan: mod.Close() - atomic.AddUint64(&modifierCache[env].modCount, ^uint64(0)) + atomic.AddUint64(&modifierCache[fmt.Sprintf("%s+%s", env, addr)].modCount, ^uint64(0)) default: break emptied } @@ -208,11 +208,11 @@ func cleanCacheHelper(env string, limit uint64) { modifierCachLock.Unlock() } -func PruneCache(env string, limit uint64) { - if modifierCache != nil && modifierCache[env] != nil { - if modifierCache[env].modCount > limit { - if _, ok := modifierCache[env]; ok { - cleanCacheHelper(env, limit) +func PruneCache(env string, addr string, limit uint64) { + if modifierCache != nil && modifierCache[fmt.Sprintf("%s+%s", env, addr)] != nil { + if modifierCache[fmt.Sprintf("%s+%s", env, addr)].modCount > limit { + if _, ok := modifierCache[fmt.Sprintf("%s+%s", env, addr)]; ok { + cleanCacheHelper(env, addr, limit) } } } @@ -221,9 +221,9 @@ func PruneCache(env string, limit uint64) { func (m *Modifier) CleanCache(limit uint64) { m.Close() if _, ok := modifierCache[m.Env]; ok { - cleanCacheHelper(m.Env, limit) + cleanCacheHelper(m.Env, m.client.Address(), limit) } else { - cleanCacheHelper(m.RawEnv, limit) + cleanCacheHelper(m.RawEnv, m.client.Address(), limit) } } From 8b53fb9dd4c5081fa776e5818c83c2f939362642 Mon Sep 17 00:00:00 2001 From: joel-rieke <53945260+joel-rieke@users.noreply.github.com> Date: Fri, 15 Dec 2023 09:21:42 -0800 Subject: [PATCH 54/55] Prepare for release. (#871) * Update to latest. --------- --- go.mod | 3 +-- go.sum | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5b1515277..ae90193a2 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect @@ -249,7 +248,7 @@ replace github.com/trimble-oss/tierceron-hat => ../tierceron-hat replace github.com/trimble-oss/tierceron-succinctly => ../tierceron-succinctly -replace github.com/dolthub/go-mysql-server => github.com/trimble-oss/go-mysql-server v0.12.0-1.10 +replace github.com/dolthub/go-mysql-server => github.com/trimble-oss/go-mysql-server v0.12.0-1.12 replace github.com/trimble-oss/tierceron-nute => ../tierceron-nute diff --git a/go.sum b/go.sum index a321d57f5..0371a0d0f 100644 --- a/go.sum +++ b/go.sum @@ -658,8 +658,6 @@ github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdM github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -822,8 +820,6 @@ github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K0 github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/trimble-oss/go-mysql-server v0.12.0-1.10 h1:sGq+oXobGEuN2oaX0+47T6d86FGQ62aWQaCXlOUw3Kc= -github.com/trimble-oss/go-mysql-server v0.12.0-1.10/go.mod h1:BWcyVjTqYp/wL9LYTRJ341vcOaJDaDRPldmyay0CbmU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchtv/twirp v5.12.1+incompatible h1:UnrJ4Z8llkdjnQbLqJBWRBwaDGojBsU5lft3DrD/SvY= github.com/twitchtv/twirp v5.12.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= From 32eefa03e9848250d93177bdfecec32ef2e6eca4 Mon Sep 17 00:00:00 2001 From: Joel Rieke Date: Fri, 15 Dec 2023 09:33:50 -0800 Subject: [PATCH 55/55] Update go.sum to match. --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 0371a0d0f..54c8ee7df 100644 --- a/go.sum +++ b/go.sum @@ -820,6 +820,8 @@ github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K0 github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/trimble-oss/go-mysql-server v0.12.0-1.12 h1:J9AY0+pgdwiAScHH4Ggb7aNy71oYobSTOmohO+32Hv0= +github.com/trimble-oss/go-mysql-server v0.12.0-1.12/go.mod h1:BWcyVjTqYp/wL9LYTRJ341vcOaJDaDRPldmyay0CbmU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchtv/twirp v5.12.1+incompatible h1:UnrJ4Z8llkdjnQbLqJBWRBwaDGojBsU5lft3DrD/SvY= github.com/twitchtv/twirp v5.12.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=