Clean up some template parsing. (#1127)

* Optimize some template path parsing.
* Support multiple services in a single project.
* Minor improvements and usage cleanup.

atrium/trcflow/core/context.go

@@ -397,7 +397,7 @@ func (tfmContext *TrcFlowMachineContext) CreateDataFlowTableTriggers(trcfc *TrcF
 }
 
 func (tfmContext *TrcFlowMachineContext) GetFlowConfiguration(trcfc *TrcFlowContext, flowTemplatePath string) (map[string]interface{}, bool) {
-	flowProject, flowService, flowConfigTemplatePath := eUtils.GetProjectService(flowTemplatePath)
+	flowProject, flowService, _, flowConfigTemplatePath := eUtils.GetProjectService(nil, flowTemplatePath)
 	flowConfigTemplateName := eUtils.GetTemplateFileName(flowConfigTemplatePath, flowService)
 	trcfc.GoMod.SectionKey = "/Restricted/"
 	trcfc.GoMod.SectionName = flowService

atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go

@@ -26,7 +26,7 @@ import (
 	trcapimgmtbase "github.com/trimble-oss/tierceron/atrium/vestibulum/trcdb/trcapimgmtbase"
 )
 
-func CommonMain(envPtr *string,
+func CommonMain(envDefaultPtr *string,
 	addrPtr *string,
 	tokenPtr *string,
 	envCtxPtr *string,
@@ -260,7 +260,7 @@ func CommonMain(envPtr *string,
 
 	//
 	if tokenNamePtr == nil || *tokenNamePtr == "" || tokenPtr == nil || *tokenPtr == "" {
-		autoErr := eUtils.AutoAuth(&trcshDriverConfigBase.DriverConfig, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, false)
+		autoErr := eUtils.AutoAuth(&trcshDriverConfigBase.DriverConfig, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envDefaultPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, false)
 		if autoErr != nil {
 			eUtils.LogErrorMessage(&trcshDriverConfigBase.DriverConfig.CoreConfig, "Auth failure: "+autoErr.Error(), false)
 			return errors.New("auth failure")
@@ -278,7 +278,7 @@ func CommonMain(envPtr *string,
 		fmt.Println("Error: Could not find plugin config")
 		return errors.New("could not find plugin config")
 	}
-	pluginConfig["env"] = *envPtr
+	pluginConfig["env"] = *envDefaultPtr
 	pluginConfig["vaddress"] = *addrPtr
 	if tokenPtr != nil {
 		pluginConfig["token"] = *tokenPtr
@@ -309,12 +309,12 @@ func CommonMain(envPtr *string,
 	} else {
 		trcshDriverConfigBase.DriverConfig.SubSectionValue = *pluginNamePtr
 	}
-	mod.Env = *envPtr
+	mod.Env = *envDefaultPtr
 	if logger != nil {
 		logger.Printf("Certify mod initialized\n")
 	}
 
-	if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") {
+	if strings.HasPrefix(*envDefaultPtr, "staging") || strings.HasPrefix(*envDefaultPtr, "prod") || strings.HasPrefix(*envDefaultPtr, "dev") {
 		supportedRegions := eUtils.GetSupportedProdRegions()
 		if *regionPtr != "" {
 			for _, supportedRegion := range supportedRegions {
@@ -332,7 +332,7 @@ func CommonMain(envPtr *string,
 	}
 
 	if *updateAPIMPtr {
-		updateAPIMError := trcapimgmtbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, startDirPtr, config, mod)
+		updateAPIMError := trcapimgmtbase.CommonMain(envDefaultPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, startDirPtr, config, mod)
 		if updateAPIMError != nil {
 			fmt.Println(updateAPIMError.Error())
 			fmt.Println("Couldn't update APIM...proceeding with build")

atrium/vestibulum/trcflow/deploy/process.go

@@ -393,7 +393,7 @@ func PluginDeployedUpdate(driverConfig *eUtils.DriverConfig, mod *helperkv.Modif
 
 	hostRegion := coreopts.BuildOptions.GetRegion(hostName)
 	mod.Regions = append(mod.Regions, hostRegion)
-	projects, services, _ := eUtils.GetProjectServices(cPath)
+	projects, services, _ := eUtils.GetProjectServices(nil, cPath)
 	for _, pluginName := range pluginNameList {
 		for i := 0; i < len(projects); i++ {
 			if services[i] == "Certify" {

atrium/vestibulum/trcflow/flumen/process.go

@@ -90,7 +90,7 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error
 		GetAdditionalFlowsByState: flowopts.BuildOptions.GetAdditionalFlowsByState,
 		FlowMap:                   map[flowcore.FlowNameType]*flowcore.TrcFlowContext{},
 	}
-	projects, services, _ := eUtils.GetProjectServices(pluginConfig["connectionPath"].([]string))
+	projects, services, _ := eUtils.GetProjectServices(nil, pluginConfig["connectionPath"].([]string))
 	var sourceDatabaseConfigs []map[string]interface{}
 	var vaultDatabaseConfig map[string]interface{}
 	var spiralDatabaseConfig map[string]interface{}
@@ -198,7 +198,7 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error
 	flowStateReceiverMap := map[string]chan flowcorehelper.FlowStateUpdate{}
 
 	for _, template := range templateList {
-		source, service, tableTemplateName := eUtils.GetProjectService(template)
+		source, service, _, tableTemplateName := eUtils.GetProjectService(nil, template)
 		tableName := eUtils.GetTemplateFileName(tableTemplateName, service)
 		if tableName != flowcorehelper.TierceronFlowConfigurationTableName {
 			driverConfigBasis.VersionFilter = append(driverConfigBasis.VersionFilter, tableName)

atrium/vestibulum/trcsh/deployutil/deployutil.go

@@ -39,7 +39,7 @@ func LoadPluginDeploymentScript(trcshDriverConfig *capauth.TrcshDriverConfig, tr
 			tempEnv := trcshDriverConfig.DriverConfig.EnvBasis
 			envParts := strings.Split(trcshDriverConfig.DriverConfig.EnvBasis, "-")
 			mod.Env = envParts[0]
-			fmt.Printf("Loading deployment details for %s and env %s", deployment, mod.Env)
+			fmt.Printf("Loading deployment details for %s and env %s\n", deployment, mod.Env)
 			deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment))
 			mod.Env = tempEnv
 			if err != nil {
@@ -56,9 +56,7 @@ func LoadPluginDeploymentScript(trcshDriverConfig *capauth.TrcshDriverConfig, tr
 			if trcProjectService, ok := trcshDriverConfig.DriverConfig.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") {
 				var content []byte
 				trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/")
-				trcshDriverConfig.DriverConfig.ZeroConfig = true
 				contentArray, _, _, err := vcutils.ConfigTemplate(&trcshDriverConfig.DriverConfig, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true)
-				trcshDriverConfig.DriverConfig.ZeroConfig = false
 				if err != nil {
 					eUtils.LogErrorObject(&trcshDriverConfig.DriverConfig.CoreConfig, err, false)
 					return nil, err

atrium/vestibulum/trcshbase/trcsh.go

@@ -108,8 +108,9 @@ func TrcshInitConfig(env string, region string, pathParam string, outputMemCache
 			MemFs: &trcshMemFs.TrcshMemFs{
 				BillyFs: memfs.New(),
 			},
-			Regions:   regions,
-			PathParam: pathParam, // Make available to trcplgtool
+			ZeroConfig: true,
+			Regions:    regions,
+			PathParam:  pathParam, // Make available to trcplgtool
 		},
 	}
 	return trcshDriverConfig, nil
@@ -287,14 +288,14 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string,
 		memprotectopts.MemProtect(nil, secretIDPtr)
 		memprotectopts.MemProtect(nil, appRoleIDPtr)
 
-		config, err := TrcshInitConfig(*envPtr, *regionPtr, pathParam, true)
+		trcshDriverConfig, err := TrcshInitConfig(*envPtr, *regionPtr, pathParam, true)
 		if err != nil {
 			fmt.Printf("trcsh config setup failure: %s\n", err.Error())
 			os.Exit(124)
 		}
 
 		//Open deploy script and parse it.
-		ProcessDeploy(nil, config, "", "", *trcPathPtr, *projectServicePtr, secretIDPtr, appRoleIDPtr, true, dronePtr)
+		ProcessDeploy(nil, trcshDriverConfig, "", "", *trcPathPtr, *projectServicePtr, secretIDPtr, appRoleIDPtr, true, dronePtr)
 	} else {
 		agentToken := os.Getenv("AGENT_TOKEN")
 		agentEnv := os.Getenv("AGENT_ENV")
@@ -485,7 +486,7 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error {
 	return nil
 }
 
-func roleBasedRunner(env string,
+func roleBasedRunner(
 	region string,
 	trcshDriverConfig *capauth.TrcshDriverConfig,
 	control string,
@@ -497,7 +498,6 @@ func roleBasedRunner(env string,
 	*configCount -= 1
 	trcshDriverConfig.DriverConfig.AppRoleConfig = "config.yml"
 	trcshDriverConfig.DriverConfig.FileFilter = nil
-	trcshDriverConfig.DriverConfig.EnvBasis = env
 	trcshDriverConfig.DriverConfig.CoreConfig.WantCerts = false
 	trcshDriverConfig.DriverConfig.IsShellSubProcess = true
 	trcshDriverConfig.DriverConfig.CoreConfig.Log.Printf("Role runner init: %s\n", control)
@@ -513,25 +513,26 @@ func roleBasedRunner(env string,
 		trcshDriverConfig.DriverConfig.EndDir = "."
 	}
 	configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":")
-	tokenName := "config_token_" + env
+	tokenName := "config_token_" + trcshDriverConfig.DriverConfig.EnvBasis
 	tokenConfig := ""
-	configEnv := env
+	envDefaultPtr := trcshDriverConfig.DriverConfig.EnvBasis
 	var err error
 	trcshDriverConfig.DriverConfig.CoreConfig.Log.Printf("Role runner complete: %s\n", control)
 
 	switch control {
 	case "trcplgtool":
 		tokenConfig := token
-		err = trcplgtoolbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, &region, nil, deployArgLines, trcshDriverConfig)
+		envDefaultPtr = trcshDriverConfig.DriverConfig.Env
+		err = trcplgtoolbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, &region, nil, deployArgLines, trcshDriverConfig)
 	case "trcconfig":
 		if trcshDriverConfig.DriverConfig.EnvBasis == "itdev" || trcshDriverConfig.DriverConfig.EnvBasis == "staging" || trcshDriverConfig.DriverConfig.EnvBasis == "prod" ||
 			trcshDriverConfig.DriverConfig.Env == "itdev" || trcshDriverConfig.DriverConfig.Env == "staging" || trcshDriverConfig.DriverConfig.Env == "prod" {
 			trcshDriverConfig.DriverConfig.OutputMemCache = false
 		}
-		err = trcconfigbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, &region, nil, deployArgLines, &trcshDriverConfig.DriverConfig)
+		err = trcconfigbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, &region, nil, deployArgLines, &trcshDriverConfig.DriverConfig)
 	case "trcsub":
 		trcshDriverConfig.DriverConfig.EndDir = trcshDriverConfig.DriverConfig.EndDir + "/trc_templates"
-		err = trcsubbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], nil, deployArgLines, &trcshDriverConfig.DriverConfig)
+		err = trcsubbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], nil, deployArgLines, &trcshDriverConfig.DriverConfig)
 	}
 	ResetModifier(&trcshDriverConfig.DriverConfig) //Resetting modifier cache to avoid token conflicts.
 
@@ -576,7 +577,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig,
 			trcshDriverConfig.DriverConfig.Token = token
 		}
 	case "trcconfig":
-		err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount)
+		err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount)
 		if err != nil {
 			os.Exit(1)
 		}
@@ -651,7 +652,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig,
 			trcshDriverConfig.FeatherCtx.Log = trcshDriverConfig.DriverConfig.CoreConfig.Log
 		}
 
-		err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, *gTrcshConfig.CToken, argsOrig, deployArgLines, configCount)
+		err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, *gTrcshConfig.CToken, argsOrig, deployArgLines, configCount)
 		if err != nil {
 			os.Exit(1)
 		}
@@ -694,7 +695,6 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig,
 func processDroneCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig,
 	onceKubeInit *sync.Once,
 	PipeOS billy.File,
-	env string,
 	region string,
 	trcshDriverConfig *capauth.TrcshDriverConfig,
 	control string,
@@ -704,7 +704,7 @@ func processDroneCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig,
 	deployArgLines []string,
 	configCount *int) error {
 
-	err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount)
+	err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount)
 	return err
 }
 
@@ -1048,7 +1048,6 @@ collaboratorReRun:
 					trcKubeDeploymentConfig,
 					&onceKubeInit,
 					PipeOS,
-					trcshDriverConfig.DriverConfig.Env,
 					region,
 					trcshDriverConfig,
 					control,

cmd/trcctl/trcctl.go

@@ -41,6 +41,7 @@ func main() {
 	flagset.Usage = func() {
 		fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0])
 		flagset.PrintDefaults()
+		fmt.Fprintf(flagset.Output(), "\nexample: trcctl {pub, sub, init, config, x} {flags}\n")
 	}
 	envPtr := flagset.String("env", "", "Environment to be seeded") //If this is blank -> use context otherwise override context.
 	tokenPtr := flagset.String("token", "", "Vault access token")
@@ -108,9 +109,9 @@ func main() {
 		case "init":
 			trcinitbase.CommonMain(envPtr, &addrPtr, &envContext, flagset, os.Args)
 		case "config":
-			trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, nil, os.Args, nil)
+			trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, flagset, os.Args, nil)
 		case "x":
-			trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil, nil, os.Args)
+			trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil, flagset, os.Args)
 		}
 	}
 }

pkg/cli/trcconfigbase/trcconfig.go

@@ -72,7 +72,7 @@ var (
 	ENDDIR_DEFAULT = "."
 )
 
-func CommonMain(envPtr *string,
+func CommonMain(envDefaultPtr *string,
 	addrPtr *string,
 	tokenPtr *string,
 	envCtxPtr *string,
@@ -96,6 +96,7 @@ func CommonMain(envPtr *string,
 		ConfigWg:      sync.WaitGroup{},
 		Mutex:         &sync.Mutex{},
 	}
+	var envPtr *string = nil
 
 	if flagset == nil {
 		flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError)
@@ -104,7 +105,7 @@ func CommonMain(envPtr *string,
 			flagset.PrintDefaults()
 		}
 
-		flagset.String("env", "dev", "Environment to configure")
+		envPtr = flagset.String("env", "dev", "Environment to configure")
 		flagset.String("addr", "", "API endpoint for the vault")
 		flagset.String("token", "", "Vault access token")
 		flagset.String("secretID", "", "Secret for app role ID")
@@ -172,6 +173,9 @@ func CommonMain(envPtr *string,
 			*wantCertsPtr = true
 		}
 	}
+	if envPtr == nil {
+		envPtr = envDefaultPtr
+	}
 	if !isShell {
 		if _, err := os.Stat(*startDirPtr); os.IsNotExist(err) {
 			fmt.Println("Missing required template folder: " + *startDirPtr)
@@ -213,6 +217,7 @@ func CommonMain(envPtr *string,
 			Insecure:   *insecurePtr,
 			StartDir:   append([]string{}, *startDirPtr),
 			EndDir:     *endDirPtr,
+			ZeroConfig: *zcPtr,
 		}
 
 		appRoleConfigPtr = new(string)
@@ -493,7 +498,7 @@ func CommonMain(envPtr *string,
 			StartDir:          driverConfigBase.StartDir,
 			EndDir:            driverConfigBase.EndDir,
 			WantKeystore:      *keyStorePtr,
-			ZeroConfig:        *zcPtr,
+			ZeroConfig:        driverConfigBase.ZeroConfig,
 			GenAuth:           false,
 			OutputMemCache:    driverConfigBase.OutputMemCache,
 			MemFs:             driverConfigBase.MemFs,