diff --git a/atrium/trcflow/core/context.go b/atrium/trcflow/core/context.go index 3b2ef3d73..b389692d0 100644 --- a/atrium/trcflow/core/context.go +++ b/atrium/trcflow/core/context.go @@ -397,7 +397,7 @@ func (tfmContext *TrcFlowMachineContext) CreateDataFlowTableTriggers(trcfc *TrcF } func (tfmContext *TrcFlowMachineContext) GetFlowConfiguration(trcfc *TrcFlowContext, flowTemplatePath string) (map[string]interface{}, bool) { - flowProject, flowService, flowConfigTemplatePath := eUtils.GetProjectService(flowTemplatePath) + flowProject, flowService, _, flowConfigTemplatePath := eUtils.GetProjectService(nil, flowTemplatePath) flowConfigTemplateName := eUtils.GetTemplateFileName(flowConfigTemplatePath, flowService) trcfc.GoMod.SectionKey = "/Restricted/" trcfc.GoMod.SectionName = flowService diff --git a/atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go b/atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go index 0b7df1747..9eb5f47e2 100644 --- a/atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go +++ b/atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go @@ -26,7 +26,7 @@ import ( trcapimgmtbase "github.com/trimble-oss/tierceron/atrium/vestibulum/trcdb/trcapimgmtbase" ) -func CommonMain(envPtr *string, +func CommonMain(envDefaultPtr *string, addrPtr *string, tokenPtr *string, envCtxPtr *string, @@ -260,7 +260,7 @@ func CommonMain(envPtr *string, // if tokenNamePtr == nil || *tokenNamePtr == "" || tokenPtr == nil || *tokenPtr == "" { - autoErr := eUtils.AutoAuth(&trcshDriverConfigBase.DriverConfig, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, false) + autoErr := eUtils.AutoAuth(&trcshDriverConfigBase.DriverConfig, secretIDPtr, appRoleIDPtr, tokenPtr, tokenNamePtr, envDefaultPtr, addrPtr, envCtxPtr, *appRoleConfigPtr, false) if autoErr != nil { eUtils.LogErrorMessage(&trcshDriverConfigBase.DriverConfig.CoreConfig, "Auth failure: "+autoErr.Error(), false) return errors.New("auth failure") @@ -278,7 +278,7 @@ func CommonMain(envPtr *string, fmt.Println("Error: Could not find plugin config") return errors.New("could not find plugin config") } - pluginConfig["env"] = *envPtr + pluginConfig["env"] = *envDefaultPtr pluginConfig["vaddress"] = *addrPtr if tokenPtr != nil { pluginConfig["token"] = *tokenPtr @@ -309,12 +309,12 @@ func CommonMain(envPtr *string, } else { trcshDriverConfigBase.DriverConfig.SubSectionValue = *pluginNamePtr } - mod.Env = *envPtr + mod.Env = *envDefaultPtr if logger != nil { logger.Printf("Certify mod initialized\n") } - if strings.HasPrefix(*envPtr, "staging") || strings.HasPrefix(*envPtr, "prod") || strings.HasPrefix(*envPtr, "dev") { + if strings.HasPrefix(*envDefaultPtr, "staging") || strings.HasPrefix(*envDefaultPtr, "prod") || strings.HasPrefix(*envDefaultPtr, "dev") { supportedRegions := eUtils.GetSupportedProdRegions() if *regionPtr != "" { for _, supportedRegion := range supportedRegions { @@ -332,7 +332,7 @@ func CommonMain(envPtr *string, } if *updateAPIMPtr { - updateAPIMError := trcapimgmtbase.CommonMain(envPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, startDirPtr, config, mod) + updateAPIMError := trcapimgmtbase.CommonMain(envDefaultPtr, addrPtr, tokenPtr, nil, secretIDPtr, appRoleIDPtr, tokenNamePtr, regionPtr, startDirPtr, config, mod) if updateAPIMError != nil { fmt.Println(updateAPIMError.Error()) fmt.Println("Couldn't update APIM...proceeding with build") diff --git a/atrium/vestibulum/trcflow/deploy/process.go b/atrium/vestibulum/trcflow/deploy/process.go index d6e4bc2f7..9647c9d34 100644 --- a/atrium/vestibulum/trcflow/deploy/process.go +++ b/atrium/vestibulum/trcflow/deploy/process.go @@ -393,7 +393,7 @@ func PluginDeployedUpdate(driverConfig *eUtils.DriverConfig, mod *helperkv.Modif hostRegion := coreopts.BuildOptions.GetRegion(hostName) mod.Regions = append(mod.Regions, hostRegion) - projects, services, _ := eUtils.GetProjectServices(cPath) + projects, services, _ := eUtils.GetProjectServices(nil, cPath) for _, pluginName := range pluginNameList { for i := 0; i < len(projects); i++ { if services[i] == "Certify" { diff --git a/atrium/vestibulum/trcflow/flumen/process.go b/atrium/vestibulum/trcflow/flumen/process.go index 44279aae0..c68978075 100644 --- a/atrium/vestibulum/trcflow/flumen/process.go +++ b/atrium/vestibulum/trcflow/flumen/process.go @@ -90,7 +90,7 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error GetAdditionalFlowsByState: flowopts.BuildOptions.GetAdditionalFlowsByState, FlowMap: map[flowcore.FlowNameType]*flowcore.TrcFlowContext{}, } - projects, services, _ := eUtils.GetProjectServices(pluginConfig["connectionPath"].([]string)) + projects, services, _ := eUtils.GetProjectServices(nil, pluginConfig["connectionPath"].([]string)) var sourceDatabaseConfigs []map[string]interface{} var vaultDatabaseConfig map[string]interface{} var spiralDatabaseConfig map[string]interface{} @@ -198,7 +198,7 @@ func ProcessFlows(pluginConfig map[string]interface{}, logger *log.Logger) error flowStateReceiverMap := map[string]chan flowcorehelper.FlowStateUpdate{} for _, template := range templateList { - source, service, tableTemplateName := eUtils.GetProjectService(template) + source, service, _, tableTemplateName := eUtils.GetProjectService(nil, template) tableName := eUtils.GetTemplateFileName(tableTemplateName, service) if tableName != flowcorehelper.TierceronFlowConfigurationTableName { driverConfigBasis.VersionFilter = append(driverConfigBasis.VersionFilter, tableName) diff --git a/atrium/vestibulum/trcsh/deployutil/deployutil.go b/atrium/vestibulum/trcsh/deployutil/deployutil.go index 238244aba..5985da689 100644 --- a/atrium/vestibulum/trcsh/deployutil/deployutil.go +++ b/atrium/vestibulum/trcsh/deployutil/deployutil.go @@ -39,7 +39,7 @@ func LoadPluginDeploymentScript(trcshDriverConfig *capauth.TrcshDriverConfig, tr tempEnv := trcshDriverConfig.DriverConfig.EnvBasis envParts := strings.Split(trcshDriverConfig.DriverConfig.EnvBasis, "-") mod.Env = envParts[0] - fmt.Printf("Loading deployment details for %s and env %s", deployment, mod.Env) + fmt.Printf("Loading deployment details for %s and env %s\n", deployment, mod.Env) deploymentConfig, err := mod.ReadData(fmt.Sprintf("super-secrets/Index/TrcVault/trcplugin/%s/Certify", deployment)) mod.Env = tempEnv if err != nil { @@ -56,9 +56,7 @@ func LoadPluginDeploymentScript(trcshDriverConfig *capauth.TrcshDriverConfig, tr if trcProjectService, ok := trcshDriverConfig.DriverConfig.DeploymentConfig["trcprojectservice"]; ok && strings.Contains(trcProjectService.(string), "/") { var content []byte trcProjectServiceSlice := strings.Split(trcProjectService.(string), "/") - trcshDriverConfig.DriverConfig.ZeroConfig = true contentArray, _, _, err := vcutils.ConfigTemplate(&trcshDriverConfig.DriverConfig, mod, fmt.Sprintf("./trc_templates/%s/deploy/deploy.trc.tmpl", trcProjectService.(string)), true, trcProjectServiceSlice[0], trcProjectServiceSlice[1], false, true) - trcshDriverConfig.DriverConfig.ZeroConfig = false if err != nil { eUtils.LogErrorObject(&trcshDriverConfig.DriverConfig.CoreConfig, err, false) return nil, err diff --git a/atrium/vestibulum/trcshbase/trcsh.go b/atrium/vestibulum/trcshbase/trcsh.go index fb8636f51..523d08ddc 100644 --- a/atrium/vestibulum/trcshbase/trcsh.go +++ b/atrium/vestibulum/trcshbase/trcsh.go @@ -108,8 +108,9 @@ func TrcshInitConfig(env string, region string, pathParam string, outputMemCache MemFs: &trcshMemFs.TrcshMemFs{ BillyFs: memfs.New(), }, - Regions: regions, - PathParam: pathParam, // Make available to trcplgtool + ZeroConfig: true, + Regions: regions, + PathParam: pathParam, // Make available to trcplgtool }, } return trcshDriverConfig, nil @@ -287,14 +288,14 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, memprotectopts.MemProtect(nil, secretIDPtr) memprotectopts.MemProtect(nil, appRoleIDPtr) - config, err := TrcshInitConfig(*envPtr, *regionPtr, pathParam, true) + trcshDriverConfig, err := TrcshInitConfig(*envPtr, *regionPtr, pathParam, true) if err != nil { fmt.Printf("trcsh config setup failure: %s\n", err.Error()) os.Exit(124) } //Open deploy script and parse it. - ProcessDeploy(nil, config, "", "", *trcPathPtr, *projectServicePtr, secretIDPtr, appRoleIDPtr, true, dronePtr) + ProcessDeploy(nil, trcshDriverConfig, "", "", *trcPathPtr, *projectServicePtr, secretIDPtr, appRoleIDPtr, true, dronePtr) } else { agentToken := os.Getenv("AGENT_TOKEN") agentEnv := os.Getenv("AGENT_ENV") @@ -485,7 +486,7 @@ func featherCtlCb(featherCtx *cap.FeatherContext, agentName string) error { return nil } -func roleBasedRunner(env string, +func roleBasedRunner( region string, trcshDriverConfig *capauth.TrcshDriverConfig, control string, @@ -497,7 +498,6 @@ func roleBasedRunner(env string, *configCount -= 1 trcshDriverConfig.DriverConfig.AppRoleConfig = "config.yml" trcshDriverConfig.DriverConfig.FileFilter = nil - trcshDriverConfig.DriverConfig.EnvBasis = env trcshDriverConfig.DriverConfig.CoreConfig.WantCerts = false trcshDriverConfig.DriverConfig.IsShellSubProcess = true trcshDriverConfig.DriverConfig.CoreConfig.Log.Printf("Role runner init: %s\n", control) @@ -513,25 +513,26 @@ func roleBasedRunner(env string, trcshDriverConfig.DriverConfig.EndDir = "." } configRoleSlice := strings.Split(*gTrcshConfig.ConfigRole, ":") - tokenName := "config_token_" + env + tokenName := "config_token_" + trcshDriverConfig.DriverConfig.EnvBasis tokenConfig := "" - configEnv := env + envDefaultPtr := trcshDriverConfig.DriverConfig.EnvBasis var err error trcshDriverConfig.DriverConfig.CoreConfig.Log.Printf("Role runner complete: %s\n", control) switch control { case "trcplgtool": tokenConfig := token - err = trcplgtoolbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, trcshDriverConfig) + envDefaultPtr = trcshDriverConfig.DriverConfig.Env + err = trcplgtoolbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, trcshDriverConfig) case "trcconfig": if trcshDriverConfig.DriverConfig.EnvBasis == "itdev" || trcshDriverConfig.DriverConfig.EnvBasis == "staging" || trcshDriverConfig.DriverConfig.EnvBasis == "prod" || trcshDriverConfig.DriverConfig.Env == "itdev" || trcshDriverConfig.DriverConfig.Env == "staging" || trcshDriverConfig.DriverConfig.Env == "prod" { trcshDriverConfig.DriverConfig.OutputMemCache = false } - err = trcconfigbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, &trcshDriverConfig.DriverConfig) + err = trcconfigbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &tokenConfig, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], &tokenName, ®ion, nil, deployArgLines, &trcshDriverConfig.DriverConfig) case "trcsub": trcshDriverConfig.DriverConfig.EndDir = trcshDriverConfig.DriverConfig.EndDir + "/trc_templates" - err = trcsubbase.CommonMain(&configEnv, &trcshDriverConfig.DriverConfig.VaultAddress, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], nil, deployArgLines, &trcshDriverConfig.DriverConfig) + err = trcsubbase.CommonMain(&envDefaultPtr, &trcshDriverConfig.DriverConfig.VaultAddress, &gTrcshConfig.EnvContext, &configRoleSlice[1], &configRoleSlice[0], nil, deployArgLines, &trcshDriverConfig.DriverConfig) } ResetModifier(&trcshDriverConfig.DriverConfig) //Resetting modifier cache to avoid token conflicts. @@ -576,7 +577,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, trcshDriverConfig.DriverConfig.Token = token } case "trcconfig": - err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount) + err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) } @@ -651,7 +652,7 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, trcshDriverConfig.FeatherCtx.Log = trcshDriverConfig.DriverConfig.CoreConfig.Log } - err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, *gTrcshConfig.CToken, argsOrig, deployArgLines, configCount) + err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, *gTrcshConfig.CToken, argsOrig, deployArgLines, configCount) if err != nil { os.Exit(1) } @@ -694,7 +695,6 @@ func processPluginCmds(trcKubeDeploymentConfig **kube.TrcKubeConfig, func processDroneCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, onceKubeInit *sync.Once, PipeOS billy.File, - env string, region string, trcshDriverConfig *capauth.TrcshDriverConfig, control string, @@ -704,7 +704,7 @@ func processDroneCmds(trcKubeDeploymentConfig *kube.TrcKubeConfig, deployArgLines []string, configCount *int) error { - err := roleBasedRunner(env, region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount) + err := roleBasedRunner(region, trcshDriverConfig, control, isAgentToken, token, argsOrig, deployArgLines, configCount) return err } @@ -1048,7 +1048,6 @@ collaboratorReRun: trcKubeDeploymentConfig, &onceKubeInit, PipeOS, - trcshDriverConfig.DriverConfig.Env, region, trcshDriverConfig, control, diff --git a/cmd/trcctl/trcctl.go b/cmd/trcctl/trcctl.go index 2072bfa1f..45a1f8fa6 100644 --- a/cmd/trcctl/trcctl.go +++ b/cmd/trcctl/trcctl.go @@ -41,6 +41,7 @@ func main() { flagset.Usage = func() { fmt.Fprintf(flagset.Output(), "Usage of %s:\n", os.Args[0]) flagset.PrintDefaults() + fmt.Fprintf(flagset.Output(), "\nexample: trcctl {pub, sub, init, config, x} {flags}\n") } envPtr := flagset.String("env", "", "Environment to be seeded") //If this is blank -> use context otherwise override context. tokenPtr := flagset.String("token", "", "Vault access token") @@ -108,9 +109,9 @@ func main() { case "init": trcinitbase.CommonMain(envPtr, &addrPtr, &envContext, flagset, os.Args) case "config": - trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, nil, os.Args, nil) + trcconfigbase.CommonMain(envPtr, &addrPtr, tokenPtr, &envContext, secretIDPtr, appRoleIDPtr, tokenNamePtr, nil, flagset, os.Args, nil) case "x": - trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil, nil, os.Args) + trcxbase.CommonMain(nil, xutil.GenerateSeedsFromVault, envPtr, &addrPtr, &envContext, nil, flagset, os.Args) } } } diff --git a/pkg/cli/trcconfigbase/trcconfig.go b/pkg/cli/trcconfigbase/trcconfig.go index 07cedafe8..7f1a7f8d8 100644 --- a/pkg/cli/trcconfigbase/trcconfig.go +++ b/pkg/cli/trcconfigbase/trcconfig.go @@ -72,7 +72,7 @@ var ( ENDDIR_DEFAULT = "." ) -func CommonMain(envPtr *string, +func CommonMain(envDefaultPtr *string, addrPtr *string, tokenPtr *string, envCtxPtr *string, @@ -96,6 +96,7 @@ func CommonMain(envPtr *string, ConfigWg: sync.WaitGroup{}, Mutex: &sync.Mutex{}, } + var envPtr *string = nil if flagset == nil { flagset = flag.NewFlagSet(argLines[0], flag.ExitOnError) @@ -104,7 +105,7 @@ func CommonMain(envPtr *string, flagset.PrintDefaults() } - flagset.String("env", "dev", "Environment to configure") + envPtr = flagset.String("env", "dev", "Environment to configure") flagset.String("addr", "", "API endpoint for the vault") flagset.String("token", "", "Vault access token") flagset.String("secretID", "", "Secret for app role ID") @@ -172,6 +173,9 @@ func CommonMain(envPtr *string, *wantCertsPtr = true } } + if envPtr == nil { + envPtr = envDefaultPtr + } if !isShell { if _, err := os.Stat(*startDirPtr); os.IsNotExist(err) { fmt.Println("Missing required template folder: " + *startDirPtr) @@ -213,6 +217,7 @@ func CommonMain(envPtr *string, Insecure: *insecurePtr, StartDir: append([]string{}, *startDirPtr), EndDir: *endDirPtr, + ZeroConfig: *zcPtr, } appRoleConfigPtr = new(string) @@ -493,7 +498,7 @@ func CommonMain(envPtr *string, StartDir: driverConfigBase.StartDir, EndDir: driverConfigBase.EndDir, WantKeystore: *keyStorePtr, - ZeroConfig: *zcPtr, + ZeroConfig: driverConfigBase.ZeroConfig, GenAuth: false, OutputMemCache: driverConfigBase.OutputMemCache, MemFs: driverConfigBase.MemFs, diff --git a/pkg/cli/trcconfigbase/utils/configinator.go b/pkg/cli/trcconfigbase/utils/configinator.go index f9977be5e..63f92a301 100644 --- a/pkg/cli/trcconfigbase/utils/configinator.go +++ b/pkg/cli/trcconfigbase/utils/configinator.go @@ -71,24 +71,72 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi templatePaths := []string{} endPaths := []string{} + var trcProjectService string = "" + var dosProjectService string = "" + var trcService string = "" + var dosService string = "" + + if projectService, ok := driverConfig.DeploymentConfig["trcprojectservice"]; ok && len(driverConfig.ServicesWanted) == 0 && strings.Contains(projectService.(string), "/") || len(driverConfig.ServicesWanted) == 1 { + if ok && len(driverConfig.ServicesWanted) == 0 { + trcProjectService = projectService.(string) + } else { + trcProjectService = driverConfig.ServicesWanted[0] + } + if !strings.HasSuffix(trcProjectService, "/") { + trcProjectService = trcProjectService + "/" + } + dosProjectService = strings.Replace(trcProjectService, "/", "\\", 1) + + if len(driverConfig.StartDir) > 1 { + trcProjectServiceParts := strings.Split(trcProjectService, "/") + project := trcProjectServiceParts[0] + "/" + trcService = "/" + trcProjectServiceParts[1] + "/" + dosService = strings.Replace(trcService, "/", "\\", 1) + startDirFiltered := []string{} + for _, startDir := range driverConfig.StartDir { + if strings.HasSuffix(startDir, project) { + startDirFiltered = append(startDirFiltered, startDir) + } + } + driverConfig.StartDir = startDirFiltered + } + } + //templatePaths for _, startDir := range driverConfig.StartDir { //get files from directory tp, ep := getDirFiles(startDir, driverConfig.EndDir) - if trcProjectService, ok := driverConfig.DeploymentConfig["trcprojectservice"]; ok && len(driverConfig.ServicesWanted) == 0 && strings.Contains(trcProjectService.(string), "/") || len(driverConfig.ServicesWanted) == 1 { - if trcProjectService == nil { - // Chewbacca: Fix duplicate /'s - trcProjectService = driverConfig.ServicesWanted[0] - } + if len(trcProjectService) > 0 { epScrubbed := []string{} + tpScrubbed := []string{} // Do some scrubbing... - for _, e := range ep { - e = strings.Replace(e, trcProjectService.(string), "/", 1) - projectService := strings.Replace(trcProjectService.(string), "/", "\\", 1) - e = strings.Replace(e, projectService, "\\", 1) - epScrubbed = append(epScrubbed, e) + for ie, e := range ep { + matched := false + if strings.Contains(e, trcProjectService) { + e = strings.Replace(e, trcProjectService, "/", 1) + matched = true + } else if strings.Contains(e, trcService) { + e = strings.Replace(e, trcService, "/", 1) + matched = true + } else if strings.Contains(e, dosProjectService) { + e = strings.Replace(e, dosProjectService, "\\", 1) + matched = true + } else { + if strings.Contains(e, dosService) { + e = strings.Replace(e, dosService, "\\", 1) + matched = true + } + } + if matched { + epScrubbed = append(epScrubbed, e) + tpScrubbed = append(tpScrubbed, tp[ie]) + } + } + if len(epScrubbed) > 0 { + // Only overwrite if something + ep = epScrubbed + tp = tpScrubbed } - ep = epScrubbed } templatePaths = append(templatePaths, tp...) @@ -116,7 +164,7 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi if !driverConfig.CoreConfig.WantCerts && strings.Contains(templatePath, "Common") { continue } - _, service, _ := eUtils.GetProjectService(templatePath) //This checks for nested project names + _, service, _, _ := eUtils.GetProjectService(driverConfig, templatePath) //This checks for nested project names driverConfig.VersionFilter = append(driverConfig.VersionFilter, service) //Adds nested project name to filter otherwise it will be not found. } @@ -227,7 +275,7 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi mod.Env = driverConfig.Env mod.Version = version //check for template_files directory here - project, service, templatePath := GetProjectService(driverConfig, templatePath) + project, service, _, templatePath := eUtils.GetProjectService(driverConfig, templatePath) var isCert bool if service != "" { @@ -249,18 +297,6 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi goto wait } - if strings.HasSuffix(templatePath, ".tmpl") { - if !driverConfig.ZeroConfig { - if strings.HasSuffix(templatePath, "nc.properties.tmpl") { - goto wait - } - } else { - if !strings.HasSuffix(templatePath, "nc.properties.tmpl") { - goto wait - } - } - } - var configuredTemplate string var certData map[int]string certLoaded := false @@ -279,7 +315,7 @@ func GenerateConfigsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.Confi goto wait } else { var ctErr error - configuredTemplate, certData, certLoaded, ctErr = ConfigTemplate(driverConfig, mod, templatePath, driverConfig.SecretMode, project, service, driverConfig.CoreConfig.WantCerts, false) + configuredTemplate, certData, certLoaded, ctErr = ConfigTemplate(driverConfig, mod, templatePath, driverConfig.SecretMode, project, service, driverConfig.CoreConfig.WantCerts, driverConfig.ZeroConfig) if ctErr != nil { if !strings.Contains(ctErr.Error(), "Missing .certData") { if !driverConfig.CoreConfig.WantCerts || strings.Contains(templatePath, "Common") { diff --git a/pkg/cli/trcconfigbase/utils/templateHelper.go b/pkg/cli/trcconfigbase/utils/templateHelper.go index abe368a12..05f856d53 100644 --- a/pkg/cli/trcconfigbase/utils/templateHelper.go +++ b/pkg/cli/trcconfigbase/utils/templateHelper.go @@ -19,44 +19,21 @@ import ( "gopkg.in/yaml.v2" ) -// GetProjectService - returns project, service, and path to template on filesystem. -// templateFile - full path to template file -// returns project, service, templatePath -func GetProjectService(driverConfig *eUtils.DriverConfig, templateFile string) (string, string, string) { - templateFile = strings.ReplaceAll(templateFile, "\\", "/") - splitDir := strings.Split(templateFile, "/") - var project, service string - offsetBase := 0 - - for i, component := range splitDir { - if component == coreopts.BuildOptions.GetFolderPrefix(driverConfig.StartDir)+"_templates" { - offsetBase = i - break - } - } - - project = splitDir[offsetBase+1] - service = splitDir[offsetBase+2] - - // Clean up service naming (Everything after '.' removed) - dotIndex := strings.Index(service, ".") - if dotIndex > 0 && dotIndex <= len(service) { - service = service[0:dotIndex] - } - - return project, service, templateFile -} - // GetTemplate makes a request to the vault for the template found in ///template-file // Returns the template data in base64 and the template's extension. Returns any errors generated by vault func GetTemplate(driverConfig *eUtils.DriverConfig, modifier *helperkv.Modifier, templatePath string) (string, error) { // Get template data from information in request. // ./trc_templates/Project/Service/configfile.yml.tmpl - project, service, templateFile := GetProjectService(driverConfig, templatePath) + project, service, serviceIndex, templateFile := eUtils.GetProjectService(driverConfig, templatePath) // templateFile currently has full path, but we don't want all that... Scrub it down. splitDir := strings.Split(templateFile, "/") - templateFile = splitDir[len(splitDir)-1] + if serviceIndex < len(splitDir)-1 { + templateFile = strings.Join(splitDir[serviceIndex+1:], "/") + } else { + // Just last entry... + templateFile = splitDir[len(splitDir)-1] + } if strings.Contains(templateFile, ".tmpl") { templateFile = templateFile[0 : len(templateFile)-len(".tmpl")] diff --git a/pkg/cli/trcsubbase/trcsub.go b/pkg/cli/trcsubbase/trcsub.go index dac081e40..a13496dd9 100644 --- a/pkg/cli/trcsubbase/trcsub.go +++ b/pkg/cli/trcsubbase/trcsub.go @@ -23,7 +23,7 @@ import ( // The file is saved under the data key, and the extension under the ext key // Vault automatically encodes the file into base64 -func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, +func CommonMain(envDefaultPtr *string, addrPtr *string, envCtxPtr *string, secretIDPtr *string, appRoleIDPtr *string, flagset *flag.FlagSet, @@ -32,6 +32,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, if memonly.IsMemonly() { memprotectopts.MemProtectInit(nil) } + var envPtr *string = nil exitOnFailure := false if flagset == nil { @@ -41,7 +42,7 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, fmt.Fprintf(flagset.Output(), "Usage of %s:\n", argLines[0]) flagset.PrintDefaults() } - flagset.String("env", "dev", "Environment to configure") + envPtr = flagset.String("env", "dev", "Environment to configure") flagset.String("addr", "", "API endpoint for the vault") flagset.String("secretID", "", "Secret for app role ID") flagset.String("appRoleID", "", "Public app role ID") @@ -58,6 +59,9 @@ func CommonMain(envPtr *string, addrPtr *string, envCtxPtr *string, templatePathsPtr := flagset.String("templatePaths", "", "Specifies which specific templates to download.") flagset.Parse(argLines[1:]) + if envPtr == nil { + envPtr = envDefaultPtr + } if len(*filterTemplatePtr) == 0 && !*projectInfoPtr && *templatePathsPtr == "" { fmt.Printf("Must specify either -projectInfo or -templateFilter flag \n") diff --git a/pkg/core/util/util.go b/pkg/core/util/util.go index 83696a4ba..2f2daacc4 100644 --- a/pkg/core/util/util.go +++ b/pkg/core/util/util.go @@ -310,7 +310,8 @@ func GetPluginToolConfig(driverConfig *eUtils.DriverConfig, mod *helperkv.Modifi driverConfig.CoreConfig.Log.Println("GetPluginToolConfig loading plugin data.") for _, templatePath := range templatePaths { - project, service, _ := eUtils.GetProjectService(templatePath) + // TODO: Chewbacca -- could pass in driverConfig but we didn't before... + project, service, _, _ := eUtils.GetProjectService(nil, templatePath) driverConfig.CoreConfig.Log.Println("GetPluginToolConfig project: " + project + " plugin: " + driverConfig.SubSectionValue + " service: " + service) if pluginPath, pathOk := pluginToolConfig["pluginpath"]; pathOk && len(pluginPath.(string)) != 0 { diff --git a/pkg/trcx/xutil/xmanager.go b/pkg/trcx/xutil/xmanager.go index 6c2a498df..d47bdd60b 100644 --- a/pkg/trcx/xutil/xmanager.go +++ b/pkg/trcx/xutil/xmanager.go @@ -1,944 +1,944 @@ -package xutil - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/hashicorp/vault/api" - "github.com/trimble-oss/tierceron/buildopts/coreopts" - vcutils "github.com/trimble-oss/tierceron/pkg/cli/trcconfigbase/utils" - "github.com/trimble-oss/tierceron/pkg/core" - "github.com/trimble-oss/tierceron/pkg/trcx/extract" - xencrypt "github.com/trimble-oss/tierceron/pkg/trcx/xencrypt" - eUtils "github.com/trimble-oss/tierceron/pkg/utils" - helperkv "github.com/trimble-oss/tierceron/pkg/vaulthelper/kv" - "gopkg.in/yaml.v2" -) - -var templateResultChan = make(chan *extract.TemplateResultData, 5) - -func GenerateSeedSectionFromVaultRaw(driverConfig *eUtils.DriverConfig, templateFromVault bool, templatePaths []string) ([]byte, bool, map[string]interface{}, map[string]map[string]map[string]string, map[string]map[string]map[string]string, string, error) { - var wg sync.WaitGroup - // Initialize global variables - valueCombinedSection := map[string]map[string]map[string]string{} - valueCombinedSection["values"] = map[string]map[string]string{} - - secretCombinedSection := map[string]map[string]map[string]string{} - secretCombinedSection["super-secrets"] = map[string]map[string]string{} - - // Declare local variables - templateCombinedSection := map[string]interface{}{} - sliceTemplateSection := []interface{}{} - sliceValueSection := []map[string]map[string]map[string]string{} - sliceSecretSection := []map[string]map[string]map[string]string{} - var sectionPath string - - maxDepth := -1 - service := "" - if len(driverConfig.ServiceFilter) > 0 { - service = driverConfig.ServiceFilter[0] - } - - //This checks whether indexed section is available in current directory. - if len(driverConfig.SectionKey) > 0 && len(driverConfig.ProjectSections) > 0 { - projectFound := false - for _, projectSection := range driverConfig.ProjectSections { - for _, templatePath := range templatePaths { - if strings.Contains(templatePath, projectSection) { - projectFound = true - goto projectFound - } - } - projectFound: - if !projectFound { - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Unable to find indexed project in local templates.", 1) - } - } - } - - multiService := false - var mod *helperkv.Modifier - - filteredTemplatePaths := templatePaths[:0] - if len(driverConfig.FileFilter) != 0 { - for _, filter := range driverConfig.FileFilter { - if !strings.HasSuffix(filter, ".tmpl") { - filter = filter + ".tmpl" - } - for _, templatePath := range templatePaths { - if strings.HasSuffix(templatePath, filter) { - filteredTemplatePaths = append(filteredTemplatePaths, templatePath) - } - } - } - } - if len(filteredTemplatePaths) > 0 { - templatePaths = filteredTemplatePaths - filteredTemplatePaths = filteredTemplatePaths[:0] - } - - envVersion := strings.Split(driverConfig.Env, "_") - if len(envVersion) != 2 { - // Make it so. - envVersion = eUtils.SplitEnv(driverConfig.Env) - } - env := envVersion[0] - version := envVersion[1] - - if driverConfig.Token != "" && driverConfig.Token != "novault" { - var err error - mod, err = helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - } - - mod.Env = env - mod.Version = version - if len(driverConfig.ProjectSections) > 0 { - mod.ProjectIndex = driverConfig.ProjectSections - mod.EnvBasis = strings.Split(driverConfig.EnvBasis, "_")[0] - mod.SectionName = driverConfig.SectionName - mod.SubSectionValue = driverConfig.SubSectionValue - } - } - - if len(filteredTemplatePaths) > 0 { - filteredTemplatePaths = eUtils.RemoveDuplicates(filteredTemplatePaths) - templatePaths = filteredTemplatePaths - } - - if driverConfig.GenAuth && mod != nil { - _, err := mod.ReadData("apiLogins/meta") - if err != nil { - eUtils.LogInfo(&driverConfig.CoreConfig, "Cannot genAuth with provided token.") - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) - } - } - - if driverConfig.Token != "novault" && mod.Version != "0" { //If version isn't latest or is a flag - var noCertPaths []string - var certPaths []string - for _, templatePath := range templatePaths { //Seperate cert vs normal paths - if !strings.Contains(templatePath, "Common") { - noCertPaths = append(noCertPaths, templatePath) - } else { - certPaths = append(certPaths, templatePath) - } - } - - if driverConfig.CoreConfig.WantCerts { //Remove unneeded template paths - templatePaths = certPaths - } else { - templatePaths = noCertPaths - } - - project := "" - if len(driverConfig.VersionFilter) > 0 { - project = driverConfig.VersionFilter[0] - } - for _, templatePath := range templatePaths { - _, service, _ := eUtils.GetProjectService(templatePath) //This checks for nested project names - - driverConfig.VersionFilter = append(driverConfig.VersionFilter, service) //Adds nested project name to filter otherwise it will be not found. - } - - if driverConfig.CoreConfig.WantCerts { //For cert version history - driverConfig.VersionFilter = append(driverConfig.VersionFilter, "Common") - } - - driverConfig.VersionFilter = eUtils.RemoveDuplicates(driverConfig.VersionFilter) - mod.VersionFilter = driverConfig.VersionFilter - versionMetadataMap := eUtils.GetProjectVersionInfo(driverConfig, mod) - - if versionMetadataMap == nil { - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, fmt.Sprintf("No version data found - this filter was applied during search: %v\n", driverConfig.VersionFilter), 1) - } else if version == "versionInfo" { //Version flag - var masterKey string - first := true - for key := range versionMetadataMap { - passed := false - if driverConfig.CoreConfig.WantCerts { - for _, service := range mod.VersionFilter { - if !passed && strings.Contains(key, "Common") && strings.Contains(key, service) && !strings.Contains(key, project) && !strings.HasSuffix(key, "Common") { - if len(key) > 0 { - keySplit := strings.Split(key, "/") - driverConfig.VersionInfo(versionMetadataMap[key], false, keySplit[len(keySplit)-1], first) - passed = true - first = false - } - } - } - } else { - if len(key) > 0 && len(masterKey) < 1 { - masterKey = key - driverConfig.VersionInfo(versionMetadataMap[masterKey], false, "", false) - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Version info provided.", 1) - } - } - } - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Version info provided.", 1) - } else { //Version bound check - if version != "0" { - versionNumbers := eUtils.GetProjectVersions(driverConfig, versionMetadataMap) - eUtils.BoundCheck(driverConfig, versionNumbers, version) - } - } - } - - //Receiver for configs - go func(dc *eUtils.DriverConfig) { - for { - select { - case tResult := <-templateResultChan: - if dc.Env == tResult.Env && dc.SubSectionValue == tResult.SubSectionValue { - sliceTemplateSection = append(sliceTemplateSection, tResult.InterfaceTemplateSection) - sliceValueSection = append(sliceValueSection, tResult.ValueSection) - sliceSecretSection = append(sliceSecretSection, tResult.SecretSection) - sectionPath = tResult.SectionPath - - if tResult.TemplateDepth > maxDepth { - maxDepth = tResult.TemplateDepth - //templateCombinedSection = interfaceTemplateSection - } - wg.Done() - } else { - go func(tResult *extract.TemplateResultData) { - templateResultChan <- tResult - }(tResult) - } - } - } - }(driverConfig) - - commonPathFound := false - for _, tPath := range templatePaths { - if strings.Contains(tPath, "Common") { - commonPathFound = true - } - } - - commonPaths := []string{} - if driverConfig.Token != "" && commonPathFound { - var commonMod *helperkv.Modifier - var err error - commonMod, err = helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, driverConfig.EnvBasis, driverConfig.Regions, true, driverConfig.CoreConfig.Log) - commonMod.Env = driverConfig.Env - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - } - envVersion := strings.Split(driverConfig.Env, "_") - if len(envVersion) == 1 { - envVersion = append(envVersion, "0") - } - commonMod.Env = envVersion[0] - commonMod.Version = envVersion[1] - driverConfig.Env = envVersion[0] + "_" + envVersion[1] - commonMod.Version = commonMod.Version + "***X-Mode" - - commonPaths, err = vcutils.GetPathsFromProject(&driverConfig.CoreConfig, commonMod, []string{"Common"}, []string{}) - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - } - if len(commonPaths) > 0 && strings.Contains(commonPaths[len(commonPaths)-1], "!=!") { - commonPaths = commonPaths[:len(commonPaths)-1] - } - commonMod.Release() - } - - // Configure each template in directory - if driverConfig.Token != "novault" { - // - // Checking for existence of values for service in vault. - // - if strings.Contains(driverConfig.EnvBasis, ".*") || len(driverConfig.ProjectSections) > 0 { - anyServiceFound := false - serviceFound := false - var acceptedTemplatePaths []string - for _, templatePath := range templatePaths { - _, _, templatePath = vcutils.GetProjectService(driverConfig, templatePath) - _, _, indexed, _ := helperkv.PreCheckEnvironment(mod.Env) - //This checks whether a enterprise env has the relevant project otherwise env gets skipped when generating seed files. - if (strings.Contains(mod.Env, ".") || len(driverConfig.ProjectSections) > 0) && !serviceFound { - var listValues *api.Secret - var err error - if driverConfig.SectionKey == "/Index/" && len(driverConfig.ProjectSections) > 0 { - listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0]+"/"+driverConfig.SectionName+"/"+driverConfig.SubSectionValue+"/", driverConfig.CoreConfig.Log) - } else if len(driverConfig.ProjectSections) > 0 { //If eid -> look inside Index and grab all environments - listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0]+"/"+driverConfig.SectionName, driverConfig.CoreConfig.Log) - if listValues == nil { - listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0], driverConfig.CoreConfig.Log) - } - } else if indexed { - listValues, err = mod.ListEnv("super-secrets/"+mod.Env+"/", driverConfig.CoreConfig.Log) - } else { - listValues, err = mod.ListEnv("values/"+mod.Env+"/", driverConfig.CoreConfig.Log) //Fix values to add to project to directory - } - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - } else if listValues == nil { - //eUtils.LogInfo(config, "No values were returned under values/.") - } else { - serviceSlice := make([]string, 0) - for _, valuesPath := range listValues.Data { - for _, serviceInterface := range valuesPath.([]interface{}) { - serviceFace := serviceInterface.(string) - if version != "0" { - versionMap := eUtils.GetProjectVersionInfo(driverConfig, mod) //("super-secrets/" + strings.Split(config.EnvBasis, ".")[0] + config.SectionKey + config.ProjectSections[0] + "/" + config.SectionName + "/" + config.SubSectionValue + "/" + serviceFace) - versionNumbers := eUtils.GetProjectVersions(driverConfig, versionMap) - eUtils.BoundCheck(driverConfig, versionNumbers, version) - } - serviceSlice = append(serviceSlice, serviceFace) - } - } - for _, listedService := range serviceSlice { - if service == "" && strings.Contains(templatePath, strings.TrimSuffix(listedService, "/")) { - serviceFound = true - } else if strings.TrimSuffix(listedService, "/") == service { - serviceFound = true - } - } - } - } - if serviceFound { //Exit for irrelevant enterprises - acceptedTemplatePaths = append(acceptedTemplatePaths, templatePath) - anyServiceFound = true - serviceFound = false - } - } - - if !anyServiceFound { //Exit for irrelevant enterprises - var errmsg error - if driverConfig.SubSectionValue != "" { - errmsg = errors.New("No relevant services were found for this environment: " + mod.Env + " for this value: " + driverConfig.SubSectionValue) - } else { - errmsg = errors.New("No relevant services were found for this environment: " + mod.Env) - } - eUtils.LogErrorObject(&driverConfig.CoreConfig, errmsg, false) - return nil, false, nil, nil, nil, "", errmsg - } - - if len(acceptedTemplatePaths) > 0 { - // template paths further trimmed by vault. - templatePaths = acceptedTemplatePaths - } - } - } - - var iFilterTemplatePaths []string - if len(driverConfig.ServiceFilter) > 0 { - for _, iFilter := range driverConfig.ServiceFilter { - for _, tPath := range templatePaths { - if strings.Contains(tPath, "/"+iFilter+"/") || strings.HasSuffix(tPath, "/"+iFilter+".yml.tmpl") { - iFilterTemplatePaths = append(iFilterTemplatePaths, tPath) - } - } - } - templatePaths = iFilterTemplatePaths - } - if driverConfig.Token != "novault" { - mod.Release() - } - - // Configure each template in directory - for _, templatePath := range templatePaths { - wg.Add(1) - go func(tp string, multiService bool, dc *eUtils.DriverConfig, cPaths []string) { - var project, service, env, version, innerProject string - var errSeed error - project = "" - service = "" - env = "" - version = "" - innerProject = "Not Found" - - // Map Subsections - var templateResult extract.TemplateResultData - var cds *vcutils.ConfigDataStore - var goMod *helperkv.Modifier - - templateResult.ValueSection = map[string]map[string]map[string]string{} - templateResult.ValueSection["values"] = map[string]map[string]string{} - - templateResult.SecretSection = map[string]map[string]map[string]string{} - templateResult.SecretSection["super-secrets"] = map[string]map[string]string{} - envVersion := eUtils.SplitEnv(dc.Env) - env = envVersion[0] - version = envVersion[1] - //check for template_files directory here - project, service, tp = vcutils.GetProjectService(dc, tp) - useCache := true - - if dc.Token != "" && dc.Token != "novault" { - var err error - goMod, err = helperkv.NewModifier(dc.Insecure, dc.Token, dc.VaultAddress, env, dc.Regions, useCache, dc.CoreConfig.Log) - goMod.Env = dc.Env - if err != nil { - if useCache && goMod != nil { - goMod.Release() - } - eUtils.LogErrorObject(&dc.CoreConfig, err, false) - wg.Done() - return - } - - goMod.Env = env - goMod.Version = version - goMod.ProjectIndex = dc.ProjectSections - if len(goMod.ProjectIndex) > 0 { - goMod.EnvBasis = strings.Split(dc.EnvBasis, "_")[0] - goMod.SectionKey = dc.SectionKey - goMod.SectionName = dc.SectionName - goMod.SubSectionValue = dc.SubSectionValue - } - - relativeTemplatePathParts := strings.Split(tp, coreopts.BuildOptions.GetFolderPrefix(dc.StartDir)+"_templates") - templatePathParts := strings.Split(relativeTemplatePathParts[1], ".") - goMod.TemplatePath = "templates" + templatePathParts[0] - - cds = new(vcutils.ConfigDataStore) - goMod.Version = goMod.Version + "***X-Mode" - if len(dc.CoreConfig.DynamicPathFilter) > 0 { - goMod.SectionPath = "super-secrets/" + dc.CoreConfig.DynamicPathFilter - } else { - // TODO: Deprecated... - // 1-800-ROIT??? Not sure how certs play into this. - if goMod.SectionName != "" && (goMod.SubSectionValue != "" || goMod.SectionKey == "/Restricted/" || goMod.SectionKey == "/Protected/") { - switch goMod.SectionKey { - case "/Index/": - goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + goMod.SectionName + "/" + goMod.SubSectionValue + "/" + service + dc.SubSectionName - case "/Restricted/": - if service != dc.SectionName { //TODO: Revisit why we need this comparison - goMod.SectionPath = "super-secrets" + goMod.SectionKey + service + "/" + dc.SectionName - } else { - goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + dc.SectionName - } - case "/Protected/": - if service != dc.SectionName { - goMod.SectionPath = "super-secrets" + goMod.SectionKey + service + "/" + dc.SectionName - } - default: - goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + goMod.SectionName + "/" + goMod.SubSectionValue - } - } - } - if dc.Token != "novault" { - if dc.CoreConfig.WantCerts { - var formattedTPath string - tempList := make([]string, 0) - // TODO: Chebacca Monday! - tPath := strings.Split(tp, coreopts.BuildOptions.GetFolderPrefix(dc.StartDir)+"_")[1] - tPathSplit := strings.Split(tPath, ".") - if len(tPathSplit) > 2 { - formattedTPath = tPathSplit[0] + "." + tPathSplit[1] - } else { - wg.Done() - return - } - if len(cPaths) > 0 { - for _, cPath := range cPaths { - if cPath == formattedTPath { - tempList = append(tempList, cPath) - } - } - } - cPaths = tempList - } - cds.Init(&dc.CoreConfig, goMod, dc.SecretMode, true, project, cPaths, service) - } - if len(goMod.VersionFilter) >= 1 && strings.Contains(goMod.VersionFilter[len(goMod.VersionFilter)-1], "!=!") { - // TODO: should this be before cds.Init??? - innerProject = strings.Split(goMod.VersionFilter[len(goMod.VersionFilter)-1], "!=!")[1] - goMod.VersionFilter = goMod.VersionFilter[:len(goMod.VersionFilter)-1] - if innerProject != "Not Found" { - project = innerProject - service = project - } - } - - } - - _, _, _, templateResult.TemplateDepth, errSeed = extract.ToSeed(dc, goMod, - cds, - tp, - project, - service, - templateFromVault, - &(templateResult.InterfaceTemplateSection), - &(templateResult.ValueSection), - &(templateResult.SecretSection), - ) - if len(dc.CoreConfig.DynamicPathFilter) > 0 { - // Pass explicit desitination indiciated in gomod. - templateResult.SectionPath = goMod.SectionPath - } - - if useCache && goMod != nil { - goMod.Release() - } - if errSeed != nil { - eUtils.LogAndSafeExit(&dc.CoreConfig, errSeed.Error(), -1) - wg.Done() - return - } - - templateResult.Env = env + "_" + version - templateResult.SubSectionValue = dc.SubSectionValue - templateResultChan <- &templateResult - }(templatePath, multiService, driverConfig, commonPaths) - } - wg.Wait() - - // Combine values of slice - CombineSection(&driverConfig.CoreConfig, sliceTemplateSection, maxDepth, templateCombinedSection) - CombineSection(&driverConfig.CoreConfig, sliceValueSection, -1, valueCombinedSection) - CombineSection(&driverConfig.CoreConfig, sliceSecretSection, -1, secretCombinedSection) - - var authYaml []byte - var errA error - - // Add special auth section. - if driverConfig.GenAuth { - if mod != nil { - authMod, authErr := helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) - eUtils.LogAndSafeExit(&driverConfig.CoreConfig, authErr.Error(), -1) - - connInfo, err := authMod.ReadData("apiLogins/meta") - authMod.Release() - if err == nil { - authSection := map[string]interface{}{} - authSection["apiLogins"] = map[string]interface{}{} - authSection["apiLogins"].(map[string]interface{})["meta"] = connInfo - authYaml, errA = yaml.Marshal(authSection) - if errA != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, errA, false) - } - } else { - return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Attempt to gen auth for reduced privilege token failed. No permissions to gen auth.", 1) - } - } else { - authConfigurations := map[string]interface{}{} - authConfigurations["authEndpoint"] = "" - authConfigurations["pass"] = "" - authConfigurations["sessionDB"] = "" - authConfigurations["user"] = "" - authConfigurations["trcAPITokenSecret"] = "" - - authSection := map[string]interface{}{} - authSection["apiLogins"] = map[string]interface{}{} - authSection["apiLogins"].(map[string]interface{})["meta"] = authConfigurations - authYaml, errA = yaml.Marshal(authSection) - if errA != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, errA, false) - } - } - } - return authYaml, multiService, templateCombinedSection, valueCombinedSection, secretCombinedSection, sectionPath, nil -} - -// GenerateSeedsFromVaultRaw configures the templates in trc_templates and writes them to trcx -func GenerateSeedsFromVaultRaw(driverConfig *eUtils.DriverConfig, fromVault bool, templatePaths []string) (string, bool, string, error) { - var projectSectionTemp []string //Used for seed file pathing; errors for -novault generation if not empty - if len(driverConfig.Trcxe) > 2 { - projectSectionTemp = driverConfig.ProjectSections - driverConfig.ProjectSections = []string{} - } - authYaml, multiService, templateCombinedSection, valueCombinedSection, secretCombinedSection, endPath, generateErr := GenerateSeedSectionFromVaultRaw(driverConfig, fromVault, templatePaths) - if generateErr != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, generateErr, false) - return "", false, "", nil - } - - if len(driverConfig.Trcxe) > 1 { //Validate first then replace fields - driverConfig.ProjectSections = projectSectionTemp - valValidateError := xencrypt.FieldValidator(driverConfig.Trcxe[0]+","+driverConfig.Trcxe[1], secretCombinedSection, valueCombinedSection) - if valValidateError != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, valValidateError, false) - return "", false, "", valValidateError - } - - encryptSecretErr := xencrypt.SetEncryptionSecret(driverConfig) - if encryptSecretErr != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, encryptSecretErr, false) - return "", false, "", encryptSecretErr - } - - encryption, encryptErr := xencrypt.GetEncryptors(secretCombinedSection) - if encryptErr != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, encryptErr, false) - return "", false, "", encryptErr - } - - if driverConfig.Trcxr { - xencrypt.FieldReader(xencrypt.CreateEncryptedReadMap(driverConfig.Trcxe[1]), secretCombinedSection, valueCombinedSection, encryption) - } else { - fieldChangedMap, encryptedChangedMap, promptErr := xencrypt.PromptUserForFields(driverConfig.Trcxe[0], driverConfig.Trcxe[1], encryption) - if promptErr != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, promptErr, false) - return "", false, "", promptErr - } - xencrypt.FieldReplacer(fieldChangedMap, encryptedChangedMap, secretCombinedSection, valueCombinedSection) - } - } - - if driverConfig.CoreConfig.WantCerts && !fromVault { - return "", false, "", nil - } - - // Create seed file structure - template, errT := yaml.Marshal(templateCombinedSection) - value, errV := yaml.Marshal(valueCombinedSection) - secret, errS := yaml.Marshal(secretCombinedSection) - - if errT != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, errT, false) - } - - if errV != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, errV, false) - } - - if errS != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, errS, false) - } - templateData := string(template) - // Remove single quotes generated by Marshal - templateData = strings.ReplaceAll(templateData, "'", "") - seedData := templateData + "\n\n\n" + string(value) + "\n\n\n" + string(secret) + "\n\n\n" + string(authYaml) - - return endPath, multiService, seedData, nil -} - -// GenerateSeedsFromVault configures the templates in trc_templates and writes them to trcx -func GenerateSeedsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.ConfigContext, driverConfig *eUtils.DriverConfig) (interface{}, error) { - if driverConfig.Clean { //Clean flag in trcx - if strings.HasSuffix(driverConfig.Env, "_0") { - envVersion := eUtils.SplitEnv(driverConfig.Env) - driverConfig.Env = envVersion[0] - } - _, err1 := os.Stat(driverConfig.EndDir + driverConfig.Env) - err := os.RemoveAll(driverConfig.EndDir + driverConfig.Env) - - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) - } - - if err1 == nil { - eUtils.LogInfo(&driverConfig.CoreConfig, "Seed removed from"+driverConfig.EndDir+driverConfig.Env) - } - return nil, nil - } - - // Get files from directory - tempTemplatePaths := []string{} - for _, startDir := range driverConfig.StartDir { - //get files from directory - tp := GetDirFiles(startDir) - tempTemplatePaths = append(tempTemplatePaths, tp...) - } - - if len(tempTemplatePaths) == 0 { - eUtils.LogErrorMessage(&driverConfig.CoreConfig, "No files found in "+coreopts.BuildOptions.GetFolderPrefix(driverConfig.StartDir)+"_templates", true) - } - - //Duplicate path remover - keys := make(map[string]bool) - templatePaths := []string{} - for _, path := range tempTemplatePaths { - if _, value := keys[path]; !value { - keys[path] = true - templatePaths = append(templatePaths, path) - } - } - - if driverConfig.Token != "novault" { //Filter unneeded templates - var err error - // TODO: Redo/deleted the indexedEnv work... - // Get filtered using mod and templates. - templatePathsAccepted, err := eUtils.GetAcceptedTemplatePaths(driverConfig, nil, templatePaths) - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) - } - templatePaths = templatePathsAccepted - } else { - templatePathsAccepted := []string{} - for _, project := range driverConfig.ProjectSections { - for _, templatePath := range templatePaths { - if strings.Contains(templatePath, project) { - templatePathsAccepted = append(templatePathsAccepted, templatePath) - } - } - } - if len(templatePathsAccepted) > 0 { - templatePaths = templatePathsAccepted - } - } - endPath, multiService, seedData, errGenerateSeeds := GenerateSeedsFromVaultRaw(driverConfig, false, templatePaths) - if errGenerateSeeds != nil { - eUtils.LogInfo(&driverConfig.CoreConfig, errGenerateSeeds.Error()) - return errGenerateSeeds, nil - } - - if endPath == "" && !multiService && seedData == "" && !driverConfig.CoreConfig.WantCerts { - return nil, nil - } - - suffixRemoved := "" - envVersion := eUtils.SplitEnv(driverConfig.Env) - driverConfig.Env = envVersion[0] - if envVersion[1] != "0" { - suffixRemoved = "_" + envVersion[1] - } - - envBasePath, pathPart, pathInclude, _ := helperkv.PreCheckEnvironment(driverConfig.Env) - - if suffixRemoved != "" { - driverConfig.Env = driverConfig.Env + suffixRemoved - } - - if multiService { - if strings.HasPrefix(driverConfig.Env, "local") { - endPath = driverConfig.EndDir + "local/local_seed.yml" - } else { - if pathInclude { - endPath = driverConfig.EndDir + envBasePath + "/" + pathPart + "/" + driverConfig.Env + "_seed.yml" - } else { - endPath = driverConfig.EndDir + envBasePath + "/" + driverConfig.Env + "_seed.yml" - } - } - } else { - if pathInclude { - endPath = driverConfig.EndDir + envBasePath + "/" + pathPart + "/" + driverConfig.Env + "_seed.yml" - } else if len(driverConfig.ProjectSections) > 0 { - envBasePath, _, _, _ := helperkv.PreCheckEnvironment(driverConfig.EnvBasis) - sectionNamePath := "/" - subSectionValuePath := "" - switch driverConfig.SectionKey { - case "/Index/": - sectionNamePath = "/" + driverConfig.SectionName + "/" - subSectionValuePath = driverConfig.SubSectionValue - case "/Restricted/": - fallthrough - case "/Protected/": - sectionNamePath = "/" + driverConfig.SectionName + "/" - subSectionValuePath = driverConfig.Env - } - - endPath = driverConfig.EndDir + envBasePath + driverConfig.SectionKey + driverConfig.ProjectSections[0] + sectionNamePath + subSectionValuePath + driverConfig.SubSectionName + "_seed.yml" - } else if len(driverConfig.CoreConfig.DynamicPathFilter) > 0 { - destPath := endPath - if len(driverConfig.SectionKey) > 0 { - destPath = strings.Replace(endPath, driverConfig.SectionName, "/", 1) - } - destPath = strings.Replace(destPath, "super-secrets/", "", 1) - endPath = driverConfig.EndDir + envBasePath + "/" + destPath + "_seed.yml" - } else { - endPath = driverConfig.EndDir + envBasePath + "/" + driverConfig.Env + "_seed.yml" - } - } - //generate template or certificate - if driverConfig.CoreConfig.WantCerts { - var certData map[int]string - certLoaded := false - - for _, templatePath := range tempTemplatePaths { - - project, service, templatePath := vcutils.GetProjectService(driverConfig, templatePath) - - envVersion := eUtils.SplitEnv(driverConfig.Env) - - certMod, err := helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, driverConfig.Env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) - - if err != nil { - eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) - } - certMod.Env = envVersion[0] - certMod.Version = envVersion[1] - - var ctErr error - _, certData, certLoaded, ctErr = vcutils.ConfigTemplate(driverConfig, certMod, templatePath, driverConfig.SecretMode, project, service, driverConfig.CoreConfig.WantCerts, false) - if ctErr != nil { - if !strings.Contains(ctErr.Error(), "Missing .certData") { - eUtils.CheckError(&driverConfig.CoreConfig, ctErr, true) - } - } - - if driverConfig.Token == "novault" { - extractedValues, parseErr := eUtils.Parse(templatePath, project, service) - if parseErr != nil { - eUtils.CheckError(&driverConfig.CoreConfig, parseErr, true) - } - if okSourcePath, okDestPath := extractedValues["certSourcePath"], extractedValues["certDestPath"]; okSourcePath != nil && okDestPath != nil { - certData[0] = extractedValues["certSourcePath"].(string) - certData[1] = "" - certData[2] = extractedValues["certSourcePath"].(string) - } else { - continue - } - } - - if len(certData) == 0 { - if certLoaded { - eUtils.LogInfo(&driverConfig.CoreConfig, "Could not load cert "+templatePath) - continue - } else { - continue - } - } - - certPath := certData[2] - eUtils.LogInfo(&driverConfig.CoreConfig, "Writing certificate: "+certPath+".") - - if strings.Contains(certPath, "ENV") { - if len(certMod.Env) >= 5 && (certMod.Env)[:5] == "local" { - envParts := strings.SplitN(certMod.Env, "/", 3) - certPath = strings.Replace(certPath, "ENV", envParts[1], 1) - } else { - certPath = strings.Replace(certPath, "ENV", certMod.Env, 1) - } - } - if certMod != nil { - certMod.Release() - } - - certDestination := driverConfig.EndDir + "/" + certPath - certDestination = strings.ReplaceAll(certDestination, "//", "/") - writeToFile(&driverConfig.CoreConfig, certData[1], certDestination) - eUtils.LogInfo(&driverConfig.CoreConfig, "certificate written to "+certDestination) - } - return nil, nil - } - - if driverConfig.Diff { - if !strings.Contains(driverConfig.Env, "_") { - driverConfig.Env = driverConfig.Env + "_0" - } - driverConfig.Update(configCtx, &seedData, driverConfig.Env+"||"+driverConfig.Env+"_seed.yml") - } else { - writeToFile(&driverConfig.CoreConfig, seedData, endPath) - // Print that we're done - if strings.Contains(driverConfig.Env, "_0") { - driverConfig.Env = strings.Split(driverConfig.Env, "_")[0] - } - - eUtils.LogInfo(&driverConfig.CoreConfig, "Seed created and written to "+endPath) - } - - return nil, nil -} - -func writeToFile(config *core.CoreConfig, data string, path string) { - byteData := []byte(data) - //Ensure directory has been created - dirPath := filepath.Dir(path) - err := os.MkdirAll(dirPath, os.ModePerm) - eUtils.CheckError(config, err, true) - //create new file - newFile, err := os.Create(path) - eUtils.CheckError(config, err, true) - defer newFile.Close() - //write to file - _, err = newFile.Write(byteData) - eUtils.CheckError(config, err, true) - err = newFile.Sync() - eUtils.CheckError(config, err, true) -} - -func GetDirFiles(dir string) []string { - files, err := os.ReadDir(dir) - filePaths := []string{} - //endPaths := []string{} - if err != nil { - //this is a file - return []string{dir} - } - for _, file := range files { - //add this directory to path names - filename := file.Name() - if strings.HasSuffix(filename, ".DS_Store") { - continue - } - extension := filepath.Ext(filename) - filePath := dir + file.Name() - if !strings.HasSuffix(dir, "/") { - filePath = dir + "/" + file.Name() - } - if extension == "" { - //if subfolder add / - filePath += "/" - } - //recurse to next level - newPaths := GetDirFiles(filePath) - filePaths = append(filePaths, newPaths...) - } - return filePaths -} - -// MergeMaps - merges 2 maps recursively. -func MergeMaps(x1, x2 interface{}) interface{} { - switch x1 := x1.(type) { - case map[string]interface{}: - x2, ok := x2.(map[string]interface{}) - if !ok { - return x1 - } - for k, v2 := range x2 { - if v1, ok := x1[k]; ok { - x1[k] = MergeMaps(v1, v2) - } else { - x1[k] = v2 - } - } - case nil: - x2, ok := x2.(map[string]interface{}) - if ok { - return x2 - } - } - return x1 -} - -// Combines the values in a slice, creating a singular map from multiple -// Input: -// - slice to combine -// - template slice to combine -// - depth of map (-1 for value/secret sections) -func CombineSection(config *core.CoreConfig, sliceSectionInterface interface{}, maxDepth int, combinedSectionInterface interface{}) { - _, okMap := sliceSectionInterface.([]map[string]map[string]map[string]string) - - // Value/secret slice section - if maxDepth < 0 && okMap { - sliceSection := sliceSectionInterface.([]map[string]map[string]map[string]string) - combinedSectionImpl := combinedSectionInterface.(map[string]map[string]map[string]string) - for _, v := range sliceSection { - for k2, v2 := range v { - for k3, v3 := range v2 { - if _, ok := combinedSectionImpl[k2][k3]; !ok { - combinedSectionImpl[k2][k3] = map[string]string{} - } - for k4, v4 := range v3 { - combinedSectionImpl[k2][k3][k4] = v4 - } - } - } - } - - combinedSectionInterface = combinedSectionImpl - - // template slice section - } else { - if maxDepth < 0 && !okMap { - eUtils.LogInfo(config, fmt.Sprintf("Env failed to gen. MaxDepth: %d, okMap: %t\n", maxDepth, okMap)) - } - sliceSection := sliceSectionInterface.([]interface{}) - - for _, v := range sliceSection { - MergeMaps(combinedSectionInterface, v) - } - } -} +package xutil + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/hashicorp/vault/api" + "github.com/trimble-oss/tierceron/buildopts/coreopts" + vcutils "github.com/trimble-oss/tierceron/pkg/cli/trcconfigbase/utils" + "github.com/trimble-oss/tierceron/pkg/core" + "github.com/trimble-oss/tierceron/pkg/trcx/extract" + xencrypt "github.com/trimble-oss/tierceron/pkg/trcx/xencrypt" + eUtils "github.com/trimble-oss/tierceron/pkg/utils" + helperkv "github.com/trimble-oss/tierceron/pkg/vaulthelper/kv" + "gopkg.in/yaml.v2" +) + +var templateResultChan = make(chan *extract.TemplateResultData, 5) + +func GenerateSeedSectionFromVaultRaw(driverConfig *eUtils.DriverConfig, templateFromVault bool, templatePaths []string) ([]byte, bool, map[string]interface{}, map[string]map[string]map[string]string, map[string]map[string]map[string]string, string, error) { + var wg sync.WaitGroup + // Initialize global variables + valueCombinedSection := map[string]map[string]map[string]string{} + valueCombinedSection["values"] = map[string]map[string]string{} + + secretCombinedSection := map[string]map[string]map[string]string{} + secretCombinedSection["super-secrets"] = map[string]map[string]string{} + + // Declare local variables + templateCombinedSection := map[string]interface{}{} + sliceTemplateSection := []interface{}{} + sliceValueSection := []map[string]map[string]map[string]string{} + sliceSecretSection := []map[string]map[string]map[string]string{} + var sectionPath string + + maxDepth := -1 + service := "" + if len(driverConfig.ServiceFilter) > 0 { + service = driverConfig.ServiceFilter[0] + } + + //This checks whether indexed section is available in current directory. + if len(driverConfig.SectionKey) > 0 && len(driverConfig.ProjectSections) > 0 { + projectFound := false + for _, projectSection := range driverConfig.ProjectSections { + for _, templatePath := range templatePaths { + if strings.Contains(templatePath, projectSection) { + projectFound = true + goto projectFound + } + } + projectFound: + if !projectFound { + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Unable to find indexed project in local templates.", 1) + } + } + } + + multiService := false + var mod *helperkv.Modifier + + filteredTemplatePaths := templatePaths[:0] + if len(driverConfig.FileFilter) != 0 { + for _, filter := range driverConfig.FileFilter { + if !strings.HasSuffix(filter, ".tmpl") { + filter = filter + ".tmpl" + } + for _, templatePath := range templatePaths { + if strings.HasSuffix(templatePath, filter) { + filteredTemplatePaths = append(filteredTemplatePaths, templatePath) + } + } + } + } + if len(filteredTemplatePaths) > 0 { + templatePaths = filteredTemplatePaths + filteredTemplatePaths = filteredTemplatePaths[:0] + } + + envVersion := strings.Split(driverConfig.Env, "_") + if len(envVersion) != 2 { + // Make it so. + envVersion = eUtils.SplitEnv(driverConfig.Env) + } + env := envVersion[0] + version := envVersion[1] + + if driverConfig.Token != "" && driverConfig.Token != "novault" { + var err error + mod, err = helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + } + + mod.Env = env + mod.Version = version + if len(driverConfig.ProjectSections) > 0 { + mod.ProjectIndex = driverConfig.ProjectSections + mod.EnvBasis = strings.Split(driverConfig.EnvBasis, "_")[0] + mod.SectionName = driverConfig.SectionName + mod.SubSectionValue = driverConfig.SubSectionValue + } + } + + if len(filteredTemplatePaths) > 0 { + filteredTemplatePaths = eUtils.RemoveDuplicates(filteredTemplatePaths) + templatePaths = filteredTemplatePaths + } + + if driverConfig.GenAuth && mod != nil { + _, err := mod.ReadData("apiLogins/meta") + if err != nil { + eUtils.LogInfo(&driverConfig.CoreConfig, "Cannot genAuth with provided token.") + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) + } + } + + if driverConfig.Token != "novault" && mod.Version != "0" { //If version isn't latest or is a flag + var noCertPaths []string + var certPaths []string + for _, templatePath := range templatePaths { //Seperate cert vs normal paths + if !strings.Contains(templatePath, "Common") { + noCertPaths = append(noCertPaths, templatePath) + } else { + certPaths = append(certPaths, templatePath) + } + } + + if driverConfig.CoreConfig.WantCerts { //Remove unneeded template paths + templatePaths = certPaths + } else { + templatePaths = noCertPaths + } + + project := "" + if len(driverConfig.VersionFilter) > 0 { + project = driverConfig.VersionFilter[0] + } + for _, templatePath := range templatePaths { + _, service, _, _ := eUtils.GetProjectService(nil, templatePath) //This checks for nested project names + + driverConfig.VersionFilter = append(driverConfig.VersionFilter, service) //Adds nested project name to filter otherwise it will be not found. + } + + if driverConfig.CoreConfig.WantCerts { //For cert version history + driverConfig.VersionFilter = append(driverConfig.VersionFilter, "Common") + } + + driverConfig.VersionFilter = eUtils.RemoveDuplicates(driverConfig.VersionFilter) + mod.VersionFilter = driverConfig.VersionFilter + versionMetadataMap := eUtils.GetProjectVersionInfo(driverConfig, mod) + + if versionMetadataMap == nil { + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, fmt.Sprintf("No version data found - this filter was applied during search: %v\n", driverConfig.VersionFilter), 1) + } else if version == "versionInfo" { //Version flag + var masterKey string + first := true + for key := range versionMetadataMap { + passed := false + if driverConfig.CoreConfig.WantCerts { + for _, service := range mod.VersionFilter { + if !passed && strings.Contains(key, "Common") && strings.Contains(key, service) && !strings.Contains(key, project) && !strings.HasSuffix(key, "Common") { + if len(key) > 0 { + keySplit := strings.Split(key, "/") + driverConfig.VersionInfo(versionMetadataMap[key], false, keySplit[len(keySplit)-1], first) + passed = true + first = false + } + } + } + } else { + if len(key) > 0 && len(masterKey) < 1 { + masterKey = key + driverConfig.VersionInfo(versionMetadataMap[masterKey], false, "", false) + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Version info provided.", 1) + } + } + } + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Version info provided.", 1) + } else { //Version bound check + if version != "0" { + versionNumbers := eUtils.GetProjectVersions(driverConfig, versionMetadataMap) + eUtils.BoundCheck(driverConfig, versionNumbers, version) + } + } + } + + //Receiver for configs + go func(dc *eUtils.DriverConfig) { + for { + select { + case tResult := <-templateResultChan: + if dc.Env == tResult.Env && dc.SubSectionValue == tResult.SubSectionValue { + sliceTemplateSection = append(sliceTemplateSection, tResult.InterfaceTemplateSection) + sliceValueSection = append(sliceValueSection, tResult.ValueSection) + sliceSecretSection = append(sliceSecretSection, tResult.SecretSection) + sectionPath = tResult.SectionPath + + if tResult.TemplateDepth > maxDepth { + maxDepth = tResult.TemplateDepth + //templateCombinedSection = interfaceTemplateSection + } + wg.Done() + } else { + go func(tResult *extract.TemplateResultData) { + templateResultChan <- tResult + }(tResult) + } + } + } + }(driverConfig) + + commonPathFound := false + for _, tPath := range templatePaths { + if strings.Contains(tPath, "Common") { + commonPathFound = true + } + } + + commonPaths := []string{} + if driverConfig.Token != "" && commonPathFound { + var commonMod *helperkv.Modifier + var err error + commonMod, err = helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, driverConfig.EnvBasis, driverConfig.Regions, true, driverConfig.CoreConfig.Log) + commonMod.Env = driverConfig.Env + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + } + envVersion := strings.Split(driverConfig.Env, "_") + if len(envVersion) == 1 { + envVersion = append(envVersion, "0") + } + commonMod.Env = envVersion[0] + commonMod.Version = envVersion[1] + driverConfig.Env = envVersion[0] + "_" + envVersion[1] + commonMod.Version = commonMod.Version + "***X-Mode" + + commonPaths, err = vcutils.GetPathsFromProject(&driverConfig.CoreConfig, commonMod, []string{"Common"}, []string{}) + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + } + if len(commonPaths) > 0 && strings.Contains(commonPaths[len(commonPaths)-1], "!=!") { + commonPaths = commonPaths[:len(commonPaths)-1] + } + commonMod.Release() + } + + // Configure each template in directory + if driverConfig.Token != "novault" { + // + // Checking for existence of values for service in vault. + // + if strings.Contains(driverConfig.EnvBasis, ".*") || len(driverConfig.ProjectSections) > 0 { + anyServiceFound := false + serviceFound := false + var acceptedTemplatePaths []string + for _, templatePath := range templatePaths { + _, _, _, templatePath = eUtils.GetProjectService(driverConfig, templatePath) + _, _, indexed, _ := helperkv.PreCheckEnvironment(mod.Env) + //This checks whether a enterprise env has the relevant project otherwise env gets skipped when generating seed files. + if (strings.Contains(mod.Env, ".") || len(driverConfig.ProjectSections) > 0) && !serviceFound { + var listValues *api.Secret + var err error + if driverConfig.SectionKey == "/Index/" && len(driverConfig.ProjectSections) > 0 { + listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0]+"/"+driverConfig.SectionName+"/"+driverConfig.SubSectionValue+"/", driverConfig.CoreConfig.Log) + } else if len(driverConfig.ProjectSections) > 0 { //If eid -> look inside Index and grab all environments + listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0]+"/"+driverConfig.SectionName, driverConfig.CoreConfig.Log) + if listValues == nil { + listValues, err = mod.ListEnv("super-secrets/"+strings.Split(driverConfig.EnvBasis, ".")[0]+driverConfig.SectionKey+driverConfig.ProjectSections[0], driverConfig.CoreConfig.Log) + } + } else if indexed { + listValues, err = mod.ListEnv("super-secrets/"+mod.Env+"/", driverConfig.CoreConfig.Log) + } else { + listValues, err = mod.ListEnv("values/"+mod.Env+"/", driverConfig.CoreConfig.Log) //Fix values to add to project to directory + } + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + } else if listValues == nil { + //eUtils.LogInfo(config, "No values were returned under values/.") + } else { + serviceSlice := make([]string, 0) + for _, valuesPath := range listValues.Data { + for _, serviceInterface := range valuesPath.([]interface{}) { + serviceFace := serviceInterface.(string) + if version != "0" { + versionMap := eUtils.GetProjectVersionInfo(driverConfig, mod) //("super-secrets/" + strings.Split(config.EnvBasis, ".")[0] + config.SectionKey + config.ProjectSections[0] + "/" + config.SectionName + "/" + config.SubSectionValue + "/" + serviceFace) + versionNumbers := eUtils.GetProjectVersions(driverConfig, versionMap) + eUtils.BoundCheck(driverConfig, versionNumbers, version) + } + serviceSlice = append(serviceSlice, serviceFace) + } + } + for _, listedService := range serviceSlice { + if service == "" && strings.Contains(templatePath, strings.TrimSuffix(listedService, "/")) { + serviceFound = true + } else if strings.TrimSuffix(listedService, "/") == service { + serviceFound = true + } + } + } + } + if serviceFound { //Exit for irrelevant enterprises + acceptedTemplatePaths = append(acceptedTemplatePaths, templatePath) + anyServiceFound = true + serviceFound = false + } + } + + if !anyServiceFound { //Exit for irrelevant enterprises + var errmsg error + if driverConfig.SubSectionValue != "" { + errmsg = errors.New("No relevant services were found for this environment: " + mod.Env + " for this value: " + driverConfig.SubSectionValue) + } else { + errmsg = errors.New("No relevant services were found for this environment: " + mod.Env) + } + eUtils.LogErrorObject(&driverConfig.CoreConfig, errmsg, false) + return nil, false, nil, nil, nil, "", errmsg + } + + if len(acceptedTemplatePaths) > 0 { + // template paths further trimmed by vault. + templatePaths = acceptedTemplatePaths + } + } + } + + var iFilterTemplatePaths []string + if len(driverConfig.ServiceFilter) > 0 { + for _, iFilter := range driverConfig.ServiceFilter { + for _, tPath := range templatePaths { + if strings.Contains(tPath, "/"+iFilter+"/") || strings.HasSuffix(tPath, "/"+iFilter+".yml.tmpl") { + iFilterTemplatePaths = append(iFilterTemplatePaths, tPath) + } + } + } + templatePaths = iFilterTemplatePaths + } + if driverConfig.Token != "novault" { + mod.Release() + } + + // Configure each template in directory + for _, templatePath := range templatePaths { + wg.Add(1) + go func(tp string, multiService bool, dc *eUtils.DriverConfig, cPaths []string) { + var project, service, env, version, innerProject string + var errSeed error + project = "" + service = "" + env = "" + version = "" + innerProject = "Not Found" + + // Map Subsections + var templateResult extract.TemplateResultData + var cds *vcutils.ConfigDataStore + var goMod *helperkv.Modifier + + templateResult.ValueSection = map[string]map[string]map[string]string{} + templateResult.ValueSection["values"] = map[string]map[string]string{} + + templateResult.SecretSection = map[string]map[string]map[string]string{} + templateResult.SecretSection["super-secrets"] = map[string]map[string]string{} + envVersion := eUtils.SplitEnv(dc.Env) + env = envVersion[0] + version = envVersion[1] + //check for template_files directory here + project, service, _, tp = eUtils.GetProjectService(dc, tp) + useCache := true + + if dc.Token != "" && dc.Token != "novault" { + var err error + goMod, err = helperkv.NewModifier(dc.Insecure, dc.Token, dc.VaultAddress, env, dc.Regions, useCache, dc.CoreConfig.Log) + goMod.Env = dc.Env + if err != nil { + if useCache && goMod != nil { + goMod.Release() + } + eUtils.LogErrorObject(&dc.CoreConfig, err, false) + wg.Done() + return + } + + goMod.Env = env + goMod.Version = version + goMod.ProjectIndex = dc.ProjectSections + if len(goMod.ProjectIndex) > 0 { + goMod.EnvBasis = strings.Split(dc.EnvBasis, "_")[0] + goMod.SectionKey = dc.SectionKey + goMod.SectionName = dc.SectionName + goMod.SubSectionValue = dc.SubSectionValue + } + + relativeTemplatePathParts := strings.Split(tp, coreopts.BuildOptions.GetFolderPrefix(dc.StartDir)+"_templates") + templatePathParts := strings.Split(relativeTemplatePathParts[1], ".") + goMod.TemplatePath = "templates" + templatePathParts[0] + + cds = new(vcutils.ConfigDataStore) + goMod.Version = goMod.Version + "***X-Mode" + if len(dc.CoreConfig.DynamicPathFilter) > 0 { + goMod.SectionPath = "super-secrets/" + dc.CoreConfig.DynamicPathFilter + } else { + // TODO: Deprecated... + // 1-800-ROIT??? Not sure how certs play into this. + if goMod.SectionName != "" && (goMod.SubSectionValue != "" || goMod.SectionKey == "/Restricted/" || goMod.SectionKey == "/Protected/") { + switch goMod.SectionKey { + case "/Index/": + goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + goMod.SectionName + "/" + goMod.SubSectionValue + "/" + service + dc.SubSectionName + case "/Restricted/": + if service != dc.SectionName { //TODO: Revisit why we need this comparison + goMod.SectionPath = "super-secrets" + goMod.SectionKey + service + "/" + dc.SectionName + } else { + goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + dc.SectionName + } + case "/Protected/": + if service != dc.SectionName { + goMod.SectionPath = "super-secrets" + goMod.SectionKey + service + "/" + dc.SectionName + } + default: + goMod.SectionPath = "super-secrets" + goMod.SectionKey + project + "/" + goMod.SectionName + "/" + goMod.SubSectionValue + } + } + } + if dc.Token != "novault" { + if dc.CoreConfig.WantCerts { + var formattedTPath string + tempList := make([]string, 0) + // TODO: Chebacca Monday! + tPath := strings.Split(tp, coreopts.BuildOptions.GetFolderPrefix(dc.StartDir)+"_")[1] + tPathSplit := strings.Split(tPath, ".") + if len(tPathSplit) > 2 { + formattedTPath = tPathSplit[0] + "." + tPathSplit[1] + } else { + wg.Done() + return + } + if len(cPaths) > 0 { + for _, cPath := range cPaths { + if cPath == formattedTPath { + tempList = append(tempList, cPath) + } + } + } + cPaths = tempList + } + cds.Init(&dc.CoreConfig, goMod, dc.SecretMode, true, project, cPaths, service) + } + if len(goMod.VersionFilter) >= 1 && strings.Contains(goMod.VersionFilter[len(goMod.VersionFilter)-1], "!=!") { + // TODO: should this be before cds.Init??? + innerProject = strings.Split(goMod.VersionFilter[len(goMod.VersionFilter)-1], "!=!")[1] + goMod.VersionFilter = goMod.VersionFilter[:len(goMod.VersionFilter)-1] + if innerProject != "Not Found" { + project = innerProject + service = project + } + } + + } + + _, _, _, templateResult.TemplateDepth, errSeed = extract.ToSeed(dc, goMod, + cds, + tp, + project, + service, + templateFromVault, + &(templateResult.InterfaceTemplateSection), + &(templateResult.ValueSection), + &(templateResult.SecretSection), + ) + if len(dc.CoreConfig.DynamicPathFilter) > 0 { + // Pass explicit desitination indiciated in gomod. + templateResult.SectionPath = goMod.SectionPath + } + + if useCache && goMod != nil { + goMod.Release() + } + if errSeed != nil { + eUtils.LogAndSafeExit(&dc.CoreConfig, errSeed.Error(), -1) + wg.Done() + return + } + + templateResult.Env = env + "_" + version + templateResult.SubSectionValue = dc.SubSectionValue + templateResultChan <- &templateResult + }(templatePath, multiService, driverConfig, commonPaths) + } + wg.Wait() + + // Combine values of slice + CombineSection(&driverConfig.CoreConfig, sliceTemplateSection, maxDepth, templateCombinedSection) + CombineSection(&driverConfig.CoreConfig, sliceValueSection, -1, valueCombinedSection) + CombineSection(&driverConfig.CoreConfig, sliceSecretSection, -1, secretCombinedSection) + + var authYaml []byte + var errA error + + // Add special auth section. + if driverConfig.GenAuth { + if mod != nil { + authMod, authErr := helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) + eUtils.LogAndSafeExit(&driverConfig.CoreConfig, authErr.Error(), -1) + + connInfo, err := authMod.ReadData("apiLogins/meta") + authMod.Release() + if err == nil { + authSection := map[string]interface{}{} + authSection["apiLogins"] = map[string]interface{}{} + authSection["apiLogins"].(map[string]interface{})["meta"] = connInfo + authYaml, errA = yaml.Marshal(authSection) + if errA != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, errA, false) + } + } else { + return nil, false, nil, nil, nil, "", eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "Attempt to gen auth for reduced privilege token failed. No permissions to gen auth.", 1) + } + } else { + authConfigurations := map[string]interface{}{} + authConfigurations["authEndpoint"] = "" + authConfigurations["pass"] = "" + authConfigurations["sessionDB"] = "" + authConfigurations["user"] = "" + authConfigurations["trcAPITokenSecret"] = "" + + authSection := map[string]interface{}{} + authSection["apiLogins"] = map[string]interface{}{} + authSection["apiLogins"].(map[string]interface{})["meta"] = authConfigurations + authYaml, errA = yaml.Marshal(authSection) + if errA != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, errA, false) + } + } + } + return authYaml, multiService, templateCombinedSection, valueCombinedSection, secretCombinedSection, sectionPath, nil +} + +// GenerateSeedsFromVaultRaw configures the templates in trc_templates and writes them to trcx +func GenerateSeedsFromVaultRaw(driverConfig *eUtils.DriverConfig, fromVault bool, templatePaths []string) (string, bool, string, error) { + var projectSectionTemp []string //Used for seed file pathing; errors for -novault generation if not empty + if len(driverConfig.Trcxe) > 2 { + projectSectionTemp = driverConfig.ProjectSections + driverConfig.ProjectSections = []string{} + } + authYaml, multiService, templateCombinedSection, valueCombinedSection, secretCombinedSection, endPath, generateErr := GenerateSeedSectionFromVaultRaw(driverConfig, fromVault, templatePaths) + if generateErr != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, generateErr, false) + return "", false, "", nil + } + + if len(driverConfig.Trcxe) > 1 { //Validate first then replace fields + driverConfig.ProjectSections = projectSectionTemp + valValidateError := xencrypt.FieldValidator(driverConfig.Trcxe[0]+","+driverConfig.Trcxe[1], secretCombinedSection, valueCombinedSection) + if valValidateError != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, valValidateError, false) + return "", false, "", valValidateError + } + + encryptSecretErr := xencrypt.SetEncryptionSecret(driverConfig) + if encryptSecretErr != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, encryptSecretErr, false) + return "", false, "", encryptSecretErr + } + + encryption, encryptErr := xencrypt.GetEncryptors(secretCombinedSection) + if encryptErr != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, encryptErr, false) + return "", false, "", encryptErr + } + + if driverConfig.Trcxr { + xencrypt.FieldReader(xencrypt.CreateEncryptedReadMap(driverConfig.Trcxe[1]), secretCombinedSection, valueCombinedSection, encryption) + } else { + fieldChangedMap, encryptedChangedMap, promptErr := xencrypt.PromptUserForFields(driverConfig.Trcxe[0], driverConfig.Trcxe[1], encryption) + if promptErr != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, promptErr, false) + return "", false, "", promptErr + } + xencrypt.FieldReplacer(fieldChangedMap, encryptedChangedMap, secretCombinedSection, valueCombinedSection) + } + } + + if driverConfig.CoreConfig.WantCerts && !fromVault { + return "", false, "", nil + } + + // Create seed file structure + template, errT := yaml.Marshal(templateCombinedSection) + value, errV := yaml.Marshal(valueCombinedSection) + secret, errS := yaml.Marshal(secretCombinedSection) + + if errT != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, errT, false) + } + + if errV != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, errV, false) + } + + if errS != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, errS, false) + } + templateData := string(template) + // Remove single quotes generated by Marshal + templateData = strings.ReplaceAll(templateData, "'", "") + seedData := templateData + "\n\n\n" + string(value) + "\n\n\n" + string(secret) + "\n\n\n" + string(authYaml) + + return endPath, multiService, seedData, nil +} + +// GenerateSeedsFromVault configures the templates in trc_templates and writes them to trcx +func GenerateSeedsFromVault(ctx eUtils.ProcessContext, configCtx *eUtils.ConfigContext, driverConfig *eUtils.DriverConfig) (interface{}, error) { + if driverConfig.Clean { //Clean flag in trcx + if strings.HasSuffix(driverConfig.Env, "_0") { + envVersion := eUtils.SplitEnv(driverConfig.Env) + driverConfig.Env = envVersion[0] + } + _, err1 := os.Stat(driverConfig.EndDir + driverConfig.Env) + err := os.RemoveAll(driverConfig.EndDir + driverConfig.Env) + + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) + } + + if err1 == nil { + eUtils.LogInfo(&driverConfig.CoreConfig, "Seed removed from"+driverConfig.EndDir+driverConfig.Env) + } + return nil, nil + } + + // Get files from directory + tempTemplatePaths := []string{} + for _, startDir := range driverConfig.StartDir { + //get files from directory + tp := GetDirFiles(startDir) + tempTemplatePaths = append(tempTemplatePaths, tp...) + } + + if len(tempTemplatePaths) == 0 { + eUtils.LogErrorMessage(&driverConfig.CoreConfig, "No files found in "+coreopts.BuildOptions.GetFolderPrefix(driverConfig.StartDir)+"_templates", true) + } + + //Duplicate path remover + keys := make(map[string]bool) + templatePaths := []string{} + for _, path := range tempTemplatePaths { + if _, value := keys[path]; !value { + keys[path] = true + templatePaths = append(templatePaths, path) + } + } + + if driverConfig.Token != "novault" { //Filter unneeded templates + var err error + // TODO: Redo/deleted the indexedEnv work... + // Get filtered using mod and templates. + templatePathsAccepted, err := eUtils.GetAcceptedTemplatePaths(driverConfig, nil, templatePaths) + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + eUtils.LogAndSafeExit(&driverConfig.CoreConfig, "", 1) + } + templatePaths = templatePathsAccepted + } else { + templatePathsAccepted := []string{} + for _, project := range driverConfig.ProjectSections { + for _, templatePath := range templatePaths { + if strings.Contains(templatePath, project) { + templatePathsAccepted = append(templatePathsAccepted, templatePath) + } + } + } + if len(templatePathsAccepted) > 0 { + templatePaths = templatePathsAccepted + } + } + endPath, multiService, seedData, errGenerateSeeds := GenerateSeedsFromVaultRaw(driverConfig, false, templatePaths) + if errGenerateSeeds != nil { + eUtils.LogInfo(&driverConfig.CoreConfig, errGenerateSeeds.Error()) + return errGenerateSeeds, nil + } + + if endPath == "" && !multiService && seedData == "" && !driverConfig.CoreConfig.WantCerts { + return nil, nil + } + + suffixRemoved := "" + envVersion := eUtils.SplitEnv(driverConfig.Env) + driverConfig.Env = envVersion[0] + if envVersion[1] != "0" { + suffixRemoved = "_" + envVersion[1] + } + + envBasePath, pathPart, pathInclude, _ := helperkv.PreCheckEnvironment(driverConfig.Env) + + if suffixRemoved != "" { + driverConfig.Env = driverConfig.Env + suffixRemoved + } + + if multiService { + if strings.HasPrefix(driverConfig.Env, "local") { + endPath = driverConfig.EndDir + "local/local_seed.yml" + } else { + if pathInclude { + endPath = driverConfig.EndDir + envBasePath + "/" + pathPart + "/" + driverConfig.Env + "_seed.yml" + } else { + endPath = driverConfig.EndDir + envBasePath + "/" + driverConfig.Env + "_seed.yml" + } + } + } else { + if pathInclude { + endPath = driverConfig.EndDir + envBasePath + "/" + pathPart + "/" + driverConfig.Env + "_seed.yml" + } else if len(driverConfig.ProjectSections) > 0 { + envBasePath, _, _, _ := helperkv.PreCheckEnvironment(driverConfig.EnvBasis) + sectionNamePath := "/" + subSectionValuePath := "" + switch driverConfig.SectionKey { + case "/Index/": + sectionNamePath = "/" + driverConfig.SectionName + "/" + subSectionValuePath = driverConfig.SubSectionValue + case "/Restricted/": + fallthrough + case "/Protected/": + sectionNamePath = "/" + driverConfig.SectionName + "/" + subSectionValuePath = driverConfig.Env + } + + endPath = driverConfig.EndDir + envBasePath + driverConfig.SectionKey + driverConfig.ProjectSections[0] + sectionNamePath + subSectionValuePath + driverConfig.SubSectionName + "_seed.yml" + } else if len(driverConfig.CoreConfig.DynamicPathFilter) > 0 { + destPath := endPath + if len(driverConfig.SectionKey) > 0 { + destPath = strings.Replace(endPath, driverConfig.SectionName, "/", 1) + } + destPath = strings.Replace(destPath, "super-secrets/", "", 1) + endPath = driverConfig.EndDir + envBasePath + "/" + destPath + "_seed.yml" + } else { + endPath = driverConfig.EndDir + envBasePath + "/" + driverConfig.Env + "_seed.yml" + } + } + //generate template or certificate + if driverConfig.CoreConfig.WantCerts { + var certData map[int]string + certLoaded := false + + for _, templatePath := range tempTemplatePaths { + + project, service, _, templatePath := eUtils.GetProjectService(driverConfig, templatePath) + + envVersion := eUtils.SplitEnv(driverConfig.Env) + + certMod, err := helperkv.NewModifier(driverConfig.Insecure, driverConfig.Token, driverConfig.VaultAddress, driverConfig.Env, driverConfig.Regions, true, driverConfig.CoreConfig.Log) + + if err != nil { + eUtils.LogErrorObject(&driverConfig.CoreConfig, err, false) + } + certMod.Env = envVersion[0] + certMod.Version = envVersion[1] + + var ctErr error + _, certData, certLoaded, ctErr = vcutils.ConfigTemplate(driverConfig, certMod, templatePath, driverConfig.SecretMode, project, service, driverConfig.CoreConfig.WantCerts, false) + if ctErr != nil { + if !strings.Contains(ctErr.Error(), "Missing .certData") { + eUtils.CheckError(&driverConfig.CoreConfig, ctErr, true) + } + } + + if driverConfig.Token == "novault" { + extractedValues, parseErr := eUtils.Parse(templatePath, project, service) + if parseErr != nil { + eUtils.CheckError(&driverConfig.CoreConfig, parseErr, true) + } + if okSourcePath, okDestPath := extractedValues["certSourcePath"], extractedValues["certDestPath"]; okSourcePath != nil && okDestPath != nil { + certData[0] = extractedValues["certSourcePath"].(string) + certData[1] = "" + certData[2] = extractedValues["certSourcePath"].(string) + } else { + continue + } + } + + if len(certData) == 0 { + if certLoaded { + eUtils.LogInfo(&driverConfig.CoreConfig, "Could not load cert "+templatePath) + continue + } else { + continue + } + } + + certPath := certData[2] + eUtils.LogInfo(&driverConfig.CoreConfig, "Writing certificate: "+certPath+".") + + if strings.Contains(certPath, "ENV") { + if len(certMod.Env) >= 5 && (certMod.Env)[:5] == "local" { + envParts := strings.SplitN(certMod.Env, "/", 3) + certPath = strings.Replace(certPath, "ENV", envParts[1], 1) + } else { + certPath = strings.Replace(certPath, "ENV", certMod.Env, 1) + } + } + if certMod != nil { + certMod.Release() + } + + certDestination := driverConfig.EndDir + "/" + certPath + certDestination = strings.ReplaceAll(certDestination, "//", "/") + writeToFile(&driverConfig.CoreConfig, certData[1], certDestination) + eUtils.LogInfo(&driverConfig.CoreConfig, "certificate written to "+certDestination) + } + return nil, nil + } + + if driverConfig.Diff { + if !strings.Contains(driverConfig.Env, "_") { + driverConfig.Env = driverConfig.Env + "_0" + } + driverConfig.Update(configCtx, &seedData, driverConfig.Env+"||"+driverConfig.Env+"_seed.yml") + } else { + writeToFile(&driverConfig.CoreConfig, seedData, endPath) + // Print that we're done + if strings.Contains(driverConfig.Env, "_0") { + driverConfig.Env = strings.Split(driverConfig.Env, "_")[0] + } + + eUtils.LogInfo(&driverConfig.CoreConfig, "Seed created and written to "+endPath) + } + + return nil, nil +} + +func writeToFile(config *core.CoreConfig, data string, path string) { + byteData := []byte(data) + //Ensure directory has been created + dirPath := filepath.Dir(path) + err := os.MkdirAll(dirPath, os.ModePerm) + eUtils.CheckError(config, err, true) + //create new file + newFile, err := os.Create(path) + eUtils.CheckError(config, err, true) + defer newFile.Close() + //write to file + _, err = newFile.Write(byteData) + eUtils.CheckError(config, err, true) + err = newFile.Sync() + eUtils.CheckError(config, err, true) +} + +func GetDirFiles(dir string) []string { + files, err := os.ReadDir(dir) + filePaths := []string{} + //endPaths := []string{} + if err != nil { + //this is a file + return []string{dir} + } + for _, file := range files { + //add this directory to path names + filename := file.Name() + if strings.HasSuffix(filename, ".DS_Store") { + continue + } + extension := filepath.Ext(filename) + filePath := dir + file.Name() + if !strings.HasSuffix(dir, "/") { + filePath = dir + "/" + file.Name() + } + if extension == "" { + //if subfolder add / + filePath += "/" + } + //recurse to next level + newPaths := GetDirFiles(filePath) + filePaths = append(filePaths, newPaths...) + } + return filePaths +} + +// MergeMaps - merges 2 maps recursively. +func MergeMaps(x1, x2 interface{}) interface{} { + switch x1 := x1.(type) { + case map[string]interface{}: + x2, ok := x2.(map[string]interface{}) + if !ok { + return x1 + } + for k, v2 := range x2 { + if v1, ok := x1[k]; ok { + x1[k] = MergeMaps(v1, v2) + } else { + x1[k] = v2 + } + } + case nil: + x2, ok := x2.(map[string]interface{}) + if ok { + return x2 + } + } + return x1 +} + +// Combines the values in a slice, creating a singular map from multiple +// Input: +// - slice to combine +// - template slice to combine +// - depth of map (-1 for value/secret sections) +func CombineSection(config *core.CoreConfig, sliceSectionInterface interface{}, maxDepth int, combinedSectionInterface interface{}) { + _, okMap := sliceSectionInterface.([]map[string]map[string]map[string]string) + + // Value/secret slice section + if maxDepth < 0 && okMap { + sliceSection := sliceSectionInterface.([]map[string]map[string]map[string]string) + combinedSectionImpl := combinedSectionInterface.(map[string]map[string]map[string]string) + for _, v := range sliceSection { + for k2, v2 := range v { + for k3, v3 := range v2 { + if _, ok := combinedSectionImpl[k2][k3]; !ok { + combinedSectionImpl[k2][k3] = map[string]string{} + } + for k4, v4 := range v3 { + combinedSectionImpl[k2][k3][k4] = v4 + } + } + } + } + + combinedSectionInterface = combinedSectionImpl + + // template slice section + } else { + if maxDepth < 0 && !okMap { + eUtils.LogInfo(config, fmt.Sprintf("Env failed to gen. MaxDepth: %d, okMap: %t\n", maxDepth, okMap)) + } + sliceSection := sliceSectionInterface.([]interface{}) + + for _, v := range sliceSection { + MergeMaps(combinedSectionInterface, v) + } + } +} diff --git a/pkg/utils/versionUtil.go b/pkg/utils/versionUtil.go index 4ba19f978..6930cfe7a 100644 --- a/pkg/utils/versionUtil.go +++ b/pkg/utils/versionUtil.go @@ -143,13 +143,13 @@ func BoundCheck(driverConfig *DriverConfig, versionNumbers []int, version string } } -func GetProjectServices(templateFiles []string) ([]string, []string, []string) { +func GetProjectServices(driverConfig *DriverConfig, templateFiles []string) ([]string, []string, []string) { projects := []string{} services := []string{} templateFilesContents := []string{} for _, templateFile := range templateFiles { - project, service, templateFileContent := GetProjectService(templateFile) + project, service, _, templateFileContent := GetProjectService(driverConfig, templateFile) projects = append(projects, project) services = append(services, service) @@ -159,33 +159,54 @@ func GetProjectServices(templateFiles []string) ([]string, []string, []string) { return projects, services, templateFilesContents } -func GetProjectService(templateFile string) (string, string, string) { +// GetProjectService - returns project, service, and path to template on filesystem. +// driverConfig - driver configuration +// templateFile - full path to template file +// returns project, service, templatePath +func GetProjectService(driverConfig *DriverConfig, templateFile string) (string, string, int, string) { templateFile = strings.ReplaceAll(strings.ReplaceAll(templateFile, "\\\\", "/"), "\\", "/") splitDir := strings.Split(templateFile, "/") var project, service string offsetBase := 0 + var startDir []string = nil + if driverConfig != nil { + startDir = driverConfig.StartDir + } + + trcTemplateParam := coreopts.BuildOptions.GetFolderPrefix(startDir) + "_templates" for i, component := range splitDir { - if component == coreopts.BuildOptions.GetFolderPrefix(nil)+"_templates" { + if component == trcTemplateParam { offsetBase = i break } } project = splitDir[offsetBase+1] - service = splitDir[offsetBase+2] - - // Clean up service naming (Everything after '.' removed) - if !strings.Contains(templateFile, "Common") { - dotIndex := strings.Index(service, ".") - if dotIndex > 0 && dotIndex <= len(service) { - service = service[0:dotIndex] + var serviceIndex int + if len(project) == 0 && len(driverConfig.DeploymentConfig) > 0 { + if projectService, ok := driverConfig.DeploymentConfig["trcprojectservice"]; ok { + projectServiceParts := strings.Split(projectService.(string), "/") + project = projectServiceParts[0] + service = projectServiceParts[1] + serviceIndex = 0 + } + } else { + serviceIndex = offsetBase + 2 + service = splitDir[serviceIndex] + + // Clean up service naming (Everything after '.' removed) + if !strings.Contains(templateFile, "Common") { + dotIndex := strings.Index(service, ".") + if dotIndex > 0 && dotIndex <= len(service) { + service = service[0:dotIndex] + } + } else if strings.Contains(service, ".mf.tmpl") { + service = strings.Split(service, ".mf.tmpl")[0] } - } else if strings.Contains(service, ".mf.tmpl") { - service = strings.Split(service, ".mf.tmpl")[0] } - return project, service, templateFile + return project, service, serviceIndex, templateFile } func GetTemplateFileName(templateFile string, service string) string {