diff --git a/Gopkg.lock b/Gopkg.lock index 830fbad518..f601b11537 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -163,6 +163,14 @@ revision = "85a78806aa1b4707d1dbace9be592cf1ece91ab3" version = "v1.1.1" +[[projects]] + digest = "1:d689318918dac8435731a16f090349f48f3758a3dbe5f967e913185bf264abaf" + name = "github.com/cloudfoundry-community/go-cfenv" + packages = ["."] + pruneopts = "UT" + revision = "f920e9562d5f951cbf11785728f67258c38a10d0" + version = "v1.17.0" + [[projects]] branch = "master" digest = "1:17f3f1af41a68b23365d83146a3504eccd759394f50a393b37ce16d9b88c4d3b" @@ -359,6 +367,22 @@ revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" version = "v1.4.0" +[[projects]] + digest = "1:8a405d0a20ac737ccd3e0ac52537887de704be70e03cc33038f4eb5a73fb3eda" + name = "github.com/govau/cf-common" + packages = ["env"] + pruneopts = "UT" + revision = "f85526034745406f35121d889ab195dc307037c2" + version = "v0.0.6" + +[[projects]] + digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + pruneopts = "UT" + revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" + version = "v1.1.2" + [[projects]] digest = "1:3bd786388cca2f0e8945a27ec836863f76762fd85da7e6da428c5a6801c6822d" name = "github.com/jessevdk/go-flags" diff --git a/manifest.yml b/manifest.yml index 25d6143dd5..ffb559cb97 100644 --- a/manifest.yml +++ b/manifest.yml @@ -14,3 +14,9 @@ applications: # Turn on backend debugging # LOG_LEVEL: debug +# User provided services can also be used to set environment properties: +# env: +# CF_UPS_NAME: stratos-properties +# services: +# - stratos-properties +# cf create-user-provided-service stratos-properties -p '{"CF_CLIENT":"stratos","CF_CLIENT_SECRET":"xxxx"}' diff --git a/src/jetstream/config/config.go b/src/jetstream/config/config.go index acd2e21c49..3cfb8a8138 100644 --- a/src/jetstream/config/config.go +++ b/src/jetstream/config/config.go @@ -13,13 +13,12 @@ import ( "strconv" "strings" + "github.com/govau/cf-common/env" log "github.com/sirupsen/logrus" ) const secretsDir = "/etc/secrets" -var loadedConfig map[string]string - // Load the given pointer to struct with values from the environment and the // /etc/secrets/ directory. // @@ -37,7 +36,7 @@ var loadedConfig map[string]string // The name will be given as defined to Getenv, and if that fails a lookup // it's name is then munged to conform to the /etc/secrets filename structure // and the file is attempted to be read. -func Load(intf interface{}) error { +func Load(intf interface{}, envLookup env.Lookup) error { value := reflect.ValueOf(intf) if value.Kind() != reflect.Ptr { @@ -60,7 +59,7 @@ func Load(intf interface{}) error { continue } - if err := setFieldValue(value, field, tag); err != nil { + if err := setFieldValue(value, field, tag, envLookup); err != nil { return err } } @@ -68,13 +67,10 @@ func Load(intf interface{}) error { return nil } -func setFieldValue(value reflect.Value, field reflect.Value, tag string) error { - val, err := GetValue(tag) - if err != nil && !isNotFoundErr(err) { - return err - } +func setFieldValue(value reflect.Value, field reflect.Value, tag string, envLookup env.Lookup) error { + val, ok := envLookup(tag) - if len(val) == 0 { + if !ok { return nil } @@ -133,49 +129,25 @@ func SetStructFieldValue(value reflect.Value, field reflect.Value, val string) e return nil } -// IsSet - is the specified config name set? -func IsSet(name string) bool { - _, err := GetValue(name) - return err == nil -} - -// GetString - Get the string value for the named configuration property -func GetString(name string) string { - v, _ := GetValue(name) - return v -} - -// GetValue tries to look up an env var of the given name and then -// tries to look up the secret file of the same -func GetValue(name string) (string, error) { - env := os.Getenv(name) - if len(env) == 0 { - env = loadedConfig[name] - } - - if len(env) == 0 { - return readSecretFileTestHarness(name) - } - - return env, nil -} - -var readSecretFileTestHarness = readSecretFile - -// readSecretFile reads a variable in the form HELLO_THERE from a file +// NewSecretsDirLookup - create a secret dir lookup +// reads a variable in the form HELLO_THERE from a file // in /etc/secrets/hello-there -func readSecretFile(name string) (string, error) { - name = strings.ToLower(strings.Replace(name, "_", "-", -1)) - filename := filepath.Join(secretsDir, name) - - contents, err := ioutil.ReadFile(filename) - if os.IsNotExist(err) { - return "", notFoundErr(filename) - } else if err != nil { - return "", fmt.Errorf("failed to read secret file %q: %v", name, err) +func NewSecretsDirLookup(secretsDir string) env.Lookup { + return func(name string) (string, bool) { + name = strings.ToLower(strings.Replace(name, "_", "-", -1)) + filename := filepath.Join(secretsDir, name) + + if _, err := os.Stat(filename); err == nil { + contents, err := ioutil.ReadFile(filename) + if err != nil { + log.Warnf("Error reading secrets file: %s, %s", filename, err) + return "", false + } + return strings.TrimSpace(string(contents)), true + } + // File does not exist + return "", false } - - return strings.TrimSpace(string(contents)), nil } type notFoundErr string @@ -192,18 +164,22 @@ func (n notFoundErr) Error() string { return fmt.Sprintf("could not find secret file: %s", string(n)) } -// LoadConfigFile - Load the configuration values in the specified config file if it exists -func LoadConfigFile(path string) error { +// NewConfigFileLookup - Load the configuration values in the specified config file if it exists +func NewConfigFileLookup(path string) env.Lookup { + + // Check if the config file exists + if _, err := os.Stat(path); err != nil { + return env.NoopLookup + } + file, err := os.Open(path) if err != nil { - if !os.IsNotExist(err) { - log.Warn("Error reading configuraion file", err) - } - return err + log.Warn("Error reading configuration file, ignoring this file: ", err) + return env.NoopLookup } defer file.Close() - loadedConfig = make(map[string]string) + loadedConfig := make(map[string]string) scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() @@ -217,17 +193,15 @@ func LoadConfigFile(path string) error { } } - if err := scanner.Err(); err != nil { - log.Warn("Error reading configuration file", err) - return err + if scanner.Err() != nil { + // Error reading configuration file, ignoring this file + return env.NoopLookup } log.Infof("Loaded configuration from file: %s", path) - return nil -} - -// SetConfigValue will update/set a value in the loaded configuration -func SetConfigValue(name, value string) { - loadedConfig[name] = value + return func(k string) (string, bool) { + v, ok := loadedConfig[k] + return v, ok + } } diff --git a/src/jetstream/datastore/database_cf_config.go b/src/jetstream/datastore/database_cf_config.go index ceffea0e59..8ec7cc790e 100644 --- a/src/jetstream/datastore/database_cf_config.go +++ b/src/jetstream/datastore/database_cf_config.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" + "github.com/govau/cf-common/env" log "github.com/sirupsen/logrus" ) @@ -27,13 +27,13 @@ type VCAPService struct { } // Discover cf db services via their 'uri' env var and apply settings to the DatabaseConfig objects -func ParseCFEnvs(db *DatabaseConfig) (bool, error) { - if config.IsSet(SERVICES_ENV) == false { +func ParseCFEnvs(db *DatabaseConfig, env *env.VarSet) (bool, error) { + if !env.IsSet(SERVICES_ENV) { return false, nil } // Extract struts from VCAP_SERVICES env - vcapServicesStr := config.GetString(SERVICES_ENV) + vcapServicesStr := env.MustString(SERVICES_ENV) var vcapServices map[string][]VCAPService err := json.Unmarshal([]byte(vcapServicesStr), &vcapServices) if err != nil { diff --git a/src/jetstream/datastore/datastore.go b/src/jetstream/datastore/datastore.go index 3eb21bd744..1b8110f8e0 100644 --- a/src/jetstream/datastore/datastore.go +++ b/src/jetstream/datastore/datastore.go @@ -8,13 +8,14 @@ import ( "strings" "time" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/goose-db-version" - + goosedbversion "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/goose-db-version" + "github.com/govau/cf-common/env" log "github.com/sirupsen/logrus" + // Mysql driver _ "github.com/go-sql-driver/mysql" "github.com/kat-co/vala" + // Sqlite driver _ "github.com/mattn/go-sqlite3" @@ -133,7 +134,7 @@ func validateRequiredDatabaseParams(username, password, database, host string, p } // GetConnection returns a database connection to either PostgreSQL or SQLite -func GetConnection(dc DatabaseConfig) (*sql.DB, error) { +func GetConnection(dc DatabaseConfig, env *env.VarSet) (*sql.DB, error) { log.Debug("GetConnection") if dc.DatabaseProvider == PGSQL { @@ -146,13 +147,12 @@ func GetConnection(dc DatabaseConfig) (*sql.DB, error) { } // SQL Lite - return GetSQLLiteConnection() + return GetSQLLiteConnection(env.MustBool("SQLITE_KEEP_DB")) } // GetSQLLiteConnection returns an SQLite DB Connection -func GetSQLLiteConnection() (*sql.DB, error) { - - if !config.IsSet("SQLITE_KEEP_DB") { +func GetSQLLiteConnection(sqliteKeepDB bool) (*sql.DB, error) { + if !sqliteKeepDB { os.Remove(SQLiteDatabaseFile) } diff --git a/src/jetstream/diagnostics.go b/src/jetstream/diagnostics.go index f492357837..ef37a1bc93 100644 --- a/src/jetstream/diagnostics.go +++ b/src/jetstream/diagnostics.go @@ -6,7 +6,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/goose-db-version" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" ) @@ -35,10 +34,11 @@ func (p *portalProxy) StoreDiagnostics() { } // Deployment information - when deployed via Helm - diagnostics.HelmName = config.GetString("HELM_NAME") - diagnostics.HelmRevision = config.GetString("HELM_REVISION") - diagnostics.HelmChartVersion = config.GetString("HELM_CHART_VERSION") - diagnostics.HelmLastModified = config.GetString("HELM_LAST_MODIFIED") + + diagnostics.HelmName = p.Env().String("HELM_NAME", "") + diagnostics.HelmRevision = p.Env().String("HELM_REVISION", "") + diagnostics.HelmChartVersion = p.Env().String("HELM_CHART_VERSION", "") + diagnostics.HelmLastModified = p.Env().String("HELM_LAST_MODIFIED", "") // Deployment type switch { @@ -46,9 +46,9 @@ func (p *portalProxy) StoreDiagnostics() { diagnostics.DeploymentType = "Kubernetes" case p.Config.IsCloudFoundry: diagnostics.DeploymentType = "Cloud Foundry" - case len(config.GetString("STRATOS_DEPLOYMENT_DOCKER")) > 0: + case len(p.Env().String("STRATOS_DEPLOYMENT_DOCKER", "")) > 0: diagnostics.DeploymentType = "Docker" - case len(config.GetString("STRATOS_DEPLOYMENT_DOCKER_AIO")) > 0: + case len(p.Env().String("STRATOS_DEPLOYMENT_DOCKER_AIO", "")) > 0: diagnostics.DeploymentType = "Docker All-in-One" default: diagnostics.DeploymentType = "Development" diff --git a/src/jetstream/main.go b/src/jetstream/main.go index 7b562d54c1..478b3e9bf2 100644 --- a/src/jetstream/main.go +++ b/src/jetstream/main.go @@ -23,7 +23,9 @@ import ( "github.com/antonlindstrom/pgstore" "github.com/cf-stratos/mysqlstore" + cfenv "github.com/cloudfoundry-community/go-cfenv" "github.com/gorilla/sessions" + "github.com/govau/cf-common/env" "github.com/labstack/echo" "github.com/labstack/echo/middleware" "github.com/nwmac/sqlitestore" @@ -74,6 +76,32 @@ func cleanup(dbc *sql.DB, ss HttpSessionStore) { log.Info("Graceful shut down complete") } +// getEnvironmentLookup return a search path for configuration settings +func getEnvironmentLookup() *env.VarSet { + // Make environment lookup + envLookup := env.NewVarSet() + + // Environment variables directly set trump all others + envLookup.AppendSource(os.LookupEnv) + + // If running in CloudFoundry, fallback to a user provided service (if set) + cfApp, err := cfenv.Current() + if err == nil { + envLookup.AppendSource(env.NewLookupFromUPS(cfApp, os.Getenv("CF_UPS_NAME"))) + } + + // Fallback to a "config.properties" files in our directory + envLookup.AppendSource(config.NewConfigFileLookup("./config.properties")) + + // Fall back to "default.config.properties" in our directory + envLookup.AppendSource(config.NewConfigFileLookup("./default.config.properties")) + + // Fallback to individual files in the "/etc/secrets" directory + envLookup.AppendSource(config.NewSecretsDirLookup("/etc/secrets")) + + return envLookup +} + func main() { log.SetFormatter(&log.TextFormatter{ForceColors: true, FullTimestamp: true, TimestampFormat: time.UnixDate}) log.SetOutput(os.Stdout) @@ -87,15 +115,18 @@ func main() { // Register time.Time in gob gob.Register(time.Time{}) + // Create common method for looking up config + envLookup := getEnvironmentLookup() + // Check to see if we are running as the database migrator - if migrateDatabase() { + if migrateDatabase(envLookup) { // End execution return } // Load the portal configuration from env vars var portalConfig interfaces.PortalConfig - portalConfig, err := loadPortalConfig(portalConfig) + portalConfig, err := loadPortalConfig(portalConfig, envLookup) if err != nil { log.Fatal(err) // calls os.Exit(1) after logging } @@ -106,11 +137,11 @@ func main() { } log.Info("Configuration loaded.") - isUpgrading := isConsoleUpgrading() + isUpgrading := isConsoleUpgrading(envLookup) if isUpgrading { log.Info("Upgrade in progress (lock file detected) ... waiting for lock file to be removed ...") - start(portalConfig, &portalProxy{}, &setupMiddleware{}, true) + start(portalConfig, &portalProxy{env: envLookup}, &setupMiddleware{}, true) } // Grab the Console Version from the executable portalConfig.ConsoleVersion = appVersion @@ -129,7 +160,7 @@ func main() { // Load database configuration var dc datastore.DatabaseConfig - dc, err = loadDatabaseConfig(dc) + dc, err = loadDatabaseConfig(dc, envLookup) if err != nil { log.Fatal(err) } @@ -143,7 +174,7 @@ func main() { // Establish a Postgresql connection pool var databaseConnectionPool *sql.DB - databaseConnectionPool, err = initConnPool(dc) + databaseConnectionPool, err = initConnPool(dc, envLookup) if err != nil { log.Fatal(err.Error()) } @@ -164,7 +195,7 @@ func main() { } for _, configPlugin := range interfaces.JetstreamConfigPlugins { - configPlugin(&portalConfig) + configPlugin(envLookup, &portalConfig) } if portalConfig.SessionStoreSecret == defaultSessionSecret { @@ -176,7 +207,7 @@ func main() { } // Initialize session store for Gorilla sessions - sessionStore, sessionStoreOptions, err := initSessionStore(databaseConnectionPool, dc.DatabaseProvider, portalConfig, SessionExpiry) + sessionStore, sessionStoreOptions, err := initSessionStore(databaseConnectionPool, dc.DatabaseProvider, portalConfig, SessionExpiry, envLookup) if err != nil { log.Fatal(err) } @@ -195,7 +226,7 @@ func main() { log.Info("Session store initialized.") // Setup the global interface for the proxy - portalProxy := newPortalProxy(portalConfig, databaseConnectionPool, sessionStore, sessionStoreOptions) + portalProxy := newPortalProxy(portalConfig, databaseConnectionPool, sessionStore, sessionStoreOptions, envLookup) log.Info("Initialization complete.") c := make(chan os.Signal, 2) @@ -285,7 +316,7 @@ func initialiseConsoleConfiguration(portalProxy *portalProxy) (*setupMiddleware, func setSSOFromConfig(portalProxy *portalProxy, configuration *interfaces.ConsoleConfig) { // For SSO, override the value loaded from the config file, so that this is what we use - if !config.IsSet("SSO_LOGIN") { + if !portalProxy.Env().IsSet("SSO_LOGIN") { portalProxy.Config.SSOLogin = configuration.UseSSO } } @@ -335,11 +366,11 @@ func getEncryptionKey(pc interfaces.PortalConfig) ([]byte, error) { return key, nil } -func initConnPool(dc datastore.DatabaseConfig) (*sql.DB, error) { +func initConnPool(dc datastore.DatabaseConfig, env *env.VarSet) (*sql.DB, error) { log.Debug("initConnPool") // initialize the database connection pool - pool, err := datastore.GetConnection(dc) + pool, err := datastore.GetConnection(dc, env) if err != nil { return nil, err } @@ -370,7 +401,7 @@ func initConnPool(dc datastore.DatabaseConfig) (*sql.DB, error) { return pool, nil } -func initSessionStore(db *sql.DB, databaseProvider string, pc interfaces.PortalConfig, sessionExpiry int) (HttpSessionStore, *sessions.Options, error) { +func initSessionStore(db *sql.DB, databaseProvider string, pc interfaces.PortalConfig, sessionExpiry int, env *env.VarSet) (HttpSessionStore, *sessions.Options, error) { log.Debug("initSessionStore") sessionsTable := "sessions" @@ -422,16 +453,10 @@ func initSessionStore(db *sql.DB, databaseProvider string, pc interfaces.PortalC return sessionStore, sessionStore.Options, err } -func loadPortalConfig(pc interfaces.PortalConfig) (interfaces.PortalConfig, error) { +func loadPortalConfig(pc interfaces.PortalConfig, env *env.VarSet) (interfaces.PortalConfig, error) { log.Debug("loadPortalConfig") - // Load config.properties if it exists, otherwise look for default.config.properties - err := config.LoadConfigFile("./config.properties") - if os.IsNotExist(err) { - config.LoadConfigFile("./default.config.properties") - } - - if err := config.Load(&pc); err != nil { + if err := config.Load(&pc, env.Lookup); err != nil { return pc, fmt.Errorf("Unable to load configuration. %v", err) } @@ -448,17 +473,17 @@ func loadPortalConfig(pc interfaces.PortalConfig) (interfaces.PortalConfig, erro return pc, nil } -func loadDatabaseConfig(dc datastore.DatabaseConfig) (datastore.DatabaseConfig, error) { +func loadDatabaseConfig(dc datastore.DatabaseConfig, env *env.VarSet) (datastore.DatabaseConfig, error) { log.Debug("loadDatabaseConfig") - parsedDBConfig, err := datastore.ParseCFEnvs(&dc) + parsedDBConfig, err := datastore.ParseCFEnvs(&dc, env) if err != nil { return dc, errors.New("Could not parse Cloud Foundry Services environment") } if parsedDBConfig { log.Info("Using Cloud Foundry DB service") - } else if err := config.Load(&dc); err != nil { + } else if err := config.Load(&dc, env.Lookup); err != nil { return dc, fmt.Errorf("Unable to load database configuration. %v", err) } @@ -508,7 +533,7 @@ func detectTLSCert(pc interfaces.PortalConfig) (string, string, error) { return certFilename, certKeyFilename, nil } -func newPortalProxy(pc interfaces.PortalConfig, dcp *sql.DB, ss HttpSessionStore, sessionStoreOptions *sessions.Options) *portalProxy { +func newPortalProxy(pc interfaces.PortalConfig, dcp *sql.DB, ss HttpSessionStore, sessionStoreOptions *sessions.Options, env *env.VarSet) *portalProxy { log.Debug("newPortalProxy") // Generate cookie name - avoids issues if the cookie domain is changed @@ -531,6 +556,7 @@ func newPortalProxy(pc interfaces.PortalConfig, dcp *sql.DB, ss HttpSessionStore SessionCookieName: cookieName, EmptyCookieMatcher: regexp.MustCompile(cookieName + "=(?:;[ ]*|$)"), AuthProviders: make(map[string]interfaces.AuthProvider), + env: env, } // Initialize built-in auth providers @@ -615,14 +641,14 @@ func start(config interfaces.PortalConfig, p *portalProxy, addSetupMiddleware *s if !isUpgrade { e.Use(errorLoggingMiddleware) } - e.Use(retryAfterUpgradeMiddleware) + e.Use(bindToEnv(retryAfterUpgradeMiddleware, p.Env())) if !isUpgrade { p.registerRoutes(e, addSetupMiddleware) } if isUpgrade { - go stopEchoWhenUpgraded(e) + go stopEchoWhenUpgraded(e, p.Env()) } var engineErr error @@ -706,7 +732,7 @@ func (p *portalProxy) registerRoutes(e *echo.Echo, addSetupMiddleware *setupMidd e.Use(middlewarePlugin.EchoMiddleware) } - staticDir, staticDirErr := getStaticFiles() + staticDir, staticDirErr := getStaticFiles(p.Env().String("UI_PATH", "./ui")) // Always serve the backend API from /pp pp := e.Group("/pp") @@ -869,13 +895,7 @@ func echoV2DefaultHTTPErrorHandler(err error, c echo.Context) { } } -func getStaticFiles() (string, error) { - - uiFolder, _ := config.GetValue("UI_PATH") - if len(uiFolder) == 0 { - uiFolder = "./ui" - } - +func getStaticFiles(uiFolder string) (string, error) { dir, err := filepath.Abs(uiFolder) if err == nil { // Check if folder exists @@ -887,13 +907,13 @@ func getStaticFiles() (string, error) { return "", errors.New("UI folder not found") } -func isConsoleUpgrading() bool { +func isConsoleUpgrading(env *env.VarSet) bool { - upgradeVolume, noUpgradeVolumeErr := config.GetValue(UpgradeVolume) - upgradeLockFile, noUpgradeLockFileNameErr := config.GetValue(UpgradeLockFileName) + upgradeVolume, noUpgradeVolumeOK := env.Lookup(UpgradeVolume) + upgradeLockFile, noUpgradeLockFileNameOK := env.Lookup(UpgradeLockFileName) // If any of those properties are not set, consider Console is running in a non-upgradeable environment - if noUpgradeVolumeErr != nil || noUpgradeLockFileNameErr != nil { + if !noUpgradeVolumeOK || !noUpgradeLockFileNameOK { return false } @@ -908,8 +928,8 @@ func isConsoleUpgrading() bool { return false } -func stopEchoWhenUpgraded(e *echo.Echo) { - for isConsoleUpgrading() { +func stopEchoWhenUpgraded(e *echo.Echo, env *env.VarSet) { + for isConsoleUpgrading(env) { time.Sleep(1 * time.Second) } log.Info("Upgrade has completed! Shutting down Upgrade web server instance") diff --git a/src/jetstream/main_test.go b/src/jetstream/main_test.go index 8065ae9af9..3e5f0d85f7 100644 --- a/src/jetstream/main_test.go +++ b/src/jetstream/main_test.go @@ -1,9 +1,10 @@ package main import ( - "os" "testing" + "github.com/govau/cf-common/env" + "github.com/cloudfoundry-incubator/stratos/src/jetstream/datastore" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" ) @@ -59,23 +60,21 @@ func (e *echoContextMock) Reset(engine.Request, engine.Response) { */ func TestLoadPortalConfig(t *testing.T) { - - os.Unsetenv("DATABASE_PROVIDER") - os.Setenv("HTTP_CLIENT_TIMEOUT_IN_SECS", "10") - os.Setenv("HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS", "35") - os.Setenv("SKIP_SSL_VALIDATION", "true") - os.Setenv("CONSOLE_PROXY_TLS_ADDRESS", ":8080") - os.Setenv("CONSOLE_CLIENT", "portal-proxy") - os.Setenv("CONSOLE_CLIENT_SECRET", "ohsosecret!") - os.Setenv("CF_CLIENT", "portal-proxy") - os.Setenv("CF_CLIENT_SECRET", "ohsosecret!") - os.Setenv("UAA_ENDPOINT", "https://login.cf.org.com:443") - os.Setenv("ALLOWED_ORIGINS", "https://localhost,https://127.0.0.1") - os.Setenv("SESSION_STORE_SECRET", "cookiesecret") - var pc interfaces.PortalConfig - result, err := loadPortalConfig(pc) + result, err := loadPortalConfig(pc, env.NewVarSet(env.WithMapLookup(map[string]string{ + "HTTP_CLIENT_TIMEOUT_IN_SECS": "10", + "HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS": "35", + "SKIP_SSL_VALIDATION": "true", + "CONSOLE_PROXY_TLS_ADDRESS": ":8080", + "CONSOLE_CLIENT": "portal-proxy", + "CONSOLE_CLIENT_SECRET": "ohsosecret!", + "CF_CLIENT": "portal-proxy", + "CF_CLIENT_SECRET": "ohsosecret!", + "UAA_ENDPOINT": "https://login.cf.org.com:443", + "ALLOWED_ORIGINS": "https://localhost,https://127.0.0.1", + "SESSION_STORE_SECRET": "cookiesecret", + }))) if err != nil { t.Errorf("Unable to load portal config from env vars: %v", err) @@ -121,19 +120,17 @@ func TestLoadPortalConfig(t *testing.T) { } func TestLoadDatabaseConfig(t *testing.T) { - - os.Unsetenv("DATABASE_PROVIDER") - os.Setenv("DB_USER", "console") - os.Setenv("DB_PASSWORD", "console") - os.Setenv("DB_DATABASE_NAME", "console-db") - os.Setenv("DB_HOST", "localhost") - os.Setenv("DB_PORT", "5432") - os.Setenv("DB_CONNECT_TIMEOUT_IN_SECS", "5") - os.Setenv("DB_SSL_MODE", "disable") - var dc datastore.DatabaseConfig - _, err := loadDatabaseConfig(dc) + _, err := loadDatabaseConfig(dc, env.NewVarSet(env.WithMapLookup(map[string]string{ + "DB_USER": "console", + "DB_PASSWORD": "console", + "DB_DATABASE_NAME": "console-db", + "DB_HOST": "localhost", + "DB_PORT": "5432", + "DB_CONNECT_TIMEOUT_IN_SECS": "5", + "DB_SSL_MODE": "disable", + }))) if err != nil { t.Errorf("Unable to load database config from env vars: %v", err) @@ -141,18 +138,17 @@ func TestLoadDatabaseConfig(t *testing.T) { } func TestLoadDatabaseConfigWithInvalidSSLMode(t *testing.T) { - - os.Setenv("DB_USER", "console") - os.Setenv("DB_PASSWORD", "console") - os.Setenv("DB_DATABASE_NAME", "console-db") - os.Setenv("DB_HOST", "localhost") - os.Setenv("DB_PORT", "5432") - os.Setenv("DATABASE_PROVIDER", "pgsql") - os.Setenv("DB_SSL_MODE", "invalid.ssl.mode") - var dc datastore.DatabaseConfig - _, err := loadDatabaseConfig(dc) + _, err := loadDatabaseConfig(dc, env.NewVarSet(env.WithMapLookup(map[string]string{ + "DB_USER": "console", + "DB_PASSWORD": "console", + "DB_DATABASE_NAME": "console-db", + "DB_HOST": "localhost", + "DB_PORT": "5432", + "DATABASE_PROVIDER": "pgsql", + "DB_SSL_MODE": "invalid.ssl.mode", + }))) if err == nil { t.Errorf("Unexpected success - should not be able to load database configs with an invalid SSL Mode specified.") diff --git a/src/jetstream/middleware.go b/src/jetstream/middleware.go index 2271759a3c..56b8d7896c 100644 --- a/src/jetstream/middleware.go +++ b/src/jetstream/middleware.go @@ -10,10 +10,10 @@ import ( "time" "github.com/gorilla/context" + "github.com/govau/cf-common/env" "github.com/labstack/echo" log "github.com/sirupsen/logrus" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" ) @@ -212,13 +212,19 @@ func errorLoggingMiddleware(h echo.HandlerFunc) echo.HandlerFunc { } } -func retryAfterUpgradeMiddleware(h echo.HandlerFunc) echo.HandlerFunc { +func bindToEnv(f func(echo.HandlerFunc, *env.VarSet) echo.HandlerFunc, e *env.VarSet) func(echo.HandlerFunc) echo.HandlerFunc { + return func(h echo.HandlerFunc) echo.HandlerFunc { + return f(h, e) + } +} + +func retryAfterUpgradeMiddleware(h echo.HandlerFunc, env *env.VarSet) echo.HandlerFunc { - upgradeVolume, noUpgradeVolumeErr := config.GetValue(UpgradeVolume) - upgradeLockFile, noUpgradeLockFileNameErr := config.GetValue(UpgradeLockFileName) + upgradeVolume, noUpgradeVolumeOK := env.Lookup(UpgradeVolume) + upgradeLockFile, noUpgradeLockFileNameOK := env.Lookup(UpgradeLockFileName) // If any of those properties are not set, disable upgrade middleware - if noUpgradeVolumeErr != nil || noUpgradeLockFileNameErr != nil { + if !noUpgradeVolumeOK || !noUpgradeLockFileNameOK { return func(c echo.Context) error { return h(c) } diff --git a/src/jetstream/migrator.go b/src/jetstream/migrator.go index e4d786c948..cb11828226 100644 --- a/src/jetstream/migrator.go +++ b/src/jetstream/migrator.go @@ -10,6 +10,7 @@ import ( "time" "bitbucket.org/liamstask/goose/lib/goose" + "github.com/govau/cf-common/env" log "github.com/sirupsen/logrus" "github.com/cloudfoundry-incubator/stratos/src/jetstream/datastore" @@ -38,7 +39,7 @@ func dbConfFromFlags() (dbconf *goose.DBConf, err error) { return goose.NewDBConf(*flagPath, *flagEnv, *flagPgSchema) } -func migrateDatabase() bool { +func migrateDatabase(env *env.VarSet) bool { flag.Usage = usage flag.Parse() @@ -53,7 +54,7 @@ func migrateDatabase() bool { return true } - if !parseCloudFoundry() { + if !parseCloudFoundry(env) { return false } @@ -72,10 +73,10 @@ func migrateDatabase() bool { return true } -func parseCloudFoundry() bool { +func parseCloudFoundry(env *env.VarSet) bool { if *flagCloudFoundry { - dbEnv, err := parseCloudFoundryEnv() + dbEnv, err := parseCloudFoundryEnv(env) if err != nil { log.Fatal("Failed to parse Cloud Foundry Environment Variables") } @@ -189,14 +190,14 @@ var usagePrefix = ` stratos db migration cli ` -func parseCloudFoundryEnv() (string, error) { +func parseCloudFoundryEnv(env *env.VarSet) (string, error) { var dbEnv string fmt.Println("Attempting to parse VCAP_SERVICES") var dbConfig datastore.DatabaseConfig - parsedDBConfig, err := datastore.ParseCFEnvs(&dbConfig) + parsedDBConfig, err := datastore.ParseCFEnvs(&dbConfig, env) if err != nil { return "", errors.New("Could not parse Cloud Foundry Services environment") } @@ -204,13 +205,13 @@ func parseCloudFoundryEnv() (string, error) { if parsedDBConfig { exportDatabaseConfig(dbConfig) - switch dbType := os.Getenv(DB_TYPE); dbType { + switch dbType := env.String(DB_TYPE, "unknown"); dbType { case TYPE_POSTGRES: dbEnv = "cf_postgres" - fmt.Printf("Migrating postgresql instance on %s\n", os.Getenv(DB_HOST)) + fmt.Printf("Migrating postgresql instance on %s\n", env.String(DB_HOST, "")) case TYPE_MYSQL: dbEnv = "cf_mysql" - fmt.Printf("Migrating mysql instance on %s\n", os.Getenv(DB_HOST)) + fmt.Printf("Migrating mysql instance on %s\n", env.String(DB_HOST, "")) default: // Database service not found or type not recognized return "", nil diff --git a/src/jetstream/mock_server_test.go b/src/jetstream/mock_server_test.go index 07a7e20f74..da42192ef4 100644 --- a/src/jetstream/mock_server_test.go +++ b/src/jetstream/mock_server_test.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/securecookie" "github.com/gorilla/sessions" + "github.com/govau/cf-common/env" "github.com/labstack/echo" sqlmock "gopkg.in/DATA-DOG/go-sqlmock.v1" @@ -137,7 +138,7 @@ func setupPortalProxy(db *sql.DB) *portalProxy { CFAdminIdentifier: CFAdminIdentifier, } - pp := newPortalProxy(pc, db, nil, nil) + pp := newPortalProxy(pc, db, nil, nil, env.NewVarSet()) pp.SessionStore = setupMockPGStore(db) initialisedEndpoint := initCFPlugin(pp) pp.Plugins = make(map[string]interfaces.StratosPlugin) diff --git a/src/jetstream/plugins/cfapppush/deploy.go b/src/jetstream/plugins/cfapppush/deploy.go index 9916728a8a..ac6c696d4e 100644 --- a/src/jetstream/plugins/cfapppush/deploy.go +++ b/src/jetstream/plugins/cfapppush/deploy.go @@ -184,7 +184,7 @@ func (cfAppPush *CFAppPush) deploy(echoContext echo.Context) error { return err } - dialTimeout := os.Getenv("CF_DIAL_TIMEOUT") + dialTimeout := cfAppPush.portalProxy.Env().String("CF_DIAL_TIMEOUT", "") pushConfig.OutputWriter = socketWriter pushConfig.DialTimeout = dialTimeout diff --git a/src/jetstream/plugins/cloudfoundryhosting/main.go b/src/jetstream/plugins/cloudfoundryhosting/main.go index 76d3746511..4e1a952ba0 100644 --- a/src/jetstream/plugins/cloudfoundryhosting/main.go +++ b/src/jetstream/plugins/cloudfoundryhosting/main.go @@ -10,11 +10,11 @@ import ( "strings" "github.com/gorilla/sessions" + "github.com/govau/cf-common/env" "github.com/labstack/echo" uuid "github.com/satori/go.uuid" log "github.com/sirupsen/logrus" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" ) @@ -42,9 +42,10 @@ func init() { } // ConfigInit updates the config if needed -func ConfigInit(jetstreamConfig *interfaces.PortalConfig) { +func ConfigInit(envLookup *env.VarSet, jetstreamConfig *interfaces.PortalConfig) { + // Check we are deployed in Cloud Foundry - if !config.IsSet(VCapApplication) { + if !envLookup.IsSet(VCapApplication) { return } isSQLite := jetstreamConfig.DatabaseProviderName == SQLiteProviderName @@ -64,9 +65,9 @@ func ConfigInit(jetstreamConfig *interfaces.PortalConfig) { // Else, if not default and is SQLlite - add the App Index to the secret // This makes sure we use a different Session Secret per App Instance IF using SQLite // Since this is not a shared database across application instances - if isSQLite && config.IsSet("CF_INSTANCE_INDEX") { - appInstanceIndex, err := config.GetValue("CF_INSTANCE_INDEX") - if err == nil { + if isSQLite && envLookup.IsSet("CF_INSTANCE_INDEX") { + appInstanceIndex, ok := envLookup.Lookup("CF_INSTANCE_INDEX") + if ok { jetstreamConfig.SessionStoreSecret = jetstreamConfig.SessionStoreSecret + "_" + appInstanceIndex log.Infof("Updated session secret for Cloud Foundry App Instance: %s", appInstanceIndex) } @@ -81,7 +82,7 @@ func Init(portalProxy interfaces.PortalProxy) (interfaces.StratosPlugin, error) // GetMiddlewarePlugin gets the middleware plugin for this plugin func (ch *CFHosting) GetMiddlewarePlugin() (interfaces.MiddlewarePlugin, error) { - if config.IsSet(VCapApplication) { + if ch.portalProxy.Env().IsSet(VCapApplication) { return ch, nil } return nil, errors.New("Not running as a Cloud Foundry application") @@ -100,7 +101,7 @@ func (ch *CFHosting) GetRoutePlugin() (interfaces.RoutePlugin, error) { // Init performs plugin initialization func (ch *CFHosting) Init() error { // Determine if we are running CF by presence of env var "VCAP_APPLICATION" and configure appropriately - if config.IsSet(VCapApplication) { + if ch.portalProxy.Env().IsSet(VCapApplication) { log.Info("Detected that Console is deployed as a Cloud Foundry Application") // Record that we are deployed in Cloud Foundry @@ -115,30 +116,24 @@ func (ch *CFHosting) Init() error { ch.portalProxy.GetConfig().ConsoleConfig.ConsoleAdminScope = ch.portalProxy.GetConfig().CFAdminIdentifier // Allow Console Application manifest to override the Admin Scope if really desired - if config.IsSet("STRATOS_ADMIN_SCOPE") { - stratosAdminScope, err := config.GetValue("STRATOS_ADMIN_SCOPE") - if err == nil { - ch.portalProxy.GetConfig().ConsoleConfig.ConsoleAdminScope = stratosAdminScope - log.Infof("Overriden Console Admin Scope to: %s", stratosAdminScope) - } + stratosAdminScope, ok := ch.portalProxy.Env().Lookup("STRATOS_ADMIN_SCOPE") + if ok { + ch.portalProxy.GetConfig().ConsoleConfig.ConsoleAdminScope = stratosAdminScope + log.Infof("Overriden Console Admin Scope to: %s", stratosAdminScope) } // Need to run as HTTP on the port we were told to use ch.portalProxy.GetConfig().HTTPS = false - if config.IsSet("PORT") { - port, err := config.GetValue("PORT") - if err != nil { - log.Warnf("Unable to read Port") - } else { - - ch.portalProxy.GetConfig().TLSAddress = ":" + port - log.Infof("Updated Console address to: %s", ch.portalProxy.GetConfig().TLSAddress) - } + port, ok := ch.portalProxy.Env().Lookup("PORT") + if ok { + ch.portalProxy.GetConfig().TLSAddress = ":" + port + log.Infof("Updated Console address to: %s", ch.portalProxy.GetConfig().TLSAddress) } + // Get the cf_api value from the JSON var appData interfaces.VCapApplicationData - vCapApp, _ := config.GetValue(VCapApplication) + vCapApp, _ := ch.portalProxy.Env().Lookup(VCapApplication) data := []byte(vCapApp) err := json.Unmarshal(data, &appData) if err != nil { @@ -149,13 +144,13 @@ func (ch *CFHosting) Init() error { log.Infof("CF API URL: %s", appData.API) // Allow the URL to be overridden by an application environment variable - if config.IsSet(CFApiURLOverride) { - apiURL, _ := config.GetValue(CFApiURLOverride) - appData.API = apiURL - log.Infof("Overriden CF API URL from environment variable %s", apiURL) + if ch.portalProxy.Env().IsSet(CFApiURLOverride) { + apiUrl, _ := ch.portalProxy.Env().Lookup(CFApiURLOverride) + appData.API = apiUrl + log.Infof("Overriden CF API URL from environment variable %s", apiUrl) } - if config.IsSet(CFApiForceSecure) { + if ch.portalProxy.Env().IsSet(CFApiForceSecure) { // Force the API URL protocol to be https appData.API = strings.Replace(appData.API, "http://", "https://", 1) log.Infof("Ensuring that CF API URL is accessed over HTTPS") @@ -164,13 +159,9 @@ func (ch *CFHosting) Init() error { } disableEndpointDashboard := true - if config.IsSet(ForceEndpointDashboard) { + if ch.portalProxy.Env().IsSet(ForceEndpointDashboard) { // Force the Endpoint Dashboard to be visible? - if forceStr, err := config.GetValue(ForceEndpointDashboard); err == nil { - if force, err := strconv.ParseBool(forceStr); err == nil { - disableEndpointDashboard = !force - } - } + disableEndpointDashboard = !ch.portalProxy.Env().MustBool(ForceEndpointDashboard) } if disableEndpointDashboard { @@ -197,19 +188,9 @@ func (ch *CFHosting) Init() error { ch.portalProxy.GetConfig().ConsoleConfig.UAAEndpoint = url log.Infof("Cloud Foundry UAA is: %s", ch.portalProxy.GetConfig().ConsoleConfig.UAAEndpoint) - skipSsl, err := config.GetValue("SKIP_SSL_VALIDATION") - if err != nil { - // Not set in the environment and failed to read from the Secrets file - ch.portalProxy.GetConfig().ConsoleConfig.SkipSSLValidation = false - } - - skipSslBool, err := strconv.ParseBool(skipSsl) - if err != nil { - // Not set in the environment and failed to read from the Secrets file - ch.portalProxy.GetConfig().ConsoleConfig.SkipSSLValidation = false - } else { - ch.portalProxy.GetConfig().ConsoleConfig.SkipSSLValidation = skipSslBool - } + // Not set in the environment and failed to read from the Secrets file + // CHECK is this necessary to set here? + ch.portalProxy.GetConfig().ConsoleConfig.SkipSSLValidation = ch.portalProxy.Env().MustBool("SKIP_SSL_VALIDATION") // Save to Console DB err = ch.portalProxy.SaveConsoleConfig(ch.portalProxy.GetConfig().ConsoleConfig, nil) @@ -218,7 +199,7 @@ func (ch *CFHosting) Init() error { return fmt.Errorf("Failed to save console configuration due to %s", err) } - if !config.IsSet(SkipAutoRegister) { + if !ch.portalProxy.Env().IsSet(SkipAutoRegister) { log.Info("Setting AUTO_REG_CF_URL config to ", appData.API) ch.portalProxy.GetConfig().AutoRegisterCFUrl = appData.API } else { diff --git a/src/jetstream/plugins/metrics/main.go b/src/jetstream/plugins/metrics/main.go index 2d0a3c7ff0..801d74e24c 100644 --- a/src/jetstream/plugins/metrics/main.go +++ b/src/jetstream/plugins/metrics/main.go @@ -10,7 +10,6 @@ import ( "net/url" "strings" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/tokens" "github.com/labstack/echo" @@ -115,11 +114,7 @@ func (m *MetricsSpecification) GetType() string { } func (m *MetricsSpecification) GetClientId() string { - if clientId, err := config.GetValue(CLIENT_ID_KEY); err == nil { - return clientId - } - - return "metrics" + return m.portalProxy.Env().String(CLIENT_ID_KEY, "metrics") } func (m *MetricsSpecification) Register(echoContext echo.Context) error { diff --git a/src/jetstream/portal_config.go b/src/jetstream/portal_config.go index 1da0ae3357..ce0a94b928 100644 --- a/src/jetstream/portal_config.go +++ b/src/jetstream/portal_config.go @@ -2,8 +2,13 @@ package main import ( "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" + "github.com/govau/cf-common/env" ) func (p *portalProxy) GetConfig() *interfaces.PortalConfig { return &p.Config } + +func (p *portalProxy) Env() *env.VarSet { + return p.env +} diff --git a/src/jetstream/portal_proxy.go b/src/jetstream/portal_proxy.go index bc14a039df..9d7adc64f9 100644 --- a/src/jetstream/portal_proxy.go +++ b/src/jetstream/portal_proxy.go @@ -7,6 +7,7 @@ import ( "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" "github.com/gorilla/sessions" + "github.com/govau/cf-common/env" ) type portalProxy struct { @@ -19,6 +20,7 @@ type portalProxy struct { SessionCookieName string EmptyCookieMatcher *regexp.Regexp // Used to detect and remove empty Cookies sent by certain browsers AuthProviders map[string]interfaces.AuthProvider + env *env.VarSet } // HttpSessionStore - Interface for a store that can manage HTTP Sessions diff --git a/src/jetstream/repository/interfaces/plugin.go b/src/jetstream/repository/interfaces/plugin.go index a84dd69576..355581dedd 100644 --- a/src/jetstream/repository/interfaces/plugin.go +++ b/src/jetstream/repository/interfaces/plugin.go @@ -1,5 +1,9 @@ package interfaces +import ( + "github.com/govau/cf-common/env" +) + // StratosPlugin is the interface for a Jetstream plugin type StratosPlugin interface { Init() error @@ -9,7 +13,7 @@ type StratosPlugin interface { } // JetstreamConfigInit is the function signature for the config plugin init function -type JetstreamConfigInit func(*PortalConfig) +type JetstreamConfigInit func(*env.VarSet, *PortalConfig) // JetstreamConfigPlugins is the array of config plugins var JetstreamConfigPlugins []JetstreamConfigInit diff --git a/src/jetstream/repository/interfaces/portal_proxy.go b/src/jetstream/repository/interfaces/portal_proxy.go index c1d783fe8b..28dde69af3 100644 --- a/src/jetstream/repository/interfaces/portal_proxy.go +++ b/src/jetstream/repository/interfaces/portal_proxy.go @@ -6,6 +6,7 @@ import ( "net/url" "github.com/gorilla/sessions" + "github.com/govau/cf-common/env" "github.com/labstack/echo" ) @@ -42,6 +43,7 @@ type PortalProxy interface { GetCNSITokenRecordWithDisconnected(cnsiGUID string, userGUID string) (TokenRecord, bool) GetCNSIUser(cnsiGUID string, userGUID string) (*ConnectedUser, bool) GetConfig() *PortalConfig + Env() *env.VarSet ListEndpointsByUser(userGUID string) ([]*ConnectedEndpoint, error) // UAA Token diff --git a/src/jetstream/setup_console.go b/src/jetstream/setup_console.go index eecd96a92b..2c25f8c190 100644 --- a/src/jetstream/setup_console.go +++ b/src/jetstream/setup_console.go @@ -14,7 +14,6 @@ import ( "github.com/labstack/echo" log "github.com/sirupsen/logrus" - "github.com/cloudfoundry-incubator/stratos/src/jetstream/config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/console_config" "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" ) @@ -151,30 +150,32 @@ func (p *portalProxy) setupConsoleUpdate(c echo.Context) error { func (p *portalProxy) initialiseConsoleConfig(consoleRepo console_config.Repository) (*interfaces.ConsoleConfig, error) { log.Debug("initialiseConsoleConfig") + var err error + consoleConfig := new(interfaces.ConsoleConfig) - uaaEndpoint, err := config.GetValue("UAA_ENDPOINT") - if err != nil { + uaaEndpoint, found := p.Env().Lookup("UAA_ENDPOINT") + if !found { return consoleConfig, errors.New("UAA_Endpoint not found") } - consoleClient, err := config.GetValue("CONSOLE_CLIENT") - if err != nil { + consoleClient, found := p.Env().Lookup("CONSOLE_CLIENT") + if !found { return consoleConfig, errors.New("CONSOLE_CLIENT not found") } - consoleClientSecret, err := config.GetValue("CONSOLE_CLIENT_SECRET") + consoleClientSecret, found := p.Env().Lookup("CONSOLE_CLIENT_SECRET") if err != nil { // Special case, mostly this is blank, so assume its blank consoleClientSecret = "" } - consoleAdminScope, err := config.GetValue("CONSOLE_ADMIN_SCOPE") - if err != nil { + consoleAdminScope, found := p.Env().Lookup("CONSOLE_ADMIN_SCOPE") + if !found { return consoleConfig, errors.New("CONSOLE_ADMIN_SCOPE not found") } - skipSslValidation, err := config.GetValue("SKIP_SSL_VALIDATION") - if err != nil { + skipSslValidation, found := p.Env().Lookup("SKIP_SSL_VALIDATION") + if !found { return consoleConfig, errors.New("SKIP_SSL_VALIDATION not found") }