[!] switch configuration to connection string from separate parameters, closes #255

docker-compose.yml

@@ -52,10 +52,7 @@ services:
       dockerfile: ./docker/Dockerfile
     image: cybertecpostgresql/pgwatch3:latest
     environment:
-      PW3_PGHOST: postgres
-      PW3_PGUSER: pgwatch3
-      PW3_PGDATABASE: pgwatch3
-      PW3_DATASTORE: postgres
+      PW3_CONFIG: postgresql://pgwatch3@postgres:5432/pgwatch3
       PW3_PG_METRIC_STORE_CONN_STR: postgresql://pgwatch3@postgres:5432/pgwatch3_metrics
     ports:
       - "8080:8080"
@@ -88,9 +85,9 @@ services:
         -f /tmp/00_helpers/get_table_bloat_approx_sql/12/metric.sql
         -f /tmp/00_helpers/get_wal_size/10/metric.sql
         -f /tmp/00_helpers/get_sequences/10/metric.sql 
-        -c "INSERT INTO pgwatch3.monitored_db (md_unique_name, md_preset_config_name, md_config, md_hostname, md_port, md_dbname, md_user, md_password)
-          SELECT 'test', 'exhaustive', null, 'postgres', '5432', 'pgwatch3', 'pgwatch3', 'pgwatch3admin'
-          WHERE NOT EXISTS (SELECT * FROM pgwatch3.monitored_db WHERE (md_unique_name, md_hostname, md_dbname) = ('test', 'localhost', 'pgwatch3'))"
+        -c "INSERT INTO pgwatch3.monitored_db (md_name, md_preset_config_name, md_connstr)
+          SELECT 'test', 'exhaustive', 'postgresql://pgwatch3:pgwatch3admin@localhost/pgwatch3'
+          WHERE NOT EXISTS (SELECT * FROM pgwatch3.monitored_db WHERE md_name = 'test')"
     volumes:
       - "./src/metrics/sql/00_helpers:/tmp/00_helpers"
     depends_on:

docs/ENV_VARIABLES.md

@@ -9,18 +9,11 @@ Some variables influence multiple components. Command line parameters override e
 
 ## Gatherer daemon
 
-- **PW3_PGHOST** Config DB host. Default: localhost
-- **PW3_PGPORT** Config DB port. Default: 5432
-- **PW3_PGDATABASE** Config DB name. Default: pgwatch3
-- **PW3_PGUSER** Config DB user. Default: pgwatch3
-- **PW3_PGPASSWORD** Config DB password. Default: pgwatch3admin
-- **PW3_PGSSL** Config DB SSL connection only. Default: False
 - **PW3_GROUP** Logical grouping/sharding key to monitor a subset of configured hosts. Default: -
-- **PW3_DATASTORE** Backend for metric storage - [postgres|prometheus|json]. Default: postgres
-- **PW3_VERBOSE** Logging vebosity. By default warning and errors are logged. Use [-v|-vv] to include [info|debug]. Default: -
-- **PW3_PG_METRIC_STORE_CONN_STR** Postgres metric store connection string. Required when PW3_DATASTORE=postgres. Default: -
+- **PW3_PG_METRIC_STORE_CONN_STR** Postgres metric store connection string. Default: -
+- **PW3_JSON_STORAGE_FILE** File to store metric values. Default: -
 - **PW3_PG_RETENTION_DAYS** Effective when PW3_DATASTORE=postgres. Default: 14
-- **PW3_CONFIG** File mode. File or folder of YAML (.yaml/.yml) files containing info on which DBs to monitor and where to store metrics
+- **PW3_CONFIG** Connection string (`postgresql://user:pwd@host/db`), file or folder of YAML (.yaml/.yml) files containing info on which DBs to monitor and where to store metrics
 - **PW3_METRICS_FOLDER** File mode. Folder of metrics definitions
 - **PW3_BATCHING_MAX_DELAY_MS** Max milliseconds to wait for a batched metrics flush. Default: 250
 - **PW3_ADHOC_CONN_STR** Ad-hoc mode. Monitor a single Postgres DB / instance specified by a standard Libpq connection string
@@ -50,33 +43,6 @@ Some variables influence multiple components. Command line parameters override e
 - **PW3_NO_HELPER_FUNCTIONS** Ignore metric definitions using helper functions (in form get_smth()) and don't also roll out any helpers automatically. Default: false
 - **PW3_TRY_CREATE_LISTED_EXTS_IF_MISSING** Try creating the listed extensions (comma sep.) on first connect for all monitored DBs when missing. Main usage - pg_stat_statements. Default: ""
 
-
-## Web UI
-
-- **PW3_WEBHOST** Network interface to listen on. Default: 0.0.0.0
-- **PW3_WEBPORT** Port. Default: 8080
-- **PW3_WEBSSL** Use HTTPS with self-signed certificates, Default: False
-- **PW3_WEBCERT** Enables use of own certificates for custom deployments. Default: '/pgwatch3/persistent-config/self-signed-ssl.pem'
-- **PW3_WEBKEY** Enables use of own certificates for custom deployments. Default: '/pgwatch3/persistent-config/self-signed-ssl.key'
-- **PW3_WEBCERTCHAIN** Path to certificate chain file for custom deployments. Default: -
-- **PW3_WEBNOANONYMOUS** Require user/password to edit data. Default: False
-- **PW3_WEBUSER** Admin login. Default: pgwatch3
-- **PW3_WEBPASSWORD** Admin password. Default: pgwatch3admin
-- **PW3_WEBNOCOMPONENTLOGS** Don't expose Docker component logs. Default: False
-- **PW3_WEBNOSTATSSUMMARY** Don't expose summary metrics and "top queries" on monitored DBs. Default: False
-- **PW3_VERBOSE** Logging vebosity. By default warning and errors are logged. Use [-v|-vv] to include [info|debug]. Default: -
-- **PW3_PGHOST** Config DB host. Default: localhost
-- **PW3_PGPORT** Config DB port. Default: 5432
-- **PW3_PGDATABASE** Config DB name. Default: pgwatch3
-- **PW3_PGUSER** Config DB user. Default: pgwatch3
-- **PW3_PGPASSWORD** Config DB password. Default: pgwatch3admin
-- **PW3_PGSSL** Config DB SSL connection only. Default: False
-- **PW3_GRAFANA_BASEURL** For linking to Grafana "Query details" dashboard from "Stat_stmt. overview". Default: http://0.0.0.0:3000
-- **PW3_AES_GCM_KEYPHRASE** Keyphrase for encryption/decpyption of connect string passwords.
-- **PW3_AES_GCM_KEYPHRASE_FILE** File containing a keyphrase for encryption/decpyption of connect string passwords.
-- **PW3_PG_METRIC_STORE_CONN_STR** Postgres metric store connection string. Required when PW3_DATASTORE=postgres. Default: -
-
-
 ## Grafana
 
 - **PW3_GRAFANANOANONYMOUS** Can be set to require login even for viewing dashboards. Default: -

src/api.go

@@ -98,23 +98,20 @@ func (uiapi uiapihandler) GetDatabases() (res string, err error) {
 
 // DeleteDatabase removes the database from the list of monitored databases
 func (uiapi uiapihandler) DeleteDatabase(database string) error {
-	_, err := configDb.Exec(context.TODO(), "DELETE FROM pgwatch3.monitored_db WHERE md_unique_name = $1", database)
+	_, err := configDb.Exec(context.TODO(), "DELETE FROM pgwatch3.monitored_db WHERE md_name = $1", database)
 	return err
 }
 
 // AddDatabase adds the database to the list of monitored databases
 func (uiapi uiapihandler) AddDatabase(params []byte) error {
 	sql := `INSERT INTO pgwatch3.monitored_db(
-md_unique_name, md_preset_config_name, md_config, md_hostname, 
-md_port, md_dbname, md_user, md_password, md_is_superuser, md_is_enabled)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`
+		md_name, md_connstr, md_preset_config_name, md_config, md_is_superuser, md_is_enabled)
+		VALUES ($1, $2, $3, $4, $5, $6)`
 	var m map[string]any
 	err := json.Unmarshal(params, &m)
 	if err == nil {
-		_, err = configDb.Exec(context.TODO(), sql, m["md_unique_name"], m["md_preset_config_name"],
-			m["md_config"], m["md_hostname"], m["md_port"],
-			m["md_dbname"], m["md_user"], m["md_password"],
-			m["md_is_superuser"], m["md_is_enabled"])
+		_, err = configDb.Exec(context.TODO(), sql, m["md_name"], m["md_connstr"],
+			m["md_preset_config_name"], m["md_config"], m["md_is_superuser"], m["md_is_enabled"])
 	}
 	return err
 }
@@ -137,7 +134,7 @@ func (uiapi uiapihandler) UpdateDatabase(database string, params []byte) error {
 	if err != nil {
 		return err
 	}
-	sql := fmt.Sprintf(`UPDATE pgwatch3.monitored_db SET %s WHERE md_unique_name = $1`, strings.Join(fields, ","))
+	sql := fmt.Sprintf(`UPDATE pgwatch3.monitored_db SET %s WHERE md_name = $1`, strings.Join(fields, ","))
 	values = append([]any{database}, values...)
 	_, err = configDb.Exec(context.TODO(), sql, values...)
 	return err

src/config/cmdparser.go

@@ -3,19 +3,24 @@ package config
 import (
 	"os"
 
+	"github.com/jackc/pgx/v5"
 	flags "github.com/jessevdk/go-flags"
 )
 
+type Kind int
+
+const (
+	ConfigPgURL Kind = iota
+	ConfigFile
+	ConfigFolder
+	ConfigError
+)
+
 // ConnectionOpts specifies the database connection options
 type ConnectionOpts struct {
-	Host                      string `long:"host" mapstructure:"host" description:"PG config DB host" default:"localhost" env:"PW3_PGHOST"`
-	Port                      string `short:"p" long:"port" mapstructure:"port" description:"PG config DB port" default:"5432" env:"PW3_PGPORT"`
-	Dbname                    string `short:"d" long:"dbname" mapstructure:"dbname" description:"PG config DB dbname" default:"pgwatch3" env:"PW3_PGDATABASE"`
-	User                      string `short:"u" long:"user" mapstructure:"user" description:"PG config DB user" default:"pgwatch3" env:"PW3_PGUSER"`
-	Password                  string `long:"password" mapstructure:"password" description:"PG config DB password" env:"PW3_PGPASSWORD"`
-	PgRequireSSL              bool   `long:"pg-require-ssl" mapstructure:"pg-require-ssl" description:"PG config DB SSL connection only" env:"PW3_PGSSL"`
+	Config                    string `short:"c" long:"config" mapstructure:"config" description:"File or folder of YAML files containing info on which DBs to monitor and where to store metrics" env:"PW3_CONFIG"`
 	ServersRefreshLoopSeconds int    `long:"servers-refresh-loop-seconds" mapstructure:"servers-refresh-loop-seconds" description:"Sleep time for the main loop" env:"PW3_SERVERS_REFRESH_LOOP_SECONDS" default:"120"`
-	Init                      bool   `long:"init" description:"Initialize database schema to the latest version and exit. Can be used with --upgrade"`
+	Init                      bool   `long:"init" description:"Initialize configuration database schema to the latest version and exit. Can be used with --upgrade"`
 }
 
 // MetricStoreOpts specifies the storage configuration to store metrics data
@@ -64,7 +69,6 @@ type CmdOptions struct {
 	Logging                      LoggingOpts    `group:"Logging" mapstructure:"Logging"`
 	WebUI                        WebUIOpts      `group:"WebUI" mapstructure:"WebUI"`
 	Start                        StartOpts      `group:"Start" mapstructure:"Start"`
-	Config                       string         `short:"c" long:"config" mapstructure:"config" description:"File or folder of YAML files containing info on which DBs to monitor and where to store metrics" env:"PW3_CONFIG"`
 	BatchingDelayMs              int64          `long:"batching-delay-ms" mapstructure:"batching-delay-ms" description:"Max milliseconds to wait for a batched metrics flush. [Default: 250]" default:"250" env:"PW3_BATCHING_MAX_DELAY_MS"`
 	AdHocConnString              string         `long:"adhoc-conn-str" mapstructure:"adhoc-conn-str" description:"Ad-hoc mode: monitor a single Postgres DB specified by a standard Libpq connection string" env:"PW3_ADHOC_CONN_STR"`
 	AdHocDBType                  string         `long:"adhoc-dbtype" mapstructure:"adhoc-dbtype" description:"Ad-hoc mode: postgres|postgres-continuous-discovery" default:"postgres" env:"PW3_ADHOC_DBTYPE"`
@@ -99,6 +103,20 @@ func (c CmdOptions) VersionOnly() bool {
 	return len(os.Args) == 2 && c.Version
 }
 
+func (c CmdOptions) GetConfigKind() (_ Kind, err error) {
+	if _, err := pgx.ParseConfig(c.Connection.Config); err == nil {
+		return Kind(ConfigPgURL), nil
+	}
+	var fi os.FileInfo
+	if fi, err = os.Stat(c.Connection.Config); err == nil {
+		if fi.IsDir() {
+			return Kind(ConfigFolder), nil
+		}
+		return Kind(ConfigFile), nil
+	}
+	return Kind(ConfigError), err
+}
+
 // NewCmdOptions returns a new instance of CmdOptions with default values
 func NewCmdOptions(args ...string) *CmdOptions {
 	cmdOpts := new(CmdOptions)

src/config/config.go

@@ -28,6 +28,11 @@ func checkFolderExistsAndReadable(path string) bool {
 	return err == nil
 }
 
+const (
+	defaultMetricsDefinitionPathPkg    = "/etc/pgwatch3/metrics" // prebuilt packages / Docker default location
+	defaultMetricsDefinitionPathDocker = "/pgwatch3/metrics"     // prebuilt packages / Docker default location
+)
+
 func validateConfig(conf *CmdOptions) error {
 	if conf.Connection.ServersRefreshLoopSeconds <= 1 {
 		return errors.New("--servers-refresh-loop-seconds must be greater than 1")
@@ -36,6 +41,10 @@ func validateConfig(conf *CmdOptions) error {
 		return errors.New("--max-parallel-connections-per-db must be >= 1")
 	}
 
+	if conf.Metric.MetricsFolder > "" && !checkFolderExistsAndReadable(conf.Metric.MetricsFolder) {
+		return fmt.Errorf("Could not read --metrics-folder path %s", conf.Metric.MetricsFolder)
+	}
+
 	if err := validateAesGcmConfig(conf); err != nil {
 		return err
 	}
@@ -78,14 +87,17 @@ func validateAdHocConfig(conf *CmdOptions) error {
 		if len(conf.AdHocConnString)*len(conf.AdHocConfig) == 0 {
 			return errors.New("--adhoc-conn-str and --adhoc-config params both need to be specified for Ad-hoc mode to work")
 		}
-		if conf.Config > "" {
+		if len(conf.Connection.Config) > 0 {
 			return errors.New("Conflicting flags! --adhoc-conn-str and --config cannot be both set")
 		}
-		if conf.Metric.MetricsFolder > "" && !checkFolderExistsAndReadable(conf.Metric.MetricsFolder) {
-			return fmt.Errorf("--metrics-folder \"%s\" not readable, trying 1st default paths and then Config DB to fetch metric definitions", conf.Metric.MetricsFolder)
-		}
-		if conf.Connection.User > "" && conf.Connection.Password > "" {
-			return errors.New("Conflicting flags! --adhoc-conn-str and --user/--password cannot be both set")
+		if conf.Metric.MetricsFolder == "" {
+			if checkFolderExistsAndReadable(defaultMetricsDefinitionPathPkg) {
+				conf.Metric.MetricsFolder = defaultMetricsDefinitionPathPkg
+			} else if checkFolderExistsAndReadable(defaultMetricsDefinitionPathDocker) {
+				conf.Metric.MetricsFolder = defaultMetricsDefinitionPathDocker
+			} else {
+				return errors.New("--adhoc-conn-str requires --metrics-folder")
+			}
 		}
 		if conf.AdHocDBType != DbTypePg && conf.AdHocDBType != DbTypePgCont {
 			return fmt.Errorf("--adhoc-dbtype can be of: [ %s (single DB) | %s (all non-template DB-s on an instance) ]. Default: %s", DbTypePg, DbTypePgCont, DbTypePg)