Refactor .vars/--var to .env/--env (#5981)

* Refactor `.vars`/`--var` to `.env`/`--env`

* Review

cli/cmd/start/start.go

@@ -28,8 +28,8 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 	var noUI bool
 	var noOpen bool
 	var logFormat string
-	var env []string
-	var vars []string
+	var envVars, envVarsOld []string
+	var environment string
 	var allowedOrigins []string
 	var tlsCertPath string
 	var tlsKeyPath string
@@ -113,18 +113,9 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 				return fmt.Errorf("invalid log format %q", logFormat)
 			}
 
-			// Backwards compatibility for --env (see comment on the flag definition for details)
-			environment := "dev"
-			for _, v := range env {
-				if strings.Contains(v, "=") {
-					vars = append(vars, v)
-				} else {
-					environment = v
-				}
-			}
-
 			// Parser variables from "a=b" format to map
-			varsMap, err := parseVariables(vars)
+			envVars = append(envVars, envVarsOld...)
+			envVarsMap, err := parseVariables(envVars)
 			if err != nil {
 				return err
 			}
@@ -161,7 +152,7 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 				OlapDSN:        olapDSN,
 				ProjectPath:    projectPath,
 				LogFormat:      parsedLogFormat,
-				Variables:      varsMap,
+				Variables:      envVarsMap,
 				LocalURL:       localURL,
 				AllowedOrigins: allowedOrigins,
 			})
@@ -182,23 +173,27 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 	}
 
 	startCmd.Flags().SortFlags = false
+	startCmd.Flags().StringSliceVarP(&envVars, "env", "e", []string{}, "Set environment variables")
+	startCmd.Flags().StringVar(&environment, "environment", "dev", `Environment name`)
+	startCmd.Flags().BoolVar(&reset, "reset", false, "Clear and re-ingest source data")
 	startCmd.Flags().BoolVar(&noOpen, "no-open", false, "Do not open browser")
+	startCmd.Flags().BoolVar(&verbose, "verbose", false, "Sets the log level to debug")
+	startCmd.Flags().BoolVar(&readonly, "readonly", false, "Show only dashboards in UI")
 	startCmd.Flags().IntVar(&httpPort, "port", 9009, "Port for HTTP")
 	startCmd.Flags().IntVar(&grpcPort, "port-grpc", 49009, "Port for gRPC (internal)")
-	startCmd.Flags().BoolVar(&readonly, "readonly", false, "Show only dashboards in UI")
 	startCmd.Flags().BoolVar(&noUI, "no-ui", false, "Serve only the backend")
-	startCmd.Flags().BoolVar(&verbose, "verbose", false, "Sets the log level to debug")
 	startCmd.Flags().BoolVar(&debug, "debug", false, "Collect additional debug info")
-	startCmd.Flags().BoolVar(&reset, "reset", false, "Clear and re-ingest source data")
 	startCmd.Flags().StringVar(&logFormat, "log-format", "console", "Log format (options: \"console\", \"json\")")
 	startCmd.Flags().StringVar(&tlsCertPath, "tls-cert", "", "Path to TLS certificate")
 	startCmd.Flags().StringVar(&tlsKeyPath, "tls-key", "", "Path to TLS key file")
-
-	// --env was previously used for variables, but is now used to set the environment name. We maintain backwards compatibility by keeping --env as a slice var, and setting any value containing an equals sign as a variable.
-	startCmd.Flags().StringSliceVarP(&env, "env", "e", []string{}, `Environment name (default "dev")`)
-	startCmd.Flags().StringSliceVarP(&vars, "var", "v", []string{}, "Set project variables")
 	startCmd.Flags().StringSliceVarP(&allowedOrigins, "allowed-origins", "", []string{}, "Override allowed origins for CORS")
 
+	// Deprecated support for "--var": replaced by "--env".
+	startCmd.Flags().StringSliceVarP(&envVarsOld, "var", "v", []string{}, "Set environment variables")
+	if err := startCmd.Flags().MarkHidden("var"); err != nil {
+		panic(err)
+	}
+
 	// We have deprecated the ability configure the OLAP database via the CLI. This should now be done via rill.yaml.
 	// Keeping these for backwards compatibility for a while.
 	startCmd.Flags().StringVar(&olapDSN, "db", local.DefaultOLAPDSN, "Database DSN")

docs/docs/build/credentials/credentials.md

@@ -16,7 +16,7 @@ At a high level, configuring credentials and credentials management in Rill can
 When reading from a source (or using a different OLAP engine), Rill will attempt to use existing credentials that have been configured on your machine.
 1. Credentials that have been configured in your local environment via the CLI (for [AWS](../../reference/connectors/s3.md#local-credentials) / [Azure](../../reference/connectors/azure.md#local-credentials) / [Google Cloud](../../reference/connectors/gcs.md#local-credentials))
 2. Credentials that have been passed in directly through the connection string or DSN (typically for databases - see [Source YAML](../../reference/project-files/sources.md) and [Connector YAML](../../reference/project-files/connectors.md) for more details)
-3. Credentials that have been passed in as a [variable](../../deploy/templating.md) when starting Rill Developer via `rill start --var key=value`
+3. Credentials that have been passed in as a [variable](../../deploy/templating.md) when starting Rill Developer via `rill start --env key=value`
 4. Credentials that have been specified in your *`<RILL_PROJECT_HOME>/.env`* file
 
 For more details, please refer to the corresponding [connector](../../reference/connectors/connectors.md) or [OLAP engine](../../reference/olap-engines/olap-engines.md) page.

docs/docs/build/models/environments.md

@@ -15,45 +15,25 @@ There could be a few reasons to have separate logic for sources, models, and das
 
 ## Default dev and prod environments
 
-Rill comes with a default `dev` and `prod` property defined, corresponding to Rill Developer and Rill Cloud respectively. Any environment specific YAML overrides or custom templated SQL logic can reference these environments without any additional configuration needed.
+Rill comes with default `dev` and `prod` properties defined, corresponding to Rill Developer and Rill Cloud respectively. You can use these keys to set environment-specific YAML overrides or SQL logic.
 
-:::tip Shortcut to specify dev and prod in YAML files
-
-For the built-in `dev` and `prod` environments **specifically**, Rill provides a shorthand where you can specify properties for these environments directly under `dev` / `prod` without first nesting it under a parent `env` key. For example, if you had the following `rill.yaml`:
-```yaml
-env:
-  dev:
-    sources:
-      path: s3://path/to/bucket/Y=2024/M=01/*.parquet
-  prod:
-    models:
-      materialize: true
-```
-
-This would be exactly equivalent to (within the same `rill.yaml`):
+For example, the following `rill.yaml` file explicitly sets the default materialization setting for models to `false` in development and `true` in production:
 ```yaml
 dev:
-  sources:
-    path: s3://path/to/bucket/Y=2024/M=01/*.parquet
+  models:
+    materialize: false
+
 prod:
   models:
     materialize: true
 ```
 
-For other custom environments that you are defining manually, you will still need to pass them in using the standard environment YAML syntax:
-```yaml
-env:
-  custom_env:
-    property: value
-```
-:::
-
 ### Specifying a custom environment
 
-When using Rill Developer, you can specify a custom environment for your local instance (instead of defaulting to `dev`) by using the following command:
+When using Rill Developer, instead of defaulting to `dev`, you can run your project in production mode using the following command:
 
 ```bash
-rill start --env <name_of_environment>
+rill start --environment prod
 ```
 
 ## Specifying environment specific YAML overrides
@@ -64,20 +44,18 @@ Environment overrides can be applied to source properties in the [YAML configura
 type: source
 connector: s3
 path: s3://path/to/bucket/*.parquet
-env:
-  dev:
-    path: s3://path/to/bucket/Y=2024/M=01/*.parquet
+dev:
+  path: s3://path/to/bucket/Y=2024/M=01/*.parquet
 ```
 
 Similarly, if you wanted to set a project-wide default in `rill.yaml` where models are [materialized](/reference/project-files/models.md#model-materialization) only on Rill Cloud (i.e. `prod) and dashboards use a different default [theme](../dashboards/customize.md#changing-themes--colors) in production compared to locally, you could do this by:
 
 ```yaml
-env:
-  prod:
-    models:
-      materialize: true
-    dashboards:
-      theme: <name_of_theme>
+prod:
+  models:
+    materialize: true
+  explores:
+    theme: <name_of_theme>
 ```
 
 :::info Hierarchy of inheritance and property overrides

docs/docs/deploy/performance.md

@@ -53,9 +53,8 @@ There are different ways this can be achieved and the method also depends heavil
 type: source
 connector: s3
 path: s3://bucket/path/**/*.parquet
-env:
-  dev:
-    path: s3://bucket/path/year=2023/month=12/**/*.parquet
+dev:
+  path: s3://bucket/path/year=2023/month=12/**/*.parquet
 ```
 
 By leveraging the [environment YAML syntax](/build/models/environments.md), this ensures that only data from December 2023 will be read in from this S3 source when using Rill Developer locally while the full range of data will still be used in production (on Rill Cloud). However, if this data was **not** partitioned, then we could simply leverage DuckDB's ability to read from S3 files directly and _apply a filter post-download_ on the source. Taking this same example and using some [templating](templating.md), the `source.yaml` could be rewritten to something like the following:

docs/docs/deploy/templating.md

@@ -25,8 +25,8 @@ For the most part, templating should be used in [SQL models](../build/models/mod
 Templating can be used in conjunction with variables to apply more advanced logic to your sources and models. 
 
 Variables can be set in Rill through one of the following methods:
-1. Defining the corresponding key-value pair under the top-level `vars` key in `rill.yaml`
-2. Manually passing in the variable when starting Rill (i.e. `rill start --var <var_name>=<value>`)
+1. Defining the corresponding key-value pair under the top-level `env` key in `rill.yaml`
+2. Manually passing in the variable when starting Rill (i.e. `rill start --env <var_name>=<value>`)
 3. Specifying the key-value pair for your variable in your `<RILL_PROJECT_HOME>/.env` file and/or using `rill env set` to set the variable via the CLI (and then [pushing / pulling as appropriate](../build/credentials/credentials.md#variables))
 
 For more information, please check our [reference documentation](/reference/project-files/rill-yaml.md#setting-variables).
@@ -45,7 +45,7 @@ To avoid this scenario, whenever you use templating in a model's SQL, it is <u>s
 # models/my_model.sql
 SELECT *
 FROM {{ ref "my_source" }}
-WHERE my_value = '{{ .vars.my_value }}'
+WHERE my_value = '{{ .env.my_value }}'
 ```
 
 In this example, the `ref` tag ensures that the model `my_model` will not be created until **after** a source named `my_source` has finished ingesting.
@@ -127,7 +127,7 @@ Our last example will highlight how the same templating concepts can be applied
 Let's say that we wanted to apply a filter on the resulting model based on the `original_language` of the movie and also limit the number of records that we retrieve, which will be based on the `language` and `local_limit` variables we have defined. Taking a quick look at our project's `rill.yaml` file, we can see the following configuration (to return only English movies and apply a limit of 5):
 
 ```yaml
-vars:
+env:
   local_limit: 5
   language: "en"
 ```
@@ -136,8 +136,8 @@ Furthermore, our `model.sql` file contains the following SQL:
 
 ```sql
 SELECT * FROM {{ ref "data_source" }}
-WHERE original_language = '{{ .vars.language }}'
-{{if dev}} LIMIT {{ .vars.local_limit }} {{end}}
+WHERE original_language = '{{ .env.language }}'
+{{if dev}} LIMIT {{ .env.local_limit }} {{end}}
 ```
 
 :::warning When applying templated logic to model SQL, make sure to leverage the `ref` function
@@ -152,7 +152,7 @@ If we simply run Rill Developer using `rill start`, our model will look like the
 
 **Now**, just to illustrate what a local override might look like, let's say we stop Rill Developer and then restart Rill via the CLI with the following command:
 ```bash
-rill start --var language="es" --var local_limit=100
+rill start --env language="es" --env local_limit=100
 ```
 
 Even though we have defaults set in `rill.yaml` (and this will be used by any downstream models and dashboards on Rill Cloud), we will instead see these local overrides come into effect with our templated logic to return Spanish movies and the model limit is now 100 rows.

docs/docs/reference/cli/start.md

@@ -25,7 +25,7 @@ rill start [<path>] [flags]
       --tls-cert string           Path to TLS certificate
       --tls-key string            Path to TLS key file
   -e, --env strings               Environment name (default "dev")
-  -v, --var strings               Set project variables
+  -v, --env strings               Set project variables
       --allowed-origins strings   Override allowed origins for CORS
 ```
 

docs/docs/reference/connectors/mysql.md

@@ -27,11 +27,11 @@ When connecting to MySQL, an appropriate Data Source Name (DSN) will need to be
 
 ## Local credentials
 
-When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify a connection string when running Rill using the `--var` flag.
+When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify a connection string when running Rill using the `--env` flag.
 An example of passing the connection DSN to Rill via the terminal:
 
 ```bash
-rill start --var connector.mysql.dsn="mysql_user:mysql_password@tcp(localhost:3306)/mysql_db"
+rill start --env connector.mysql.dsn="mysql_user:mysql_password@tcp(localhost:3306)/mysql_db"
 ```
 
 Alternatively, you can include the connection string directly in the source YAML definition by adding the `database_url` parameter. 

docs/docs/reference/connectors/postgres.md

@@ -25,11 +25,11 @@ postgresql://postgres_user:postgres_pass@localhost:5432/postgres_db
 
 ## Local credentials
 
-When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify a connection string when running Rill using the `--var` flag.
+When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify a connection string when running Rill using the `--env` flag.
 An example of using this syntax in terminal:
 
 ```bash
-rill start --var connector.postgres.database_url="postgresql://postgres:postgres@localhost:5432/postgres"
+rill start --env connector.postgres.database_url="postgresql://postgres:postgres@localhost:5432/postgres"
 ```
 
 Alternatively, you can include the connection string directly in the source YAML definition by adding the `database_url` parameter. 

docs/docs/reference/connectors/salesforce.md

@@ -15,9 +15,9 @@ sidebar_position: 12
 
 ## Local credentials
 
-When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify credentials when running Rill using the `--var` flag. For example, you could run the following command via the terminal (when starting Rill):
+When using Rill Developer on your local machine (i.e. `rill start`), you have the option to specify credentials when running Rill using the `--env` flag. For example, you could run the following command via the terminal (when starting Rill):
 ```bash
-rill start --var connector.salesforce.username="user@example.com" --var connector.salesforce.password="MyPasswordMyToken"
+rill start --env connector.salesforce.username="user@example.com" --env connector.salesforce.password="MyPasswordMyToken"
 ```
 
 Alternatively, you can also include the credentials directly in the underlying source YAML by adding the `username` and `password` parameters. For example, your source YAML may contain the following properties (also can be configured through the UI during source creation):

docs/docs/reference/connectors/snowflake.md

@@ -18,7 +18,7 @@ sidebar_position: 11
 When using Rill Developer on your local machine (i.e. `rill start`), Rill will use the credentials passed via the Snowflake connection string in one of several ways:
 1. As defined in the [source YAML configuration](../../reference/project-files/sources.md#properties) directly via the `dsn` property
 2. As defined in the optional _Snowflake Connection String_ field from within the UI source creation workflow (this is equivalent to setting the `dsn` property in the underlying source YAML file)
-3. As defined from the CLI when running `rill start --var connector.snowflake.dsn=...`
+3. As defined from the CLI when running `rill start --env connector.snowflake.dsn=...`
 
 :::warning Beware of committing credentials to Git
 

docs/docs/reference/olap-engines/druid.md

@@ -59,7 +59,7 @@ Please see our [Using Multiple OLAP Engines](multiple-olap.md) page.
 
 When using Rill for local development, there are two options to configure Rill to enable Druid as an OLAP engine:
 - You can set `connector.druid.dsn` in your project's `.env` file or try pulling existing credentials locally using `rill env pull` if the project has already been deployed to Rill Cloud
-- You can pass in `connector.druid.dsn` as a variable to `rill start` directly (e.g. `rill start --var connector.druid.dsn=...`)
+- You can pass in `connector.druid.dsn` as a variable to `rill start` directly (e.g. `rill start --env connector.druid.dsn=...`)
 
 :::tip Getting DSN errors in dashboards after setting `.env`?
 

docs/docs/reference/olap-engines/multiple-olap.md

@@ -35,7 +35,7 @@ Before getting started, you'll need to first configure the appopriate connection
 
 **For Rill Developer:**
 - You can set these variables in your project's `.env` file or try pulling existing credentials locally using `rill env pull` if the project has already been deployed to Rill Cloud
-- Alternatively, you can pass in these connector DSN variables to `rill start` directly when starting Rill (e.g. `rill start --var connector.druid.dsn=... --var connector.clickhouse.dsn=...`)
+- Alternatively, you can pass in these connector DSN variables to `rill start` directly when starting Rill (e.g. `rill start --env connector.druid.dsn=... --env connector.clickhouse.dsn=...`)
 
 :::tip Getting DSN errors in dashboards after setting `.env`?
 

docs/docs/reference/olap-engines/pinot.md

@@ -54,7 +54,7 @@ Please see our [Using Multiple OLAP Engines](multiple-olap.md) page.
 
 When using Rill for local development, there are two options to configure Rill to enable Pinot as an OLAP engine:
 - You can set `connector.pinot.dsn` in your project's `.env` file or try pulling existing credentials locally using `rill env pull` if the project has already been deployed to Rill Cloud
-- You can pass in `connector.pinot.dsn` as a variable to `rill start` directly (e.g. `rill start --var connector.pinot.dsn=...`)
+- You can pass in `connector.pinot.dsn` as a variable to `rill start` directly (e.g. `rill start --env connector.pinot.dsn=...`)
 
 :::tip Getting DSN errors in dashboards after setting `.env`?
 

docs/docs/reference/project-files/rill-yaml.md

@@ -81,9 +81,9 @@ As a general rule of thumb, properties that have been specified at a more _granu
 
 Primarily useful for [templating](/deploy/templating.md), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. 
 
-To define a variable in `rill.yaml`, pass in the appropriate key-value pair for the variable under the `vars` key:
+To define a variable in `rill.yaml`, pass in the appropriate key-value pair for the variable under the `env` key:
 ```yaml
-vars:
+env:
   numeric_var: 10
   string_var: "string_value"
 ```
@@ -93,7 +93,7 @@ vars:
 Variables also follow an order of precedence and can be overriden locally. By default, any variables defined will be inherited from `rill.yaml`. However, if you manually pass in a variable when starting Rill Developer locally via the CLI, this value will be used instead for the current instance of your running project:
 
 ```bash
-rill start --var numeric_var=100 --var string_var="different_value"
+rill start --env numeric_var=100 --env string_var="different_value"
 ```
 
 :::