Fix environment-specific YAML overrides not working for common proper…

…ties (such as `sql:`) (#4998)
rilldata · May 31, 2024 · 412d470 · 412d470
1 parent 53ec621
commit 412d470
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/runtime/compilers/rillv1/parse_node.go b/runtime/compilers/rillv1/parse_node.go
@@ -112,12 +112,6 @@ func (p *Parser) parseStem(paths []string, ymlPath, yml, sqlPath, sql string) (*
 	// Handle YAML config
 	templatingEnabled := true
 	if cfg != nil {
-		// Copy basic properties
-		res.Name = cfg.Name
-		res.Connector = cfg.Connector
-		res.SQL = cfg.SQL
-		res.SQLPath = ymlPath
-
 		// Handle "dev:" and "prod:" shorthands (copy to to cfg.Env)
 		if !cfg.Dev.IsZero() {
 			if cfg.Env == nil {
@@ -135,8 +129,20 @@ func (p *Parser) parseStem(paths []string, ymlPath, yml, sqlPath, sql string) (*
 		// Set environment-specific override
 		if envOverride := cfg.Env[p.Environment]; !envOverride.IsZero() {
 			res.YAMLOverride = &envOverride
+
+			// Apply the override immediately in case it changes any of the commonYAML fields
+			err := res.YAMLOverride.Decode(&cfg)
+			if err != nil {
+				return nil, pathError{path: ymlPath, err: newYAMLError(err)}
+			}
 		}
 
+		// Copy basic properties
+		res.Name = cfg.Name
+		res.Connector = cfg.Connector
+		res.SQL = cfg.SQL
+		res.SQLPath = ymlPath
+
 		// Handle templating config
 		if cfg.ParserConfig.Templating != nil {
 			templatingEnabled = *cfg.ParserConfig.Templating

diff --git a/runtime/compilers/rillv1/parser_test.go b/runtime/compilers/rillv1/parser_test.go
@@ -1083,9 +1083,11 @@ env:
 		`sources/s1.yaml`: `
 connector: s3
 path: hello
+sql: SELECT 10
 env:
   test:
     path: world
+    sql: SELECT 20 # Override a property from commonYAML
     refresh:
       cron: "0 0 * * *"
 `,
@@ -1097,7 +1099,7 @@ env:
 		SourceSpec: &runtimev1.SourceSpec{
 			SourceConnector: "s3",
 			SinkConnector:   "duckdb",
-			Properties:      must(structpb.NewStruct(map[string]any{"path": "hello"})),
+			Properties:      must(structpb.NewStruct(map[string]any{"path": "hello", "sql": "SELECT 10"})),
 			RefreshSchedule: &runtimev1.Schedule{RefUpdate: true},
 		},
 	}
@@ -1108,7 +1110,7 @@ env:
 		SourceSpec: &runtimev1.SourceSpec{
 			SourceConnector: "s3",
 			SinkConnector:   "duckdb",
-			Properties:      must(structpb.NewStruct(map[string]any{"path": "world", "limit": 10000})),
+			Properties:      must(structpb.NewStruct(map[string]any{"path": "world", "limit": 10000, "sql": "SELECT 20"})),
 			RefreshSchedule: &runtimev1.Schedule{RefUpdate: true, Cron: "0 0 * * *"},
 		},
 	}