Fix template usage with custom environments

internal/cli/common/reader.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"github.com/redpanda-data/benthos/v4/internal/config"
+	"github.com/redpanda-data/benthos/v4/internal/docs"
 	"github.com/redpanda-data/benthos/v4/internal/filepath/ifs"
 
 	"github.com/urfave/cli/v2"
@@ -22,10 +23,14 @@ func ReadConfig(c *cli.Context, cliOpts *CLIOpts, streamsMode bool) (mainPath st
 			}
 		}
 	}
+
+	lintConf := docs.NewLintConfig(cliOpts.Environment)
+
 	opts := []config.OptFunc{
 		config.OptSetFullSpec(cliOpts.MainConfigSpecCtor),
 		config.OptAddOverrides(cliOpts.RootFlags.GetSet(c)...),
 		config.OptTestSuffix("_benthos_test"),
+		config.OptSetLintConfig(lintConf),
 	}
 	if streamsMode {
 		opts = append(opts, config.OptSetStreamPaths(c.Args().Slice()...))

internal/cli/common/run_flags.go

@@ -189,7 +189,7 @@ func PreApplyEnvFilesAndTemplates(c *cli.Context, opts *CLIOpts) error {
 	if err != nil {
 		return fmt.Errorf("failed to resolve template glob pattern: %w", err)
 	}
-	lints, err := template.InitTemplates(templatesPaths...)
+	lints, err := template.InitTemplates(opts.Environment, opts.BloblEnvironment, templatesPaths...)
 	if err != nil {
 		return fmt.Errorf("template file read error: %w", err)
 	}

internal/cli/template/lint.go

@@ -7,6 +7,8 @@ import (
 	"github.com/fatih/color"
 	"github.com/urfave/cli/v2"
 
+	"github.com/redpanda-data/benthos/v4/internal/bloblang"
+	"github.com/redpanda-data/benthos/v4/internal/bundle"
 	"github.com/redpanda-data/benthos/v4/internal/cli/common"
 	"github.com/redpanda-data/benthos/v4/internal/docs"
 	ifilepath "github.com/redpanda-data/benthos/v4/internal/filepath"
@@ -74,7 +76,7 @@ type pathLint struct {
 }
 
 func lintFile(path string) (pathLints []pathLint) {
-	conf, lints, err := template.ReadConfigFile(path)
+	conf, lints, err := template.ReadConfigFile(bundle.GlobalEnvironment, path)
 	if err != nil {
 		pathLints = append(pathLints, pathLint{
 			source: path,
@@ -90,7 +92,7 @@ func lintFile(path string) (pathLints []pathLint) {
 		})
 	}
 
-	testErrors, err := conf.Test()
+	testErrors, err := conf.Test(bundle.GlobalEnvironment, bloblang.GlobalEnvironment())
 	if err != nil {
 		pathLints = append(pathLints, pathLint{
 			source: path,

internal/template/config.go

@@ -102,12 +102,12 @@ func (c Config) ComponentSpec() (docs.ComponentSpec, error) {
 	}, nil
 }
 
-func (c Config) compile() (*compiled, error) {
+func (c Config) compile(env *bloblang.Environment) (*compiled, error) {
 	spec, err := c.ComponentSpec()
 	if err != nil {
 		return nil, err
 	}
-	mapping, err := bloblang.GlobalEnvironment().NewMapping(c.Mapping)
+	mapping, err := env.NewMapping(c.Mapping)
 	if err != nil {
 		var perr *parser.Error
 		if errors.As(err, &perr) {
@@ -149,8 +149,8 @@ func diffYAMLNodesAsJSON(expNode *yaml.Node, actNode any) (string, error) {
 
 // Test ensures that the template compiles, and executes any unit test
 // definitions within the config.
-func (c Config) Test() ([]string, error) {
-	compiled, err := c.compile()
+func (c Config) Test(env *bundle.Environment, benv *bloblang.Environment) ([]string, error) {
+	compiled, err := c.compile(benv)
 	if err != nil {
 		return nil, err
 	}
@@ -164,7 +164,7 @@ func (c Config) Test() ([]string, error) {
 
 		var yNode yaml.Node
 		if err := yNode.Encode(outConf); err == nil {
-			for _, lint := range docs.LintYAML(docs.NewLintContext(docs.NewLintConfig(bundle.GlobalEnvironment)), docs.Type(c.Type), &yNode) {
+			for _, lint := range docs.LintYAML(docs.NewLintContext(docs.NewLintConfig(env)), docs.Type(c.Type), &yNode) {
 				failures = append(failures, fmt.Sprintf("test '%v': lint error in resulting config: %v", test.Name, lint.Error()))
 			}
 		} else {
@@ -186,7 +186,7 @@ func (c Config) Test() ([]string, error) {
 
 // ReadConfigYAML attempts to read a YAML byte slice as a template configuration
 // file.
-func ReadConfigYAML(templateBytes []byte) (conf Config, lints []docs.Lint, err error) {
+func ReadConfigYAML(env *bundle.Environment, templateBytes []byte) (conf Config, lints []docs.Lint, err error) {
 	if err = yaml.Unmarshal(templateBytes, &conf); err != nil {
 		return
 	}
@@ -196,17 +196,17 @@ func ReadConfigYAML(templateBytes []byte) (conf Config, lints []docs.Lint, err e
 		return
 	}
 
-	lints = ConfigSpec().LintYAML(docs.NewLintContext(docs.NewLintConfig(bundle.GlobalEnvironment)), &node)
+	lints = ConfigSpec().LintYAML(docs.NewLintContext(docs.NewLintConfig(env)), &node)
 	return
 }
 
 // ReadConfigFile attempts to read a template configuration file.
-func ReadConfigFile(path string) (conf Config, lints []docs.Lint, err error) {
+func ReadConfigFile(env *bundle.Environment, path string) (conf Config, lints []docs.Lint, err error) {
 	var templateBytes []byte
 	if templateBytes, err = ifs.ReadFile(ifs.OS(), path); err != nil {
 		return
 	}
-	return ReadConfigYAML(templateBytes)
+	return ReadConfigYAML(env, templateBytes)
 }
 
 //------------------------------------------------------------------------------

internal/template/template.go

@@ -5,6 +5,7 @@ import (
 
 	"gopkg.in/yaml.v3"
 
+	"github.com/redpanda-data/benthos/v4/internal/bloblang"
 	"github.com/redpanda-data/benthos/v4/internal/bloblang/mapping"
 	"github.com/redpanda-data/benthos/v4/internal/bundle"
 	"github.com/redpanda-data/benthos/v4/internal/component/cache"
@@ -20,23 +21,24 @@ import (
 
 // InitTemplates parses and registers native templates, as well as templates
 // at paths provided, and returns any linting errors that occur.
-func InitTemplates(templatesPaths ...string) ([]string, error) {
+func InitTemplates(env *bundle.Environment, bloblEnv *bloblang.Environment, templatesPaths ...string) ([]string, error) {
 	var lints []string
 	for _, tPath := range templatesPaths {
-		tmplConf, tLints, err := ReadConfigFile(tPath)
+		tmplConf, tLints, err := ReadConfigFile(env, tPath)
 		if err != nil {
 			return nil, fmt.Errorf("template %v: %w", tPath, err)
 		}
 		for _, l := range tLints {
 			lints = append(lints, fmt.Sprintf("template file %v: %v", tPath, l))
 		}
 
-		tmpl, err := tmplConf.compile()
+		tmpl, err := tmplConf.compile(bloblEnv)
 		if err != nil {
 			return nil, fmt.Errorf("template %v: %w", tPath, err)
 		}
 
-		if err := registerTemplate(bundle.GlobalEnvironment, tmpl); err != nil {
+		fmt.Println("Registering " + tmpl.spec.Name)
+		if err := registerTemplate(env, tmpl); err != nil {
 			return nil, fmt.Errorf("template %v: %w", tPath, err)
 		}
 	}
@@ -86,13 +88,13 @@ func (c *compiled) Render(node any) (any, error) {
 
 // RegisterTemplateYAML attempts to register a new template component to the
 // specified environment.
-func RegisterTemplateYAML(env *bundle.Environment, template []byte) error {
-	tmplConf, _, err := ReadConfigYAML(template)
+func RegisterTemplateYAML(env *bundle.Environment, bloblEnv *bloblang.Environment, template []byte) error {
+	tmplConf, _, err := ReadConfigYAML(env, template)
 	if err != nil {
 		return err
 	}
 
-	tmpl, err := tmplConf.compile()
+	tmpl, err := tmplConf.compile(bloblEnv)
 	if err != nil {
 		return err
 	}

internal/template/template_test.go

@@ -24,7 +24,7 @@ func TestCacheTemplate(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: foo_memory
 type: cache
 
@@ -56,7 +56,7 @@ func TestInputTemplate(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: generate_a_foo
 type: input
 
@@ -116,7 +116,7 @@ func TestOutputTemplate(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: write_inproc
 type: output
 
@@ -190,7 +190,7 @@ func TestProcessorTemplate(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: append_foo
 type: processor
 
@@ -225,7 +225,7 @@ func TestProcessorTemplateOddIndentation(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: meow
 type: processor
 
@@ -267,7 +267,7 @@ func TestRateLimitTemplate(t *testing.T) {
 	mgr, err := manager.New(manager.NewResourceConfig())
 	require.NoError(t, err)
 
-	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), []byte(`
+	require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), []byte(`
 name: foo
 type: rate_limit
 

public/service/environment.go

@@ -695,7 +695,7 @@ func (e *Environment) GetScannerConfig(name string) (*ConfigView, bool) {
 // document, to the environment such that it may be used similarly to any other
 // component plugin.
 func (e *Environment) RegisterTemplateYAML(yamlStr string) error {
-	return template.RegisterTemplateYAML(e.internal, []byte(yamlStr))
+	return template.RegisterTemplateYAML(e.internal, e.getBloblangParserEnv(), []byte(yamlStr))
 }
 
 // XFormatConfigJSON returns a byte slice of the Benthos configuration spec

public/service/stream_template_tester.go

@@ -47,7 +47,7 @@ func (s *StreamTemplateTester) RunYAML(yamlBytes []byte) (lints []Lint, err erro
 		return
 	}
 
-	testErrors, err := conf.Test()
+	testErrors, err := conf.Test(s.env.internal, s.env.getBloblangParserEnv())
 	if err != nil {
 		lints = append(lints, Lint{Line: 1, Type: LintFailedRead, What: err.Error()})
 		return