core: unify data files into config (#376)

Previously there isn't a good way to unify json and yaml files with the
cue configuration.  This is a problem for use cases where data can be
generated idempotentialy prior to rendering the platform configuration.

The first use case is to explore unifying configuration with decrypted
sops values, which isn't typical since Holos is designed to handle
secrets with ExternalSecret resources, but does fit into the use case of
executing a command to produce data idempotently, then make the data
available to the platform configuration.

Other use cases this feature is intended to support are the prior
experiment where we fetch top level platform configuration from an rpc
service, and the future goal of integrating with data provided by
Terraform.

.cspell.json

@@ -74,6 +74,7 @@
     "creds",
     "crossplane",
     "crunchydata",
+    "ctxt",
     "cuecontext",
     "cuelang",
     "customresourcedefinition",

api/core/v1alpha5/types.go

@@ -303,6 +303,9 @@ type Component struct {
 	// Path represents the path of the component relative to the platform root.
 	// Injected as the tag variable "holos_component_path".
 	Path string `json:"path" yaml:"path"`
+	// Instances represents additional cue instance paths to unify with Path.
+	// Useful to unify data files into a component BuildPlan.
+	Instances []Instance `json:"instances,omitempty" yaml:"instances,omitempty"`
 	// WriteTo represents the holos render component --write-to flag.  If empty,
 	// the default value for the --write-to flag is used.
 	WriteTo string `json:"writeTo,omitempty" yaml:"writeTo,omitempty"`
@@ -319,3 +322,23 @@ type Component struct {
 	// `cli.holos.run/description` to customize the log message of each BuildPlan.
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }
+
+// Instance represents a data instance to unify with the configuration.  Useful
+// to unify json and yaml files with cue configuration files.
+type Instance struct {
+	// Kind is a discriminator.
+	Kind string `json:"kind" yaml:"kind" cue:"\"extractYAML\""`
+	// Ignored unless kind is extractYAML.
+	ExtractYAML ExtractYAML `json:"extractYAML" yaml:"extractYAML"`
+}
+
+// ExtractYAML represents a cue data instance encoded as yaml.  Holos extracts data of
+// this kind using cue [encoding/yaml].
+//
+// If Path refers to a directory, all files in the directory are extracted
+// non-recursively.  Otherwise, path must refer to a file.
+//
+// [yaml.Extract]: https://pkg.go.dev/cuelang.org/go@v0.11.0/encoding/yaml#Extract
+type ExtractYAML struct {
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+}

doc/md/api/core.md

@@ -22,12 +22,14 @@ Package core contains schemas for a [Platform](<#Platform>) and [BuildPlan](<#Bu
 - [type Chart](<#Chart>)
 - [type Command](<#Command>)
 - [type Component](<#Component>)
+- [type ExtractYAML](<#ExtractYAML>)
 - [type File](<#File>)
 - [type FileContent](<#FileContent>)
 - [type FileContentMap](<#FileContentMap>)
 - [type FilePath](<#FilePath>)
 - [type Generator](<#Generator>)
 - [type Helm](<#Helm>)
+- [type Instance](<#Instance>)
 - [type InternalLabel](<#InternalLabel>)
 - [type Join](<#Join>)
 - [type Kind](<#Kind>)
@@ -169,6 +171,9 @@ type Component struct {
     // Path represents the path of the component relative to the platform root.
     // Injected as the tag variable "holos_component_path".
     Path string `json:"path" yaml:"path"`
+    // Instances represents additional cue instance paths to unify with Path.
+    // Useful to unify data files into a component BuildPlan.
+    Instances []Instance `json:"instances,omitempty" yaml:"instances,omitempty"`
     // WriteTo represents the holos render component --write-to flag.  If empty,
     // the default value for the --write-to flag is used.
     WriteTo string `json:"writeTo,omitempty" yaml:"writeTo,omitempty"`
@@ -187,6 +192,19 @@ type Component struct {
 }
 ```
 
+<a name="ExtractYAML"></a>
+## type ExtractYAML {#ExtractYAML}
+
+ExtractYAML represents a cue data instance encoded as yaml. Holos extracts data of this kind using cue [encoding/yaml](<https://pkg.go.dev/encoding/yaml/>).
+
+If Path refers to a directory, all files in the directory are extracted non\-recursively. Otherwise, path must refer to a file.
+
+```go
+type ExtractYAML struct {
+    Path string `json:"path,omitempty" yaml:"path,omitempty"`
+}
+```
+
 <a name="File"></a>
 ## type File {#File}
 
@@ -279,6 +297,20 @@ type Helm struct {
 }
 ```
 
+<a name="Instance"></a>
+## type Instance {#Instance}
+
+Instance represents a data instance to unify with the configuration. Useful to unify json and yaml files with cue configuration files.
+
+```go
+type Instance struct {
+    // Kind is a discriminator.
+    Kind string `json:"kind" yaml:"kind" cue:"\"extractYAML\""`
+    // Ignored unless kind is extractYAML.
+    ExtractYAML ExtractYAML `json:"extractYAML" yaml:"extractYAML"`
+}
+```
+
 <a name="InternalLabel"></a>
 ## type InternalLabel {#InternalLabel}
 

internal/builder/instance.go

@@ -5,17 +5,66 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/load"
+	"cuelang.org/go/encoding/yaml"
 	"github.com/holos-run/holos/internal/errors"
 	"github.com/holos-run/holos/internal/holos"
 	"github.com/holos-run/holos/internal/util"
 )
 
-func LoadInstance(path string, tags []string) (*Instance, error) {
+// loadYamlFiles loads data files represented by paths.  The files are unified
+// into one value.  If a path element is a directory, all files in the directory
+// are loaded non-recursively.
+//
+// Attribution: https://github.com/cue-lang/cue/issues/3504
+func loadYamlFiles(ctxt *cue.Context, paths []string) (cue.Value, error) {
+	value := ctxt.CompileString("")
+	files := make([]string, 0, 10*len(paths))
+
+	for _, path := range paths {
+		info, err := os.Stat(path)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+
+		if !info.IsDir() {
+			files = append(files, path)
+			continue
+		}
+
+		entries, err := os.ReadDir(path)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			files = append(files, filepath.Join(path, entry.Name()))
+		}
+	}
+
+	for _, file := range files {
+		f, err := yaml.Extract(file, nil)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+		value = value.Unify(ctxt.BuildFile(f))
+	}
+
+	return value, nil
+}
+
+// LoadInstance loads the cue configuration instance at path.  Additional
+// instances are loaded with the [cue.Context.BuildFile] method, then unified
+// into the configuration instance.
+func LoadInstance(path string, instances []string, tags []string) (*Instance, error) {
 	root, leaf, err := util.FindRootLeaf(path)
 	if err != nil {
 		return nil, errors.Wrap(err)
@@ -26,20 +75,26 @@ func LoadInstance(path string, tags []string) (*Instance, error) {
 		ModuleRoot: root,
 		Tags:       tags,
 	}
+	ctxt := cuecontext.New()
 
-	ctx := cuecontext.New()
+	bis := load.Instances([]string{path}, cfg)
+	values, err := ctxt.BuildInstances(bis)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
 
-	instances := load.Instances([]string{leaf}, cfg)
-	values, err := ctx.BuildInstances(instances)
+	value, err := loadYamlFiles(ctxt, instances)
 	if err != nil {
 		return nil, errors.Wrap(err)
 	}
+	// TODO: https://cuelang.org/docs/howto/place-data-go-api/
+	value = value.Unify(values[0])
 
 	inst := &Instance{
 		path:  leaf,
-		ctx:   ctx,
+		ctx:   ctxt,
 		cfg:   cfg,
-		value: values[0],
+		value: value,
 	}
 
 	return inst, nil

internal/builder/v1alpha5/builder.go

@@ -107,6 +107,22 @@ func (c *Component) Path() string {
 	return util.DotSlash(c.Component.Path)
 }
 
+func (c *Component) Instances() ([]string, error) {
+	if c == nil {
+		return nil, nil
+	}
+	instances := make([]string, 0, len(c.Component.Instances))
+	for _, instance := range c.Component.Instances {
+		switch instance.Kind {
+		case "extractYAML":
+			instances = append(instances, instance.ExtractYAML.Path)
+		default:
+			return nil, errors.Format("unsupported instance kind: %s", instance.Kind)
+		}
+	}
+	return instances, nil
+}
+
 var _ holos.BuildPlan = &BuildPlan{}
 var _ task = generatorTask{}
 var _ task = transformersTask{}

internal/cli/render/render.go

@@ -43,6 +43,8 @@ func newPlatform(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 	cmd.Flags().IntVar(&concurrency, "concurrency", runtime.NumCPU(), "number of components to render concurrently")
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var instances holos.StringSlice
+	cmd.Flags().Var(&instances, "instance", "cue instances to unify with the platform")
 	var selector holos.Selector
 	cmd.Flags().VarP(&selector, "selector", "l", "label selector (e.g. label==string,label!=string)")
 	tagMap := make(holos.TagMap)
@@ -57,7 +59,7 @@ func newPlatform(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 			log.WarnContext(ctx, fmt.Sprintf(msg, platform))
 		}
 
-		inst, err := builder.LoadInstance(platform, tagMap.Tags())
+		inst, err := builder.LoadInstance(platform, instances, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -107,12 +109,14 @@ func newComponent(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 	cmd.Flags().VarP(&tagMap, "inject", "t", tagHelp)
 	var concurrency int
 	cmd.Flags().IntVar(&concurrency, "concurrency", runtime.NumCPU(), "number of concurrent build steps")
+	var instances holos.StringSlice
+	cmd.Flags().Var(&instances, "instance", "cue instances to unify with the platform")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		ctx := cmd.Root().Context()
 		path := args[0]
 
-		inst, err := builder.LoadInstance(path, tagMap.Tags())
+		inst, err := builder.LoadInstance(path, instances, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}

internal/cli/show.go

@@ -30,13 +30,15 @@ func newShowPlatformCmd() (cmd *cobra.Command) {
 
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var instances holos.StringSlice
+	cmd.Flags().Var(&instances, "instance", "cue instances to unify with the platform")
 	var format string
 	cmd.Flags().StringVar(&format, "format", "yaml", "yaml or json format")
 	tagMap := make(holos.TagMap)
 	cmd.Flags().VarP(&tagMap, "inject", "t", "set the value of a cue @tag field from a key=value pair")
 
 	cmd.RunE = func(c *cobra.Command, args []string) (err error) {
-		inst, err := builder.LoadInstance(platform, tagMap.Tags())
+		inst, err := builder.LoadInstance(platform, instances, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -64,6 +66,8 @@ func newShowBuildPlanCmd() (cmd *cobra.Command) {
 
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var instances holos.StringSlice
+	cmd.Flags().Var(&instances, "instance", "cue instances to unify with the platform")
 	var format string
 	cmd.Flags().StringVar(&format, "format", "yaml", "yaml or json format")
 	var selector holos.Selector
@@ -75,7 +79,7 @@ func newShowBuildPlanCmd() (cmd *cobra.Command) {
 
 	cmd.RunE = func(c *cobra.Command, args []string) (err error) {
 		path := platform
-		inst, err := builder.LoadInstance(path, tagMap.Tags())
+		inst, err := builder.LoadInstance(path, instances, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -122,7 +126,11 @@ func makeBuildFunc(encoder holos.OrderedEncoder, opts holos.BuildOpts) func(cont
 				return errors.Wrap(err)
 			}
 			tags = append(tags, opts.Tags...)
-			inst, err := builder.LoadInstance(component.Path(), tags)
+			instances, err := component.Instances()
+			if err != nil {
+				return errors.Wrap(err)
+			}
+			inst, err := builder.LoadInstance(component.Path(), instances, tags)
 			if err != nil {
 				return errors.Wrap(err)
 			}

internal/generate/platforms/cue.mod/gen/github.com/holos-run/holos/api/core/v1alpha5/types_go_gen.cue

@@ -334,6 +334,10 @@ package core
 	// Injected as the tag variable "holos_component_path".
 	path: string @go(Path)
 
+	// Instances represents additional cue instance paths to unify with Path.
+	// Useful to unify data files into a component BuildPlan.
+	instances?: [...#Instance] @go(Instances,[]Instance)
+
 	// WriteTo represents the holos render component --write-to flag.  If empty,
 	// the default value for the --write-to flag is used.
 	writeTo?: string @go(WriteTo)
@@ -353,3 +357,24 @@ package core
 	// `cli.holos.run/description` to customize the log message of each BuildPlan.
 	annotations?: {[string]: string} @go(Annotations,map[string]string)
 }
+
+// Instance represents a data instance to unify with the configuration.  Useful
+// to unify json and yaml files with cue configuration files.
+#Instance: {
+	// Kind is a discriminator.
+	kind: string & "extractYAML" @go(Kind)
+
+	// Ignored unless kind is extractYAML.
+	extractYAML: #ExtractYAML @go(ExtractYAML)
+}
+
+// ExtractYAML represents a cue data instance encoded as yaml.  Holos extracts data of
+// this kind using cue [encoding/yaml].
+//
+// If Path refers to a directory, all files in the directory are extracted
+// non-recursively.  Otherwise, path must refer to a file.
+//
+// [yaml.Extract]: https://pkg.go.dev/cuelang.org/go@v0.11.0/encoding/yaml#Extract
+#ExtractYAML: {
+	path?: string @go(Path)
+}

internal/holos/interface.go

@@ -21,6 +21,7 @@ type Platform interface {
 type Component interface {
 	Describe() string
 	Path() string
+	Instances() ([]string, error)
 	Tags() ([]string, error)
 	WriteTo() string
 	Labels() Labels

internal/util/util.go

@@ -42,6 +42,9 @@ func FindCueMod(path string) (root string, err error) {
 	return root, nil
 }
 
+// FindRootLeaf returns the root path containing the cue.mod and the leaf path
+// relative to the root for the given target path.  FindRootLeaf calls
+// [filepath.Clean] on the returned paths.
 func FindRootLeaf(target string) (root string, leaf string, err error) {
 	if root, err = FindCueMod(target); err != nil {
 		return "", "", err

service/buf.lock

@@ -4,5 +4,5 @@ deps:
   - remote: buf.build
     owner: bufbuild
     repository: protovalidate
-    commit: 5a7b106cbb87462d9a8c9ffecdbd2e38
-    digest: shake256:2f7efa5a904668219f039d4f6eeb51e871f8f7f5966055a10663cba335bd65f76cac84da3fa758ab7b5dcb489ec599521390ce3951d119fb56df1fc2def16bb0
+    commit: a3320276596649bcad929ac829d451f4
+    digest: shake256:a6e5f64fd3fd47e3e8568e9753f59a1566f56c11ec055baf65463d3bca3499f6f16c2d6f5628fa41cfd0f4fa7e72abe65be4efd77d269749492472ed4cc4070d