cmd/cue/cmd: support binary file type

- add special binary import mode
- allow specifying custom extensions for import
- encoder/decoder for binary

Note that there are no default file extensions for binary.

Change-Id: Ieeb90edc38b47a05e25e736479730e857345d853
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/9570
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Paul Jolly <paul@myitcv.org.uk>

cmd/cue/cmd/common.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"golang.org/x/text/language"
@@ -357,8 +358,12 @@ type config struct {
 	outMode filetypes.Mode
 
 	fileFilter     string
+	reFile         *regexp.Regexp
+	encoding       build.Encoding
 	interpretation build.Interpretation
 
+	overrideDefault bool
+
 	noMerge bool // do not merge individual data files.
 
 	loadCfg *load.Config
@@ -378,12 +383,21 @@ func newBuildPlan(cmd *Command, args []string, cfg *config) (p *buildPlan, err e
 	if err := p.parseFlags(); err != nil {
 		return nil, err
 	}
+	re, err := regexp.Compile(p.cfg.fileFilter)
+	if err != nil {
+		return nil, err
+	}
+	cfg.reFile = re
 
 	cfg.loadCfg.Tags = flagInject.StringArray(cmd)
 
 	return p, nil
 }
 
+func (p *buildPlan) matchFile(file string) bool {
+	return p.cfg.reFile.MatchString(file)
+}
+
 type decoderInfo struct {
 	file *build.File
 	d    *encoding.Decoder // may be nil if delayed
@@ -407,9 +421,21 @@ func (d *decoderInfo) close() {
 // returned slices. It is up to the caller to Close any of the decoders that are
 // returned.
 func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderInfo, err error) {
-	for _, f := range b.OrphanedFiles {
+	files := b.OrphanedFiles
+	if p.cfg.overrideDefault {
+		files = append(files, b.UnknownFiles...)
+	}
+	for _, f := range files {
+		if !p.matchFile(f.Filename) && f.Filename != "-" {
+			continue
+		}
+		if p.cfg.overrideDefault {
+			f.Encoding = p.cfg.encoding
+			f.Interpretation = p.cfg.interpretation
+		}
 		switch f.Encoding {
-		case build.Protobuf, build.YAML, build.JSON, build.JSONL, build.Text:
+		case build.Protobuf, build.YAML, build.JSON, build.JSONL,
+			build.Text, build.Binary:
 			if f.Interpretation == build.ProtobufJSON {
 				// Need a schema.
 				values = append(values, &decoderInfo{f, nil})

cmd/cue/cmd/flags.go

@@ -44,6 +44,7 @@ const (
 	flagFiles       flagName = "files"
 	flagProtoPath   flagName = "proto_path"
 	flagProtoEnum   flagName = "proto_enum"
+	flagExt         flagName = "ext"
 	flagWithContext flagName = "with-context"
 	flagOut         flagName = "out"
 	flagOutFile     flagName = "outfile"

cmd/cue/cmd/help.go

@@ -189,8 +189,10 @@ var filetypeHelp = &cobra.Command{
     textproto    .textproto     Text-based protocol buffers.
     proto        .proto         Protocol Buffer definitions.
     go           .go            Go source files.
-    text         .txt           Raw text file; the evaluated
-                                value must be of type string.
+    text         .txt           Raw text file; the evaluated value
+                                must be of type string.
+    binary                      Raw binary file; the evaluated value
+                                must be of type string or bytes.
 
 OpenAPI, JSON Schema and Protocol Buffer definitions are
 always interpreted as schema. YAML and JSON are always

cmd/cue/cmd/import.go

@@ -55,6 +55,8 @@ the following modes:
    json       Look for JSON files (.json, .jsonl, .ldjson).
    yaml       Look for YAML files (.yaml .yml).
    text       Look for text files (.txt).
+   binary     Look for files with extensions specified by --ext
+              and interpret them as binary.
    jsonschema Interpret JSON, YAML or CUE files as JSON Schema.
    openapi    Interpret JSON, YAML or CUE files as OpenAPI.
    auto       Look for JSON or YAML files and interpret them as
@@ -65,7 +67,9 @@ the following modes:
    proto      Convert Protocol buffer definition files and
               transitive dependencies.
 
-For user-specified files the modes only affect the
+Using the --ext flag in combination with a mode causes matched files to be
+interpreted as the format indicated by the mode, overriding any other meaning
+attributed to that extension.
 
 auto mode
 
@@ -101,6 +105,11 @@ subdirectories as well all dependencies.
 The module root is implicitly added as an import path.
 
 
+Binary mode
+
+Loads matched files as binary.
+
+
 JSON/YAML mode
 
 The -f option allows overwriting of existing files. This only
@@ -249,6 +258,7 @@ Example:
 	cmd.Flags().Bool(string(flagFiles), false, "split multiple entries into different files")
 	cmd.Flags().Bool(string(flagDryrun), false, "only run simulation")
 	cmd.Flags().BoolP(string(flagRecursive), "R", false, "recursively parse string values")
+	cmd.Flags().StringArray(string(flagExt), nil, "match files with these extensions")
 
 	return cmd
 }
@@ -261,11 +271,18 @@ func runImport(cmd *Command, args []string) (err error) {
 		interpretation: build.Auto,
 		loadCfg:        &load.Config{DataFiles: true},
 	}
+
 	var mode string
+	extensions := flagExt.StringArray(cmd)
 	if len(args) >= 1 && !strings.ContainsAny(args[0], `/\:.`) {
 		c.interpretation = ""
+		if len(extensions) > 0 {
+			c.overrideDefault = true
+		}
+
 		mode = args[0]
 		args = args[1:]
+		c.encoding = build.Encoding(mode)
 		switch mode {
 		case "proto":
 			c.fileFilter = `\.proto$`
@@ -275,14 +292,24 @@ func runImport(cmd *Command, args []string) (err error) {
 			c.fileFilter = `\.(yaml|yml)$`
 		case "text":
 			c.fileFilter = `\.txt$`
+		case "binary":
+			if len(extensions) == 0 {
+				return errors.Newf(token.NoPos,
+					"use of --ext flag required in binary mode")
+			}
 		case "auto", "openapi", "jsonschema":
 			c.interpretation = build.Interpretation(mode)
+			c.encoding = "yaml"
 		case "data":
 			// default mode for encoding/ no interpretation.
+			c.encoding = ""
 		default:
 			return errors.Newf(token.NoPos, "unknown mode %q", mode)
 		}
 	}
+	if len(extensions) > 0 {
+		c.fileFilter = `\.(` + strings.Join(extensions, "|") + `)$`
+	}
 
 	b, err := parseArgs(cmd, args, c)
 	exitOnErr(cmd, err, true)

cmd/cue/cmd/orphans.go

@@ -17,7 +17,6 @@ package cmd
 import (
 	"fmt"
 	"path/filepath"
-	"regexp"
 	"strconv"
 
 	"cuelang.org/go/cue"
@@ -89,13 +88,8 @@ func (b *buildPlan) placeOrphans(i *build.Instance, a []*decoderInfo) error {
 
 	var files []*ast.File
 
-	re, err := regexp.Compile(b.cfg.fileFilter)
-	if err != nil {
-		return err
-	}
-
 	for _, di := range a {
-		if !i.User && !re.MatchString(filepath.Base(di.file.Filename)) {
+		if !i.User && !b.matchFile(filepath.Base(di.file.Filename)) {
 			continue
 		}
 

cmd/cue/cmd/testdata/script/import_binary.txt

@@ -0,0 +1,33 @@
+cue import binary --ext crt .
+cmp x.cue out/expect.cue
+
+cue export bin.cue --out binary
+cmp stdout out/bin
+
+# TODO: txtarscript should distinguish final newline
+cue export str.cue --out binary
+cmp stdout out/str
+
+-- x.crt --
+1234
+-- y.crt2 --
+// Skip this file, wrong extension.
+-- bin.cue --
+'''
+  foo
+
+  '''
+-- str.cue --
+"""
+  foo
+
+  """
+-- out/bin --
+foo
+-- out/str --
+foo
+-- out/expect.cue --
+'''
+	1234
+
+	'''

cue/build/file.go

@@ -35,6 +35,7 @@ const (
 	YAML        Encoding = "yaml"
 	JSONL       Encoding = "jsonl"
 	Text        Encoding = "text"
+	Binary      Encoding = "binary"
 	Protobuf    Encoding = "proto"
 	TextProto   Encoding = "textproto"
 	BinaryProto Encoding = "pb"

internal/encoding/encoder.go

@@ -219,6 +219,17 @@ func NewEncoder(f *build.File, cfg *Config) (*Encoder, error) {
 			return err
 		}
 
+	case build.Binary:
+		e.concrete = true
+		e.encValue = func(v cue.Value) error {
+			b, err := v.Bytes()
+			if err != nil {
+				return err
+			}
+			_, err = w.Write(b)
+			return err
+		}
+
 	default:
 		return nil, fmt.Errorf("unsupported encoding %q", f.Encoding)
 	}

internal/encoding/encoding.go

@@ -31,6 +31,7 @@ import (
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/cue/format"
+	"cuelang.org/go/cue/literal"
 	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/json"
@@ -251,6 +252,11 @@ func NewDecoder(f *build.File, cfg *Config) *Decoder {
 		b, err := ioutil.ReadAll(r)
 		i.err = err
 		i.expr = ast.NewString(string(b))
+	case build.Binary:
+		b, err := ioutil.ReadAll(r)
+		i.err = err
+		s := literal.Bytes.WithTabIndent(1).Quote(string(b))
+		i.expr = ast.NewLit(token.STRING, s)
 	case build.Protobuf:
 		paths := &protobuf.Config{
 			Paths:   cfg.ProtoPath,

internal/filetypes/types.cue

@@ -201,6 +201,10 @@ tags: {
 		encoding: "text"
 		form:     "data"
 	}
+	binary: {
+		encoding: "binary"
+		form:     "data"
+	}
 	go: {
 		encoding:       "code"
 		interpretation: ""
@@ -310,6 +314,11 @@ encodings: text: {
 	stream: false
 }
 
+encodings: binary: {
+	forms.data
+	stream: false
+}
+
 encodings: toml: {
 	forms.data
 	stream: false