Skip to content
This repository has been archived by the owner on Nov 18, 2021. It is now read-only.

Commit

Permalink
cmd/cue/cmd: support binary file type
Browse files Browse the repository at this point in the history
- add special binary import mode
- allow specifying custom extensions for import
- encoder/decoder for binary

Note that there are no default file extensions for binary.

Change-Id: Ieeb90edc38b47a05e25e736479730e857345d853
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/9570
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Paul Jolly <paul@myitcv.org.uk>
  • Loading branch information
mpvl committed Apr 30, 2021
1 parent aa70399 commit b9e7d90
Show file tree
Hide file tree
Showing 11 changed files with 124 additions and 14 deletions.
30 changes: 28 additions & 2 deletions cmd/cue/cmd/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"strings"

"golang.org/x/text/language"
Expand Down Expand Up @@ -357,8 +358,12 @@ type config struct {
outMode filetypes.Mode

fileFilter string
reFile *regexp.Regexp
encoding build.Encoding
interpretation build.Interpretation

overrideDefault bool

noMerge bool // do not merge individual data files.

loadCfg *load.Config
Expand All @@ -378,12 +383,21 @@ func newBuildPlan(cmd *Command, args []string, cfg *config) (p *buildPlan, err e
if err := p.parseFlags(); err != nil {
return nil, err
}
re, err := regexp.Compile(p.cfg.fileFilter)
if err != nil {
return nil, err
}
cfg.reFile = re

cfg.loadCfg.Tags = flagInject.StringArray(cmd)

return p, nil
}

func (p *buildPlan) matchFile(file string) bool {
return p.cfg.reFile.MatchString(file)
}

type decoderInfo struct {
file *build.File
d *encoding.Decoder // may be nil if delayed
Expand All @@ -407,9 +421,21 @@ func (d *decoderInfo) close() {
// returned slices. It is up to the caller to Close any of the decoders that are
// returned.
func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderInfo, err error) {
for _, f := range b.OrphanedFiles {
files := b.OrphanedFiles
if p.cfg.overrideDefault {
files = append(files, b.UnknownFiles...)
}
for _, f := range files {
if !p.matchFile(f.Filename) && f.Filename != "-" {
continue
}
if p.cfg.overrideDefault {
f.Encoding = p.cfg.encoding
f.Interpretation = p.cfg.interpretation
}
switch f.Encoding {
case build.Protobuf, build.YAML, build.JSON, build.JSONL, build.Text:
case build.Protobuf, build.YAML, build.JSON, build.JSONL,
build.Text, build.Binary:
if f.Interpretation == build.ProtobufJSON {
// Need a schema.
values = append(values, &decoderInfo{f, nil})
Expand Down
1 change: 1 addition & 0 deletions cmd/cue/cmd/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const (
flagFiles flagName = "files"
flagProtoPath flagName = "proto_path"
flagProtoEnum flagName = "proto_enum"
flagExt flagName = "ext"
flagWithContext flagName = "with-context"
flagOut flagName = "out"
flagOutFile flagName = "outfile"
Expand Down
6 changes: 4 additions & 2 deletions cmd/cue/cmd/help.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,10 @@ var filetypeHelp = &cobra.Command{
textproto .textproto Text-based protocol buffers.
proto .proto Protocol Buffer definitions.
go .go Go source files.
text .txt Raw text file; the evaluated
value must be of type string.
text .txt Raw text file; the evaluated value
must be of type string.
binary Raw binary file; the evaluated value
must be of type string or bytes.
OpenAPI, JSON Schema and Protocol Buffer definitions are
always interpreted as schema. YAML and JSON are always
Expand Down
29 changes: 28 additions & 1 deletion cmd/cue/cmd/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ the following modes:
json Look for JSON files (.json, .jsonl, .ldjson).
yaml Look for YAML files (.yaml .yml).
text Look for text files (.txt).
binary Look for files with extensions specified by --ext
and interpret them as binary.
jsonschema Interpret JSON, YAML or CUE files as JSON Schema.
openapi Interpret JSON, YAML or CUE files as OpenAPI.
auto Look for JSON or YAML files and interpret them as
Expand All @@ -65,7 +67,9 @@ the following modes:
proto Convert Protocol buffer definition files and
transitive dependencies.
For user-specified files the modes only affect the
Using the --ext flag in combination with a mode causes matched files to be
interpreted as the format indicated by the mode, overriding any other meaning
attributed to that extension.
auto mode
Expand Down Expand Up @@ -101,6 +105,11 @@ subdirectories as well all dependencies.
The module root is implicitly added as an import path.
Binary mode
Loads matched files as binary.
JSON/YAML mode
The -f option allows overwriting of existing files. This only
Expand Down Expand Up @@ -249,6 +258,7 @@ Example:
cmd.Flags().Bool(string(flagFiles), false, "split multiple entries into different files")
cmd.Flags().Bool(string(flagDryrun), false, "only run simulation")
cmd.Flags().BoolP(string(flagRecursive), "R", false, "recursively parse string values")
cmd.Flags().StringArray(string(flagExt), nil, "match files with these extensions")

return cmd
}
Expand All @@ -261,11 +271,18 @@ func runImport(cmd *Command, args []string) (err error) {
interpretation: build.Auto,
loadCfg: &load.Config{DataFiles: true},
}

var mode string
extensions := flagExt.StringArray(cmd)
if len(args) >= 1 && !strings.ContainsAny(args[0], `/\:.`) {
c.interpretation = ""
if len(extensions) > 0 {
c.overrideDefault = true
}

mode = args[0]
args = args[1:]
c.encoding = build.Encoding(mode)
switch mode {
case "proto":
c.fileFilter = `\.proto$`
Expand All @@ -275,14 +292,24 @@ func runImport(cmd *Command, args []string) (err error) {
c.fileFilter = `\.(yaml|yml)$`
case "text":
c.fileFilter = `\.txt$`
case "binary":
if len(extensions) == 0 {
return errors.Newf(token.NoPos,
"use of --ext flag required in binary mode")
}
case "auto", "openapi", "jsonschema":
c.interpretation = build.Interpretation(mode)
c.encoding = "yaml"
case "data":
// default mode for encoding/ no interpretation.
c.encoding = ""
default:
return errors.Newf(token.NoPos, "unknown mode %q", mode)
}
}
if len(extensions) > 0 {
c.fileFilter = `\.(` + strings.Join(extensions, "|") + `)$`
}

b, err := parseArgs(cmd, args, c)
exitOnErr(cmd, err, true)
Expand Down
8 changes: 1 addition & 7 deletions cmd/cue/cmd/orphans.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package cmd
import (
"fmt"
"path/filepath"
"regexp"
"strconv"

"cuelang.org/go/cue"
Expand Down Expand Up @@ -89,13 +88,8 @@ func (b *buildPlan) placeOrphans(i *build.Instance, a []*decoderInfo) error {

var files []*ast.File

re, err := regexp.Compile(b.cfg.fileFilter)
if err != nil {
return err
}

for _, di := range a {
if !i.User && !re.MatchString(filepath.Base(di.file.Filename)) {
if !i.User && !b.matchFile(filepath.Base(di.file.Filename)) {
continue
}

Expand Down
33 changes: 33 additions & 0 deletions cmd/cue/cmd/testdata/script/import_binary.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
cue import binary --ext crt .
cmp x.cue out/expect.cue

cue export bin.cue --out binary
cmp stdout out/bin

# TODO: txtarscript should distinguish final newline
cue export str.cue --out binary
cmp stdout out/str

-- x.crt --
1234
-- y.crt2 --
// Skip this file, wrong extension.
-- bin.cue --
'''
foo

'''
-- str.cue --
"""
foo

"""
-- out/bin --
foo
-- out/str --
foo
-- out/expect.cue --
'''
1234

'''
1 change: 1 addition & 0 deletions cue/build/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
YAML Encoding = "yaml"
JSONL Encoding = "jsonl"
Text Encoding = "text"
Binary Encoding = "binary"
Protobuf Encoding = "proto"
TextProto Encoding = "textproto"
BinaryProto Encoding = "pb"
Expand Down
11 changes: 11 additions & 0 deletions internal/encoding/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,17 @@ func NewEncoder(f *build.File, cfg *Config) (*Encoder, error) {
return err
}

case build.Binary:
e.concrete = true
e.encValue = func(v cue.Value) error {
b, err := v.Bytes()
if err != nil {
return err
}
_, err = w.Write(b)
return err
}

default:
return nil, fmt.Errorf("unsupported encoding %q", f.Encoding)
}
Expand Down
6 changes: 6 additions & 0 deletions internal/encoding/encoding.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"cuelang.org/go/cue/build"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/format"
"cuelang.org/go/cue/literal"
"cuelang.org/go/cue/parser"
"cuelang.org/go/cue/token"
"cuelang.org/go/encoding/json"
Expand Down Expand Up @@ -251,6 +252,11 @@ func NewDecoder(f *build.File, cfg *Config) *Decoder {
b, err := ioutil.ReadAll(r)
i.err = err
i.expr = ast.NewString(string(b))
case build.Binary:
b, err := ioutil.ReadAll(r)
i.err = err
s := literal.Bytes.WithTabIndent(1).Quote(string(b))
i.expr = ast.NewLit(token.STRING, s)
case build.Protobuf:
paths := &protobuf.Config{
Paths: cfg.ProtoPath,
Expand Down
9 changes: 9 additions & 0 deletions internal/filetypes/types.cue
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,10 @@ tags: {
encoding: "text"
form: "data"
}
binary: {
encoding: "binary"
form: "data"
}
go: {
encoding: "code"
interpretation: ""
Expand Down Expand Up @@ -310,6 +314,11 @@ encodings: text: {
stream: false
}

encodings: binary: {
forms.data
stream: false
}

encodings: toml: {
forms.data
stream: false
Expand Down
4 changes: 2 additions & 2 deletions internal/filetypes/types.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit b9e7d90

Please sign in to comment.