diff --git a/src/cmd/rocker/main.go b/src/cmd/rocker/main.go index bbdea3a2..99eb8302 100644 --- a/src/cmd/rocker/main.go +++ b/src/cmd/rocker/main.go @@ -192,7 +192,11 @@ func buildCommand(c *cli.Context) { } } - cliVars := template.VarsFromStrings(c.StringSlice("var")) + cliVars, err := template.VarsFromStrings(c.StringSlice("var")) + if err != nil { + log.Fatal(err) + } + vars := template.Vars{}.Merge(cliVars) // obtain git info about current directory diff --git a/src/rocker/build/builder.go b/src/rocker/build/builder.go index d1b1b33c..82c50359 100644 --- a/src/rocker/build/builder.go +++ b/src/rocker/build/builder.go @@ -140,7 +140,7 @@ func (builder *Builder) Build() (imageID string, err error) { } defer fd.Close() - data, err := template.ProcessConfigTemplate(builder.Rockerfile, fd, builder.Vars.ToMapOfInterface(), map[string]interface{}{}) + data, err := template.Process(builder.Rockerfile, fd, builder.Vars.ToMapOfInterface(), map[string]interface{}{}) if err != nil { return err } diff --git a/src/rocker/template/README.md b/src/rocker/template/README.md index 5937aba9..575aa90e 100644 --- a/src/rocker/template/README.md +++ b/src/rocker/template/README.md @@ -32,6 +32,47 @@ This template will yield: www.grammarly.com ``` +### {{ json *anything* }} or {{ *anything* | json }} +Marshals given input to JSON. + +Example: +``` +ENV={{ .Env | json }} +``` + +This template will yield: +``` +ENV={"USER":"johnsnow","DOCKER_MACHINE_NAME":"dev","PATH":"/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin",...} +``` + +### {{ yaml *anything* }} or {{ *anything* | yaml }} +Marshals given input to YAML. + +Example: +``` +{{ .Env | yaml }} +``` + +This template will yield: +``` +USER: johnsnow +DOCKER_MACHINE_NAME: dev +PATH: /usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin +``` + +### {{ shell *string* }} or {{ *string* | shell }} +Escapes given string so it can be substituted to a shell command. + +Example: +```Dockerfile +RUN echo {{ "hello\nworld" | shell }} +``` + +This template will yield: +```Dockerfile +RUN echo $'hello\nworld' +``` + ### {{ dump *anything* }} Pretty-prints any variable. Useful for debugging. @@ -73,6 +114,26 @@ Example: HOME={{ .Env.HOME }} ``` +# Load file content to a variable +This template engine also supports loading files content to a variables. `rocker` and `rocker-compose` support this through a command line parameters: + +```bash +rocker build -var key=@key.pem +rocker-compose run -var key=@key.pem +``` + +If the file path is relative, it will be resolved according to the current working directory. + +**Usage options:** + +``` +key=@relative/file/path.txt +key=@../another/relative/file/path.txt +key=@/absolute/file/path.txt +key=@~/.bash_history +key=\@keep_value_as_is +``` + # Development Please install pre-push git hook that will run tests before every push: diff --git a/src/rocker/template/shellarg.go b/src/rocker/template/shellarg.go new file mode 100644 index 00000000..57c662f5 --- /dev/null +++ b/src/rocker/template/shellarg.go @@ -0,0 +1,52 @@ +/*- + * Copyright 2015 Grammarly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package template + +import ( + "regexp" + "strings" +) + +var ( + complexShellArgRegex = regexp.MustCompile("(?i:[^a-z\\d_\\/:=-])") + leadingSingleQuotesRegex = regexp.MustCompile("^(?:'')+") +) + +// EscapeShellarg escapes any string so it can be safely passed to a shell +func EscapeShellarg(value string) string { + // Nothing to escape, return as is + if !complexShellArgRegex.MatchString(value) { + return value + } + + // escape all single quotes + value = "'" + strings.Replace(value, "'", "'\\''", -1) + "'" + + // remove duplicated single quotes at the beginning + value = leadingSingleQuotesRegex.ReplaceAllString(value, "") + + // remove non-escaped single-quote if there are enclosed between 2 escaped + value = strings.Replace(value, "\\'''", "\\'", -1) + + // if the string contains new lines, then use bash $'string' representation + // to have the newline escape character + if strings.Contains(value, "\n") { + value = "$" + strings.Replace(value, "\n", "\\n", -1) + } + + return value +} diff --git a/src/rocker/template/shellarg_test.go b/src/rocker/template/shellarg_test.go new file mode 100644 index 00000000..6c6fea92 --- /dev/null +++ b/src/rocker/template/shellarg_test.go @@ -0,0 +1,51 @@ +/*- + * Copyright 2015 Grammarly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEscapeShellarg_Basic(t *testing.T) { + t.Parallel() + assert.Equal(t, "Testing", EscapeShellarg("Testing")) + assert.Equal(t, "'Testing;'", EscapeShellarg("Testing;")) +} + +func TestEscapeShellarg_Advanced(t *testing.T) { + t.Parallel() + + assertions := map[string]string{ + "hello\\nworld": "'hello\\nworld'", + "hello:world": "hello:world", + "--hello=world": "--hello=world", + "hello\\tworld": "'hello\\tworld'", + "hello\nworld": "$'hello\\nworld'", + "\thello\nworld'": "$'\thello\\nworld'\\'", + "hello world": "'hello world'", + "hello\\\\'": "'hello\\\\'\\'", + "'\\\\'world": "\\''\\\\'\\''world'", + "world\\": "'world\\'", + "'single'": "\\''single'\\'", + } + + for k, v := range assertions { + assert.Equal(t, v, EscapeShellarg(k)) + } +} diff --git a/src/rocker/template/template.go b/src/rocker/template/template.go index bcd38f7b..f9deef3b 100644 --- a/src/rocker/template/template.go +++ b/src/rocker/template/template.go @@ -18,6 +18,7 @@ package template import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -27,12 +28,13 @@ import ( "strings" "text/template" + "github.com/go-yaml/yaml" "github.com/kr/pretty" ) -// ProcessConfigTemplate renders config through the template processor. +// Process renders config through the template processor. // vars and additional functions are acceptable. -func ProcessConfigTemplate(name string, reader io.Reader, vars Vars, funcs map[string]interface{}) (*bytes.Buffer, error) { +func Process(name string, reader io.Reader, vars Vars, funcs map[string]interface{}) (*bytes.Buffer, error) { var buf bytes.Buffer // read template @@ -43,7 +45,7 @@ func ProcessConfigTemplate(name string, reader io.Reader, vars Vars, funcs map[s // merge OS environment variables with the given Vars map // todo: maybe, we need to make it configurable - vars["Env"] = VarsFromStrings(os.Environ()) + vars["Env"] = ParseKvPairs(os.Environ()) // Populate functions funcMap := map[string]interface{}{ @@ -51,6 +53,9 @@ func ProcessConfigTemplate(name string, reader io.Reader, vars Vars, funcs map[s "replace": replace, "dump": dump, "assert": assertFn, + "json": jsonFn, + "shell": EscapeShellarg, + "yaml": yamlFn, } for k, f := range funcs { funcMap[k] = f @@ -155,6 +160,22 @@ func assertFn(v interface{}) (string, error) { return "", fmt.Errorf("Assertion failed") } +func jsonFn(v interface{}) (string, error) { + data, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(data), nil +} + +func yamlFn(v interface{}) (string, error) { + data, err := yaml.Marshal(v) + if err != nil { + return "", err + } + return string(data), nil +} + func interfaceToInt(v interface{}) (int, error) { switch v.(type) { case int: diff --git a/src/rocker/template/template_test.go b/src/rocker/template/template_test.go index 1c008c20..254c7419 100644 --- a/src/rocker/template/template_test.go +++ b/src/rocker/template/template_test.go @@ -35,8 +35,8 @@ var ( } ) -func TestProcessConfigTemplate_Basic(t *testing.T) { - result, err := ProcessConfigTemplate("test", strings.NewReader("this is a test {{.mykey}}"), configTemplateVars, map[string]interface{}{}) +func TestProcess_Basic(t *testing.T) { + result, err := Process("test", strings.NewReader("this is a test {{.mykey}}"), configTemplateVars, map[string]interface{}{}) if err != nil { t.Fatal(err) } @@ -44,7 +44,7 @@ func TestProcessConfigTemplate_Basic(t *testing.T) { assert.Equal(t, "this is a test myval", result.String(), "template should be rendered") } -func TestProcessConfigTemplate_Seq(t *testing.T) { +func TestProcess_Seq(t *testing.T) { assert.Equal(t, "[1 2 3 4 5]", processTemplate(t, "{{ seq 1 5 1 }}")) assert.Equal(t, "[0 1 2 3 4]", processTemplate(t, "{{ seq 0 4 1 }}")) assert.Equal(t, "[1 3 5]", processTemplate(t, "{{ seq 1 5 2 }}")) @@ -75,35 +75,49 @@ func TestProcessConfigTemplate_Seq(t *testing.T) { assert.Equal(t, "[1 2 3 4 5]", processTemplate(t, "{{ seq .n }}")) } -func TestProcessConfigTemplate_Replace(t *testing.T) { +func TestProcess_Replace(t *testing.T) { assert.Equal(t, "url-com-", processTemplate(t, `{{ replace "url.com." "." "-" }}`)) assert.Equal(t, "url", processTemplate(t, `{{ replace "url" "*" "l" }}`)) assert.Equal(t, "krl", processTemplate(t, `{{ replace "url" "u" "k" }}`)) } -func TestProcessConfigTemplate_Env(t *testing.T) { +func TestProcess_Env(t *testing.T) { env := os.Environ() kv := strings.SplitN(env[0], "=", 2) assert.Equal(t, kv[1], processTemplate(t, fmt.Sprintf("{{ .Env.%s }}", kv[0]))) } -func TestProcessConfigTemplate_Dump(t *testing.T) { +func TestProcess_Dump(t *testing.T) { assert.Equal(t, `map[string]string{"foo":"bar"}`, processTemplate(t, "{{ dump .data }}")) } -func TestProcessConfigTemplate_AssertSuccess(t *testing.T) { +func TestProcess_AssertSuccess(t *testing.T) { assert.Equal(t, "output", processTemplate(t, "{{ assert true }}output")) } -func TestProcessConfigTemplate_AssertFail(t *testing.T) { +func TestProcess_AssertFail(t *testing.T) { tpl := "{{ assert .Version }}lololo" - _, err := ProcessConfigTemplate("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) + _, err := Process("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) errStr := "Error executing template test, error: template: test:1:3: executing \"test\" at : error calling assert: Assertion failed" assert.Equal(t, errStr, err.Error()) } +func TestProcess_Json(t *testing.T) { + assert.Equal(t, "key: {\"foo\":\"bar\"}", processTemplate(t, "key: {{ .data | json }}")) +} + +func TestProcess_Shellarg(t *testing.T) { + assert.Equal(t, "echo 'hello world'", processTemplate(t, "echo {{ \"hello world\" | shell }}")) +} + +func TestProcess_Yaml(t *testing.T) { + assert.Equal(t, "key: foo: bar\n", processTemplate(t, "key: {{ .data | yaml }}")) + assert.Equal(t, "key: myval\n", processTemplate(t, "key: {{ .mykey | yaml }}")) + assert.Equal(t, "key: |-\n hello\n world\n", processTemplate(t, "key: {{ \"hello\\nworld\" | yaml }}")) +} + func processTemplate(t *testing.T, tpl string) string { - result, err := ProcessConfigTemplate("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) + result, err := Process("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) if err != nil { t.Fatal(err) } diff --git a/src/rocker/template/testdata/content.txt b/src/rocker/template/testdata/content.txt new file mode 100644 index 00000000..ce013625 --- /dev/null +++ b/src/rocker/template/testdata/content.txt @@ -0,0 +1 @@ +hello diff --git a/src/rocker/template/vars.go b/src/rocker/template/vars.go index 462b8d5b..91463394 100644 --- a/src/rocker/template/vars.go +++ b/src/rocker/template/vars.go @@ -19,6 +19,10 @@ package template import ( "encoding/json" "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" "regexp" "sort" "strings" @@ -67,24 +71,50 @@ func (vars Vars) MarshalJSON() ([]byte, error) { } // UnmarshalJSON unserialize Vars from JSON string -func (vars *Vars) UnmarshalJSON(data []byte) error { +func (vars *Vars) UnmarshalJSON(data []byte) (err error) { // try unmarshal map to keep backward compatibility maps := map[string]interface{}{} - if err := json.Unmarshal(data, &maps); err == nil { + if err = json.Unmarshal(data, &maps); err == nil { *vars = (Vars)(maps) return nil } // unmarshal slice of strings strings := []string{} - if err := json.Unmarshal(data, &strings); err != nil { + if err = json.Unmarshal(data, &strings); err != nil { + return err + } + if *vars, err = VarsFromStrings(strings); err != nil { return err } - *vars = VarsFromStrings(strings) return nil } -// VarsFromStrings parses Vars from a slice of strings e.g. []string{"KEY=VALUE"} -func VarsFromStrings(pairs []string) (vars Vars) { +// VarsFromStrings parses Vars through ParseKvPairs and then loads content from files +// for vars values with "@" prefix +func VarsFromStrings(pairs []string) (vars Vars, err error) { + vars = ParseKvPairs(pairs) + for k, v := range vars { + // We care only about strings + switch v := v.(type) { + case string: + // Read variable content from a file if "@" prefix is given + if strings.HasPrefix(v, "@") { + f := v[1:] + if vars[k], err = loadFileContent(f); err != nil { + return vars, fmt.Errorf("Failed to read file '%s' for variable %s, error: %s", f, k, err) + } + } + // Unescape "\@" + if strings.HasPrefix(v, "\\@") { + vars[k] = v[1:] + } + } + } + return vars, nil +} + +// ParseKvPairs parses Vars from a slice of strings e.g. []string{"KEY=VALUE"} +func ParseKvPairs(pairs []string) (vars Vars) { vars = make(Vars) for _, varPair := range pairs { tmp := strings.SplitN(varPair, "=", 2) @@ -93,6 +123,24 @@ func VarsFromStrings(pairs []string) (vars Vars) { return vars } +func loadFileContent(f string) (string, error) { + if f == "~" || strings.HasPrefix(f, "~/") { + f = strings.Replace(f, "~", os.Getenv("HOME"), 1) + } + if !filepath.IsAbs(f) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + f = path.Join(wd, f) + } + data, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + return string(data), nil +} + // Code borrowed from https://github.com/docker/docker/blob/df0e0c76831bed08cf5e08ac9a1abebf6739da23/builder/support.go var ( // `\\\\+|[^\\]|\b|\A` - match any number of "\\" (ie, properly-escaped backslashes), or a single non-backslash character, or a word boundary, or beginning-of-line diff --git a/src/rocker/template/vars_test.go b/src/rocker/template/vars_test.go index 85856148..7f77feb7 100644 --- a/src/rocker/template/vars_test.go +++ b/src/rocker/template/vars_test.go @@ -18,6 +18,9 @@ package template import ( "encoding/json" + "fmt" + "os" + "path" "testing" "github.com/stretchr/testify/assert" @@ -87,7 +90,10 @@ func TestVarsFromStrings(t *testing.T) { } for _, a := range tests { - result := VarsFromStrings(a.input) + result, err := VarsFromStrings(a.input) + if err != nil { + t.Fatal(err) + } assert.Equal(t, len(a.input), len(result), "resulting number of strings not match number of vars keys") } } @@ -178,3 +184,46 @@ func TestVarsJsonMarshal(t *testing.T) { assert.Equal(t, "qwe", v4["asd"], "bad decoded vars element") assert.Equal(t, "bar", v4["foo"], "bad decoded vars element") } + +func TestVarsFileContent(t *testing.T) { + t.Parallel() + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + // Test absolute + result, err := VarsFromStrings([]string{fmt.Sprintf("FOO=@%s/testdata/content.txt", wd)}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "hello\n", result["FOO"]) + + // Test relative + result2, err := VarsFromStrings([]string{"FOO=@testdata/content.txt"}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "hello\n", result2["FOO"]) + + // Test escaped @ + result3, err := VarsFromStrings([]string{"FOO=\\@testdata/content.txt"}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "@testdata/content.txt", result3["FOO"]) + + // Test HOME + os.Setenv("HOME", path.Join(wd, "testdata")) + + result4, err := VarsFromStrings([]string{"FOO=@~/content.txt"}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "hello\n", result4["FOO"]) +}