17 embed (#18)

* Added support for file-inclusion.

This allows three forms of file-inclusion (where "file" means either reading a file from the local filesystem, or reading the output of executing a command):

* Literal/Complete file inclusion.
* Include lines from a file that match a regexp.
* Include lines between two regular expressions.

This closes #17

README.md

@@ -148,6 +148,15 @@ You can freely use the built-in golang template facilities, for example please s
 
 > As an alternative you can consider the `envsubst` binary contained in your system's `gettext{-base}` package.
 
+**NOTE**: This sub-command also allows file-inclusion, in three different ways:
+
+* Including files literally.
+* Including lines from a file which match a particular regular expression.
+* Including the region from a file which is bounded by two regular expressions.
+
+See `sysbox help env-template` for further details, and examples.  You'll also
+see it is possible to execute arbitrary commands and read their output.  This facility was inspired by the [embedmd](https://github.com/campoy/embedmd) utility, and added in [#17](https://github.com/skx/sysbox/issues/17).
+
 
 ## exec-stdin
 

cmd_env_template.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
+	"regexp"
 	"strings"
 	"text/template"
 
@@ -24,11 +26,15 @@ func (et *envTemplateCommand) Info() (string, string) {
 Details:
 
 This command is a slight reworking of the standard 'envsubst' command,
-which might not be available upon systems by default.
+which might not be available upon systems by default, along with extra
+support for file-inclusion (which supports the inclusion of other files,
+along with extra behavior such as 'grep' and inserting regions of files
+between matches of a start/end pair of regular expressions).
 
-The intention is that you can substitute environmental variables into
-simple (golang) template-files.
+The basic use-case of this sub-command is to allow substituting
+environmental variables into simple (golang) template-files.
 
+However there are extra facilities, as noted above.
 
 Examples:
 
@@ -51,18 +57,43 @@ and process variables.  For example splitting $PATH into parts:
       {{$k}} {{$v}}
     {{end}}
 
+
+Inclusion Examples:
+
+The basic case of including a file could be handled like so:
+
+   Before
+   {{include "/etc/passwd"}}
+   After
+
+You can also include only lines matching a particular regular
+expression:
+
+   {{grep "/etc/passwd" "^(root|nobody):"}}
+
+Or lines between a pair of marker (regular expressions):
+
+   {{between "/etc/passwd" "^root" "^bin"}}
+
+If the input-file contains a '|' prefix it will instead read the output
+of running the named command - so you shouldn't process user-submitted
+templates, as that is a potential security-risk.
+
+NOTE: Using 'between' includes the lines that match too, not just the region
+between them.  If you regard this as a bug please file an issue.
+
 `
 
 }
 
-// Execute is invoked if the user specifies `with-lock` as the subcommand.
+// Execute is invoked if the user specifies `env-template` as the subcommand.
 func (et *envTemplateCommand) Execute(args []string) int {
 
 	//
 	// Ensure we have an argument
 	//
 	if len(args) < 1 {
-		fmt.Printf("You must specify the template to expand\n")
+		fmt.Printf("You must specify the template to expand.\n")
 		return 1
 	}
 
@@ -78,23 +109,145 @@ func (et *envTemplateCommand) Execute(args []string) int {
 	return fail
 }
 
+// runCommand returns the output of running the given command
+func (et *envTemplateCommand) runCommand(command string) ([]byte, error) {
+
+	// Build up the thing to run, using a shell so that
+	// we can handle pipes/redirection.
+	toRun := []string{"/bin/bash", "-c", command}
+
+	// Run the command
+	cmd := exec.Command(toRun[0], toRun[1:]...)
+
+	// Get the output
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return []byte{}, fmt.Errorf("error running command '%s' %s", command, err.Error())
+	}
+
+	// Strip trailing newline.
+	return output, nil
+}
+
 // expandFile does the file expansion
 func (et *envTemplateCommand) expandFile(path string) error {
 
 	// Load the file
-	content, err := ioutil.ReadFile(path)
+	var err error
+	var content []byte
+	content, err = ioutil.ReadFile(path)
 	if err != nil {
 		return err
 	}
 
 	//
-	// Define a helper-function that users can call to get
-	// the variables they've set.
+	// Define a helper-function that are available within the
+	// templates we process.
 	//
 	funcMap := template.FuncMap{
+		"between": func(in string, begin string, end string) string {
+
+			// Read the named file/command-output here.
+			if strings.HasPrefix(in, "|") {
+
+				content, err = et.runCommand(strings.TrimPrefix(in, "|"))
+			} else {
+				content, err = ioutil.ReadFile(in)
+			}
+
+			if err != nil {
+				return fmt.Sprintf("error reading %s: %s", in, err.Error())
+			}
+
+			// temporary holder
+			res := []string{}
+
+			// found the open?
+			var found bool
+
+			// for each line
+			for _, line := range strings.Split(string(content), "\n") {
+
+				// in the section we care about?
+				var matched bool
+				matched, err = regexp.MatchString(begin, line)
+				if err != nil {
+					return fmt.Sprintf("error matching %s: %s", begin, err.Error())
+				}
+
+				// if we matched add the line
+				if matched || found {
+					res = append(res, line)
+				}
+
+				// if we matched, or we're in a match
+				// then skip
+				if matched {
+					found = true
+					continue
+				}
+
+				// are we closing a match?
+				if found {
+					matched, err = regexp.MatchString(end, line)
+					if err != nil {
+						return fmt.Sprintf("error matching %s: %s", end, err.Error())
+					}
+
+					if matched {
+						found = false
+					}
+				}
+			}
+			return strings.Join(res, "\n")
+
+		},
 		"env": func(s string) string {
 			return (os.Getenv(s))
 		},
+		"grep": func(in string, pattern string) string {
+
+			// Read the named file/command-output here.
+			if strings.HasPrefix(in, "|") {
+
+				content, err = et.runCommand(strings.TrimPrefix(in, "|"))
+			} else {
+				content, err = ioutil.ReadFile(in)
+			}
+
+			if err != nil {
+				return fmt.Sprintf("error reading %s: %s", in, err.Error())
+			}
+
+			var matched bool
+			res := []string{}
+			for _, line := range strings.Split(string(content), "\n") {
+				matched, err = regexp.MatchString(pattern, line)
+				if err != nil {
+					return fmt.Sprintf("error matching %s: %s", pattern, err.Error())
+				}
+				if matched {
+					res = append(res, line)
+				}
+			}
+			return strings.Join(res, "\n")
+
+		},
+		"include": func(in string) string {
+
+			// Read the named file/command-output here.
+			if strings.HasPrefix(in, "|") {
+
+				content, err = et.runCommand(strings.TrimPrefix(in, "|"))
+			} else {
+				content, err = ioutil.ReadFile(in)
+			}
+
+			if err != nil {
+				return fmt.Sprintf("error reading %s: %s", in, err.Error())
+			}
+			return (string(content))
+		},
 		"split": func(in string, delim string) []string {
 			return strings.Split(in, delim)
 		},