Pull templates from store when not present

Implements a user-experience changes:

faas-cli new - when the templates folder is not present the
language will be looked up in the template store using the
default URL or an overriden one.

faas-cli publish - when no template folder is available then
a pull will be carried out if a configuration section is
present with a template given within in

Filtering - whilst there is no way to avoid pulling all the
templates wihtin a repository, only the requested/needed ones
are expanded by commands like faas-cli new.

Tested mainly with existing-unit tests, faas-cli new / publish
with and without a templates folder.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

README.md

@@ -216,9 +216,9 @@ Find templates with: `faas-cli template store list`
 
 > Note: You can set your own custom store location with `--url` flag or set `OPENFAAS_TEMPLATE_STORE_URL` environmental variable
 
-To pull templates from the store just write the name of the template you want `faas-cli template store pull go` or the repository and name `faas-cli template store pull openfaas/go`
+To pull templates from the store just write the name of the template you want `faas-cli template store pull golang-middleware` or the repository and name `faas-cli template store pull openfaas/golang-middleware`
 
-To get more detail on a template just use the `template store describe` command and pick a template of your choice, example with `go` would look like this `faas-cli template store describe go`
+To get more detail on a template just use the `template store describe` command and pick a template of your choice, example with `go` would look like this `faas-cli template store describe golang-middleware`
 
 > Note: This feature is still in experimental stage and in the future the CLI verbs might be changed
 

commands/build.go

@@ -5,7 +5,6 @@ package commands
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"strings"
 	"sync"
@@ -155,6 +154,8 @@ func parseBuildArgs(args []string) (map[string]string, error) {
 
 func runBuild(cmd *cobra.Command, args []string) error {
 
+	templateName := "" // templateName may not be known at this point
+
 	var services stack.Services
 	if len(yamlFile) > 0 {
 		parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter, envsubst)
@@ -178,7 +179,7 @@ func runBuild(cmd *cobra.Command, args []string) error {
 		}
 	} else {
 		templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository)
-		if pullErr := pullTemplates(templateAddress); pullErr != nil {
+		if pullErr := pullTemplates(templateAddress, templateName); pullErr != nil {
 			return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr)
 		}
 	}
@@ -304,20 +305,19 @@ func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool
 }
 
 // pullTemplates pulls templates from specified git remote. templateURL may be a pinned repository.
-func pullTemplates(templateURL string) error {
-	var err error
-	exists, err := os.Stat("./template")
-	if err != nil || exists == nil {
-		log.Println("No templates found in current directory.")
+func pullTemplates(templateURL, templateName string) error {
+
+	if _, err := os.Stat("./template"); err != nil && os.IsNotExist(err) {
+
+		fmt.Printf("No templates found in current directory.\n")
 
 		templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL)
-		err = fetchTemplates(templateURL, refName, false)
-		if err != nil {
-			log.Println("Unable to download templates from Github.")
+		if err := fetchTemplates(templateURL, refName, templateName, false); err != nil {
 			return err
 		}
 	}
-	return err
+
+	return nil
 }
 
 func combineBuildOpts(YAMLBuildOpts []string, buildFlagBuildOpts []string) []string {

commands/fetch_templates.go

@@ -5,7 +5,6 @@ package commands
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -20,21 +19,22 @@ const DefaultTemplateRepository = "https://github.com/openfaas/templates.git"
 const templateDirectory = "./template/"
 
 // fetchTemplates fetch code templates using git clone.
-func fetchTemplates(templateURL string, refName string, overwrite bool) error {
+func fetchTemplates(templateURL, refName, templateName string, overwrite bool) error {
 	if len(templateURL) == 0 {
 		return fmt.Errorf("pass valid templateURL")
 	}
 
 	dir, err := os.MkdirTemp("", "openfaas-templates-*")
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
+
 	if !pullDebug {
-		defer os.RemoveAll(dir) // clean up
+		defer os.RemoveAll(dir)
 	}
 
-	log.Printf("Attempting to expand templates from %s\n", templateURL)
 	pullDebugPrint(fmt.Sprintf("Temp files in %s", dir))
+
 	args := map[string]string{"dir": dir, "repo": templateURL}
 	cmd := versioncontrol.GitCloneDefault
 
@@ -47,7 +47,7 @@ func fetchTemplates(templateURL string, refName string, overwrite bool) error {
 		return err
 	}
 
-	preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, overwrite)
+	preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, templateName, overwrite)
 	if err != nil {
 		return err
 	}
@@ -56,7 +56,7 @@ func fetchTemplates(templateURL string, refName string, overwrite bool) error {
 		log.Printf("Cannot overwrite the following %d template(s): %v\n", len(preExistingLanguages), preExistingLanguages)
 	}
 
-	log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL)
+	fmt.Printf("Wrote %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL)
 
 	return err
 }
@@ -87,7 +87,7 @@ func templateFolderExists(language string, overwrite bool) bool {
 	return true
 }
 
-func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error) {
+func moveTemplates(repoPath, templateName string, overwrite bool) ([]string, []string, error) {
 	var (
 		existingLanguages []string
 		fetchedLanguages  []string
@@ -97,7 +97,7 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error)
 	availableLanguages := make(map[string]bool)
 
 	templateDir := filepath.Join(repoPath, templateDirectory)
-	templates, err := ioutil.ReadDir(templateDir)
+	templates, err := os.ReadDir(templateDir)
 	if err != nil {
 		return nil, nil, fmt.Errorf("can't find templates in: %s", repoPath)
 	}
@@ -108,23 +108,35 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error)
 		}
 		language := file.Name()
 
-		canWrite := canWriteLanguage(availableLanguages, language, overwrite)
-		if canWrite {
-			fetchedLanguages = append(fetchedLanguages, language)
-			// Do cp here
-			languageSrc := filepath.Join(templateDir, language)
-			languageDest := filepath.Join(templateDirectory, language)
-			builder.CopyFiles(languageSrc, languageDest)
-		} else {
-			existingLanguages = append(existingLanguages, language)
-			continue
+		if len(templateName) == 0 {
+
+			canWrite := canWriteLanguage(availableLanguages, language, overwrite)
+			if canWrite {
+				fetchedLanguages = append(fetchedLanguages, language)
+				// Do cp here
+				languageSrc := filepath.Join(templateDir, language)
+				languageDest := filepath.Join(templateDirectory, language)
+				builder.CopyFiles(languageSrc, languageDest)
+			} else {
+				existingLanguages = append(existingLanguages, language)
+				continue
+			}
+		} else if language == templateName {
+
+			if canWriteLanguage(availableLanguages, language, overwrite) {
+				fetchedLanguages = append(fetchedLanguages, language)
+				// Do cp here
+				languageSrc := filepath.Join(templateDir, language)
+				languageDest := filepath.Join(templateDirectory, language)
+				builder.CopyFiles(languageSrc, languageDest)
+			}
 		}
 	}
 
 	return existingLanguages, fetchedLanguages, nil
 }
 
-func pullTemplate(repository string) error {
+func pullTemplate(repository, templateName string) error {
 	if _, err := os.Stat(repository); err != nil {
 		if !versioncontrol.IsGitRemote(repository) && !versioncontrol.IsPinnedGitRemote(repository) {
 			return fmt.Errorf("the repository URL must be a valid git repo uri")
@@ -143,8 +155,12 @@ func pullTemplate(repository string) error {
 		}
 	}
 
-	fmt.Printf("Fetch templates from repository: %s at %s\n", repository, refName)
-	if err := fetchTemplates(repository, refName, overwrite); err != nil {
+	refStr := ""
+	if len(refName) > 0 {
+		refStr = fmt.Sprintf(" @ %s", refName)
+	}
+	fmt.Printf("Fetch templates from repository: %s%s\n", repository, refStr)
+	if err := fetchTemplates(repository, refName, templateName, overwrite); err != nil {
 		return fmt.Errorf("error while fetching templates: %s", err)
 	}
 

commands/fetch_templates_test.go

@@ -21,17 +21,18 @@ func Test_PullTemplates(t *testing.T) {
 	defer os.RemoveAll(localTemplateRepository)
 	defer tearDownFetchTemplates(t)
 
+	templateName := ""
 	t.Run("pullTemplates", func(t *testing.T) {
 		defer tearDownFetchTemplates(t)
-		if err := pullTemplates(localTemplateRepository); err != nil {
-			t.Fatal(err)
+		if err := pullTemplates(localTemplateRepository, templateName); err != nil {
+			t.Fatalf("Trying to pull %s, error: %s", localTemplateRepository, err)
 		}
 	})
 
 	t.Run("fetchTemplates with master ref", func(t *testing.T) {
 		defer tearDownFetchTemplates(t)
 
-		if err := fetchTemplates(localTemplateRepository, "master", false); err != nil {
+		if err := fetchTemplates(localTemplateRepository, "master", templateName, false); err != nil {
 			t.Fatal(err)
 		}
 
@@ -40,8 +41,8 @@ func Test_PullTemplates(t *testing.T) {
 	t.Run("fetchTemplates with default ref", func(t *testing.T) {
 		defer tearDownFetchTemplates(t)
 
-		err := fetchTemplates(localTemplateRepository, "", false)
-		if err != nil {
+		templateName := ""
+		if err := fetchTemplates(localTemplateRepository, "", templateName, false); err != nil {
 			t.Error(err)
 		}
 

commands/new_function.go

@@ -5,7 +5,6 @@ package commands
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -74,7 +73,7 @@ func validateFunctionName(functionName string) error {
 
 // preRunNewFunction validates args & flags
 func preRunNewFunction(cmd *cobra.Command, args []string) error {
-	if list == true {
+	if list {
 		return nil
 	}
 
@@ -102,16 +101,18 @@ func preRunNewFunction(cmd *cobra.Command, args []string) error {
 }
 
 func runNewFunction(cmd *cobra.Command, args []string) error {
-	if list == true {
+	if list {
 		var availableTemplates []string
 
-		templateFolders, err := ioutil.ReadDir(templateDirectory)
+		templateFolders, err := os.ReadDir(templateDirectory)
 		if err != nil {
 			return fmt.Errorf(`no language templates were found.
 
 Download templates:
-  faas-cli template pull           download the default templates
-  faas-cli template store list     view the community template store`)
+  faas-cli template pull                download the default templates
+  faas-cli template store list          view the template store
+  faas-cli template store pull NAME     download the default templates
+  faas-cli new --lang NAME              Attempt to download NAME from the template store`)
 		}
 
 		for _, file := range templateFolders {
@@ -125,11 +126,34 @@ Download templates:
 		return nil
 	}
 
-	templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository)
-	pullTemplates(templateAddress)
-
 	if !stack.IsValidTemplate(language) {
-		return fmt.Errorf("template: \"%s\" was not found in the templates directory", language)
+
+		envTemplateRepoStore := os.Getenv(templateStoreURLEnvironment)
+		storeURL := getTemplateStoreURL(templateStoreURL, envTemplateRepoStore, DefaultTemplatesStore)
+
+		templatesInfo, err := getTemplateInfo(storeURL)
+		if err != nil {
+			return fmt.Errorf("error while getting templates info: %s", err)
+		}
+
+		var templateInfo *TemplateInfo
+		for _, info := range templatesInfo {
+			if info.TemplateName == language {
+				templateInfo = &info
+				break
+			}
+		}
+
+		if templateInfo == nil {
+			return fmt.Errorf("template: \"%s\" was not found in the templates folder or in the store", language)
+		}
+
+		templateName := templateInfo.TemplateName
+
+		if err := pullTemplate(templateInfo.Repository, templateName); err != nil {
+			return fmt.Errorf("error while pulling template: %s", err)
+		}
+
 	}
 
 	var fileName, outputMsg string

commands/new_function_test.go

@@ -27,13 +27,15 @@ const (
 - dockerfile
 - ruby`
 
-	LangNotExistsOutput  = `(template: \"([0-9A-Za-z-])*\" was not found in the templates directory)`
+	LangNotExistsOutput  = `(template: \"([0-9A-Za-z-])*\" was not found in the templates folder or in the store)`
 	FunctionExistsOutput = `(Function (.+)? already exists in (.+)? file)`
 	NoTemplates          = `no language templates were found.
 
 Download templates:
-  faas-cli template pull           download the default templates
-  faas-cli template store list     view the community template store`
+  faas-cli template pull                download the default templates
+  faas-cli template store list          view the template store
+  faas-cli template store pull NAME     download the default templates
+  faas-cli new --lang NAME              Attempt to download NAME from the template store`
 	InvalidFileSuffix = "when appending to a stack the suffix should be .yml or .yaml"
 )
 

commands/publish.go

@@ -139,9 +139,35 @@ func runPublish(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository)
-	if pullErr := pullTemplates(templateAddress); pullErr != nil {
-		return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr)
+	needTemplates := false
+	for _, function := range services.Functions {
+		if len(function.Language) > 0 {
+			needTemplates = true
+			break
+		}
+	}
+
+	templatesFound := false
+	if stat, err := os.Stat("./template"); err == nil && stat.IsDir() {
+		templatesFound = true
+	}
+
+	// if no templates are configured, but they exist in the configuration section,
+	// attempt to pull them first
+	if !templatesFound && needTemplates {
+		if len(services.StackConfiguration.TemplateConfigs) > 0 {
+			if err := pullStackTemplates(services.StackConfiguration.TemplateConfigs, cmd); err != nil {
+				return err
+			}
+
+		}
+	}
+
+	if needTemplates {
+		if _, err := os.Stat("./template"); os.IsNotExist(err) {
+
+			return fmt.Errorf(`the "template" directory is missing but required by at least one function`)
+		}
 	}
 
 	if resetQemu {

commands/template_pull.go

@@ -43,8 +43,11 @@ func runTemplatePull(cmd *cobra.Command, args []string) error {
 	if len(args) > 0 {
 		repository = args[0]
 	}
+
+	templateName := "" // templateName may not be known at this point
+
 	repository = getTemplateURL(repository, os.Getenv(templateURLEnvironment), DefaultTemplateRepository)
-	return pullTemplate(repository)
+	return pullTemplate(repository, templateName)
 }
 
 func pullDebugPrint(message string) {