Adds 'faas-cli template pull' command

Pull language templates from any repo that has a 'template/' directory
in the root with 'faas-cli template pull <github repo>'.

Also allows 'faas-cli new --lang' to list available languages

Signed-off-by: Eric Stoekl <ems5311@gmail.com>
Signed-off-by: Minh-Quan TRAN <account@itscaro.me>

builder/build.go

@@ -9,49 +9,51 @@ import (
 	"log"
 	"os"
 	"strings"
+
+	"github.com/openfaas/faas-cli/stack"
 )
 
 // BuildImage construct Docker image from function parameters
 func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool) {
 
-	switch language {
-	case "node", "python", "ruby", "csharp", "python3", "go":
-		tempPath := createBuildTemplate(functionName, handler, language)
+	if stack.IsValidTemplate(language) {
 
-		fmt.Printf("Building: %s with %s template. Please wait..\n", image, language)
+		var tempPath string
+		if strings.ToLower(language) == "dockerfile" {
 
-		flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy"))
+			if shrinkwrap {
+				fmt.Printf("Nothing to do for: %s.\n", functionName)
 
-		if shrinkwrap {
-			fmt.Printf("%s shrink-wrapped to %s\n", functionName, tempPath)
-		} else {
-			builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ")
+				return
+			}
 
-			ExecCommand(tempPath, builder)
-			fmt.Printf("Image: %s built.\n", image)
-		}
-		break
-	case "Dockerfile", "dockerfile":
-		if shrinkwrap {
-			fmt.Printf("Nothing to do for: %s.\n", functionName)
-		} else {
-			tempPath := handler
+			tempPath = handler
 			if _, err := os.Stat(handler); err != nil {
 				fmt.Printf("Unable to build %s, %s is an invalid path\n", image, handler)
 				fmt.Printf("Image: %s not built.\n", image)
 
-				break
+				return
 			}
 			fmt.Printf("Building: %s with Dockerfile. Please wait..\n", image)
 
-			flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy"))
+		} else {
 
-			builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ")
-			fmt.Println(strings.Join(builder, " "))
-			ExecCommand(tempPath, builder)
-			fmt.Printf("Image: %s built.\n", image)
+			tempPath = createBuildTemplate(functionName, handler, language)
+			fmt.Printf("Building: %s with %s template. Please wait..\n", image, language)
+
+			if shrinkwrap {
+				fmt.Printf("%s shrink-wrapped to %s\n", functionName, tempPath)
+
+				return
+			}
 		}
-	default:
+
+		flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy"))
+		builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ")
+		ExecCommand(tempPath, builder)
+		fmt.Printf("Image: %s built.\n", image)
+
+	} else {
 		log.Fatalf("Language template: %s not supported. Build a custom Dockerfile instead.", language)
 	}
 }

commands/build.go

@@ -149,7 +149,7 @@ func PullTemplates(templateUrl string) error {
 	if err != nil || exists == nil {
 		log.Println("No templates found in current directory.")
 
-		err = fetchTemplates(templateUrl)
+		err = fetchTemplates(templateUrl, false)
 		if err != nil {
 			log.Println("Unable to download templates from Github.")
 			return err

commands/deploy.go

@@ -66,7 +66,7 @@ var deployCmd = &cobra.Command{
                   [--handler HANDLER_DIR]
                   [--fprocess PROCESS]
                   [--env ENVVAR=VALUE ...]
-                  [--label LABEL=VALUE ...]				  
+                  [--label LABEL=VALUE ...]
 				  [--replace=false]
 				  [--update=false]
                   [--constraint PLACEMENT_CONSTRAINT ...]
@@ -160,6 +160,28 @@ func runDeploy(cmd *cobra.Command, args []string) {
 				log.Fatalln(envErr)
 			}
 
+			// Get FProcess to use from the ./template/template.yml, if a template is being used
+			if function.Language != "" && function.Language != "Dockerfile" && function.Language != "dockerfile" {
+				pathToTemplateYAML := "./template/" + function.Language + "/template.yml"
+				if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) {
+					log.Fatalln(err.Error())
+					return
+				}
+
+				var langTemplate stack.LanguageTemplate
+				parsedLangTemplate, err := stack.ParseYAMLForLanguageTemplate(pathToTemplateYAML)
+
+				if err != nil {
+					log.Fatalln(err.Error())
+					return
+				}
+
+				if parsedLangTemplate != nil {
+					langTemplate = *parsedLangTemplate
+					function.FProcess = langTemplate.FProcess
+				}
+			}
+
 			proxy.DeployFunction(function.FProcess, services.Provider.GatewayURL, function.Name, function.Image, function.Language, replace, allEnvironment, services.Provider.Network, constraints, update, secrets, allLabels)
 		}
 	} else {

commands/fetch_templates.go

@@ -5,6 +5,7 @@ package commands
 
 import (
 	"archive/zip"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -18,113 +19,161 @@ import (
 	"github.com/openfaas/faas-cli/proxy"
 )
 
-const ZipFileName string = "master.zip"
+const (
+	defaultTemplateRepository = "https://github.com/openfaas/faas-cli"
+	templateDirectory         = "./template/"
+)
 
 // fetchTemplates fetch code templates from GitHub master zip file.
-func fetchTemplates(templateUrl string) error {
+func fetchTemplates(templateURL string, overwrite bool) error {
+	var existingLanguages []string
+	availableLanguages := make(map[string]bool)
+	var fetchedTemplates []string
+
+	if len(templateURL) == 0 {
+		templateURL = defaultTemplateRepository
+	}
 
-	err := fetchMasterZip(templateUrl)
+	archive, err := fetchMasterZip(templateURL)
+	if err != nil {
+		removeArchive(archive)
+		return err
+	}
 
-	zipFile, err := zip.OpenReader(ZipFileName)
+	zipFile, err := zip.OpenReader(archive)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Attempting to expand templates from %s\n", ZipFileName)
+	log.Printf("Attempting to expand templates from %s\n", archive)
 
 	for _, z := range zipFile.File {
-		relativePath := strings.Replace(z.Name, "faas-cli-master/", "", -1)
-		if strings.Index(relativePath, "template") == 0 {
-			fmt.Printf("Found \"%s\"\n", relativePath)
-			rc, err := z.Open()
-			if err != nil {
-				return err
-			}
+		var rc io.ReadCloser
 
-			err = createPath(relativePath, z.Mode())
-			if err != nil {
-				return err
-			}
+		relativePath := z.Name[strings.Index(z.Name, "/")+1:]
+		if strings.Index(relativePath, "template/") != 0 {
+			// Process only directories inside "template" at root
+			continue
+		}
+
+		var language string
+		if languageSplit := strings.Split(relativePath, "/"); len(languageSplit) > 2 {
+			language = languageSplit[1]
+
+			if len(languageSplit) == 3 && relativePath[len(relativePath)-1:] == "/" {
+				// template/language/
+
+				if !canWriteLanguage(&availableLanguages, language, overwrite) {
+					existingLanguages = append(existingLanguages, language)
+					continue
+				}
+				fetchedTemplates = append(fetchedTemplates, language)
+			} else {
+				// template/language/*
 
-			// If relativePath is just a directory, then skip expanding it.
-			if len(relativePath) > 1 && relativePath[len(relativePath)-1:] != string(os.PathSeparator) {
-				err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode())
-				if err != nil {
-					return err
+				if !canWriteLanguage(&availableLanguages, language, overwrite) {
+					continue
 				}
 			}
+		} else {
+			// template/
+			continue
+		}
+
+		if rc, err = z.Open(); err != nil {
+			break
+		}
+
+		if err = createPath(relativePath, z.Mode()); err != nil {
+			break
+		}
+
+		// If relativePath is just a directory, then skip expanding it.
+		if len(relativePath) > 1 && relativePath[len(relativePath)-1:] != "/" {
+			if err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()); err != nil {
+				break
+			}
 		}
 	}
 
+	if len(existingLanguages) > 0 {
+		log.Printf("Cannot overwrite the following %d directories: %v\n", len(existingLanguages), existingLanguages)
+	}
+
+	zipFile.Close()
+
+	log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedTemplates), fetchedTemplates, templateURL)
+
+	err = removeArchive(archive)
+
+	return err
+}
+
+// removeArchive removes the given file
+func removeArchive(archive string) error {
 	log.Printf("Cleaning up zip file...")
-	if _, err := os.Stat(ZipFileName); err == nil {
-		os.Remove(ZipFileName)
+	if _, err := os.Stat(archive); err == nil {
+		return os.Remove(archive)
 	} else {
 		return err
 	}
-	fmt.Println("")
-
-	return err
 }
 
-func fetchMasterZip(templateUrl string) error {
+// fetchMasterZip downloads a zip file from a repository URL
+func fetchMasterZip(templateURL string) (string, error) {
 	var err error
-	if _, err = os.Stat(ZipFileName); err != nil {
 
-		if len(templateUrl) == 0 {
-			templateUrl = "https://github.com/openfaas/faas-cli/archive/" + ZipFileName
-		}
+	templateURL = strings.TrimRight(templateURL, "/")
+	templateURL = templateURL + "/archive/master.zip"
+	archive := "master.zip"
 
+	if _, err := os.Stat(archive); err != nil {
 		timeout := 120 * time.Second
 		client := proxy.MakeHTTPClient(&timeout)
 
-		req, err := http.NewRequest(http.MethodGet, templateUrl, nil)
+		req, err := http.NewRequest(http.MethodGet, templateURL, nil)
 		if err != nil {
 			log.Println(err.Error())
-			return err
+			return "", err
 		}
-		log.Printf("HTTP GET %s\n", templateUrl)
+		log.Printf("HTTP GET %s\n", templateURL)
 		res, err := client.Do(req)
 		if err != nil {
 			log.Println(err.Error())
-			return err
+			return "", err
+		}
+		if res.StatusCode != http.StatusOK {
+			err := errors.New(fmt.Sprintf("%s is not valid, status code %d", templateURL, res.StatusCode))
+			log.Println(err.Error())
+			return "", err
 		}
 		if res.Body != nil {
 			defer res.Body.Close()
 		}
 		bytesOut, err := ioutil.ReadAll(res.Body)
 		if err != nil {
 			log.Println(err.Error())
-			return err
+			return "", err
 		}
 
-		log.Printf("Writing %dKb to %s\n", len(bytesOut)/1024, ZipFileName)
-		err = ioutil.WriteFile(ZipFileName, bytesOut, 0700)
+		log.Printf("Writing %dKb to %s\n", len(bytesOut)/1024, archive)
+		err = ioutil.WriteFile(archive, bytesOut, 0700)
 		if err != nil {
 			log.Println(err.Error())
+			return "", err
 		}
 	}
 	fmt.Println("")
-	return err
+	return archive, err
 }
 
 func writeFile(rc io.ReadCloser, size uint64, relativePath string, perms os.FileMode) error {
 	var err error
 
 	defer rc.Close()
-	fmt.Printf("Writing %d bytes to \"%s\"\n", size, relativePath)
-	if strings.HasSuffix(relativePath, "/") {
-		mkdirErr := os.MkdirAll(relativePath, perms)
-		if mkdirErr != nil {
-			return fmt.Errorf("error making directory %s got: %s", relativePath, mkdirErr)
-		}
-		return err
-	}
-
-	// Create a file instead.
 	f, err := os.OpenFile(relativePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perms)
 	if err != nil {
-		return fmt.Errorf("error writing to %s got: %s", relativePath, err)
+		return err
 	}
 	defer f.Close()
 	_, err = io.CopyN(f, rc, int64(size))
@@ -137,3 +186,29 @@ func createPath(relativePath string, perms os.FileMode) error {
 	err := os.MkdirAll(dir, perms)
 	return err
 }
+
+// Tells whether the language can be expanded from the zip or not
+// availableLanguages map keeps track of which languages we know to be okay to copy.
+// overwrite flag will allow to force copy the language template
+func canWriteLanguage(availableLanguages *map[string]bool, language string, overwrite bool) bool {
+	canWrite := false
+	if len(language) > 0 {
+		if _, ok := (*availableLanguages)[language]; ok {
+			return (*availableLanguages)[language]
+		}
+		canWrite = checkLanguage(language, overwrite)
+		(*availableLanguages)[language] = canWrite
+	}
+
+	return canWrite
+}
+
+// Takes a language input (e.g. "node"), tells whether or not it is OK to download
+func checkLanguage(language string, overwrite bool) bool {
+	dir := templateDirectory + language
+	if _, err := os.Stat(dir); err == nil && !overwrite {
+		// The directory template/language/ exists
+		return false
+	}
+	return true
+}