From 414732fd16823d9f0b63a22e33c98c014c966104 Mon Sep 17 00:00:00 2001 From: Eric Stoekl Date: Wed, 6 Sep 2017 23:12:34 -0700 Subject: [PATCH 1/4] Adds 'faas-cli template pull' command Pull language templates from any repo that has a 'template/' directory in the root with 'faas-cli template pull '. Also allows 'faas-cli new --lang' to list available languages Signed-off-by: Eric Stoekl Signed-off-by: Minh-Quan TRAN --- builder/build.go | 52 ++--- commands/build.go | 2 +- commands/deploy.go | 24 ++- commands/fetch_templates.go | 175 ++++++++++++----- commands/fetch_templates_test.go | 64 +++++-- commands/new_function.go | 41 ++-- commands/new_function_test.go | 178 ++++++++++++++++-- commands/template_pull.go | 80 ++++++++ commands/template_pull_test.go | 144 ++++++++++++++ commands/testdata/master_test.zip | Bin 0 -> 13909 bytes .../template/csharp/function/.gitignore | 0 .../new_function/template/go-armhf/.gitignore | 0 .../template/go/function/.gitignore | 0 .../template/node-arm64/function/.gitignore | 0 .../template/node-armhf/.gitignore | 0 .../template/node/function/.gitignore | 0 .../template/python-armhf/.gitignore | 0 .../template/python/function/.gitignore | 0 .../template/python3/function/.gitignore | 0 .../template/ruby/function/.gitignore | 0 guide/TEMPLATE.md | 56 ++++++ proxy/deploy.go | 10 - stack/language_template.go | 61 ++++++ stack/language_template_test.go | 77 ++++++++ stack/schema.go | 5 + template/csharp/template.yml | 2 + template/go-armhf/template.yml | 2 + template/go/template.yml | 2 + template/node-arm64/template.yml | 2 + template/node-armhf/template.yml | 2 + template/node/template.yml | 2 + template/python-armhf/template.yml | 2 + template/python/template.yml | 2 + template/python3/template.yml | 2 + template/ruby/template.yml | 2 + 35 files changed, 851 insertions(+), 138 deletions(-) create mode 100644 commands/template_pull.go create mode 100644 commands/template_pull_test.go create mode 100755 commands/testdata/master_test.zip create mode 100644 commands/testdata/new_function/template/csharp/function/.gitignore create mode 100644 commands/testdata/new_function/template/go-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/go/function/.gitignore create mode 100644 commands/testdata/new_function/template/node-arm64/function/.gitignore create mode 100644 commands/testdata/new_function/template/node-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/node/function/.gitignore create mode 100644 commands/testdata/new_function/template/python-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/python/function/.gitignore create mode 100644 commands/testdata/new_function/template/python3/function/.gitignore create mode 100644 commands/testdata/new_function/template/ruby/function/.gitignore create mode 100644 guide/TEMPLATE.md create mode 100644 stack/language_template.go create mode 100644 stack/language_template_test.go create mode 100644 template/csharp/template.yml create mode 100644 template/go-armhf/template.yml create mode 100644 template/go/template.yml create mode 100644 template/node-arm64/template.yml create mode 100644 template/node-armhf/template.yml create mode 100644 template/node/template.yml create mode 100644 template/python-armhf/template.yml create mode 100644 template/python/template.yml create mode 100644 template/python3/template.yml create mode 100644 template/ruby/template.yml diff --git a/builder/build.go b/builder/build.go index ecad52664..44c8cf06b 100644 --- a/builder/build.go +++ b/builder/build.go @@ -9,49 +9,51 @@ import ( "log" "os" "strings" + + "github.com/openfaas/faas-cli/stack" ) // BuildImage construct Docker image from function parameters func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool) { - switch language { - case "node", "python", "ruby", "csharp", "python3", "go": - tempPath := createBuildTemplate(functionName, handler, language) + if stack.IsValidTemplate(language) { - fmt.Printf("Building: %s with %s template. Please wait..\n", image, language) + var tempPath string + if strings.ToLower(language) == "dockerfile" { - flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy")) + if shrinkwrap { + fmt.Printf("Nothing to do for: %s.\n", functionName) - if shrinkwrap { - fmt.Printf("%s shrink-wrapped to %s\n", functionName, tempPath) - } else { - builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ") + return + } - ExecCommand(tempPath, builder) - fmt.Printf("Image: %s built.\n", image) - } - break - case "Dockerfile", "dockerfile": - if shrinkwrap { - fmt.Printf("Nothing to do for: %s.\n", functionName) - } else { - tempPath := handler + tempPath = handler if _, err := os.Stat(handler); err != nil { fmt.Printf("Unable to build %s, %s is an invalid path\n", image, handler) fmt.Printf("Image: %s not built.\n", image) - break + return } fmt.Printf("Building: %s with Dockerfile. Please wait..\n", image) - flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy")) + } else { - builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ") - fmt.Println(strings.Join(builder, " ")) - ExecCommand(tempPath, builder) - fmt.Printf("Image: %s built.\n", image) + tempPath = createBuildTemplate(functionName, handler, language) + fmt.Printf("Building: %s with %s template. Please wait..\n", image, language) + + if shrinkwrap { + fmt.Printf("%s shrink-wrapped to %s\n", functionName, tempPath) + + return + } } - default: + + flagStr := buildFlagString(nocache, squash, os.Getenv("http_proxy"), os.Getenv("https_proxy")) + builder := strings.Split(fmt.Sprintf("docker build %s-t %s .", flagStr, image), " ") + ExecCommand(tempPath, builder) + fmt.Printf("Image: %s built.\n", image) + + } else { log.Fatalf("Language template: %s not supported. Build a custom Dockerfile instead.", language) } } diff --git a/commands/build.go b/commands/build.go index 37f8057b2..567f7458c 100644 --- a/commands/build.go +++ b/commands/build.go @@ -149,7 +149,7 @@ func PullTemplates(templateUrl string) error { if err != nil || exists == nil { log.Println("No templates found in current directory.") - err = fetchTemplates(templateUrl) + err = fetchTemplates(templateUrl, false) if err != nil { log.Println("Unable to download templates from Github.") return err diff --git a/commands/deploy.go b/commands/deploy.go index 5efcda6f7..098ed5f80 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -66,7 +66,7 @@ var deployCmd = &cobra.Command{ [--handler HANDLER_DIR] [--fprocess PROCESS] [--env ENVVAR=VALUE ...] - [--label LABEL=VALUE ...] + [--label LABEL=VALUE ...] [--replace=false] [--update=false] [--constraint PLACEMENT_CONSTRAINT ...] @@ -160,6 +160,28 @@ func runDeploy(cmd *cobra.Command, args []string) { log.Fatalln(envErr) } + // Get FProcess to use from the ./template/template.yml, if a template is being used + if function.Language != "" && function.Language != "Dockerfile" && function.Language != "dockerfile" { + pathToTemplateYAML := "./template/" + function.Language + "/template.yml" + if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { + log.Fatalln(err.Error()) + return + } + + var langTemplate stack.LanguageTemplate + parsedLangTemplate, err := stack.ParseYAMLForLanguageTemplate(pathToTemplateYAML) + + if err != nil { + log.Fatalln(err.Error()) + return + } + + if parsedLangTemplate != nil { + langTemplate = *parsedLangTemplate + function.FProcess = langTemplate.FProcess + } + } + proxy.DeployFunction(function.FProcess, services.Provider.GatewayURL, function.Name, function.Image, function.Language, replace, allEnvironment, services.Provider.Network, constraints, update, secrets, allLabels) } } else { diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index f3187aaf0..6aab71f99 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -5,6 +5,7 @@ package commands import ( "archive/zip" + "errors" "fmt" "io" "io/ioutil" @@ -18,76 +19,133 @@ import ( "github.com/openfaas/faas-cli/proxy" ) -const ZipFileName string = "master.zip" +const ( + defaultTemplateRepository = "https://github.com/openfaas/faas-cli" + templateDirectory = "./template/" +) // fetchTemplates fetch code templates from GitHub master zip file. -func fetchTemplates(templateUrl string) error { +func fetchTemplates(templateURL string, overwrite bool) error { + var existingLanguages []string + availableLanguages := make(map[string]bool) + var fetchedTemplates []string + + if len(templateURL) == 0 { + templateURL = defaultTemplateRepository + } - err := fetchMasterZip(templateUrl) + archive, err := fetchMasterZip(templateURL) + if err != nil { + removeArchive(archive) + return err + } - zipFile, err := zip.OpenReader(ZipFileName) + zipFile, err := zip.OpenReader(archive) if err != nil { return err } - log.Printf("Attempting to expand templates from %s\n", ZipFileName) + log.Printf("Attempting to expand templates from %s\n", archive) for _, z := range zipFile.File { - relativePath := strings.Replace(z.Name, "faas-cli-master/", "", -1) - if strings.Index(relativePath, "template") == 0 { - fmt.Printf("Found \"%s\"\n", relativePath) - rc, err := z.Open() - if err != nil { - return err - } + var rc io.ReadCloser - err = createPath(relativePath, z.Mode()) - if err != nil { - return err - } + relativePath := z.Name[strings.Index(z.Name, "/")+1:] + if strings.Index(relativePath, "template/") != 0 { + // Process only directories inside "template" at root + continue + } + + var language string + if languageSplit := strings.Split(relativePath, "/"); len(languageSplit) > 2 { + language = languageSplit[1] + + if len(languageSplit) == 3 && relativePath[len(relativePath)-1:] == "/" { + // template/language/ + + if !canWriteLanguage(&availableLanguages, language, overwrite) { + existingLanguages = append(existingLanguages, language) + continue + } + fetchedTemplates = append(fetchedTemplates, language) + } else { + // template/language/* - // If relativePath is just a directory, then skip expanding it. - if len(relativePath) > 1 && relativePath[len(relativePath)-1:] != string(os.PathSeparator) { - err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()) - if err != nil { - return err + if !canWriteLanguage(&availableLanguages, language, overwrite) { + continue } } + } else { + // template/ + continue + } + + if rc, err = z.Open(); err != nil { + break + } + + if err = createPath(relativePath, z.Mode()); err != nil { + break + } + + // If relativePath is just a directory, then skip expanding it. + if len(relativePath) > 1 && relativePath[len(relativePath)-1:] != "/" { + if err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()); err != nil { + break + } } } + if len(existingLanguages) > 0 { + log.Printf("Cannot overwrite the following %d directories: %v\n", len(existingLanguages), existingLanguages) + } + + zipFile.Close() + + log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedTemplates), fetchedTemplates, templateURL) + + err = removeArchive(archive) + + return err +} + +// removeArchive removes the given file +func removeArchive(archive string) error { log.Printf("Cleaning up zip file...") - if _, err := os.Stat(ZipFileName); err == nil { - os.Remove(ZipFileName) + if _, err := os.Stat(archive); err == nil { + return os.Remove(archive) } else { return err } - fmt.Println("") - - return err } -func fetchMasterZip(templateUrl string) error { +// fetchMasterZip downloads a zip file from a repository URL +func fetchMasterZip(templateURL string) (string, error) { var err error - if _, err = os.Stat(ZipFileName); err != nil { - if len(templateUrl) == 0 { - templateUrl = "https://github.com/openfaas/faas-cli/archive/" + ZipFileName - } + templateURL = strings.TrimRight(templateURL, "/") + templateURL = templateURL + "/archive/master.zip" + archive := "master.zip" + if _, err := os.Stat(archive); err != nil { timeout := 120 * time.Second client := proxy.MakeHTTPClient(&timeout) - req, err := http.NewRequest(http.MethodGet, templateUrl, nil) + req, err := http.NewRequest(http.MethodGet, templateURL, nil) if err != nil { log.Println(err.Error()) - return err + return "", err } - log.Printf("HTTP GET %s\n", templateUrl) + log.Printf("HTTP GET %s\n", templateURL) res, err := client.Do(req) if err != nil { log.Println(err.Error()) - return err + return "", err + } + if res.StatusCode != http.StatusOK { + err := errors.New(fmt.Sprintf("%s is not valid, status code %d", templateURL, res.StatusCode)) + log.Println(err.Error()) + return "", err } if res.Body != nil { defer res.Body.Close() @@ -95,36 +153,27 @@ func fetchMasterZip(templateUrl string) error { bytesOut, err := ioutil.ReadAll(res.Body) if err != nil { log.Println(err.Error()) - return err + return "", err } - log.Printf("Writing %dKb to %s\n", len(bytesOut)/1024, ZipFileName) - err = ioutil.WriteFile(ZipFileName, bytesOut, 0700) + log.Printf("Writing %dKb to %s\n", len(bytesOut)/1024, archive) + err = ioutil.WriteFile(archive, bytesOut, 0700) if err != nil { log.Println(err.Error()) + return "", err } } fmt.Println("") - return err + return archive, err } func writeFile(rc io.ReadCloser, size uint64, relativePath string, perms os.FileMode) error { var err error defer rc.Close() - fmt.Printf("Writing %d bytes to \"%s\"\n", size, relativePath) - if strings.HasSuffix(relativePath, "/") { - mkdirErr := os.MkdirAll(relativePath, perms) - if mkdirErr != nil { - return fmt.Errorf("error making directory %s got: %s", relativePath, mkdirErr) - } - return err - } - - // Create a file instead. f, err := os.OpenFile(relativePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perms) if err != nil { - return fmt.Errorf("error writing to %s got: %s", relativePath, err) + return err } defer f.Close() _, err = io.CopyN(f, rc, int64(size)) @@ -137,3 +186,29 @@ func createPath(relativePath string, perms os.FileMode) error { err := os.MkdirAll(dir, perms) return err } + +// Tells whether the language can be expanded from the zip or not +// availableLanguages map keeps track of which languages we know to be okay to copy. +// overwrite flag will allow to force copy the language template +func canWriteLanguage(availableLanguages *map[string]bool, language string, overwrite bool) bool { + canWrite := false + if len(language) > 0 { + if _, ok := (*availableLanguages)[language]; ok { + return (*availableLanguages)[language] + } + canWrite = checkLanguage(language, overwrite) + (*availableLanguages)[language] = canWrite + } + + return canWrite +} + +// Takes a language input (e.g. "node"), tells whether or not it is OK to download +func checkLanguage(language string, overwrite bool) bool { + dir := templateDirectory + language + if _, err := os.Stat(dir); err == nil && !overwrite { + // The directory template/language/ exists + return false + } + return true +} diff --git a/commands/fetch_templates_test.go b/commands/fetch_templates_test.go index ef0b0adb5..077eaf45a 100644 --- a/commands/fetch_templates_test.go +++ b/commands/fetch_templates_test.go @@ -9,16 +9,46 @@ import ( "testing" ) -var SmallestZipFile = []byte{80, 75, 05, 06, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00} - func Test_PullTemplates(t *testing.T) { - // Remove existing master.zip file if it exists - if _, err := os.Stat("master.zip"); err == nil { - t.Log("Found a master.zip file, removing it...") + defer tearDown_fetch_templates(t) + + // Create fake server for testing. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "testdata/master_test.zip") + })) + defer ts.Close() + + err := PullTemplates(ts.URL) + if err != nil { + t.Error(err) + } +} + +func Test_fetchTemplates(t *testing.T) { + defer tearDown_fetch_templates(t) + + // Create fake server for testing. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "testdata/master_test.zip") + })) + defer ts.Close() - err := os.Remove("master.zip") + err := fetchTemplates(ts.URL+"/owner/repo", false) + if err != nil { + t.Error(err) + } +} + +// tearDown_fetch_templates_test cleans all files and directories created by the test +func tearDown_fetch_templates(t *testing.T) { + + // Remove existing archive file if it exists + if _, err := os.Stat("template-owner-repo.zip"); err == nil { + t.Log("The archive was not deleted") + + err := os.Remove("template-owner-repo.zip") if err != nil { - t.Fatal(err) + t.Log(err) } } @@ -28,21 +58,15 @@ func Test_PullTemplates(t *testing.T) { err := os.RemoveAll("template/") if err != nil { - t.Fatal(err) + t.Log(err) } + } else { + t.Logf("Directory template was not created: %s", err) } - // Create fake server for testing. - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - // Write out the minimum number of bytes to make the response a valid .zip file - w.Write(SmallestZipFile) - - })) - defer ts.Close() - - err := PullTemplates(ts.URL) - if err != nil { - t.Error(err) + // Verify the downloaded archive + archive := "template-owner-repo.zip" + if _, err := os.Stat(archive); err == nil { + t.Fatalf("The archive %s was not deleted", archive) } } diff --git a/commands/new_function.go b/commands/new_function.go index e5e58ae6e..641585a6b 100644 --- a/commands/new_function.go +++ b/commands/new_function.go @@ -7,11 +7,11 @@ import ( "fmt" "io/ioutil" "os" - "path" "strings" "github.com/morikuni/aec" "github.com/openfaas/faas-cli/builder" + "github.com/openfaas/faas-cli/stack" "github.com/spf13/cobra" ) @@ -42,15 +42,23 @@ language or type in --list for a list of languages available.`, } func runNewFunction(cmd *cobra.Command, args []string) { + if list == true { + var availableTemplates []string + + if templateFolders, err := ioutil.ReadDir(templateDirectory); err != nil { + fmt.Printf("No language templates were found. Please run 'faas-cli template pull'.") + return + } else { + for _, file := range templateFolders { + if file.IsDir() { + availableTemplates = append(availableTemplates, file.Name()) + } + } + } + fmt.Printf(`Languages available as templates: -- node -- python -- python3 -- ruby -- csharp -- Dockerfile -- go +` + printAvailableTemplates(availableTemplates) + ` Or alternatively create a folder containing a Dockerfile, then pick the "Dockerfile" lang type in your YAML file. @@ -71,8 +79,9 @@ the "Dockerfile" lang type in your YAML file. PullTemplates("") - if validTemplate(lang) == false { + if stack.IsValidTemplate(lang) == false { fmt.Printf("%s is unavailable or not supported.\n", lang) + return } if _, err := os.Stat(functionName); err == nil { @@ -134,14 +143,10 @@ functions: return } -func validTemplate(lang string) bool { - var found bool - if strings.ToLower(lang) != "dockerfile" { - found = true - } - if _, err := os.Stat(path.Join("./template/", lang)); err == nil { - found = true +func printAvailableTemplates(availableTemplates []string) string { + var result string + for _, template := range availableTemplates { + result += fmt.Sprintf("- %s\n", template) } - - return found + return result } diff --git a/commands/new_function_test.go b/commands/new_function_test.go index 642175f9f..1f8b2d73d 100644 --- a/commands/new_function_test.go +++ b/commands/new_function_test.go @@ -5,37 +5,189 @@ package commands import ( "os" + "path/filepath" + "reflect" "regexp" + "strings" "testing" + "github.com/openfaas/faas-cli/stack" "github.com/openfaas/faas-cli/test" ) -func Test_new(t *testing.T) { - //TODO activate the teardown when PR#87 is merged defer teardown() - funcName := "test-function" +const SuccessMsg = `(?m:Function created in folder)` +const InvalidYAMLMsg = `is not valid YAML` +const InvalidYAMLMap = `map is empty` +const ListOptionOutput = `Languages available as templates: +- csharp +- go +- go-armhf +- node +- node-arm64 +- node-armhf +- python +- python-armhf +- python3 +- ruby` - // Symlink template directory at root to current directory to avoid re-downloading templates - os.Symlink("../template", "template") +const LangNotExistsOutput = `(?m:bash is unavailable or not supported.)` + +type NewFunctionTest struct { + title string + funcName string + funcLang string + expectedMsg string +} + +var NewFunctionTests = []NewFunctionTest{ + { + title: "new_1", + funcName: "new-test-1", + funcLang: "ruby", + expectedMsg: SuccessMsg, + }, + { + title: "new_2", + funcName: "new-test-2", + funcLang: "dockerfile", + expectedMsg: SuccessMsg, + }, +} + +func runNewFunctionTest(t *testing.T, nft NewFunctionTest) { + funcName := nft.funcName + funcLang := nft.funcLang + var funcYAML string + funcYAML = funcName + ".yml" // Cleanup the created directory defer func() { os.RemoveAll(funcName) - os.Remove(funcName + ".yml") - os.Remove("template") + os.Remove(funcYAML) }() + cmdParameters := []string{ + "new", + funcName, + "--lang=" + funcLang, + "--gateway=" + defaultGateway, + } + stdOut := test.CaptureStdout(func() { - faasCmd.SetArgs([]string{ - "new", - funcName, - "--lang=python", + faasCmd.SetArgs(cmdParameters) + faasCmd.Execute() + }) + + // Validate new function output + if found, err := regexp.MatchString(nft.expectedMsg, stdOut); err != nil || !found { + t.Fatalf("Output is not as expected: %s\n", stdOut) + } + + if nft.expectedMsg == SuccessMsg { + + // Make sure that the folder and file was created: + if _, err := os.Stat("./" + funcName); os.IsNotExist(err) { + t.Fatalf("%s/ directory was not created", funcName) + } + + if _, err := os.Stat(funcYAML); os.IsNotExist(err) { + t.Fatalf("\"%s\" yaml file was not created", funcYAML) + } + + // Make sure that the information in the YAML file is correct: + parsedServices, err := stack.ParseYAMLFile(funcYAML, "", "") + if err != nil { + t.Fatalf("Couldn't open modified YAML file \"%s\" due to error: %v", funcYAML, err) + } + services := *parsedServices + + var testServices stack.Services + testServices.Provider = stack.Provider{Name: "faas", GatewayURL: defaultGateway} + if !reflect.DeepEqual(services.Provider, testServices.Provider) { + t.Fatalf("YAML `provider` section was not created correctly for file %s: got %v", funcYAML, services.Provider) + } + + testServices.Functions = make(map[string]stack.Function) + testServices.Functions[funcName] = stack.Function{Language: funcLang, Image: funcName, Handler: "./" + funcName} + if !reflect.DeepEqual(services.Functions[funcName], testServices.Functions[funcName]) { + t.Fatalf("YAML `functions` section was not created correctly for file %s, got %v", funcYAML, services.Functions[funcName]) + } + } + +} + +func Test_newFunctionTests(t *testing.T) { + + homeDir, _ := filepath.Abs(".") + if err := os.Chdir("testdata/new_function"); err != nil { + t.Fatalf("Error on cd to testdata dir: %v", err) + } + + for _, test := range NewFunctionTests { + t.Run(test.title, func(t *testing.T) { + runNewFunctionTest(t, test) }) + } + + if err := os.Chdir(homeDir); err != nil { + t.Fatalf("Error on cd back to commands/ directory: %v", err) + } +} + +func Test_newFunctionListCmds(t *testing.T) { + + homeDir, _ := filepath.Abs(".") + if err := os.Chdir("testdata/new_function"); err != nil { + t.Fatalf("Error on cd to testdata dir: %v", err) + } + + cmdParameters := []string{ + "new", + "--list", + } + + stdOut := test.CaptureStdout(func() { + faasCmd.SetArgs(cmdParameters) faasCmd.Execute() + }) + + // Validate new function output + if !strings.HasPrefix(stdOut, ListOptionOutput) { + t.Fatalf("Output is not as expected: %s\n", stdOut) + } + if err := os.Chdir(homeDir); err != nil { + t.Fatalf("Error on cd back to commands/ directory: %v", err) + } +} + +func Test_languageNotExists(t *testing.T) { + + homeDir, _ := filepath.Abs(".") + if err := os.Chdir("testdata/new_function"); err != nil { + t.Fatalf("Error on cd to testdata dir: %v", err) + } + + // Attempt to create a function with a non-existing language + cmdParameters := []string{ + "new", + "sampleName", + "--lang=bash", + "--gateway=" + defaultGateway, + "--list=false", + } + + stdOut := test.CaptureStdout(func() { + faasCmd.SetArgs(cmdParameters) + faasCmd.Execute() }) - if found, err := regexp.MatchString(`(?m:Function created in folder)`, stdOut); err != nil || !found { - t.Fatalf("Output is not as expected:\n%s", stdOut) + // Validate new function output + if found, err := regexp.MatchString(LangNotExistsOutput, stdOut); err != nil || !found { + t.Fatalf("Output is not as expected: %s\n", stdOut) + } + + if err := os.Chdir(homeDir); err != nil { + t.Fatalf("Error on cd back to commands/ directory: %v", err) } } diff --git a/commands/template_pull.go b/commands/template_pull.go new file mode 100644 index 000000000..64870675f --- /dev/null +++ b/commands/template_pull.go @@ -0,0 +1,80 @@ +// Copyright (c) Alex Ellis, Eric Stoekl 2017. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +package commands + +import ( + "errors" + + "fmt" + + "os" + + "regexp" + + "github.com/spf13/cobra" +) + +// Args and Flags that are to be added to commands + +const ( + repositoryRegexpMockedServer = `^http://127.0.0.1:\d+/([a-z0-9-]+)/([a-z0-9-]+)$` + repositoryRegexpGithub = `^https://github.com/([a-z0-9-]+)/([a-z0-9-]+)/?$` +) + +var ( + repository string + overwrite bool +) + +var supportedVerbs = [...]string{"pull"} + +func init() { + templatePullCmd.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite existing templates?") + + faasCmd.AddCommand(templatePullCmd) +} + +// templatePullCmd allows the user to fetch a template from a repository +var templatePullCmd = &cobra.Command{ + Use: "template pull ", + Args: func(cmd *cobra.Command, args []string) error { + msg := fmt.Sprintf(`Must use a supported verb for 'faas-cli template' +Currently supported verbs: %v`, supportedVerbs) + + if len(args) == 0 { + return errors.New(msg) + } + + if args[0] != "pull" { + return errors.New(msg) + } + + if len(args) > 1 { + var validURL = regexp.MustCompile(repositoryRegexpGithub + "|" + repositoryRegexpMockedServer) + + if !validURL.MatchString(args[1]) { + return errors.New("The repository URL must be in the format https://github.com//") + } + } + return nil + }, + Short: "Downloads templates from the specified github repo", + Long: `Downloads the compressed github repo specified by [URL], and extracts the 'template' + directory from the root of the repo, if it exists.`, + Example: "faas-cli template pull https://github.com/openfaas/faas-cli", + Run: runTemplatePull, +} + +func runTemplatePull(cmd *cobra.Command, args []string) { + repository := "" + if len(args) > 1 { + repository = args[1] + } + + fmt.Println("Fetch templates from repository: " + repository) + if err := fetchTemplates(repository, overwrite); err != nil { + fmt.Println(err) + + os.Exit(1) + } +} diff --git a/commands/template_pull_test.go b/commands/template_pull_test.go new file mode 100644 index 000000000..d2be68660 --- /dev/null +++ b/commands/template_pull_test.go @@ -0,0 +1,144 @@ +// Copyright (c) Alex Ellis, Eric Stoekl 2017. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +package commands + +import ( + "bytes" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" + "testing" +) + +func Test_templatePull(t *testing.T) { + defer tearDown_fetch_templates(t) + + ts := httpTestServer(t) + defer ts.Close() + + repository = ts.URL + "/owner/repo" + faasCmd.SetArgs([]string{"template", "pull", repository}) + faasCmd.Execute() + + // Verify created directories + if _, err := os.Stat("template"); err != nil { + t.Fatalf("The directory %s was not created", "template") + } +} + +func Test_templatePull_with_overwriting(t *testing.T) { + defer tearDown_fetch_templates(t) + + ts := httpTestServer(t) + defer ts.Close() + + repository = ts.URL + "/owner/repo" + faasCmd.SetArgs([]string{"template", "pull", repository}) + faasCmd.Execute() + + var buf bytes.Buffer + log.SetOutput(&buf) + + r := regexp.MustCompile(`(?m:Cannot overwrite the following \d+ directories:)`) + + faasCmd.SetArgs([]string{"template", "pull", repository}) + faasCmd.Execute() + + if !r.MatchString(buf.String()) { + t.Fatal(buf.String()) + } + + buf.Reset() + + faasCmd.SetArgs([]string{"template", "pull", repository, "--overwrite"}) + faasCmd.Execute() + + str := buf.String() + if r.MatchString(str) { + t.Fatal() + } + + // Verify created directories + if _, err := os.Stat("template"); err != nil { + t.Fatalf("The directory %s was not created", "template") + } +} + +func Test_templatePull_no_arg(t *testing.T) { + defer tearDown_fetch_templates(t) + var buf bytes.Buffer + + faasCmd.SetArgs([]string{"template", "pull"}) + faasCmd.SetOutput(&buf) + faasCmd.Execute() + + if strings.Contains(buf.String(), "Error: A repository URL must be specified") { + t.Fatal("Output does not contain the required string") + } +} + +func Test_templatePull_error_not_valid_url(t *testing.T) { + var buf bytes.Buffer + + faasCmd.SetArgs([]string{"template", "pull", "git@github.com:openfaas/faas-cli.git"}) + faasCmd.SetOutput(&buf) + faasCmd.Execute() + + if !strings.Contains(buf.String(), "Error: The repository URL must be in the format https://github.com//") { + t.Fatal("Output does not contain the required string", buf.String()) + } +} + +// httpTestServer returns a testing http server +func httpTestServer(t *testing.T) *httptest.Server { + const sampleMasterZipPath string = "testdata/master_test.zip" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if _, err := os.Stat(sampleMasterZipPath); os.IsNotExist(err) { + t.Error(err) + } + + fileData, err := ioutil.ReadFile(sampleMasterZipPath) + if err != nil { + t.Error(err) + } + + w.Write(fileData) + })) + + return ts +} + +func Test_repositoryUrlRegExp(t *testing.T) { + var url string + r := regexp.MustCompile(repositoryRegexpGithub) + + url = "http://github.com/owner/repo" + if r.MatchString(url) { + t.Errorf("Url %s must start with https", url) + } + + url = "https://github.com/owner/repo.git" + if r.MatchString(url) { + t.Errorf("Url %s must not end with .git or must start with https", url) + } + + url = "https://github.com/owner/repo//" + if r.MatchString(url) { + t.Errorf("Url %s must end with no or one slash", url) + } + + url = "https://github.com/owner/repo" + if !r.MatchString(url) { + t.Errorf("Url %s must be valid", url) + } + + url = "https://github.com/owner/repo/" + if !r.MatchString(url) { + t.Errorf("Url %s must be valid", url) + } +} diff --git a/commands/testdata/master_test.zip b/commands/testdata/master_test.zip new file mode 100755 index 0000000000000000000000000000000000000000..42b74f2090f63c5a2be9c0f544b3465324433dac GIT binary patch literal 13909 zcmb_?2Rzl^|G$w)gsh8XWrb^HmywaZvXgPS_PlmRkx^zD*@T9fh{y_+%7`S%p2?ADcLs(Yk z=59O|4z@gw=5Fp#7_Y7l9u~IkaIvB0u0KYz!;iT`9bFvE-J$#Wc}7e>Z{cQR4s+QJ zMm*tS69J$eK6v&41GaRwu!q8It(}}<&`r}IwoZi35Ie#R6AAu$#+ke~>jpL!Ryz(B z*2#Tf%l(D6vUNZL7ip_Gckz+d{?v~Q>cV~X{T6Qi$-rP?Y~5KWJ=pO<4Z zh*uqhKcrW`f7JApKp=dI#C8b#o6zEgktafwF=gHs(!Wz~IJh0MHE}A1RT!twH9I5} zJu0mnoahsH$}y63qo8IbHiNRh)-CW4S$3jjvU71D_QNT)$xL!eh>Hnss<-Os@IXvr zu-z%WHk|2eN>MA4K8-!&GWi81BmB2YZh}HrXeO89q3_a%=XC zN?qK!?Gwxgq8N7uCiv$*Cs-5aYz;Ga1Y5XmMvkLai~2J@(i;6{Dej8#SRCpjlwsPG zT6yxsbEgwshvQc8x?R^4%_warKBfJP0Ueb~YG->l zX-OzXO<*jnmY|oJWYnQ-_$nc^<{l+aR0_e#^Cc9iJPl?HL|2$8TS4A$^E(QcH7~eo+16Q6H5eFWYIiz01qzqYB$~T_ zoME=5J#Gi?|Ha^ualQ)wda4s=q{2mr#K+~~w&o`kp`oFw@u!r{*2B3|-L9KcG*Gs& z6=Z`8*qcKVkH!=b-nCAx3zq$n_qN;U_;rwVtY>Y87PkwY%r|@36#|bw}4IYjJz!hIlvT4?T5p?qh*kc{o|P z+d4a;ClkbizhwX!=L8Z7H!uYM3m(|o)*U6Ks5G~}n{3u-?wssw;ZoD!!S3W-W%V&r zP_lE9QHLnm*_m9jGRc>u#@a9mAwhO6bN(7YKMxGi{~|%*pKU;Jy1<<6kO3MaL)gei zD*q#7Fl!m>dSx^=wUIGLP;GsWs$8E{v!&c6-*;f4>h_Y`rIufAfoU&YOR zbVRQhD`UuB{^ZLHVxKX;X5;?2Wk$c$SatO0U(8fAcd~SV!nQ6& zwJ0^Xr6E58`^`V+Gk?eO^rO>IM$2r zwGJn9imfcMe3T4*kw#zcdMmwQ`ZMQnH1{oeSyJ$4GCjV^&tIAPDT=cU)P=tdO(eI( zYx`ku*pD8$C-z}KusNXzEjrYGb}2ZN92DYVy&u5+gdh^o3F>YEY^&xjE&^b_-QWcW zhAzPu8GKyWr|Ag2+u>o(&hGz$9V`p5$~rr`09&t;lZ%JD8`#3!!Unkwi&#JfEG+oV zEd&Ii=1>7a0X_?erJx|6prDwrrJ%X6s0E*eg^;BHN&)b$db+j_kZB6+5nTI_x?K=V}XoB?TCpiSD;JDQeGHtUt?fe&kE zbl5@*!ScTrn=c*Jt%=3C_R@Au(p9lP8r$w0uA#=FN{3z5l@E9D`_FhZ2S%}fCZ&2G zR`{p0eekmUqxW>8H>~`cu8mYP&{@}(R>iDF8ajc0KJi{ZNBQ)Bj9MpWODK;y%+baQ zJqi$cZp4__DH3pNBCz#>F}ZdN)DHEPzjlNA7`(pXQV?@7WW5*BCCjRJmZ|JoSzdFa zUHR?ip19{-B`ke`%u7q62eiw$=8Ow#l0_EU&Vz+t98Gdmuai>aJYUo$Dxq{Q*uhD! zzL%bli7~qTy4aXdkT4Fzz)SL($JlQm(cLZDuupHJp7Y)bz+Wf~lEBG|j>htFWHj

vVUM_%E3>a9|mNP(km`-LA=ss=lo}^;`N>piKs0``x4C@aYm4y77_joW);oQbxu&jhmLc;@qCV^b3%SjJ?YLvu+bfAh zLdPy*{WLW@MOPIu%l+qxBDLWqO6?oHIsTAo_ff`+8P6E5k>xrYrEom+av4$?Gr5!$ zY_=9?6DeH9?!0@f9N%k#cZ-triHyD}AWH0Tv9LJy1lcJ*&bK)A0p-~Y*FiApGWK;3 zk~1Jh>}Tza(T% znWMp!cvq9+fOAJ;bmXY|tm1Jk!?Tw~<>rlTeIrNaB&n66Yj1Rti#`tyW*57AEe14g zsdCVj)l&J>#fNLc_`DSavUi`66P;GOsiQE8jqPV_d?nU6=TYGu3cIUkyp%Ggi6L2) zV$6b7^>URp;WI=T+v_elA}3aFVb;L3Zh!uZOOd2VETwD5*MbQdA$=9sQnaNKYXv1MiJQ zDGRwhS3jA$Z)l``r%Y^x6rsXbiPl7snFfetT}q*K#A^q4krbb}$o%AaVqTRmf-o z+fTna_r1=MvK;hX+5bY{k)ZqR%O<|d_)8Iiy@t1Jq(u`*+%CXi+cRz;9<~maU^ko1oE_34apVdk5mra- z{ajI8SN^weQy23(6^&TDFOMDa`$>n-VCct%6(64c@%V$}iTYnRyA1EHf@q7flV@kC zL_TWrQVHA;N_V$dKiK zUda$%VI;-t_?0#~c9Kb4TFZ++U0Q(HI+(G_*YQ$|bqb%H{j%M)=%YrXfyIw>C|f`K zy>3qYz6ieiesut1;zBNY*@Fjv5b~*GjLM_eGWZbdBcEA0lG+XP5p%64>X6j=VQy7* zBLg8a5ie8kytUlNAQpmg*^{wdM-HCmmzZx5SHxLA+Zh?!bw%@ov@`kbXQEFg9ZA+| zjun*6Uuuynl#%#7#80EtJ!$$Tkwwq<8{T%K9}roch60If2-xoS4oF)kOQ<(s>873O zu?@f}lPGk-pYYMMTgo?pV`i=*d9S&RL)a1yt4D>jVrU#QXRCWpQk zQ{eSi4d73;3186^Qzfo@)(V;%`LaH$Tx!OcaEA2K9g#w)9>v2B!lIY7Mi*FQdYJp$ z8S33jgN2y6KUF-MxO6Z|n4p)<=G^=tT`U+_dW<0D7W=XzQOUA4-dtg8sa8uY$sJt~ zG`Rtbj%Y^nk?!lhn#U=css<8YwbI5v(3clEIC|${pNP0uesYbn0CQqZ8l`hW z;Fp7vg2NxKYMsUll)KL#v`&3ZLnJSrsZzP@|Z-Tk|a__?w zaliX^1bhRzDL0m^Rus6>xmK9)KwDoqxxf2HVvR-nW^v*BD~4isvxyrjx_IT3E57q@ z`A?T?w_l%gTZkF%mlwL0*M2rF?V$oF6yFyw5qRv%-;)nnwV~% zF(GNr;0ho_LBOnGEUW*9>5kE<21I8g5S>2)LK3pbb@(3g1%V6? zGllP52b`oxeZH1^Jmt{Ww(<71sl6B=SOGdqBfmmD9vskWa(_I`(3_vWd2N<>!w=}$ z3S;ED__*6RJ7KQ+-bKc@-3J`{8(0Q>n3B~tupRdGDD8{(ebS_U^S{DnLOQhX6#VQs z%ZO)D?eRM7YTV;^wxqbS(&)7LuIlsLG68<=u);gw`1!HOvW0>5E$(50%G_p@DI zsvei+*5MHe;69t?+lWL&H_A>Q*=Iy!1VtMC$p7lohn{sR1H@!974Qi)J|Js=SH*NCMPOLu?*ff&=^xA ze~`53f2&weTji+{o=_MBzcsUe-Q|14A02O-7qg>u;k_x5X%g2xrNj)#`Gqs1+8<2D ztvbMHxf8_KCu=p?S;AhgQb>`JR~Fl4ND*t8*$oG257t8X+ zFBv*9-DFL+;mtpr=-s0AXOurtc58FmjNK%I-PG4%oVl5@9?e~=}Z6{nbb3fwPPAXLuh_Vnx@?|r9#9_D=!c?K~;CAmJo5n#q%B>Hyr>JsxDTHwYT6`)X~QNj}?;#g_g_`|TySefS+jHc^YI6dEG*w<$td4HcD$)bE0 z;mkp24};C0=&cBZz9XVW8(u}yLYFtfRsik#LOS#pU8 z!`Ea#qtFmZ%t$*=F;F_RdkVtX=KD9{n>EAc{<(pNs6bk^6{?N_sr~p~6$mQcM$zQr zvx%&tq;~$5Vynt)rS}}*w`HT+SV2MAtu;qU6Ay28_Rtfl*xtDhS^2=At{%29s3X(~ z=xexpyKi!Dx2k;X(bOzp5xk*|@vQXEDnya7#T2z@e*0&96v1A~3@n{1_ZK@{;dN&tPb9mGk2KXZ#{J;n4}7KPMjz zYt9nfQ(RXw3a5Ybn4Lk%;(H3nmeDUwazxK?R5|$TRFBC6o=A@aGlx&PR-VtSXF8Eo z&*yfer%53GZi$MpH)k;Y{rW>oWFaGbR$V_LLfSy$#q=elDMU{3I;y>ERX*geChdhq zZ|65#E5W)GX?-%e$D4nZwEWU}2;sY%{44v@$$NUuUM!p;vV-|uJ%+_PLP^h%hE-OCAQ@(_d%;q zHgCjoRs}LIHr%)Kzog^W95PjQcWgAc#0!+HI2fMG@Vf9Oj=9b)8^~-U=cM;jhcbtU zT}|~>*TNY?92B#Ra@@ZJp3c7$#eBK`(@-%}75?}Wk5#IJSrnFE4tkk=wz$l2f-Y>9 zjV5OfM}-WU(~pzzxUpEv_rs|1OEz8k(CUGlxyaM~CPzA{+CfuBV4|X225N5}x#0|= z>(YT-6=h9_UPw`~zhElHf6zY6={4yUVEL)`4_9c`S8z2B7UmYwtq`QSa(^}0DX-T&RkgvURNVs%5;u$jVtYyAGy zXTJL9%E$3F{OLCmN=#|cR9u>PsDn7o#emf4`*BP1&YFF{)sKgWNf3fe6IPMi<~ z8keoF_?|u!s3$)9@J9`{<+JWm-|zSvixd$OhalN+8GuC!AFP;?z~9+C2X)+bo9O23 z0keQIvfH@3ySRz-^1}a zmh~%gk;KoUT|b~;(;o4~#2UnL;RDeOhxQ^x!j)Xstn}yIKd7Uojt<#OK0DUBh`TYT z2$^11SLxwhVgg{5Vyuw=%0#5yE!ucu&S(G);!v+Y;ri2>2CXj}pKI!7>6|2rr8~ zh9#O*!)WR8po}Kp7YGC&i^hu1R)$Cjj>HHDeE2lMtzg&1!Xz8kF*X8`C zDbnM2%lh9$N&z<>YfP41CDdWSPY(E(JTSPij-kCJ*nAZ5?^1u`-(Vj{2VI>*Sh!+V zC5C5!;o*H43;V!lpzZhfH{zH4N9q{~u<8f&7jbOG#ecBmb_p{{YyKG;^gJsrObqV$ zBs((o{b7?E)#u0Y)Ru+Cqc1#2)`z&0`rG}?l8XA}JO3_3gi=B|c9jXSxSJ;cmF&Pc ziAm|-;~kpLtANHlbe%0qttz}~sv70muUa&eCX~52nl;pUz@aXb0$;VYxt2@ApZu)~ z5IX2lbnxw^9J$=DU$u{I zdgc&^<)^!M^gdJSz_7Qt<`jw;0Zgtzh}q3Ka5>%oA0){Dl3p->GTK+v{9Wp3-kAbq z^jo7vH!^BCPO9aPZ~A53K2pRVXzOk>5n_UYzv4lBa2OF_ zpgslKFlcM7vb!w!5f6#yg$WP+jn8&i)K@-mSRmU1A3X8yD)S%g&ToE@u0W)5)Ym?6 zbOZ-2@W);pI}~211gNh~kOc7eCwmd>@b&~rfWG6y&q;_6KAF`A_9ob=0@#=ZgojX{ zM<5M2jQ1b#cX%v;v;@I~`U(Qc6#ZYA6n1$Nfkub&8Uab?aqvGFwn4YU+XN&d`c%R1 z{}CU2syy)bW<+@nurW*M(D24361sk$&}gp%b|-+p4?q%-0k81(isOB}7T8XQ`d$D| zhgg>`{|CA)SM__o9zaeWB05o@1t1mC0p8fl0b7{&m6o@M2N$pf?8>m^u6ze*gm~18 zccggFeL`bQb?7k&Klvg)czEj*?V*3Cy#=8h^)eW#JZ(?VT{KhU!go{zJnEg{2K-!L z*O9$6qun4PLk8U;@WUA5gFD2PWH0c&6Eo@^;D&T8tf%`&-#9jHxpqQYfp8J()g6+@ z1uz9;SR&h5J6@HbGNIm=A(^0LdoiKio}q%HUhN>ksmb>O-$On6Y{O54hz~y73_#_- ztDm;jvAZr{)H@WUdWU_3@3I_rD=@cLa&}aE+OYc~Ms40B)tBr8_CM<0UIb->_SU`k zT0?CfBMtcSU+_0p+m`+-GEC8<2;MM8GTG4l2TL$CD>pV;#Og+EAS3x+?Sl`kos7vG zc#{gKPe3M(wa+ za{Kn3r)N)WOsfIm?XAk;Z9$}RHsE$;*G%_U`6l}ACLjV6wGW6SN!dTi#)8|@c|*Di z!Gqe!L-NP~{aVb%ptbej;Ky+hjUM#Y7?3J}4_?NW1*$S)0N=4uhKFdBp)|?1GNQM* zwlaRl#JEGl3*dt{yfE;gx3aeK1u$W-a7)p;(O*Ec0|C|@T5MabL~mJbWhG+9z`A1t z3t)sdurSz(-j~|Sn7lXRj$JB%cZa6gR!h;FPg{ApSuj|-Ln{j4gSVp4?8F03yI5Gf zz%PrVKx6`_99SYk=9X3hq89w3qEJf#F;Oc40e+~tkf?>|DM3N#sZ%260)oO;q9Xqf DR2c!{ literal 0 HcmV?d00001 diff --git a/commands/testdata/new_function/template/csharp/function/.gitignore b/commands/testdata/new_function/template/csharp/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/go-armhf/.gitignore b/commands/testdata/new_function/template/go-armhf/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/go/function/.gitignore b/commands/testdata/new_function/template/go/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/node-arm64/function/.gitignore b/commands/testdata/new_function/template/node-arm64/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/node-armhf/.gitignore b/commands/testdata/new_function/template/node-armhf/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/node/function/.gitignore b/commands/testdata/new_function/template/node/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/python-armhf/.gitignore b/commands/testdata/new_function/template/python-armhf/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/python/function/.gitignore b/commands/testdata/new_function/template/python/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/python3/function/.gitignore b/commands/testdata/new_function/template/python3/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/commands/testdata/new_function/template/ruby/function/.gitignore b/commands/testdata/new_function/template/ruby/function/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/guide/TEMPLATE.md b/guide/TEMPLATE.md new file mode 100644 index 000000000..f4041dd46 --- /dev/null +++ b/guide/TEMPLATE.md @@ -0,0 +1,56 @@ +# Using template from external repository + +## Repository structure + +The external repository must have a directory named ```template``` at the root directory, in which there are directories +containing templates. The directory for each template can be freely named with alphanumeric characters and hyphen. + +Example: + +``` +template +├── csharp +│   ├── Dockerfile +│   └── template.yml +├── node +│   ├── Dockerfile +│   └── template.yml +├── node-armhf +│   ├── Dockerfile +│   └── template.yml +├── python +│   ├── Dockerfile +│   └── template.yml +├── python-armhf +│   ├── Dockerfile +│   └── template.yml +├── php5 +│   ├── Dockerfile +│   └── template.yml +├── php7 +│   ├── Dockerfile +│   └── template.yml +└── ruby + ├── Dockerfile + └── template.yml +``` + +## Download external repository + +In order to build functions using 3rd party templates, you need to add 3rd templates before the build step, with the following command: + +```bash +./faas-cli template pull https://github.com/itscaro/openfaas-template-php +``` + +If you need to update the downloaded repository, just add the flag `--overwrite` to the download command: + +```bash +./faas-cli template pull https://github.com/itscaro/openfaas-template-php --override +``` + +## List locally available languages + +```bash +./faas-cli new --list +``` diff --git a/proxy/deploy.go b/proxy/deploy.go index 743c2248e..f331d4d4d 100644 --- a/proxy/deploy.go +++ b/proxy/deploy.go @@ -24,16 +24,6 @@ func DeployFunction(fprocess string, gateway string, functionName string, image var fprocessTemplate string if len(fprocess) > 0 { fprocessTemplate = fprocess - } else if language == "python" { - fprocessTemplate = "python index.py" - } else if language == "node" { - fprocessTemplate = "node index.js" - } else if language == "ruby" { - fprocessTemplate = "ruby index.rb" - } else if language == "csharp" { - fprocessTemplate = "dotnet ./root.dll" - } else if language == "python3" { - fprocessTemplate = "python3 index.py" } gateway = strings.TrimRight(gateway, "/") diff --git a/stack/language_template.go b/stack/language_template.go new file mode 100644 index 000000000..d612807f8 --- /dev/null +++ b/stack/language_template.go @@ -0,0 +1,61 @@ +// Copyright (c) Alex Ellis 2017. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +package stack + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +func ParseYAMLForLanguageTemplate(file string) (*LanguageTemplate, error) { + var err error + var fileData []byte + + urlParsed, err := url.Parse(file) + if err == nil && len(urlParsed.Scheme) > 0 { + fmt.Println("Parsed: " + urlParsed.String()) + fileData, err = fetchYAML(urlParsed) + if err != nil { + return nil, err + } + } else { + fileData, err = ioutil.ReadFile(file) + if err != nil { + return nil, err + } + } + + return ParseYAMLDataForLanguageTemplate(fileData) +} + +// iParseYAMLDataForLanguageTemplate parses YAML data into language template +// Use the alias ParseYAMLDataForLanguageTemplate +func ParseYAMLDataForLanguageTemplate(fileData []byte) (*LanguageTemplate, error) { + var langTemplate LanguageTemplate + var err error + + err = yaml.Unmarshal(fileData, &langTemplate) + if err != nil { + fmt.Printf("Error with YAML file\n") + return nil, err + } + + return &langTemplate, err +} + +func IsValidTemplate(lang string) bool { + var found bool + if strings.ToLower(lang) == "dockerfile" { + found = true + } else if _, err := os.Stat("./template/" + lang); err == nil { + found = true + } + + return found +} diff --git a/stack/language_template_test.go b/stack/language_template_test.go new file mode 100644 index 000000000..d784a5dfc --- /dev/null +++ b/stack/language_template_test.go @@ -0,0 +1,77 @@ +// Copyright (c) Alex Ellis 2017. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +package stack + +import ( + "fmt" + "os" + "reflect" + "testing" +) + +func Test_ParseYAMLDataForLanguageTemplate(t *testing.T) { + dataProvider := []struct { + Input string + Expected *LanguageTemplate + }{ + { + ` +language: python +fprocess: python index.py +`, + &LanguageTemplate{ + Language: "python", + FProcess: "python index.py", + }, + }, + { + ` +language: python +`, + &LanguageTemplate{ + Language: "python", + }, + }, + { + ` +fprocess: python index.py +`, + &LanguageTemplate{ + FProcess: "python index.py", + }, + }, + } + + for k, i := range dataProvider { + t.Run(fmt.Sprintf("%d", k), func(t *testing.T) { + if actual, err := ParseYAMLDataForLanguageTemplate([]byte(i.Input)); err != nil { + t.Errorf("test failed, %s", err) + } else { + if !reflect.DeepEqual(actual, i.Expected) { + t.Errorf("does not match expected result;\n parsedYAML: [%+v]\n expected: [%+v]", + actual, + i.Expected, + ) + } + } + }) + } +} + +func Test_IsValidTemplate(t *testing.T) { + if !IsValidTemplate("Dockerfile") || !IsValidTemplate("dockerfile") { + t.Fatalf("Dockerfile and dockerfile must be valid") + } + + if IsValidTemplate("unknown-language") { + t.Fatalf("unknown-language must be invalid") + } + + os.MkdirAll("template/python", 0600) + defer func() { + os.RemoveAll("template") + }() + if !IsValidTemplate("python") { + t.Fatalf("python must be valid") + } +} diff --git a/stack/schema.go b/stack/schema.go index ebab9182d..e243c48b2 100644 --- a/stack/schema.go +++ b/stack/schema.go @@ -47,3 +47,8 @@ type Services struct { Functions map[string]Function `yaml:"functions,omitempty"` Provider Provider `yaml:"provider,omitempty"` } + +type LanguageTemplate struct { + Language string `yaml:"language"` + FProcess string `yaml:"fprocess"` +} diff --git a/template/csharp/template.yml b/template/csharp/template.yml new file mode 100644 index 000000000..45caa67ed --- /dev/null +++ b/template/csharp/template.yml @@ -0,0 +1,2 @@ +language: csharp +fprocess: dotnet ./root.dll diff --git a/template/go-armhf/template.yml b/template/go-armhf/template.yml new file mode 100644 index 000000000..234cf4400 --- /dev/null +++ b/template/go-armhf/template.yml @@ -0,0 +1,2 @@ +language: go-armhf +fprocess: ./handler diff --git a/template/go/template.yml b/template/go/template.yml new file mode 100644 index 000000000..5d341d5d3 --- /dev/null +++ b/template/go/template.yml @@ -0,0 +1,2 @@ +language: go +fprocess: ./handler diff --git a/template/node-arm64/template.yml b/template/node-arm64/template.yml new file mode 100644 index 000000000..97e25c5f9 --- /dev/null +++ b/template/node-arm64/template.yml @@ -0,0 +1,2 @@ +language: node-arm64 +fprocess: node index.js diff --git a/template/node-armhf/template.yml b/template/node-armhf/template.yml new file mode 100644 index 000000000..c2313ae67 --- /dev/null +++ b/template/node-armhf/template.yml @@ -0,0 +1,2 @@ +language: node-armhf +fprocess: node index.js diff --git a/template/node/template.yml b/template/node/template.yml new file mode 100644 index 000000000..1117efa75 --- /dev/null +++ b/template/node/template.yml @@ -0,0 +1,2 @@ +language: node +fprocess: node index.js diff --git a/template/python-armhf/template.yml b/template/python-armhf/template.yml new file mode 100644 index 000000000..ff8986785 --- /dev/null +++ b/template/python-armhf/template.yml @@ -0,0 +1,2 @@ +language: python-armhf +fprocess: python index.py diff --git a/template/python/template.yml b/template/python/template.yml new file mode 100644 index 000000000..5487218c4 --- /dev/null +++ b/template/python/template.yml @@ -0,0 +1,2 @@ +language: python +fprocess: python index.py diff --git a/template/python3/template.yml b/template/python3/template.yml new file mode 100644 index 000000000..ea405e8b6 --- /dev/null +++ b/template/python3/template.yml @@ -0,0 +1,2 @@ +language: python3 +fprocess: python3 index.py diff --git a/template/ruby/template.yml b/template/ruby/template.yml new file mode 100644 index 000000000..417bd6133 --- /dev/null +++ b/template/ruby/template.yml @@ -0,0 +1,2 @@ +language: ruby +fprocess: ruby index.rb From 85b2a1cd9c0febf50a4e7de55c7f7a5c9727ad58 Mon Sep 17 00:00:00 2001 From: Eric Stoekl Date: Mon, 6 Nov 2017 22:18:42 -0800 Subject: [PATCH 2/4] template pull - address issues from thread #201 1) Moves code that merges zip file contents into template/ dir into expandTemplatesFromZip() function. Removes hard-coded constant. 2) Sort list of languages found with 'faas-cli new --list' before printing 3) Fixes for typos, more descriptive comments, and remove incorrect license attribution Signed-off-by: Eric Stoekl --- commands/deploy.go | 6 ++- commands/fetch_templates.go | 68 ++++++++++++++++++++------------ commands/fetch_templates_test.go | 6 ++- commands/new_function.go | 9 +++++ commands/template_pull.go | 2 +- commands/template_pull_test.go | 2 +- stack/language_template.go | 3 +- stack/schema.go | 1 + 8 files changed, 66 insertions(+), 31 deletions(-) diff --git a/commands/deploy.go b/commands/deploy.go index 098ed5f80..4ccc29800 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -161,7 +161,7 @@ func runDeploy(cmd *cobra.Command, args []string) { } // Get FProcess to use from the ./template/template.yml, if a template is being used - if function.Language != "" && function.Language != "Dockerfile" && function.Language != "dockerfile" { + if languageExistsNotDockerfile(function.Language) { pathToTemplateYAML := "./template/" + function.Language + "/template.yml" if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { log.Fatalln(err.Error()) @@ -299,3 +299,7 @@ func compileEnvironment(envvarOpts []string, yamlEnvironment map[string]string, functionAndStack := mergeMap(yamlEnvironment, fileEnvironment) return mergeMap(functionAndStack, envvarArguments), nil } + +func languageExistsNotDockerfile(language string) bool { + return len(language) > 0 && strings.ToLower(language) != "dockerfile" +} diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index 6aab71f99..2c5b2ebe5 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -22,13 +22,11 @@ import ( const ( defaultTemplateRepository = "https://github.com/openfaas/faas-cli" templateDirectory = "./template/" + rootLanguageDirSplitCount = 3 ) // fetchTemplates fetch code templates from GitHub master zip file. func fetchTemplates(templateURL string, overwrite bool) error { - var existingLanguages []string - availableLanguages := make(map[string]bool) - var fetchedTemplates []string if len(templateURL) == 0 { templateURL = defaultTemplateRepository @@ -40,15 +38,41 @@ func fetchTemplates(templateURL string, overwrite bool) error { return err } - zipFile, err := zip.OpenReader(archive) + log.Printf("Attempting to expand templates from %s\n", archive) + + preExistingLanguages, fetchedLanguages, err := expandTemplatesFromZip(archive, overwrite) if err != nil { return err } - log.Printf("Attempting to expand templates from %s\n", archive) + if len(preExistingLanguages) > 0 { + log.Printf("Cannot overwrite the following %d directories: %v\n", len(preExistingLanguages), preExistingLanguages) + } + + log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL) + + err = removeArchive(archive) + + return err +} + +// expandTemplatesFromZip() takes a path to an archive, and whether or not +// we are allowed to overwrite pre-existing language templates. It returns +// a list of languages that already exist (could not be overwritten), and +// a list of languages that are newly downloaded. +func expandTemplatesFromZip(archive string, overwrite bool) ([]string, []string, error) { + var existingLanguages []string + var fetchedLanguages []string + availableLanguages := make(map[string]bool) + + zipFile, err := zip.OpenReader(archive) + if err != nil { + return nil, nil, err + } for _, z := range zipFile.File { var rc io.ReadCloser + var isDirectory bool relativePath := z.Name[strings.Index(z.Name, "/")+1:] if strings.Index(relativePath, "template/") != 0 { @@ -56,18 +80,21 @@ func fetchTemplates(templateURL string, overwrite bool) error { continue } - var language string - if languageSplit := strings.Split(relativePath, "/"); len(languageSplit) > 2 { - language = languageSplit[1] + if pathSplit := strings.Split(relativePath, "/"); len(pathSplit) > 2 { + language := pathSplit[1] + + // We know that this path is a directory if the last character is a "/" + isDirectory = relativePath[len(relativePath)-1:] == "/" - if len(languageSplit) == 3 && relativePath[len(relativePath)-1:] == "/" { + // Check if this is the root directory for a language (at ./template/lang) + if len(pathSplit) == rootLanguageDirSplitCount && isDirectory { // template/language/ if !canWriteLanguage(&availableLanguages, language, overwrite) { existingLanguages = append(existingLanguages, language) continue } - fetchedTemplates = append(fetchedTemplates, language) + fetchedLanguages = append(fetchedLanguages, language) } else { // template/language/* @@ -89,24 +116,15 @@ func fetchTemplates(templateURL string, overwrite bool) error { } // If relativePath is just a directory, then skip expanding it. - if len(relativePath) > 1 && relativePath[len(relativePath)-1:] != "/" { + if len(relativePath) > 1 && !isDirectory { if err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()); err != nil { - break + return nil, nil, err } } } - if len(existingLanguages) > 0 { - log.Printf("Cannot overwrite the following %d directories: %v\n", len(existingLanguages), existingLanguages) - } - zipFile.Close() - - log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedTemplates), fetchedTemplates, templateURL) - - err = removeArchive(archive) - - return err + return existingLanguages, fetchedLanguages, nil } // removeArchive removes the given file @@ -187,13 +205,13 @@ func createPath(relativePath string, perms os.FileMode) error { return err } -// Tells whether the language can be expanded from the zip or not +// canWriteLanguage() tells whether the language can be expanded from the zip or not. // availableLanguages map keeps track of which languages we know to be okay to copy. // overwrite flag will allow to force copy the language template func canWriteLanguage(availableLanguages *map[string]bool, language string, overwrite bool) bool { canWrite := false - if len(language) > 0 { - if _, ok := (*availableLanguages)[language]; ok { + if availableLanguages != nil && len(language) > 0 { + if _, found := (*availableLanguages)[language]; found { return (*availableLanguages)[language] } canWrite = checkLanguage(language, overwrite) diff --git a/commands/fetch_templates_test.go b/commands/fetch_templates_test.go index 077eaf45a..44a93f257 100644 --- a/commands/fetch_templates_test.go +++ b/commands/fetch_templates_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Alex Ellis, Eric Stoekl 2017. All rights reserved. +// Copyright (c) Alex Ellis 2017. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package commands @@ -22,6 +22,8 @@ func Test_PullTemplates(t *testing.T) { if err != nil { t.Error(err) } + + tearDown_fetch_templates(t) } func Test_fetchTemplates(t *testing.T) { @@ -37,6 +39,8 @@ func Test_fetchTemplates(t *testing.T) { if err != nil { t.Error(err) } + + tearDown_fetch_templates(t) } // tearDown_fetch_templates_test cleans all files and directories created by the test diff --git a/commands/new_function.go b/commands/new_function.go index 641585a6b..a9f9a4e41 100644 --- a/commands/new_function.go +++ b/commands/new_function.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "os" + "sort" "strings" "github.com/morikuni/aec" @@ -20,6 +21,13 @@ var ( list bool ) +// Implement interface for sorting array of strings +type StrSort []string + +func (a StrSort) Len() int { return len(a) } +func (a StrSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a StrSort) Less(i, j int) bool { return a[i] < a[j] } + func init() { newFunctionCmd.Flags().StringVar(&lang, "lang", "", "Language or template to use") newFunctionCmd.Flags().StringVarP(&gateway, "gateway", "g", defaultGateway, "Gateway URL to store in YAML stack file") @@ -145,6 +153,7 @@ functions: func printAvailableTemplates(availableTemplates []string) string { var result string + sort.Sort(StrSort(availableTemplates)) for _, template := range availableTemplates { result += fmt.Sprintf("- %s\n", template) } diff --git a/commands/template_pull.go b/commands/template_pull.go index 64870675f..0cf5ce8b0 100644 --- a/commands/template_pull.go +++ b/commands/template_pull.go @@ -1,4 +1,4 @@ -// Copyright (c) Alex Ellis, Eric Stoekl 2017. All rights reserved. +// Copyright (c) Alex Ellis 2017. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package commands diff --git a/commands/template_pull_test.go b/commands/template_pull_test.go index d2be68660..86c0a178f 100644 --- a/commands/template_pull_test.go +++ b/commands/template_pull_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Alex Ellis, Eric Stoekl 2017. All rights reserved. +// Copyright (c) Alex Ellis 2017. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package commands diff --git a/stack/language_template.go b/stack/language_template.go index d612807f8..8f2240b6a 100644 --- a/stack/language_template.go +++ b/stack/language_template.go @@ -34,8 +34,7 @@ func ParseYAMLForLanguageTemplate(file string) (*LanguageTemplate, error) { return ParseYAMLDataForLanguageTemplate(fileData) } -// iParseYAMLDataForLanguageTemplate parses YAML data into language template -// Use the alias ParseYAMLDataForLanguageTemplate +// ParseYAMLDataForLanguageTemplate parses YAML data into language template func ParseYAMLDataForLanguageTemplate(fileData []byte) (*LanguageTemplate, error) { var langTemplate LanguageTemplate var err error diff --git a/stack/schema.go b/stack/schema.go index e243c48b2..8828413f5 100644 --- a/stack/schema.go +++ b/stack/schema.go @@ -48,6 +48,7 @@ type Services struct { Provider Provider `yaml:"provider,omitempty"` } +// LanguageTemplate read from template.yml within root of a language template folder type LanguageTemplate struct { Language string `yaml:"language"` FProcess string `yaml:"fprocess"` From d77e292b581938a5955045cdcb716744d723a212 Mon Sep 17 00:00:00 2001 From: Eric Stoekl Date: Mon, 6 Nov 2017 23:23:43 -0800 Subject: [PATCH 3/4] template pull - disallow function creation when 'template.yml' is not present in language template directory Needs to add a 'template.yml' to each fake language template dir in commands/testdata/new_function/template/ in order to have current tests pass. Deletes some unnecessary .gitignore files as a result. Signed-off-by: Eric Stoekl --- commands/new_function_test.go | 8 +++++++- .../testdata/new_function/template/csharp/template.yml | 2 ++ .../testdata/new_function/template/go-armhf/.gitignore | 0 .../testdata/new_function/template/go-armhf/template.yml | 2 ++ commands/testdata/new_function/template/go/template.yml | 2 ++ .../new_function/template/node-arm64/template.yml | 2 ++ .../testdata/new_function/template/node-armhf/.gitignore | 0 .../new_function/template/node-armhf/template.yml | 2 ++ commands/testdata/new_function/template/node/template.yml | 2 ++ .../new_function/template/python-armhf/.gitignore | 0 .../new_function/template/python-armhf/template.yml | 2 ++ .../testdata/new_function/template/python/template.yml | 2 ++ .../testdata/new_function/template/python3/template.yml | 2 ++ commands/testdata/new_function/template/ruby/template.yml | 2 ++ stack/language_template.go | 7 ++++++- stack/language_template_test.go | 4 ++-- 16 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 commands/testdata/new_function/template/csharp/template.yml delete mode 100644 commands/testdata/new_function/template/go-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/go-armhf/template.yml create mode 100644 commands/testdata/new_function/template/go/template.yml create mode 100644 commands/testdata/new_function/template/node-arm64/template.yml delete mode 100644 commands/testdata/new_function/template/node-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/node-armhf/template.yml create mode 100644 commands/testdata/new_function/template/node/template.yml delete mode 100644 commands/testdata/new_function/template/python-armhf/.gitignore create mode 100644 commands/testdata/new_function/template/python-armhf/template.yml create mode 100644 commands/testdata/new_function/template/python/template.yml create mode 100644 commands/testdata/new_function/template/python3/template.yml create mode 100644 commands/testdata/new_function/template/ruby/template.yml diff --git a/commands/new_function_test.go b/commands/new_function_test.go index 1f8b2d73d..eb69e381c 100644 --- a/commands/new_function_test.go +++ b/commands/new_function_test.go @@ -30,7 +30,7 @@ const ListOptionOutput = `Languages available as templates: - python3 - ruby` -const LangNotExistsOutput = `(?m:bash is unavailable or not supported.)` +const LangNotExistsOutput = `(?m:is unavailable or not supported.)` type NewFunctionTest struct { title string @@ -52,6 +52,12 @@ var NewFunctionTests = []NewFunctionTest{ funcLang: "dockerfile", expectedMsg: SuccessMsg, }, + { + title: "invalid_1", + funcName: "new-test-invalid-1", + funcLang: "dockerfilee", + expectedMsg: LangNotExistsOutput, + }, } func runNewFunctionTest(t *testing.T, nft NewFunctionTest) { diff --git a/commands/testdata/new_function/template/csharp/template.yml b/commands/testdata/new_function/template/csharp/template.yml new file mode 100644 index 000000000..45caa67ed --- /dev/null +++ b/commands/testdata/new_function/template/csharp/template.yml @@ -0,0 +1,2 @@ +language: csharp +fprocess: dotnet ./root.dll diff --git a/commands/testdata/new_function/template/go-armhf/.gitignore b/commands/testdata/new_function/template/go-armhf/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/commands/testdata/new_function/template/go-armhf/template.yml b/commands/testdata/new_function/template/go-armhf/template.yml new file mode 100644 index 000000000..234cf4400 --- /dev/null +++ b/commands/testdata/new_function/template/go-armhf/template.yml @@ -0,0 +1,2 @@ +language: go-armhf +fprocess: ./handler diff --git a/commands/testdata/new_function/template/go/template.yml b/commands/testdata/new_function/template/go/template.yml new file mode 100644 index 000000000..5d341d5d3 --- /dev/null +++ b/commands/testdata/new_function/template/go/template.yml @@ -0,0 +1,2 @@ +language: go +fprocess: ./handler diff --git a/commands/testdata/new_function/template/node-arm64/template.yml b/commands/testdata/new_function/template/node-arm64/template.yml new file mode 100644 index 000000000..97e25c5f9 --- /dev/null +++ b/commands/testdata/new_function/template/node-arm64/template.yml @@ -0,0 +1,2 @@ +language: node-arm64 +fprocess: node index.js diff --git a/commands/testdata/new_function/template/node-armhf/.gitignore b/commands/testdata/new_function/template/node-armhf/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/commands/testdata/new_function/template/node-armhf/template.yml b/commands/testdata/new_function/template/node-armhf/template.yml new file mode 100644 index 000000000..c2313ae67 --- /dev/null +++ b/commands/testdata/new_function/template/node-armhf/template.yml @@ -0,0 +1,2 @@ +language: node-armhf +fprocess: node index.js diff --git a/commands/testdata/new_function/template/node/template.yml b/commands/testdata/new_function/template/node/template.yml new file mode 100644 index 000000000..1117efa75 --- /dev/null +++ b/commands/testdata/new_function/template/node/template.yml @@ -0,0 +1,2 @@ +language: node +fprocess: node index.js diff --git a/commands/testdata/new_function/template/python-armhf/.gitignore b/commands/testdata/new_function/template/python-armhf/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/commands/testdata/new_function/template/python-armhf/template.yml b/commands/testdata/new_function/template/python-armhf/template.yml new file mode 100644 index 000000000..ff8986785 --- /dev/null +++ b/commands/testdata/new_function/template/python-armhf/template.yml @@ -0,0 +1,2 @@ +language: python-armhf +fprocess: python index.py diff --git a/commands/testdata/new_function/template/python/template.yml b/commands/testdata/new_function/template/python/template.yml new file mode 100644 index 000000000..5487218c4 --- /dev/null +++ b/commands/testdata/new_function/template/python/template.yml @@ -0,0 +1,2 @@ +language: python +fprocess: python index.py diff --git a/commands/testdata/new_function/template/python3/template.yml b/commands/testdata/new_function/template/python3/template.yml new file mode 100644 index 000000000..ea405e8b6 --- /dev/null +++ b/commands/testdata/new_function/template/python3/template.yml @@ -0,0 +1,2 @@ +language: python3 +fprocess: python3 index.py diff --git a/commands/testdata/new_function/template/ruby/template.yml b/commands/testdata/new_function/template/ruby/template.yml new file mode 100644 index 000000000..417bd6133 --- /dev/null +++ b/commands/testdata/new_function/template/ruby/template.yml @@ -0,0 +1,2 @@ +language: ruby +fprocess: ruby index.rb diff --git a/stack/language_template.go b/stack/language_template.go index 8f2240b6a..e8c035d2c 100644 --- a/stack/language_template.go +++ b/stack/language_template.go @@ -53,7 +53,12 @@ func IsValidTemplate(lang string) bool { if strings.ToLower(lang) == "dockerfile" { found = true } else if _, err := os.Stat("./template/" + lang); err == nil { - found = true + templateYAMLPath := "./template/" + lang + "/template.yml" + + _, err := ParseYAMLForLanguageTemplate(templateYAMLPath) + if err == nil { + found = true + } } return found diff --git a/stack/language_template_test.go b/stack/language_template_test.go index d784a5dfc..081112332 100644 --- a/stack/language_template_test.go +++ b/stack/language_template_test.go @@ -71,7 +71,7 @@ func Test_IsValidTemplate(t *testing.T) { defer func() { os.RemoveAll("template") }() - if !IsValidTemplate("python") { - t.Fatalf("python must be valid") + if IsValidTemplate("python") { + t.Fatalf("python must is not valid because it does not contain template.yml") } } From 927400ade0ea85d0f3104a5b4c61292bc8769454 Mon Sep 17 00:00:00 2001 From: Eric Stoekl Date: Thu, 9 Nov 2017 08:34:50 -0800 Subject: [PATCH 4/4] Updates to 'faas-cli template pull' 1) Improve readability by adding canExpandTemplateData() function, which returns an enum telling what to do with the data found in the zip archive. 2) Add function data to ARMHF directories in template/ dir 3) More descriptive naming for test array in language_template_test.go and checkLanguage() function changed to templateFolderExists() Signed-off-by: Eric Stoekl --- commands/fetch_templates.go | 107 +++++++++++------- stack/language_template_test.go | 4 +- template/go-armhf/function/handler.go | 10 ++ template/go-armhf/main.go | 19 ++++ template/node-armhf/build.sh | 5 + template/node-armhf/function/handler.js | 5 + template/node-armhf/function/package.json | 12 ++ template/node-armhf/index.js | 31 +++++ template/node-armhf/package.json | 15 +++ template/python-armhf/function/handler.py | 2 + .../python-armhf/function/requirements.txt | 0 template/python-armhf/index.py | 15 +++ template/python-armhf/requirements.txt | 0 13 files changed, 185 insertions(+), 40 deletions(-) create mode 100644 template/go-armhf/function/handler.go create mode 100644 template/go-armhf/main.go create mode 100755 template/node-armhf/build.sh create mode 100644 template/node-armhf/function/handler.js create mode 100644 template/node-armhf/function/package.json create mode 100644 template/node-armhf/index.js create mode 100644 template/node-armhf/package.json create mode 100644 template/python-armhf/function/handler.py create mode 100644 template/python-armhf/function/requirements.txt create mode 100644 template/python-armhf/index.py create mode 100644 template/python-armhf/requirements.txt diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index 2c5b2ebe5..f351f2189 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -25,6 +25,15 @@ const ( rootLanguageDirSplitCount = 3 ) +type ExtractAction int + +const ( + ShouldExtractData ExtractAction = iota + NewTemplateFound + DirectoryAlreadyExists + SkipWritingData +) + // fetchTemplates fetch code templates from GitHub master zip file. func fetchTemplates(templateURL string, overwrite bool) error { @@ -72,7 +81,6 @@ func expandTemplatesFromZip(archive string, overwrite bool) ([]string, []string, for _, z := range zipFile.File { var rc io.ReadCloser - var isDirectory bool relativePath := z.Name[strings.Index(z.Name, "/")+1:] if strings.Index(relativePath, "template/") != 0 { @@ -80,45 +88,41 @@ func expandTemplatesFromZip(archive string, overwrite bool) ([]string, []string, continue } - if pathSplit := strings.Split(relativePath, "/"); len(pathSplit) > 2 { - language := pathSplit[1] + action, language, isDirectory := canExpandTemplateData(availableLanguages, relativePath) - // We know that this path is a directory if the last character is a "/" - isDirectory = relativePath[len(relativePath)-1:] == "/" + var expandFromZip bool - // Check if this is the root directory for a language (at ./template/lang) - if len(pathSplit) == rootLanguageDirSplitCount && isDirectory { - // template/language/ + switch action { - if !canWriteLanguage(&availableLanguages, language, overwrite) { - existingLanguages = append(existingLanguages, language) - continue - } - fetchedLanguages = append(fetchedLanguages, language) - } else { - // template/language/* + case ShouldExtractData: + expandFromZip = true + case NewTemplateFound: + expandFromZip = true + fetchedLanguages = append(fetchedLanguages, language) + case DirectoryAlreadyExists: + expandFromZip = false + existingLanguages = append(existingLanguages, language) + case SkipWritingData: + expandFromZip = false + default: + return nil, nil, errors.New(fmt.Sprintf("Don't know what to do when extracting zip: %s", archive)) - if !canWriteLanguage(&availableLanguages, language, overwrite) { - continue - } - } - } else { - // template/ - continue } - if rc, err = z.Open(); err != nil { - break - } + if expandFromZip { + if rc, err = z.Open(); err != nil { + break + } - if err = createPath(relativePath, z.Mode()); err != nil { - break - } + if err = createPath(relativePath, z.Mode()); err != nil { + break + } - // If relativePath is just a directory, then skip expanding it. - if len(relativePath) > 1 && !isDirectory { - if err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()); err != nil { - return nil, nil, err + // If relativePath is just a directory, then skip expanding it. + if len(relativePath) > 1 && !isDirectory { + if err = writeFile(rc, z.UncompressedSize64, relativePath, z.Mode()); err != nil { + return nil, nil, err + } } } } @@ -127,6 +131,33 @@ func expandTemplatesFromZip(archive string, overwrite bool) ([]string, []string, return existingLanguages, fetchedLanguages, nil } +// canExpandTemplateData() takes the map of available languages, and the +// path to a file in the zip archive. Returns what we should do with the file +// in form of ExtractAction enum, the language name, and whether it is a directory +func canExpandTemplateData(availableLanguages map[string]bool, relativePath string) (ExtractAction, string, bool) { + if pathSplit := strings.Split(relativePath, "/"); len(pathSplit) > 2 { + language := pathSplit[1] + + // We know that this path is a directory if the last character is a "/" + isDirectory := relativePath[len(relativePath)-1:] == "/" + + // Check if this is the root directory for a language (at ./template/lang) + if len(pathSplit) == rootLanguageDirSplitCount && isDirectory { + if !canWriteLanguage(availableLanguages, language, overwrite) { + return DirectoryAlreadyExists, language, isDirectory + } + return NewTemplateFound, language, isDirectory + } else { + if !canWriteLanguage(availableLanguages, language, overwrite) { + return SkipWritingData, language, isDirectory + } + return ShouldExtractData, language, isDirectory + } + } + // template/ + return SkipWritingData, "", true +} + // removeArchive removes the given file func removeArchive(archive string) error { log.Printf("Cleaning up zip file...") @@ -208,21 +239,21 @@ func createPath(relativePath string, perms os.FileMode) error { // canWriteLanguage() tells whether the language can be expanded from the zip or not. // availableLanguages map keeps track of which languages we know to be okay to copy. // overwrite flag will allow to force copy the language template -func canWriteLanguage(availableLanguages *map[string]bool, language string, overwrite bool) bool { +func canWriteLanguage(availableLanguages map[string]bool, language string, overwrite bool) bool { canWrite := false if availableLanguages != nil && len(language) > 0 { - if _, found := (*availableLanguages)[language]; found { - return (*availableLanguages)[language] + if _, found := availableLanguages[language]; found { + return availableLanguages[language] } - canWrite = checkLanguage(language, overwrite) - (*availableLanguages)[language] = canWrite + canWrite = templateFolderExists(language, overwrite) + availableLanguages[language] = canWrite } return canWrite } // Takes a language input (e.g. "node"), tells whether or not it is OK to download -func checkLanguage(language string, overwrite bool) bool { +func templateFolderExists(language string, overwrite bool) bool { dir := templateDirectory + language if _, err := os.Stat(dir); err == nil && !overwrite { // The directory template/language/ exists diff --git a/stack/language_template_test.go b/stack/language_template_test.go index 081112332..13f21b341 100644 --- a/stack/language_template_test.go +++ b/stack/language_template_test.go @@ -10,7 +10,7 @@ import ( ) func Test_ParseYAMLDataForLanguageTemplate(t *testing.T) { - dataProvider := []struct { + langTemplateTest := []struct { Input string Expected *LanguageTemplate }{ @@ -42,7 +42,7 @@ fprocess: python index.py }, } - for k, i := range dataProvider { + for k, i := range langTemplateTest { t.Run(fmt.Sprintf("%d", k), func(t *testing.T) { if actual, err := ParseYAMLDataForLanguageTemplate([]byte(i.Input)); err != nil { t.Errorf("test failed, %s", err) diff --git a/template/go-armhf/function/handler.go b/template/go-armhf/function/handler.go new file mode 100644 index 000000000..0c3f15fb1 --- /dev/null +++ b/template/go-armhf/function/handler.go @@ -0,0 +1,10 @@ +package function + +import ( + "fmt" +) + +// Handle a serverless request +func Handle(req []byte) string { + return fmt.Sprintf("Hello, Go. You said: %s", string(req)) +} diff --git a/template/go-armhf/main.go b/template/go-armhf/main.go new file mode 100644 index 000000000..2816215c9 --- /dev/null +++ b/template/go-armhf/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + + "handler/function" +) + +func main() { + input, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("Unable to read standard input: %s", err.Error()) + } + + fmt.Println(function.Handle(input)) +} diff --git a/template/node-armhf/build.sh b/template/node-armhf/build.sh new file mode 100755 index 000000000..3e0953991 --- /dev/null +++ b/template/node-armhf/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +echo "Building functions/base:node-6.9.1-alpine" +docker build -t functions/base:node-6.9.1-alpine . + diff --git a/template/node-armhf/function/handler.js b/template/node-armhf/function/handler.js new file mode 100644 index 000000000..73344f435 --- /dev/null +++ b/template/node-armhf/function/handler.js @@ -0,0 +1,5 @@ +"use strict" + +module.exports = (context, callback) => { + callback(undefined, {status: "done"}); +} diff --git a/template/node-armhf/function/package.json b/template/node-armhf/function/package.json new file mode 100644 index 000000000..3d3a3e456 --- /dev/null +++ b/template/node-armhf/function/package.json @@ -0,0 +1,12 @@ +{ + "name": "function", + "version": "1.0.0", + "description": "", + "main": "handler.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC" +} diff --git a/template/node-armhf/index.js b/template/node-armhf/index.js new file mode 100644 index 000000000..1ad234276 --- /dev/null +++ b/template/node-armhf/index.js @@ -0,0 +1,31 @@ +// Copyright (c) Alex Ellis 2017. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +"use strict" + +let getStdin = require('get-stdin'); + +let handler = require('./function/handler'); + +getStdin().then(val => { + handler(val, (err, res) => { + if (err) { + return console.error(err); + } + if(isArray(res) || isObject(res)) { + console.log(JSON.stringify(res)); + } else { + process.stdout.write(res); + } + }); +}).catch(e => { + console.error(e.stack); +}); + +let isArray = (a) => { + return (!!a) && (a.constructor === Array); +}; + +let isObject = (a) => { + return (!!a) && (a.constructor === Object); +}; diff --git a/template/node-armhf/package.json b/template/node-armhf/package.json new file mode 100644 index 000000000..d35e9625d --- /dev/null +++ b/template/node-armhf/package.json @@ -0,0 +1,15 @@ +{ + "name": "NodejsBase", + "version": "1.0.0", + "description": "", + "main": "faas_index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "get-stdin": "^5.0.1" + } +} diff --git a/template/python-armhf/function/handler.py b/template/python-armhf/function/handler.py new file mode 100644 index 000000000..1f69d4f47 --- /dev/null +++ b/template/python-armhf/function/handler.py @@ -0,0 +1,2 @@ +def handle(st): + print(st) diff --git a/template/python-armhf/function/requirements.txt b/template/python-armhf/function/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/template/python-armhf/index.py b/template/python-armhf/index.py new file mode 100644 index 000000000..415dd8e2d --- /dev/null +++ b/template/python-armhf/index.py @@ -0,0 +1,15 @@ +# Copyright (c) Alex Ellis 2017. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import sys +from function import handler + +def get_stdin(): + buf = "" + for line in sys.stdin: + buf = buf + line + return buf + +if(__name__ == "__main__"): + st = get_stdin() + handler.handle(st) diff --git a/template/python-armhf/requirements.txt b/template/python-armhf/requirements.txt new file mode 100644 index 000000000..e69de29bb