From 2ad875cfd7b33e1e71bb4f8bc89c232e29ea5456 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 10:48:03 +0000 Subject: [PATCH 1/9] Pull templates from store when not present Implements a user-experience changes: faas-cli new - when the templates folder is not present the language will be looked up in the template store using the default URL or an overriden one. faas-cli publish - when no template folder is available then a pull will be carried out if a configuration section is present with a template given within in Filtering - whilst there is no way to avoid pulling all the templates wihtin a repository, only the requested/needed ones are expanded by commands like faas-cli new. Tested mainly with existing-unit tests, faas-cli new / publish with and without a templates folder. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- README.md | 4 +- commands/build.go | 22 +++++----- commands/fetch_templates.go | 60 ++++++++++++++++++---------- commands/fetch_templates_test.go | 11 ++--- commands/new_function.go | 44 +++++++++++++++----- commands/new_function_test.go | 8 ++-- commands/publish.go | 32 +++++++++++++-- commands/template_pull.go | 5 ++- commands/template_pull_stack.go | 13 ++---- commands/template_pull_stack_test.go | 37 ----------------- 10 files changed, 132 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 6ff6c9f52..6d93eb85d 100644 --- a/README.md +++ b/README.md @@ -216,9 +216,9 @@ Find templates with: `faas-cli template store list` > Note: You can set your own custom store location with `--url` flag or set `OPENFAAS_TEMPLATE_STORE_URL` environmental variable -To pull templates from the store just write the name of the template you want `faas-cli template store pull go` or the repository and name `faas-cli template store pull openfaas/go` +To pull templates from the store just write the name of the template you want `faas-cli template store pull golang-middleware` or the repository and name `faas-cli template store pull openfaas/golang-middleware` -To get more detail on a template just use the `template store describe` command and pick a template of your choice, example with `go` would look like this `faas-cli template store describe go` +To get more detail on a template just use the `template store describe` command and pick a template of your choice, example with `go` would look like this `faas-cli template store describe golang-middleware` > Note: This feature is still in experimental stage and in the future the CLI verbs might be changed diff --git a/commands/build.go b/commands/build.go index 77ce228f8..36978b943 100644 --- a/commands/build.go +++ b/commands/build.go @@ -5,7 +5,6 @@ package commands import ( "fmt" - "log" "os" "strings" "sync" @@ -155,6 +154,8 @@ func parseBuildArgs(args []string) (map[string]string, error) { func runBuild(cmd *cobra.Command, args []string) error { + templateName := "" // templateName may not be known at this point + var services stack.Services if len(yamlFile) > 0 { parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter, envsubst) @@ -178,7 +179,7 @@ func runBuild(cmd *cobra.Command, args []string) error { } } else { templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - if pullErr := pullTemplates(templateAddress); pullErr != nil { + if pullErr := pullTemplates(templateAddress, templateName); pullErr != nil { return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr) } } @@ -304,20 +305,19 @@ func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool } // pullTemplates pulls templates from specified git remote. templateURL may be a pinned repository. -func pullTemplates(templateURL string) error { - var err error - exists, err := os.Stat("./template") - if err != nil || exists == nil { - log.Println("No templates found in current directory.") +func pullTemplates(templateURL, templateName string) error { + + if _, err := os.Stat("./template"); err != nil && os.IsNotExist(err) { + + fmt.Printf("No templates found in current directory.\n") templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL) - err = fetchTemplates(templateURL, refName, false) - if err != nil { - log.Println("Unable to download templates from Github.") + if err := fetchTemplates(templateURL, refName, templateName, false); err != nil { return err } } - return err + + return nil } func combineBuildOpts(YAMLBuildOpts []string, buildFlagBuildOpts []string) []string { diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index ad12ef949..c159f2695 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -5,7 +5,6 @@ package commands import ( "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -20,21 +19,22 @@ const DefaultTemplateRepository = "https://github.com/openfaas/templates.git" const templateDirectory = "./template/" // fetchTemplates fetch code templates using git clone. -func fetchTemplates(templateURL string, refName string, overwrite bool) error { +func fetchTemplates(templateURL, refName, templateName string, overwrite bool) error { if len(templateURL) == 0 { return fmt.Errorf("pass valid templateURL") } dir, err := os.MkdirTemp("", "openfaas-templates-*") if err != nil { - log.Fatal(err) + return err } + if !pullDebug { - defer os.RemoveAll(dir) // clean up + defer os.RemoveAll(dir) } - log.Printf("Attempting to expand templates from %s\n", templateURL) pullDebugPrint(fmt.Sprintf("Temp files in %s", dir)) + args := map[string]string{"dir": dir, "repo": templateURL} cmd := versioncontrol.GitCloneDefault @@ -47,7 +47,7 @@ func fetchTemplates(templateURL string, refName string, overwrite bool) error { return err } - preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, overwrite) + preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, templateName, overwrite) if err != nil { return err } @@ -56,7 +56,7 @@ func fetchTemplates(templateURL string, refName string, overwrite bool) error { log.Printf("Cannot overwrite the following %d template(s): %v\n", len(preExistingLanguages), preExistingLanguages) } - log.Printf("Fetched %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL) + fmt.Printf("Wrote %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL) return err } @@ -87,7 +87,7 @@ func templateFolderExists(language string, overwrite bool) bool { return true } -func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error) { +func moveTemplates(repoPath, templateName string, overwrite bool) ([]string, []string, error) { var ( existingLanguages []string fetchedLanguages []string @@ -97,7 +97,7 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error) availableLanguages := make(map[string]bool) templateDir := filepath.Join(repoPath, templateDirectory) - templates, err := ioutil.ReadDir(templateDir) + templates, err := os.ReadDir(templateDir) if err != nil { return nil, nil, fmt.Errorf("can't find templates in: %s", repoPath) } @@ -108,23 +108,35 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error) } language := file.Name() - canWrite := canWriteLanguage(availableLanguages, language, overwrite) - if canWrite { - fetchedLanguages = append(fetchedLanguages, language) - // Do cp here - languageSrc := filepath.Join(templateDir, language) - languageDest := filepath.Join(templateDirectory, language) - builder.CopyFiles(languageSrc, languageDest) - } else { - existingLanguages = append(existingLanguages, language) - continue + if len(templateName) == 0 { + + canWrite := canWriteLanguage(availableLanguages, language, overwrite) + if canWrite { + fetchedLanguages = append(fetchedLanguages, language) + // Do cp here + languageSrc := filepath.Join(templateDir, language) + languageDest := filepath.Join(templateDirectory, language) + builder.CopyFiles(languageSrc, languageDest) + } else { + existingLanguages = append(existingLanguages, language) + continue + } + } else if language == templateName { + + if canWriteLanguage(availableLanguages, language, overwrite) { + fetchedLanguages = append(fetchedLanguages, language) + // Do cp here + languageSrc := filepath.Join(templateDir, language) + languageDest := filepath.Join(templateDirectory, language) + builder.CopyFiles(languageSrc, languageDest) + } } } return existingLanguages, fetchedLanguages, nil } -func pullTemplate(repository string) error { +func pullTemplate(repository, templateName string) error { if _, err := os.Stat(repository); err != nil { if !versioncontrol.IsGitRemote(repository) && !versioncontrol.IsPinnedGitRemote(repository) { return fmt.Errorf("the repository URL must be a valid git repo uri") @@ -143,8 +155,12 @@ func pullTemplate(repository string) error { } } - fmt.Printf("Fetch templates from repository: %s at %s\n", repository, refName) - if err := fetchTemplates(repository, refName, overwrite); err != nil { + refStr := "" + if len(refName) > 0 { + refStr = fmt.Sprintf(" @ %s", refName) + } + fmt.Printf("Fetch templates from repository: %s%s\n", repository, refStr) + if err := fetchTemplates(repository, refName, templateName, overwrite); err != nil { return fmt.Errorf("error while fetching templates: %s", err) } diff --git a/commands/fetch_templates_test.go b/commands/fetch_templates_test.go index a7e3b204a..911e6bcb9 100644 --- a/commands/fetch_templates_test.go +++ b/commands/fetch_templates_test.go @@ -21,17 +21,18 @@ func Test_PullTemplates(t *testing.T) { defer os.RemoveAll(localTemplateRepository) defer tearDownFetchTemplates(t) + templateName := "" t.Run("pullTemplates", func(t *testing.T) { defer tearDownFetchTemplates(t) - if err := pullTemplates(localTemplateRepository); err != nil { - t.Fatal(err) + if err := pullTemplates(localTemplateRepository, templateName); err != nil { + t.Fatalf("Trying to pull %s, error: %s", localTemplateRepository, err) } }) t.Run("fetchTemplates with master ref", func(t *testing.T) { defer tearDownFetchTemplates(t) - if err := fetchTemplates(localTemplateRepository, "master", false); err != nil { + if err := fetchTemplates(localTemplateRepository, "master", templateName, false); err != nil { t.Fatal(err) } @@ -40,8 +41,8 @@ func Test_PullTemplates(t *testing.T) { t.Run("fetchTemplates with default ref", func(t *testing.T) { defer tearDownFetchTemplates(t) - err := fetchTemplates(localTemplateRepository, "", false) - if err != nil { + templateName := "" + if err := fetchTemplates(localTemplateRepository, "", templateName, false); err != nil { t.Error(err) } diff --git a/commands/new_function.go b/commands/new_function.go index 7c216295e..b58fa8149 100644 --- a/commands/new_function.go +++ b/commands/new_function.go @@ -5,7 +5,6 @@ package commands import ( "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -74,7 +73,7 @@ func validateFunctionName(functionName string) error { // preRunNewFunction validates args & flags func preRunNewFunction(cmd *cobra.Command, args []string) error { - if list == true { + if list { return nil } @@ -102,16 +101,18 @@ func preRunNewFunction(cmd *cobra.Command, args []string) error { } func runNewFunction(cmd *cobra.Command, args []string) error { - if list == true { + if list { var availableTemplates []string - templateFolders, err := ioutil.ReadDir(templateDirectory) + templateFolders, err := os.ReadDir(templateDirectory) if err != nil { return fmt.Errorf(`no language templates were found. Download templates: - faas-cli template pull download the default templates - faas-cli template store list view the community template store`) + faas-cli template pull download the default templates + faas-cli template store list view the template store + faas-cli template store pull NAME download the default templates + faas-cli new --lang NAME Attempt to download NAME from the template store`) } for _, file := range templateFolders { @@ -125,11 +126,34 @@ Download templates: return nil } - templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - pullTemplates(templateAddress) - if !stack.IsValidTemplate(language) { - return fmt.Errorf("template: \"%s\" was not found in the templates directory", language) + + envTemplateRepoStore := os.Getenv(templateStoreURLEnvironment) + storeURL := getTemplateStoreURL(templateStoreURL, envTemplateRepoStore, DefaultTemplatesStore) + + templatesInfo, err := getTemplateInfo(storeURL) + if err != nil { + return fmt.Errorf("error while getting templates info: %s", err) + } + + var templateInfo *TemplateInfo + for _, info := range templatesInfo { + if info.TemplateName == language { + templateInfo = &info + break + } + } + + if templateInfo == nil { + return fmt.Errorf("template: \"%s\" was not found in the templates folder or in the store", language) + } + + templateName := templateInfo.TemplateName + + if err := pullTemplate(templateInfo.Repository, templateName); err != nil { + return fmt.Errorf("error while pulling template: %s", err) + } + } var fileName, outputMsg string diff --git a/commands/new_function_test.go b/commands/new_function_test.go index d615eb5c1..078cebe09 100644 --- a/commands/new_function_test.go +++ b/commands/new_function_test.go @@ -27,13 +27,15 @@ const ( - dockerfile - ruby` - LangNotExistsOutput = `(template: \"([0-9A-Za-z-])*\" was not found in the templates directory)` + LangNotExistsOutput = `(template: \"([0-9A-Za-z-])*\" was not found in the templates folder or in the store)` FunctionExistsOutput = `(Function (.+)? already exists in (.+)? file)` NoTemplates = `no language templates were found. Download templates: - faas-cli template pull download the default templates - faas-cli template store list view the community template store` + faas-cli template pull download the default templates + faas-cli template store list view the template store + faas-cli template store pull NAME download the default templates + faas-cli new --lang NAME Attempt to download NAME from the template store` InvalidFileSuffix = "when appending to a stack the suffix should be .yml or .yaml" ) diff --git a/commands/publish.go b/commands/publish.go index fd08f746f..db5bf4940 100644 --- a/commands/publish.go +++ b/commands/publish.go @@ -139,9 +139,35 @@ func runPublish(cmd *cobra.Command, args []string) error { } } - templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - if pullErr := pullTemplates(templateAddress); pullErr != nil { - return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr) + needTemplates := false + for _, function := range services.Functions { + if len(function.Language) > 0 { + needTemplates = true + break + } + } + + templatesFound := false + if stat, err := os.Stat("./template"); err == nil && stat.IsDir() { + templatesFound = true + } + + // if no templates are configured, but they exist in the configuration section, + // attempt to pull them first + if !templatesFound && needTemplates { + if len(services.StackConfiguration.TemplateConfigs) > 0 { + if err := pullStackTemplates(services.StackConfiguration.TemplateConfigs, cmd); err != nil { + return err + } + + } + } + + if needTemplates { + if _, err := os.Stat("./template"); os.IsNotExist(err) { + + return fmt.Errorf(`the "template" directory is missing but required by at least one function`) + } } if resetQemu { diff --git a/commands/template_pull.go b/commands/template_pull.go index 6000074c7..af496f035 100644 --- a/commands/template_pull.go +++ b/commands/template_pull.go @@ -43,8 +43,11 @@ func runTemplatePull(cmd *cobra.Command, args []string) error { if len(args) > 0 { repository = args[0] } + + templateName := "" // templateName may not be known at this point + repository = getTemplateURL(repository, os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - return pullTemplate(repository) + return pullTemplate(repository, templateName) } func pullDebugPrint(message string) { diff --git a/commands/template_pull_stack.go b/commands/template_pull_stack.go index e82b6cb45..41987ded7 100644 --- a/commands/template_pull_stack.go +++ b/commands/template_pull_stack.go @@ -77,7 +77,9 @@ func pullStackTemplates(templateInfo []stack.TemplateSource, cmd *cobra.Command) return pullErr } } else { - pullErr := pullTemplate(val.Source) + + templateName := val.Name + pullErr := pullTemplate(val.Source, templateName) if pullErr != nil { return pullErr } @@ -86,15 +88,6 @@ func pullStackTemplates(templateInfo []stack.TemplateSource, cmd *cobra.Command) return nil } -func findTemplate(templateInfo []stack.TemplateSource, customName string) (specificTemplate *stack.TemplateSource) { - for _, val := range templateInfo { - if val.Name == customName { - return &val - } - } - return nil -} - // filter templates which are already available on filesystem func filterExistingTemplates(templateInfo []stack.TemplateSource, templatesDir string) ([]stack.TemplateSource, error) { var newTemplates []stack.TemplateSource diff --git a/commands/template_pull_stack_test.go b/commands/template_pull_stack_test.go index ee6509632..1770203ee 100644 --- a/commands/template_pull_stack_test.go +++ b/commands/template_pull_stack_test.go @@ -3,49 +3,12 @@ package commands import ( "os" "path/filepath" - "reflect" "testing" "github.com/openfaas/faas-cli/builder" "github.com/openfaas/faas-cli/stack" ) -func Test_findTemplate(t *testing.T) { - tests := []struct { - title string - desiredTemplate string - existingTemplates []stack.TemplateSource - expectedTemplate *stack.TemplateSource - }{ - { - title: "Desired template is found", - desiredTemplate: "powershell", - existingTemplates: []stack.TemplateSource{ - {Name: "powershell", Source: "exampleURL"}, - {Name: "rust", Source: "exampleURL"}, - }, - expectedTemplate: &stack.TemplateSource{Name: "powershell", Source: "exampleURL"}, - }, - { - title: "Desired template is not found", - desiredTemplate: "golang", - existingTemplates: []stack.TemplateSource{ - {Name: "powershell", Source: "exampleURL"}, - {Name: "rust", Source: "exampleURL"}, - }, - expectedTemplate: nil, - }, - } - for _, test := range tests { - t.Run(test.title, func(t *testing.T) { - result := findTemplate(test.existingTemplates, test.desiredTemplate) - if !reflect.DeepEqual(result, test.expectedTemplate) { - t.Errorf("Wanted template: `%s` got `%s`", test.expectedTemplate, result) - } - }) - } -} - func Test_pullAllTemplates(t *testing.T) { tests := []struct { title string From 3831ce71460a29f2c78ada2dced5f2aabba6bb78 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 10:56:58 +0000 Subject: [PATCH 2/9] Switch out ruby for dockerfile in tests The tests relied on ruby being a long term template in the templates repository, however we are moving to a model with a repository per template, and this template has gone to EOL. The Dockerfile template makes sense as a replacement and is still supported. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- commands/new_function_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/commands/new_function_test.go b/commands/new_function_test.go index 078cebe09..191212dda 100644 --- a/commands/new_function_test.go +++ b/commands/new_function_test.go @@ -272,7 +272,7 @@ func Test_languageNotExists(t *testing.T) { func Test_appendInvalidSuffix(t *testing.T) { const functionName = "samplefunc" - const functionLang = "ruby" + const functionLang = "dockerfile" templatePullLocalTemplateRepo(t) defer tearDownFetchTemplates(t) @@ -294,7 +294,7 @@ func Test_appendInvalidSuffix(t *testing.T) { func Test_appendInvalidFile(t *testing.T) { const functionName = "samplefunc" - const functionLang = "ruby" + const functionLang = "dockerfile" templatePullLocalTemplateRepo(t) defer tearDownFetchTemplates(t) @@ -318,7 +318,7 @@ func Test_duplicateFunctionName(t *testing.T) { resetForTest() const functionName = "samplefunc" - const functionLang = "ruby" + const functionLang = "dockerfile" templatePullLocalTemplateRepo(t) defer tearDownFetchTemplates(t) @@ -346,7 +346,7 @@ func Test_duplicateFunctionName(t *testing.T) { func Test_backfillTemplates(t *testing.T) { resetForTest() const functionName = "samplefunc" - const functionLang = "ruby" + const functionLang = "dockerfile" // Delete cached templates localTemplateRepository := setupLocalTemplateRepo(t) From e3c149ac7e2f8bd20efd28dec1c11b5eade8687a Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 10:58:22 +0000 Subject: [PATCH 3/9] Make test more accurate by fixing URLs Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- commands/template_pull_stack_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/template_pull_stack_test.go b/commands/template_pull_stack_test.go index 1770203ee..28b91bf37 100644 --- a/commands/template_pull_stack_test.go +++ b/commands/template_pull_stack_test.go @@ -77,8 +77,8 @@ func Test_filterExistingTemplates(t *testing.T) { defer os.RemoveAll(templatesDir) templates := []stack.TemplateSource{ - {Name: "dockerfile", Source: "https://github.com/openfaas-incubator/powershell-http-template"}, - {Name: "ruby", Source: "https://github.com/openfaas-incubator/openfaas-rust-template"}, + {Name: "dockerfile", Source: "https://github.com/openfaas/templates"}, + {Name: "ruby", Source: "https://github.com/openfaas/classic-templates"}, {Name: "perl", Source: "https://github.com/openfaas-incubator/perl-template"}, } From 04a590c12aceadba8dd34aba29300dc5586705f6 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:04:23 +0000 Subject: [PATCH 4/9] Accept stack.yaml as a default file if present Requested by @welteki, if stack.yml is present then it is taken to be the default with a --yaml/-f argument being needed. This change also looks for stack.yaml as a valid alternative name. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- commands/faas.go | 10 +++++++--- commands/faas_test.go | 6 +++--- commands/template_store_list.go | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/commands/faas.go b/commands/faas.go index 16c1355d3..9272cd08a 100644 --- a/commands/faas.go +++ b/commands/faas.go @@ -21,9 +21,11 @@ import ( ) const ( - defaultGateway = "http://127.0.0.1:8080" - defaultNetwork = "" - defaultYAML = "stack.yml" + defaultGateway = "http://127.0.0.1:8080" + defaultNetwork = "" + defaultYML = "stack.yml" + defaultYAML = "stack.yaml" + defaultSchemaVersion = "1.0" ) @@ -144,6 +146,8 @@ func checkAndSetDefaultYaml() { // Check if there is a default yaml file and set it if _, err := stat(defaultYAML); err == nil { yamlFile = defaultYAML + } else if _, err := stat(defaultYML); err == nil { + yamlFile = defaultYML } } diff --git a/commands/faas_test.go b/commands/faas_test.go index cad86fe2c..757a9bb14 100644 --- a/commands/faas_test.go +++ b/commands/faas_test.go @@ -1,7 +1,7 @@ package commands import ( - "io/ioutil" + "io" "os" "testing" ) @@ -11,7 +11,7 @@ var mockStatParams string func setupFaas(statError error) { yamlFile = "" mockStatParams = "" - faasCmd.SetOutput(ioutil.Discard) + faasCmd.SetOutput(io.Discard) stat = func(f string) (os.FileInfo, error) { mockStatParams = f @@ -19,7 +19,7 @@ func setupFaas(statError error) { } } -func TestCallsStatWithDefaulYAMLFileName(t *testing.T) { +func TestCallsStatWithDefaultYAMLFileName(t *testing.T) { setupFaas(nil) Execute([]string{"help"}) diff --git a/commands/template_store_list.go b/commands/template_store_list.go index 22aefc92e..8de486426 100644 --- a/commands/template_store_list.go +++ b/commands/template_store_list.go @@ -138,7 +138,7 @@ func getTemplateInfo(repository string) ([]TemplateInfo, error) { templatesInfo := []TemplateInfo{} if err := json.Unmarshal(body, &templatesInfo); err != nil { - return nil, fmt.Errorf("can't unmarshal text: %s", err.Error()) + return nil, fmt.Errorf("can't unmarshal text: %s, value: %s", err.Error(), string(body)) } sortTemplates(templatesInfo) From 5761ec843775eca5f85bc6bfe20e6c190390292f Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:07:11 +0000 Subject: [PATCH 5/9] Update CI test to use python3-http instead of EOL template Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- build_integration_test.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build_integration_test.sh b/build_integration_test.sh index 33c68ba2d..fd4d84913 100755 --- a/build_integration_test.sh +++ b/build_integration_test.sh @@ -1,7 +1,8 @@ #!/bin/bash cli="./bin/faas-cli" -template="python3" + +TEMPLATE_NAME="python3-http" get_package() { uname=$(uname) @@ -22,21 +23,20 @@ get_package() { case $arch in "armv6l" | "armv7l") cli="./faas-cli-armhf" - template="python3-armhf" ;; esac ;; esac echo "Using package $cli" - echo "Using template $template" + echo "Using template $TEMPLATE_NAME" } build_faas_function() { function_name=$1 - eval $cli new $function_name --lang $template + eval $cli new $function_name --lang $TEMPLATE_NAME cat << EOF > $function_name/handler.py def handle(req): @@ -120,6 +120,13 @@ EOF rm -rf got.txt want.txt $function_name* } +get_templates() { + echo "Getting templates..." + eval $cli template pull $TEMPLATE_NAME +} + +get_templates + get_package build_faas_function $FUNCTION run_and_test $FUNCTION $PORT $FUNCTION_UP_TIMEOUT From bd0300f76635519c65e1d24c506858fe10b6b387 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:08:00 +0000 Subject: [PATCH 6/9] Name template from python3 to python3-http Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- commands/generate_test.go | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/commands/generate_test.go b/commands/generate_test.go index f26e28d40..3109e6590 100644 --- a/commands/generate_test.go +++ b/commands/generate_test.go @@ -150,7 +150,7 @@ functions: handler: ./sample/url-ping image: alexellis/faas-url-ping:0.2 astronaut-finder: - lang: python3 + lang: python3-http handler: ./astronaut-finder image: astronaut-finder environment: @@ -348,43 +348,43 @@ provider: network: "func_functions" functions: fn1: - lang: python3 + lang: python3-http handler: ./fn1 image: fn1:latest fn2: - lang: python3 + lang: python3-http handler: ./fn2 image: fn2:latest fn3: - lang: python3 + lang: python3-http handler: ./fn3 image: fn3:latest fn4: - lang: python3 + lang: python3-http handler: ./fn4 image: fn4:latest fn5: - lang: python3 + lang: python3-http handler: ./fn5 image: fn5:latest fn6: - lang: python3 + lang: python3-http handler: ./fn6 image: fn6:latest fn7: - lang: python3 + lang: python3-http handler: ./fn7 image: fn7:latest fn8: - lang: python3 + lang: python3-http handler: ./fn8 image: fn8:latest fn9: - lang: python3 + lang: python3-http handler: ./fn9 image: fn9:latest fn10: - lang: python3 + lang: python3-http handler: ./fn10 image: fn10:latest`, Output: []string{ @@ -405,43 +405,43 @@ provider: network: "func_functions" functions: fn3: - lang: python3 + lang: python3-http handler: ./fn3 image: fn3:latest fn7: - lang: python3 + lang: python3-http handler: ./fn7 image: fn7:latest fn2: - lang: python3 + lang: python3-http handler: ./fn2 image: fn2:latest fn10: - lang: python3 + lang: python3-http handler: ./fn10 image: fn10:latest fn5: - lang: python3 + lang: python3-http handler: ./fn5 image: fn5:latest fn1: - lang: python3 + lang: python3-http handler: ./fn1 image: fn1:latest fn6: - lang: python3 + lang: python3-http handler: ./fn6 image: fn6:latest fn9: - lang: python3 + lang: python3-http handler: ./fn9 image: fn9:latest fn4: - lang: python3 + lang: python3-http handler: ./fn4 image: fn4:latest fn8: - lang: python3 + lang: python3-http handler: ./fn8 image: fn8:latest`, Output: []string{ @@ -462,43 +462,43 @@ provider: network: "func_functions" functions: fn3: - lang: python3 + lang: python3-http handler: ./fn3 image: fn3:latest fn7: - lang: python3 + lang: python3-http handler: ./fn7 image: fn7:latest fn2: - lang: python3 + lang: python3-http handler: ./fn2 image: fn2:latest fn10: - lang: python3 + lang: python3-http handler: ./fn10 image: fn10:latest fn5: - lang: python3 + lang: python3-http handler: ./fn5 image: fn5:latest fn1: - lang: python3 + lang: python3-http handler: ./fn1 image: fn1:latest fn6: - lang: python3 + lang: python3-http handler: ./fn6 image: fn6:latest fn9: - lang: python3 + lang: python3-http handler: ./fn9 image: fn9:latest fn4: - lang: python3 + lang: python3-http handler: ./fn4 image: fn4:latest fn8: - lang: python3 + lang: python3-http handler: ./fn8 image: fn8:latest`, Output: []string{ From 2a9b309e4461d00c4b0469332120325ae89cecd3 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:08:59 +0000 Subject: [PATCH 7/9] Fail integration test early through set mechanisms Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- build_integration_test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build_integration_test.sh b/build_integration_test.sh index fd4d84913..976643ce9 100755 --- a/build_integration_test.sh +++ b/build_integration_test.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e # Exit script immediately on first error. +set -x # Print commands and their arguments as they are executed. +set -o pipefail # Print commands and their arguments as they are + cli="./bin/faas-cli" TEMPLATE_NAME="python3-http" From 452db1e0f1899c906f23ae788a78370bd5f80557 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:14:29 +0000 Subject: [PATCH 8/9] Better error handling and Handle potential permissions issue err could be nil, so it is checked before checking for the type of error. This may not be needed, but is a better default to establish since code is often copied by new contributors when adding features. Addresses comment by @rgee0 for handling error from stat. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- builder/build.go | 2 +- builder/copy_test.go | 2 +- builder/publish.go | 2 +- commands/build.go | 18 ++++++++++++------ commands/deploy.go | 2 +- commands/new_function.go | 2 +- commands/new_function_test.go | 6 +++--- commands/plugin_get.go | 2 +- commands/publish.go | 2 +- commands/registry_login.go | 2 +- commands/template_pull_stack.go | 2 +- config/config_file.go | 6 +++--- config/config_file_test.go | 3 +-- 13 files changed, 28 insertions(+), 23 deletions(-) diff --git a/builder/build.go b/builder/build.go index 64acb0a8d..1edb416e3 100644 --- a/builder/build.go +++ b/builder/build.go @@ -34,7 +34,7 @@ func BuildImage(image string, handler string, functionName string, language stri if stack.IsValidTemplate(language) { pathToTemplateYAML := fmt.Sprintf("./template/%s/template.yml", language) - if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { + if _, err := os.Stat(pathToTemplateYAML); err != nil && os.IsNotExist(err) { return err } diff --git a/builder/copy_test.go b/builder/copy_test.go index 824f2b134..ea573a806 100644 --- a/builder/copy_test.go +++ b/builder/copy_test.go @@ -103,7 +103,7 @@ func checkDestinationFiles(dir string, numberOfFiles, mode int) error { // Check each file inside the destination folder for i := 1; i <= numberOfFiles; i++ { fileStat, err := os.Stat(fmt.Sprintf("%s/test-file-%d", dir, i)) - if os.IsNotExist(err) { + if err != nil && os.IsNotExist(err) { return err } if fileStat.Mode() != os.FileMode(mode) { diff --git a/builder/publish.go b/builder/publish.go index 8eb217425..c47efa747 100644 --- a/builder/publish.go +++ b/builder/publish.go @@ -48,7 +48,7 @@ func PublishImage(image string, handler string, functionName string, language st if stack.IsValidTemplate(language) { pathToTemplateYAML := fmt.Sprintf("./template/%s/template.yml", language) - if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { + if _, err := os.Stat(pathToTemplateYAML); err != nil && os.IsNotExist(err) { return err } diff --git a/commands/build.go b/commands/build.go index 36978b943..6c5e7dae7 100644 --- a/commands/build.go +++ b/commands/build.go @@ -179,8 +179,8 @@ func runBuild(cmd *cobra.Command, args []string) error { } } else { templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - if pullErr := pullTemplates(templateAddress, templateName); pullErr != nil { - return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr) + if err := pullTemplates(templateAddress, templateName); err != nil { + return fmt.Errorf("could not pull templates: %v", err) } } @@ -307,12 +307,18 @@ func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool // pullTemplates pulls templates from specified git remote. templateURL may be a pinned repository. func pullTemplates(templateURL, templateName string) error { - if _, err := os.Stat("./template"); err != nil && os.IsNotExist(err) { + if _, err := os.Stat("./template"); err != nil { + if os.IsNotExist(err) { - fmt.Printf("No templates found in current directory.\n") + fmt.Printf("No templates found in current directory.\n") - templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL) - if err := fetchTemplates(templateURL, refName, templateName, false); err != nil { + templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL) + if err := fetchTemplates(templateURL, refName, templateName, false); err != nil { + return err + } + } else { + + // Perhaps there was a permissions issue or something else. return err } } diff --git a/commands/deploy.go b/commands/deploy.go index 10baf2e4a..dd882a6c0 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -458,7 +458,7 @@ func deriveFprocess(function stack.Function) (string, error) { } pathToTemplateYAML := "./template/" + function.Language + "/template.yml" - if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { + if _, err := os.Stat(pathToTemplateYAML); err != nil && os.IsNotExist(err) { return "", err } diff --git a/commands/new_function.go b/commands/new_function.go index b58fa8149..894f5c7b6 100644 --- a/commands/new_function.go +++ b/commands/new_function.go @@ -207,7 +207,7 @@ Download templates: } pathToTemplateYAML := fmt.Sprintf("./template/%s/template.yml", language) - if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) { + if _, err := os.Stat(pathToTemplateYAML); err != nil && os.IsNotExist(err) { return err } diff --git a/commands/new_function_test.go b/commands/new_function_test.go index 191212dda..09fdf9df4 100644 --- a/commands/new_function_test.go +++ b/commands/new_function_test.go @@ -150,18 +150,18 @@ func runNewFunctionTest(t *testing.T, nft NewFunctionTest) { if nft.expectedMsg == SuccessMsg { // Make sure that the folder and file was created: - if _, err := os.Stat("./" + dirName); os.IsNotExist(err) { + if _, err := os.Stat("./" + dirName); err != nil && os.IsNotExist(err) { t.Fatalf("%s/ directory was not created", dirName) } // Check that the Dockerfile was created if funcLang == "Dockerfile" || funcLang == "dockerfile" { - if _, err := os.Stat("./" + dirName + "/Dockerfile"); os.IsNotExist(err) { + if _, err := os.Stat("./" + dirName + "/Dockerfile"); err != nil && os.IsNotExist(err) { t.Fatalf("Dockerfile language should create a Dockerfile for you: %s", funcName) } } - if _, err := os.Stat(funcYAML); os.IsNotExist(err) { + if _, err := os.Stat(funcYAML); err != nil && os.IsNotExist(err) { t.Fatalf("\"%s\" yaml file was not created", funcYAML) } diff --git a/commands/plugin_get.go b/commands/plugin_get.go index 9f1d17290..d7a0566b5 100644 --- a/commands/plugin_get.go +++ b/commands/plugin_get.go @@ -99,7 +99,7 @@ func runPluginGetCmd(cmd *cobra.Command, args []string) error { } } - if _, err := os.Stat(pluginDir); os.IsNotExist(err) { + if _, err := os.Stat(pluginDir); err != nil && os.IsNotExist(err) { if err := os.MkdirAll(pluginDir, 0755); err != nil && os.ErrExist != err { return fmt.Errorf("failed to create plugin directory %s: %w", pluginDir, err) } diff --git a/commands/publish.go b/commands/publish.go index db5bf4940..0c697e7bd 100644 --- a/commands/publish.go +++ b/commands/publish.go @@ -164,7 +164,7 @@ func runPublish(cmd *cobra.Command, args []string) error { } if needTemplates { - if _, err := os.Stat("./template"); os.IsNotExist(err) { + if _, err := os.Stat("./template"); err != nil && os.IsNotExist(err) { return fmt.Errorf(`the "template" directory is missing but required by at least one function`) } diff --git a/commands/registry_login.go b/commands/registry_login.go index 21e438b7f..8f9bb4a07 100644 --- a/commands/registry_login.go +++ b/commands/registry_login.go @@ -174,7 +174,7 @@ func generateECRRegistryAuth(accountID, region string) ([]byte, error) { func writeFileToFassCLITmp(fileBytes []byte) error { path := "./credentials" - if _, err := os.Stat(path); os.IsNotExist(err) { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { err := os.Mkdir(path, 0744) if err != nil { return err diff --git a/commands/template_pull_stack.go b/commands/template_pull_stack.go index 41987ded7..bee0ed137 100644 --- a/commands/template_pull_stack.go +++ b/commands/template_pull_stack.go @@ -93,7 +93,7 @@ func filterExistingTemplates(templateInfo []stack.TemplateSource, templatesDir s var newTemplates []stack.TemplateSource for _, info := range templateInfo { templatePath := fmt.Sprintf("%s/%s", templatesDir, info.Name) - if _, err := os.Stat(templatePath); os.IsNotExist(err) { + if _, err := os.Stat(templatePath); err != nil && os.IsNotExist(err) { newTemplates = append(newTemplates, info) } } diff --git a/config/config_file.go b/config/config_file.go index 72b9698f0..97add8bd8 100644 --- a/config/config_file.go +++ b/config/config_file.go @@ -135,7 +135,7 @@ func EnsureFile() (string, error) { return "", err } - if _, err := os.Stat(filePath); os.IsNotExist(err) { + if _, err := os.Stat(filePath); err != nil && os.IsNotExist(err) { file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return "", err @@ -155,7 +155,7 @@ func fileExists() bool { } filePath := path.Clean(filepath.Join(dirPath, DefaultFile)) - if _, err := os.Stat(filePath); os.IsNotExist(err) { + if _, err := os.Stat(filePath); err != nil && os.IsNotExist(err) { return false } @@ -185,7 +185,7 @@ func (configFile *ConfigFile) save() error { func (configFile *ConfigFile) load() error { conf := &ConfigFile{} - if _, err := os.Stat(configFile.FilePath); os.IsNotExist(err) { + if _, err := os.Stat(configFile.FilePath); err != nil && os.IsNotExist(err) { return fmt.Errorf("can't load config from non existent filePath") } diff --git a/config/config_file_test.go b/config/config_file_test.go index 338c5e62e..75490c744 100644 --- a/config/config_file_test.go +++ b/config/config_file_test.go @@ -234,8 +234,7 @@ func Test_EnsureFile(t *testing.T) { if err != nil { t.Error(err.Error()) } - _, err = os.Stat(cfg) - if os.IsNotExist(err) { + if _, err = os.Stat(cfg); err != nil && os.IsNotExist(err) { t.Errorf("expected config at %s", cfg) } } From 8fc6ee00f9571306cd4c67530e88bd532afe9e19 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 23 Jan 2025 11:40:16 +0000 Subject: [PATCH 9/9] Fix bash test Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- build_integration_test.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build_integration_test.sh b/build_integration_test.sh index 976643ce9..3c90c4c66 100755 --- a/build_integration_test.sh +++ b/build_integration_test.sh @@ -43,9 +43,14 @@ build_faas_function() { eval $cli new $function_name --lang $TEMPLATE_NAME cat << EOF > $function_name/handler.py -def handle(req): - - return "Function output from integration testing: Hello World!" +def handle(event, context): + return { + "statusCode": 200, + "body": {"message": "Hello from OpenFaaS!"}, + "headers": { + "Content-Type": "application/json" + } + } EOF eval $cli build -f $function_name.yml @@ -126,7 +131,7 @@ EOF get_templates() { echo "Getting templates..." - eval $cli template pull $TEMPLATE_NAME + eval $cli template store pull $TEMPLATE_NAME } get_templates