From c9c980edd0a315dc24305f6052ab9979ac5feaa5 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Sun, 29 Oct 2023 15:30:16 +0100 Subject: [PATCH] feat: allow URL and local files for Store locations Enable local files and remote files over HTTP(S) as store locations for the Function and Template stores. This enables the following commands to work as expected ```sh faas-cli store list --url $HOME/Downloads/functions.json faas-cli store list --url ~/Downloads/functions.json faas-cli store list --url https://internal/organization.com/openfaas/functions.json faas-cli template store list --url $HOME/Downloads/templates.json faas-cli template store list --url ~/Downloads/templates.json faas-cli template store list --url https://internal.organization.com/openfaas/templates.json ``` Additional locations such as ipfs, blob storage (s3, Google Cloud Storge, etc) and other could also be supported via a small patch to the `ReadJSON` implementation. Signed-off-by: Lucas Roesler --- commands/store.go | 50 +----- commands/store_deploy.go | 2 +- commands/store_describe.go | 3 +- commands/store_list.go | 3 +- commands/template_store_describe.go | 2 +- commands/template_store_list.go | 43 +---- commands/template_store_pull.go | 2 +- proxy/function_store.go | 100 ++++++++--- .../alexellis/go-execute/pkg/v2/exec.go | 156 ++++++++++++++++++ 9 files changed, 252 insertions(+), 109 deletions(-) create mode 100644 vendor/github.com/alexellis/go-execute/pkg/v2/exec.go diff --git a/commands/store.go b/commands/store.go index 5e7062810..23a19930a 100644 --- a/commands/store.go +++ b/commands/store.go @@ -4,14 +4,8 @@ package commands import ( - "encoding/json" - "fmt" - "io" - "net/http" "strings" - "time" - "github.com/openfaas/faas-cli/proxy" storeV2 "github.com/openfaas/faas-cli/schema/store/v2" "github.com/spf13/cobra" ) @@ -34,12 +28,13 @@ var ( const ( defaultStore = "https://raw.githubusercontent.com/openfaas/store/master/functions.json" maxDescriptionLen = 40 + storeAddressDoc = `Alternative path to Function Store metadata. It may be an http(s) URL or a local path to a JSON file.` ) var platformValue string func init() { - storeCmd.PersistentFlags().StringVarP(&storeAddress, "url", "u", defaultStore, "Alternative Store URL starting with http(s)://") + storeCmd.PersistentFlags().StringVarP(&storeAddress, "url", "u", defaultStore, storeAddressDoc) storeCmd.PersistentFlags().StringVarP(&platformValue, "platform", "p", Platform, "Target platform for store") faasCmd.AddCommand(storeCmd) @@ -51,47 +46,6 @@ var storeCmd = &cobra.Command{ Long: "Allows browsing and deploying OpenFaaS functions from a store", } -func storeList(store string) ([]storeV2.StoreFunction, error) { - - var storeData storeV2.Store - - store = strings.TrimRight(store, "/") - - timeout := 60 * time.Second - tlsInsecure := false - - client := proxy.MakeHTTPClient(&timeout, tlsInsecure) - - res, err := client.Get(store) - if err != nil { - return nil, fmt.Errorf("cannot connect to OpenFaaS store at URL: %s", store) - } - - if res.Body != nil { - defer res.Body.Close() - } - - switch res.StatusCode { - case http.StatusOK: - bytesOut, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store) - } - - jsonErr := json.Unmarshal(bytesOut, &storeData) - if jsonErr != nil { - return nil, fmt.Errorf("cannot parse result from OpenFaaS store at URL: %s\n%s", store, jsonErr.Error()) - } - default: - bytesOut, err := io.ReadAll(res.Body) - if err == nil { - return nil, fmt.Errorf("server returned unexpected status code: %d - %s", res.StatusCode, string(bytesOut)) - } - } - - return storeData.Functions, nil -} - func filterStoreList(functions []storeV2.StoreFunction, platform string) []storeV2.StoreFunction { var filteredList []storeV2.StoreFunction diff --git a/commands/store_deploy.go b/commands/store_deploy.go index 737a12c42..f882b0f71 100644 --- a/commands/store_deploy.go +++ b/commands/store_deploy.go @@ -71,7 +71,7 @@ func preRunEStoreDeploy(cmd *cobra.Command, args []string) error { func runStoreDeploy(cmd *cobra.Command, args []string) error { targetPlatform := getTargetPlatform(platformValue) - storeItems, err := storeList(storeAddress) + storeItems, err := proxy.FunctionStoreList(storeAddress) if err != nil { return err } diff --git a/commands/store_describe.go b/commands/store_describe.go index 640219613..c3e3e96f4 100644 --- a/commands/store_describe.go +++ b/commands/store_describe.go @@ -9,6 +9,7 @@ import ( "text/tabwriter" "github.com/mitchellh/go-wordwrap" + "github.com/openfaas/faas-cli/proxy" storeV2 "github.com/openfaas/faas-cli/schema/store/v2" "github.com/spf13/cobra" ) @@ -33,7 +34,7 @@ func runStoreDescribe(cmd *cobra.Command, args []string) error { } targetPlatform := getTargetPlatform(platformValue) - storeItems, err := storeList(storeAddress) + storeItems, err := proxy.FunctionStoreList(storeAddress) if err != nil { return err } diff --git a/commands/store_list.go b/commands/store_list.go index a72eda11e..3c4c15134 100644 --- a/commands/store_list.go +++ b/commands/store_list.go @@ -9,6 +9,7 @@ import ( "strings" "text/tabwriter" + "github.com/openfaas/faas-cli/proxy" storeV2 "github.com/openfaas/faas-cli/schema/store/v2" "github.com/spf13/cobra" ) @@ -33,7 +34,7 @@ var storeListCmd = &cobra.Command{ func runStoreList(cmd *cobra.Command, args []string) error { targetPlatform := getTargetPlatform(platformValue) - storeList, err := storeList(storeAddress) + storeList, err := proxy.FunctionStoreList(storeAddress) if err != nil { return err } diff --git a/commands/template_store_describe.go b/commands/template_store_describe.go index 4e699a492..77832f86c 100644 --- a/commands/template_store_describe.go +++ b/commands/template_store_describe.go @@ -13,7 +13,7 @@ import ( ) func init() { - templateStoreDescribeCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates") + templateStoreDescribeCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc) templateStoreCmd.AddCommand(templateStoreDescribeCmd) } diff --git a/commands/template_store_list.go b/commands/template_store_list.go index 22aefc92e..5eaf47c6b 100644 --- a/commands/template_store_list.go +++ b/commands/template_store_list.go @@ -6,17 +6,14 @@ package commands import ( "bytes" "context" - "encoding/json" "fmt" - "io" - "net/http" "os" "sort" "strings" "text/tabwriter" - "time" + "github.com/openfaas/faas-cli/proxy" "github.com/spf13/cobra" ) @@ -24,6 +21,7 @@ const ( // DefaultTemplatesStore is the URL where the official store can be found DefaultTemplatesStore = "https://raw.githubusercontent.com/openfaas/store/master/templates.json" mainPlatform = "x86_64" + templateStoreDoc = `Alternative path to the template store metadata. It may be an http(s) URL or a local path to a JSON file.` ) var ( @@ -35,7 +33,7 @@ var ( func init() { templateStoreListCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Shows additional language and platform") - templateStoreListCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates") + templateStoreListCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc) templateStoreListCmd.Flags().StringVarP(&inputPlatform, "platform", "p", mainPlatform, "Shows the platform if the output is verbose") templateStoreListCmd.Flags().BoolVarP(&recommended, "recommended", "r", false, "Shows only recommended templates") templateStoreListCmd.Flags().BoolVarP(&official, "official", "o", false, "Shows only official templates") @@ -107,42 +105,13 @@ func runTemplateStoreList(cmd *cobra.Command, args []string) error { } func getTemplateInfo(repository string) ([]TemplateInfo, error) { - req, reqErr := http.NewRequest(http.MethodGet, repository, nil) - if reqErr != nil { - return nil, fmt.Errorf("error while trying to create request to take template info: %s", reqErr.Error()) - } - - reqContext, cancel := context.WithTimeout(req.Context(), 5*time.Second) - defer cancel() - req = req.WithContext(reqContext) - - client := http.DefaultClient - res, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("error while requesting template list: %s", err.Error()) - } - - if res.Body == nil { - return nil, fmt.Errorf("error empty response body from: %s", templateStoreURL) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code wanted: %d got: %d", http.StatusOK, res.StatusCode) - } - - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("error while reading response: %s", err.Error()) - } - templatesInfo := []TemplateInfo{} - if err := json.Unmarshal(body, &templatesInfo); err != nil { - return nil, fmt.Errorf("can't unmarshal text: %s", err.Error()) + err := proxy.ReadJSON(context.TODO(), repository, &templatesInfo) + if err != nil { + return nil, fmt.Errorf("cannot read templates info from: %s", repository) } sortTemplates(templatesInfo) - return templatesInfo, nil } diff --git a/commands/template_store_pull.go b/commands/template_store_pull.go index dfa25e081..7eec55f43 100644 --- a/commands/template_store_pull.go +++ b/commands/template_store_pull.go @@ -11,7 +11,7 @@ import ( ) func init() { - templateStorePullCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates") + templateStorePullCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc) templatePull, _, _ := faasCmd.Find([]string{"template", "pull"}) templateStoreCmd.PersistentFlags().AddFlagSet(templatePull.Flags()) diff --git a/proxy/function_store.go b/proxy/function_store.go index e29509f25..1364322fe 100644 --- a/proxy/function_store.go +++ b/proxy/function_store.go @@ -1,10 +1,13 @@ package proxy import ( + "context" "encoding/json" "fmt" "io" "net/http" + "os" + "os/user" "strings" "time" @@ -23,36 +26,95 @@ func FunctionStoreList(store string) ([]v2.StoreFunction, error) { store = strings.TrimRight(store, "/") + err := ReadJSON(context.TODO(), store, &storeResults) + if err != nil { + return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store) + } + + return storeResults.Functions, nil +} + +// ReadJSON reads a JSON file from a URL or local file +func ReadJSON(ctx context.Context, location string, dest interface{}) error { + var body io.ReadCloser + var err error + timeout := 60 * time.Second tlsInsecure := false - client := MakeHTTPClient(&timeout, tlsInsecure) + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() - res, err := client.Get(store) - if err != nil { - return nil, fmt.Errorf("cannot connect to OpenFaaS store at URL: %s", store) - } + scheme := determineScheme(location) + switch scheme { + case "http", "https": + client := MakeHTTPClient(&timeout, tlsInsecure) - if res.Body != nil { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return fmt.Errorf("cannot create request to: %s", location) + } + + res, err := client.Do(req) + if err != nil { + return fmt.Errorf("cannot connect to: %s", location) + } defer res.Body.Close() - } - switch res.StatusCode { - case http.StatusOK: - bytesOut, err := io.ReadAll(res.Body) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("server returned unexpected status code: %d", res.StatusCode) + } + + body = res.Body + case "file": + location, err = expandTilde(location) if err != nil { - return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store) + return err } - jsonErr := json.Unmarshal(bytesOut, &storeResults) - if jsonErr != nil { - return nil, fmt.Errorf("cannot parse result from OpenFaaS store at URL: %s\n%s", store, jsonErr.Error()) + body, err = os.Open(location) + if err != nil { + return fmt.Errorf("cannot read file: %s", location) } + + // Add more schemes such as s3:// or gs:// default: - bytesOut, err := io.ReadAll(res.Body) - if err == nil { - return nil, fmt.Errorf("server returned unexpected status code: %d - %s", res.StatusCode, string(bytesOut)) - } + return fmt.Errorf("unsupported scheme: %s", scheme) } - return storeResults.Functions, nil + + if body != nil { + defer body.Close() + } + + data, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("cannot read data from: %s", location) + } + + return json.Unmarshal(data, dest) +} + +func determineScheme(location string) string { + location = strings.ToLower(location) + if strings.HasPrefix(location, "http://") { + return "http" + } + if strings.HasPrefix(location, "https://") { + return "https" + } + return "file" +} + +// expandTilde expands a path with a leading tilde to the home directory +func expandTilde(location string) (string, error) { + if !strings.HasPrefix(location, "~") { + return location, nil + } + + usr, err := user.Current() + if err != nil { + return "", err + } + + return strings.Replace(location, "~", usr.HomeDir, 1), nil } diff --git a/vendor/github.com/alexellis/go-execute/pkg/v2/exec.go b/vendor/github.com/alexellis/go-execute/pkg/v2/exec.go new file mode 100644 index 000000000..1ce9c646b --- /dev/null +++ b/vendor/github.com/alexellis/go-execute/pkg/v2/exec.go @@ -0,0 +1,156 @@ +package execute + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" +) + +type ExecTask struct { + // Command is the command to execute. This can be the path to an executable + // or the executable with arguments. The arguments are detected by looking for + // a space. + // + // Examples: + // - Just a binary executable: `/bin/ls` + // - Binary executable with arguments: `/bin/ls -la /` + Command string + // Args are the arguments to pass to the command. These are ignored if the + // Command contains arguments. + Args []string + // Shell run the command in a bash shell. + // Note that the system must have `/bin/bash` installed. + Shell bool + // Env is a list of environment variables to add to the current environment, + // these are used to override any existing environment variables. + Env []string + // Cwd is the working directory for the command + Cwd string + + // Stdin connect a reader to stdin for the command + // being executed. + Stdin io.Reader + + // StreamStdio prints stdout and stderr directly to os.Stdout/err as + // the command runs. + StreamStdio bool + + // PrintCommand prints the command before executing + PrintCommand bool +} + +type ExecResult struct { + Stdout string + Stderr string + ExitCode int + Cancelled bool +} + +func (et ExecTask) Execute(ctx context.Context) (ExecResult, error) { + argsSt := "" + if len(et.Args) > 0 { + argsSt = strings.Join(et.Args, " ") + } + + if et.PrintCommand { + fmt.Println("exec: ", et.Command, argsSt) + } + + // don't try to run if the context is already cancelled + if ctx.Err() != nil { + return ExecResult{ + // the exec package returns -1 for cancelled commands + ExitCode: -1, + Cancelled: ctx.Err() == context.Canceled, + }, ctx.Err() + } + + var command string + var commandArgs []string + if et.Shell { + command = "/bin/bash" + if len(et.Args) == 0 { + // use Split and Join to remove any extra whitespace? + startArgs := strings.Split(et.Command, " ") + script := strings.Join(startArgs, " ") + commandArgs = append([]string{"-c"}, script) + + } else { + script := strings.Join(et.Args, " ") + commandArgs = append([]string{"-c"}, fmt.Sprintf("%s %s", et.Command, script)) + } + } else { + if strings.Contains(et.Command, " ") { + parts := strings.Split(et.Command, " ") + command = parts[0] + commandArgs = parts[1:] + } else { + command = et.Command + commandArgs = et.Args + } + } + + cmd := exec.CommandContext(ctx, command, commandArgs...) + cmd.Dir = et.Cwd + + if len(et.Env) > 0 { + overrides := map[string]bool{} + for _, env := range et.Env { + key := strings.Split(env, "=")[0] + overrides[key] = true + cmd.Env = append(cmd.Env, env) + } + + for _, env := range os.Environ() { + key := strings.Split(env, "=")[0] + + if _, ok := overrides[key]; !ok { + cmd.Env = append(cmd.Env, env) + } + } + } + if et.Stdin != nil { + cmd.Stdin = et.Stdin + } + + stdoutBuff := bytes.Buffer{} + stderrBuff := bytes.Buffer{} + + var stdoutWriters io.Writer + var stderrWriters io.Writer + + if et.StreamStdio { + stdoutWriters = io.MultiWriter(os.Stdout, &stdoutBuff) + stderrWriters = io.MultiWriter(os.Stderr, &stderrBuff) + } else { + stdoutWriters = &stdoutBuff + stderrWriters = &stderrBuff + } + + cmd.Stdout = stdoutWriters + cmd.Stderr = stderrWriters + + startErr := cmd.Start() + if startErr != nil { + return ExecResult{}, startErr + } + + exitCode := 0 + execErr := cmd.Wait() + if execErr != nil { + if exitError, ok := execErr.(*exec.ExitError); ok { + exitCode = exitError.ExitCode() + } + } + + return ExecResult{ + Stdout: stdoutBuff.String(), + Stderr: stderrBuff.String(), + ExitCode: exitCode, + Cancelled: ctx.Err() == context.Canceled, + }, ctx.Err() +}