From d3bf35db20fa2ee29f2e0a2c502393c33dff2adc Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sat, 4 Nov 2023 09:25:59 +0100 Subject: [PATCH 1/7] feat(amazonsagemaker): Add AmazonSageMaker AI provider Co-authored-by: NAME 18630245+zaremb@users.noreply.github.com Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- README.md | 52 ++++++++++++ cmd/auth/add.go | 8 +- cmd/auth/auth.go | 1 + pkg/ai/amazonsagemaker.go | 161 ++++++++++++++++++++++++++++++++++++++ pkg/ai/iai.go | 8 ++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 pkg/ai/amazonsagemaker.go diff --git a/README.md b/README.md index 3759295654..6756c7efd6 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,58 @@ k8sgpt analyze -e -b amazonbedrock You're right, I don't have enough context to determine if a StatefulSet is correctly configured to use a non-existent service. A StatefulSet manages Pods with persistent storage, and the Pods are created from the same spec. The service name referenced in the StatefulSet configuration would need to match an existing Kubernetes service for the Pods to connect to. Without more details on the specific StatefulSet and environment, I can't confirm whether the configuration is valid or not. ``` + + + +
+Amazon SageMaker Provider + +Prerequisites + +1. **AWS CLI Configuration**: Make sure you have the AWS Command Line Interface (CLI) configured on your machine. If you haven't already configured the AWS CLI, you can follow the official AWS documentation for instructions on how to do it: [AWS CLI Configuration Guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). + +2. **SageMaker Instance**: You need to have an Amazon SageMaker instance set up. If you don't have one already, you can follow the step-by-step instructions provided in this repository for creating a SageMaker instance: [llm-sagemaker-jumpstart-cdk](https://github.com/zaremb/llm-sagemaker-jumpstart-cdk). + +3. **Backend Configuration**: +To add amazonsagemaker backend two parameters are required: + +* Amazon SageMaker endpoint name: You'll need the name of the SageMaker endpoint. +* AWS region where your SageMaker instance is created + +To add amazonsagemaker as a backend run: + +```bash +k8sgpt auth add --backend amazonsagemaker --providerRegion eu-west-1 --endpointname endpoint-xxxxxxxxxx +``` + +***Note**: +TODO: Currently access key will be requested in the CLI, you can enter anything into this. + +To make amazonsagemaer as a default backend run: + +```bash +k8sgpt auth default -p amazonsagemaker +``` + +#### AmazonSageMaker Usage + +```bash +./k8sgpt analyze -e -b amazonsagemaker + 100% |███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| (1/1, 14 it/min) +AI Provider: amazonsagemaker + +0 default/nginx(nginx) +- Error: Back-off pulling image "nginxx" + Error: Back-off pulling image "nginxx" + +Solution: + +1. Check if the image exists in the registry by running `docker image ls nginxx`. +2. If the image is not found, try pulling it by running `docker pull nginxx`. +3. If the image is still not available, check if there are any network issues by running `docker network inspect` and `docker network list`. +4. If the issue persists, try restarting the Docker daemon by running `sudo service docker restart`. +``` +
diff --git a/cmd/auth/add.go b/cmd/auth/add.go index 89ca4da73f..de97107d83 100644 --- a/cmd/auth/add.go +++ b/cmd/auth/add.go @@ -41,7 +41,10 @@ var addCmd = &cobra.Command{ _ = cmd.MarkFlagRequired("engine") _ = cmd.MarkFlagRequired("baseurl") } - + if strings.ToLower(backend) == "amazonsagemaker" { + _ = cmd.MarkFlagRequired("endpointname") + _ = cmd.MarkFlagRequired("providerRegion") + } }, Run: func(cmd *cobra.Command, args []string) { @@ -108,6 +111,7 @@ var addCmd = &cobra.Command{ Model: model, Password: password, BaseURL: baseURL, + EndpointName: endpointName, Engine: engine, Temperature: temperature, ProviderRegion: providerRegion, @@ -138,6 +142,8 @@ func init() { addCmd.Flags().StringVarP(&password, "password", "p", "", "Backend AI password") // add flag for url addCmd.Flags().StringVarP(&baseURL, "baseurl", "u", "", "URL AI provider, (e.g `http://localhost:8080/v1`)") + // add flag for endpointName + addCmd.Flags().StringVarP(&endpointName, "endpointname", "n", "", "Endpoint Name, (e.g `endpoint-xxxxxxxxxxxx`)") // add flag for temperature addCmd.Flags().Float32VarP(&temperature, "temperature", "t", 0.7, "The sampling temperature, value ranges between 0 ( output be more deterministic) and 1 (more random)") // add flag for azure open ai engine/deployment name diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index b66f027a0a..687f23eec2 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -22,6 +22,7 @@ var ( backend string password string baseURL string + endpointName string model string engine string temperature float32 diff --git a/pkg/ai/amazonsagemaker.go b/pkg/ai/amazonsagemaker.go new file mode 100644 index 0000000000..96c3ab381d --- /dev/null +++ b/pkg/ai/amazonsagemaker.go @@ -0,0 +1,161 @@ +/* +Copyright 2023 The K8sGPT Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ai + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + "encoding/json" + "log" + + "github.com/fatih/color" + "github.com/k8sgpt-ai/k8sgpt/pkg/cache" + "github.com/k8sgpt-ai/k8sgpt/pkg/util" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sagemakerruntime" +) + +type SageMakerAIClient struct { + client *sagemakerruntime.SageMakerRuntime + language string + model string + temperature float32 + endpoint string +} + +const ( + // SageMaker completion parameters + maxNewTokens = 256 + top_P = 0.9 +) + +type Generation struct { + Role string `json:"role"` + Content string `json:"content"` +} + +func (c *SageMakerAIClient) Configure(config IAIConfig, language string) error { + + // Create a new AWS session + sess := session.Must(session.NewSessionWithOptions(session.Options{ + Config: aws.Config{Region: aws.String(config.GetProviderRegion())}, + SharedConfigState: session.SharedConfigEnable, + })) + + c.language = language + // Create a new SageMaker runtime client + c.client = sagemakerruntime.New(sess) + c.model = config.GetModel() + c.endpoint = config.GetEndpointName() + c.temperature = config.GetTemperature() + return nil +} + +func (c *SageMakerAIClient) GetCompletion(ctx context.Context, prompt string, promptTmpl string) (string, error) { + // Create a completion request + + if len(promptTmpl) == 0 { + promptTmpl = PromptMap["default"] + } + // TODO: extract all paramseters to config + data := map[string]interface{}{ + "inputs": []interface{}{ + []interface{}{ + map[string]interface{}{ + "role": "system", + "content": "DEFAULT_PROMPT", + }, + map[string]interface{}{ + "role": "user", + "content": fmt.Sprintf(promptTmpl, c.language, prompt), + }, + }, + }, + "parameters": map[string]interface{}{ + "max_new_tokens": maxNewTokens, + "top_p": top_P, + "temperature": c.temperature, + }, + } + // Convert data to []byte + bytesData, err := json.Marshal(data) + if err != nil { + fmt.Println("Error:", err) + log.Fatal(err) + return "", err + } + + // Create an input object + input := &sagemakerruntime.InvokeEndpointInput{ + Body: bytesData, + EndpointName: aws.String(c.endpoint), + ContentType: aws.String("application/json"), // Set the content type as per your model's requirements + Accept: aws.String("application/json"), // Set the accept type as per your model's requirements + CustomAttributes: aws.String("accept_eula=true"), + } + + // Call the InvokeEndpoint function + result, err := c.client.InvokeEndpoint(input) + if err != nil { + log.Fatal(err) + return "", err + } + + // Define a slice of Generations + var generations []struct { + Generation Generation `json:"generation"` + } + + err = json.Unmarshal([]byte(string(result.Body)), &generations) + if err != nil { + log.Fatal(err) + return "", err + } + + // Access the content + content := generations[0].Generation.Content + return content, nil +} + +func (a *SageMakerAIClient) Parse(ctx context.Context, prompt []string, cache cache.ICache, promptTmpl string) (string, error) { + // parse the text with the AI backend + inputKey := strings.Join(prompt, " ") + // Check for cached data + sEnc := base64.StdEncoding.EncodeToString([]byte(inputKey)) + cacheKey := util.GetCacheKey(a.GetName(), a.language, sEnc) + + response, err := a.GetCompletion(ctx, inputKey, promptTmpl) + if err != nil { + color.Red("error getting completion: %v", err) + return "", err + } + + err = cache.Store(cacheKey, base64.StdEncoding.EncodeToString([]byte(response))) + + if err != nil { + color.Red("error storing value to cache: %v", err) + return "", nil + } + + return response, nil +} + +func (a *SageMakerAIClient) GetName() string { + return "amazonsagemaker" +} diff --git a/pkg/ai/iai.go b/pkg/ai/iai.go index 430e04ca4e..16104e561c 100644 --- a/pkg/ai/iai.go +++ b/pkg/ai/iai.go @@ -27,6 +27,7 @@ var ( &NoOpAIClient{}, &CohereClient{}, &AmazonBedRockClient{}, + &SageMakerAIClient{}, } Backends = []string{ "openai", @@ -35,6 +36,7 @@ var ( "noopai", "cohere", "amazonbedrock", + "amazonsagemaker", } ) @@ -49,6 +51,7 @@ type IAIConfig interface { GetPassword() string GetModel() string GetBaseURL() string + GetEndpointName() string GetEngine() string GetTemperature() float32 GetProviderRegion() string @@ -74,6 +77,7 @@ type AIProvider struct { Model string `mapstructure:"model"` Password string `mapstructure:"password" yaml:"password,omitempty"` BaseURL string `mapstructure:"baseurl" yaml:"baseurl,omitempty"` + EndpointName string `mapstructure:"endpointname" yaml:"endpointname,omitempty"` Engine string `mapstructure:"engine" yaml:"engine,omitempty"` Temperature float32 `mapstructure:"temperature" yaml:"temperature,omitempty"` ProviderRegion string `mapstructure:"providerregion" yaml:"providerregion,omitempty"` @@ -83,6 +87,10 @@ func (p *AIProvider) GetBaseURL() string { return p.BaseURL } +func (p *AIProvider) GetEndpointName() string { + return p.EndpointName +} + func (p *AIProvider) GetPassword() string { return p.Password } From b9d43b8a06c6f29843518474ed6f75fadafd29ce Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sat, 4 Nov 2023 09:25:59 +0100 Subject: [PATCH 2/7] feat(amazonsagemaker): Add AmazonSageMaker AI provider Co-authored-by: Mateusz Zaremba <18630245+zaremb@users.noreply.github.com> Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6756c7efd6..cd26a36b07 100644 --- a/README.md +++ b/README.md @@ -425,7 +425,7 @@ k8sgpt auth add --backend amazonsagemaker --providerRegion eu-west-1 --endpointn ***Note**: TODO: Currently access key will be requested in the CLI, you can enter anything into this. -To make amazonsagemaer as a default backend run: +To make amazonsagemaker as a default backend run: ```bash k8sgpt auth default -p amazonsagemaker From 7f76d7f57c8c61dafd84a1fa73277ce0c5a1218d Mon Sep 17 00:00:00 2001 From: Mateusz Zaremba <18630245+zaremb@users.noreply.github.com> Date: Sat, 4 Nov 2023 19:22:14 +0000 Subject: [PATCH 3/7] feat(auth): add top p and max tokens to auth and use them in sagemaker backend Signed-off-by: Mateusz Zaremba <18630245+zaremb@users.noreply.github.com> --- cmd/auth/add.go | 8 ++++ cmd/auth/auth.go | 2 + pkg/ai/amazonsagemaker.go | 78 +++++++++++++++++++++++---------------- pkg/ai/iai.go | 14 +++++++ 4 files changed, 71 insertions(+), 31 deletions(-) diff --git a/cmd/auth/add.go b/cmd/auth/add.go index de97107d83..4401e0a348 100644 --- a/cmd/auth/add.go +++ b/cmd/auth/add.go @@ -115,6 +115,9 @@ var addCmd = &cobra.Command{ Engine: engine, Temperature: temperature, ProviderRegion: providerRegion, + TopP: topP, + MaxTokens: maxTokens, + } if providerIndex == -1 { @@ -144,6 +147,11 @@ func init() { addCmd.Flags().StringVarP(&baseURL, "baseurl", "u", "", "URL AI provider, (e.g `http://localhost:8080/v1`)") // add flag for endpointName addCmd.Flags().StringVarP(&endpointName, "endpointname", "n", "", "Endpoint Name, (e.g `endpoint-xxxxxxxxxxxx`)") + // Maybe we could switch p to topp instead of password? + // add flag for topP + addCmd.Flags().Float32VarP(&topP, "topp", "c", 0.5, "Probability Cutoff: Set a threshold (0.0-1.0) to limit word choices. Higher values add randomness, lower values increase predictability.") + // max tokens + addCmd.Flags().IntVarP(&maxTokens, "maxtokens", "l", 2048, "Specify a maximum output length. Adjust (1-...) to control text length. Higher values produce longer output, lower values limit length") // add flag for temperature addCmd.Flags().Float32VarP(&temperature, "temperature", "t", 0.7, "The sampling temperature, value ranges between 0 ( output be more deterministic) and 1 (more random)") // add flag for azure open ai engine/deployment name diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index 687f23eec2..a7d6f52178 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -27,6 +27,8 @@ var ( engine string temperature float32 providerRegion string + topP float32 + maxTokens int ) var configAI ai.AIConfiguration diff --git a/pkg/ai/amazonsagemaker.go b/pkg/ai/amazonsagemaker.go index 96c3ab381d..06067836ce 100644 --- a/pkg/ai/amazonsagemaker.go +++ b/pkg/ai/amazonsagemaker.go @@ -37,19 +37,33 @@ type SageMakerAIClient struct { model string temperature float32 endpoint string + topP float32 + maxTokens int } -const ( - // SageMaker completion parameters - maxNewTokens = 256 - top_P = 0.9 -) +type Generations []struct { + Generation struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"generation"` +} + +type Request struct { + Inputs [][]Message `json:"inputs"` + Parameters Parameters `json:"parameters"` +} -type Generation struct { +type Message struct { Role string `json:"role"` Content string `json:"content"` } +type Parameters struct { + MaxNewTokens int `json:"max_new_tokens"` + TopP float64 `json:"top_p"` + Temperature float64 `json:"temperature"` +} + func (c *SageMakerAIClient) Configure(config IAIConfig, language string) error { // Create a new AWS session @@ -64,6 +78,8 @@ func (c *SageMakerAIClient) Configure(config IAIConfig, language string) error { c.model = config.GetModel() c.endpoint = config.GetEndpointName() c.temperature = config.GetTemperature() + c.maxTokens = config.GetMaxTokens() + c.topP = config.GetTopP() return nil } @@ -73,31 +89,30 @@ func (c *SageMakerAIClient) GetCompletion(ctx context.Context, prompt string, pr if len(promptTmpl) == 0 { promptTmpl = PromptMap["default"] } - // TODO: extract all paramseters to config - data := map[string]interface{}{ - "inputs": []interface{}{ - []interface{}{ - map[string]interface{}{ - "role": "system", - "content": "DEFAULT_PROMPT", - }, - map[string]interface{}{ - "role": "user", - "content": fmt.Sprintf(promptTmpl, c.language, prompt), - }, + + request := Request{ + Inputs: [][]Message{ + { + {Role: "system", Content: "DEFAULT_PROMPT"}, + {Role: "user", Content: fmt.Sprintf(promptTmpl, c.language, prompt)}, }, }, - "parameters": map[string]interface{}{ - "max_new_tokens": maxNewTokens, - "top_p": top_P, - "temperature": c.temperature, + // Parameters: Parameters{ + // MaxNewTokens: sageMakerMaxTokens, // TODO: move to config params + // TopP: sageMakerTopP, // TODO: move to config params + // Temperature: float64(c.temperature), + // }, + Parameters: Parameters{ + MaxNewTokens: int(c.maxTokens), // TODO: move to config params + TopP: float64(c.topP), // TODO: move to config params + Temperature: float64(c.temperature), }, } - // Convert data to []byte - bytesData, err := json.Marshal(data) + + + // Convert request to []byte + bytesData, err := json.Marshal(request) if err != nil { - fmt.Println("Error:", err) - log.Fatal(err) return "", err } @@ -117,16 +132,17 @@ func (c *SageMakerAIClient) GetCompletion(ctx context.Context, prompt string, pr return "", err } - // Define a slice of Generations - var generations []struct { - Generation Generation `json:"generation"` - } + // // Define a slice of Generations + var generations Generations err = json.Unmarshal([]byte(string(result.Body)), &generations) if err != nil { - log.Fatal(err) return "", err } + // Check for length of generations + if len(generations) != 1 { + return "", fmt.Errorf("Expected exactly one generation, but got %d", len(generations)) + } // Access the content content := generations[0].Generation.Content diff --git a/pkg/ai/iai.go b/pkg/ai/iai.go index 16104e561c..540842d479 100644 --- a/pkg/ai/iai.go +++ b/pkg/ai/iai.go @@ -55,6 +55,8 @@ type IAIConfig interface { GetEngine() string GetTemperature() float32 GetProviderRegion() string + GetTopP() float32 + GetMaxTokens() int } func NewClient(provider string) IAI { @@ -81,6 +83,10 @@ type AIProvider struct { Engine string `mapstructure:"engine" yaml:"engine,omitempty"` Temperature float32 `mapstructure:"temperature" yaml:"temperature,omitempty"` ProviderRegion string `mapstructure:"providerregion" yaml:"providerregion,omitempty"` + TopP float32 `mapstructure:"topp" yaml:"topp,omitempty"` + MaxTokens int `mapstructure:"maxtokens" yaml:"maxtokens,omitempty"` + + } func (p *AIProvider) GetBaseURL() string { @@ -91,6 +97,14 @@ func (p *AIProvider) GetEndpointName() string { return p.EndpointName } +func (p *AIProvider) GetTopP() float32 { + return p.TopP +} + +func (p *AIProvider) GetMaxTokens() int { + return p.MaxTokens +} + func (p *AIProvider) GetPassword() string { return p.Password } From 725928769dcae23581c7d4625b44c697ed854b4f Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sun, 5 Nov 2023 11:59:42 +0100 Subject: [PATCH 4/7] feat: Updates SageMaker docs, validate topP, ident Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- README.md | 21 ++++++++++++++++----- cmd/auth/add.go | 10 ++++++---- pkg/ai/amazonsagemaker.go | 19 +++++++------------ pkg/ai/iai.go | 6 ++---- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index cd26a36b07..7b55b6be85 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,7 @@ In addition to this you will need to set the follow local environmental variable k8sgpt auth add --backend amazonbedrock --model anthropic.claude-v2 ``` -TODO: Currently access key will be requested in the CLI, you can enter anything into this. +TODO: Currently access key will be requested in the CLI, you can enter anything into this. #### Usage @@ -404,17 +404,18 @@ k8sgpt analyze -e -b amazonbedrock
Amazon SageMaker Provider -Prerequisites +#### Prerequisites 1. **AWS CLI Configuration**: Make sure you have the AWS Command Line Interface (CLI) configured on your machine. If you haven't already configured the AWS CLI, you can follow the official AWS documentation for instructions on how to do it: [AWS CLI Configuration Guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). 2. **SageMaker Instance**: You need to have an Amazon SageMaker instance set up. If you don't have one already, you can follow the step-by-step instructions provided in this repository for creating a SageMaker instance: [llm-sagemaker-jumpstart-cdk](https://github.com/zaremb/llm-sagemaker-jumpstart-cdk). -3. **Backend Configuration**: +#### Backend Configuration + To add amazonsagemaker backend two parameters are required: -* Amazon SageMaker endpoint name: You'll need the name of the SageMaker endpoint. -* AWS region where your SageMaker instance is created +* `--endpointname` Amazon SageMaker endpoint name. +* `--providerRegion` AWS region where SageMaker instance is created. `k8sgpt` uses this region to connect to SageMaker (not the one defined with AWS CLI or environment variables ) To add amazonsagemaker as a backend run: @@ -425,6 +426,16 @@ k8sgpt auth add --backend amazonsagemaker --providerRegion eu-west-1 --endpointn ***Note**: TODO: Currently access key will be requested in the CLI, you can enter anything into this. +#### Optional params + +Optionally, when adding the backend and later by changing the configuration file, you can set the following parameters: + +`-l, --maxtokens int` Specify a maximum output length. Adjust (1-...) to control text length. Higher values produce longer output, lower values limit length (default 2048) + +`-t, --temperature float32` The sampling temperature, value ranges between 0 ( output be more deterministic) and 1 (more random) (default 0.7) + +`-c, --topp float32` Probability Cutoff: Set a threshold (0.0-1.0) to limit word choices. Higher values add randomness, lower values increase predictability. (default 0.5) + To make amazonsagemaker as a default backend run: ```bash diff --git a/cmd/auth/add.go b/cmd/auth/add.go index 4401e0a348..7a4f69c084 100644 --- a/cmd/auth/add.go +++ b/cmd/auth/add.go @@ -93,6 +93,10 @@ var addCmd = &cobra.Command{ color.Red("Error: temperature ranges from 0 to 1.") os.Exit(1) } + if topP > 1.0 || topP < 0.0 { + color.Red("Error: topP ranges from 0 to 1.") + os.Exit(1) + } if ai.NeedPassword(backend) && password == "" { fmt.Printf("Enter %s Key: ", backend) @@ -115,9 +119,8 @@ var addCmd = &cobra.Command{ Engine: engine, Temperature: temperature, ProviderRegion: providerRegion, - TopP: topP, - MaxTokens: maxTokens, - + TopP: topP, + MaxTokens: maxTokens, } if providerIndex == -1 { @@ -147,7 +150,6 @@ func init() { addCmd.Flags().StringVarP(&baseURL, "baseurl", "u", "", "URL AI provider, (e.g `http://localhost:8080/v1`)") // add flag for endpointName addCmd.Flags().StringVarP(&endpointName, "endpointname", "n", "", "Endpoint Name, (e.g `endpoint-xxxxxxxxxxxx`)") - // Maybe we could switch p to topp instead of password? // add flag for topP addCmd.Flags().Float32VarP(&topP, "topp", "c", 0.5, "Probability Cutoff: Set a threshold (0.0-1.0) to limit word choices. Higher values add randomness, lower values increase predictability.") // max tokens diff --git a/pkg/ai/amazonsagemaker.go b/pkg/ai/amazonsagemaker.go index 06067836ce..9cfc19f101 100644 --- a/pkg/ai/amazonsagemaker.go +++ b/pkg/ai/amazonsagemaker.go @@ -37,8 +37,8 @@ type SageMakerAIClient struct { model string temperature float32 endpoint string - topP float32 - maxTokens int + topP float32 + maxTokens int } type Generations []struct { @@ -49,8 +49,8 @@ type Generations []struct { } type Request struct { - Inputs [][]Message `json:"inputs"` - Parameters Parameters `json:"parameters"` + Inputs [][]Message `json:"inputs"` + Parameters Parameters `json:"parameters"` } type Message struct { @@ -97,19 +97,14 @@ func (c *SageMakerAIClient) GetCompletion(ctx context.Context, prompt string, pr {Role: "user", Content: fmt.Sprintf(promptTmpl, c.language, prompt)}, }, }, - // Parameters: Parameters{ - // MaxNewTokens: sageMakerMaxTokens, // TODO: move to config params - // TopP: sageMakerTopP, // TODO: move to config params - // Temperature: float64(c.temperature), - // }, + Parameters: Parameters{ - MaxNewTokens: int(c.maxTokens), // TODO: move to config params - TopP: float64(c.topP), // TODO: move to config params + MaxNewTokens: int(c.maxTokens), + TopP: float64(c.topP), Temperature: float64(c.temperature), }, } - // Convert request to []byte bytesData, err := json.Marshal(request) if err != nil { diff --git a/pkg/ai/iai.go b/pkg/ai/iai.go index 540842d479..491af5706b 100644 --- a/pkg/ai/iai.go +++ b/pkg/ai/iai.go @@ -83,10 +83,8 @@ type AIProvider struct { Engine string `mapstructure:"engine" yaml:"engine,omitempty"` Temperature float32 `mapstructure:"temperature" yaml:"temperature,omitempty"` ProviderRegion string `mapstructure:"providerregion" yaml:"providerregion,omitempty"` - TopP float32 `mapstructure:"topp" yaml:"topp,omitempty"` - MaxTokens int `mapstructure:"maxtokens" yaml:"maxtokens,omitempty"` - - + TopP float32 `mapstructure:"topp" yaml:"topp,omitempty"` + MaxTokens int `mapstructure:"maxtokens" yaml:"maxtokens,omitempty"` } func (p *AIProvider) GetBaseURL() string { From 6979f3f7d9c0768d6366634fafbb25a55b39cad8 Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sun, 5 Nov 2023 12:44:07 +0100 Subject: [PATCH 5/7] feat: list of passwordlessProviders Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- README.md | 5 ----- pkg/ai/iai.go | 9 ++++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7b55b6be85..ab2fccc2f1 100644 --- a/README.md +++ b/README.md @@ -386,8 +386,6 @@ In addition to this you will need to set the follow local environmental variable k8sgpt auth add --backend amazonbedrock --model anthropic.claude-v2 ``` -TODO: Currently access key will be requested in the CLI, you can enter anything into this. - #### Usage ``` @@ -423,9 +421,6 @@ To add amazonsagemaker as a backend run: k8sgpt auth add --backend amazonsagemaker --providerRegion eu-west-1 --endpointname endpoint-xxxxxxxxxx ``` -***Note**: -TODO: Currently access key will be requested in the CLI, you can enter anything into this. - #### Optional params Optionally, when adding the backend and later by changing the configuration file, you can set the following parameters: diff --git a/pkg/ai/iai.go b/pkg/ai/iai.go index 491af5706b..a603c6b3c9 100644 --- a/pkg/ai/iai.go +++ b/pkg/ai/iai.go @@ -122,6 +122,13 @@ func (p *AIProvider) GetProviderRegion() string { return p.ProviderRegion } +var passwordlessProviders = []string{"localai", "amazonsagemaker", "amazonbedrock"} + func NeedPassword(backend string) bool { - return backend != "localai" + for _, b := range passwordlessProviders { + if b == backend { + return false + } + } + return true } From 3e3aad8574cf7d00b0c6d1a41d31f7c591d3c8b2 Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sun, 5 Nov 2023 12:52:32 +0100 Subject: [PATCH 6/7] feat: returns err Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- pkg/ai/amazonsagemaker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ai/amazonsagemaker.go b/pkg/ai/amazonsagemaker.go index 9cfc19f101..2a312c3e10 100644 --- a/pkg/ai/amazonsagemaker.go +++ b/pkg/ai/amazonsagemaker.go @@ -161,7 +161,7 @@ func (a *SageMakerAIClient) Parse(ctx context.Context, prompt []string, cache ca if err != nil { color.Red("error storing value to cache: %v", err) - return "", nil + return "", err } return response, nil From 603ba08029498248ac7727f1ec822c5af793d6cc Mon Sep 17 00:00:00 2001 From: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> Date: Sun, 5 Nov 2023 20:31:43 +0100 Subject: [PATCH 7/7] fix: remove log.Fatal(err) Signed-off-by: Damian Kuroczko <7778327+dkuroczk@users.noreply.github.com> --- pkg/ai/amazonsagemaker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/ai/amazonsagemaker.go b/pkg/ai/amazonsagemaker.go index 2a312c3e10..e85be89188 100644 --- a/pkg/ai/amazonsagemaker.go +++ b/pkg/ai/amazonsagemaker.go @@ -20,7 +20,6 @@ import ( "strings" "encoding/json" - "log" "github.com/fatih/color" "github.com/k8sgpt-ai/k8sgpt/pkg/cache" @@ -123,7 +122,6 @@ func (c *SageMakerAIClient) GetCompletion(ctx context.Context, prompt string, pr // Call the InvokeEndpoint function result, err := c.client.InvokeEndpoint(input) if err != nil { - log.Fatal(err) return "", err }