From 8fca1d7b23202086455b210813c4e5f09dd0cb15 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 11:10:11 +0545 Subject: [PATCH 1/7] feat: add AsGoGetterURL() to connection --- models/connections.go | 121 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/models/connections.go b/models/connections.go index 28c09d07..024f4289 100644 --- a/models/connections.go +++ b/models/connections.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "net/url" "regexp" "time" @@ -9,6 +10,50 @@ import ( "github.com/google/uuid" ) +// List of all connection types +const ( + ConnectionTypeAWS = "AWS" + ConnectionTypeAzure = "Azure" + ConnectionTypeAzureDevops = "Azure Devops" + ConnectionTypeDiscord = "Discord" + ConnectionTypeDynatrace = "Dynatrace" + ConnectionTypeElasticSearch = "ElasticSearch" + ConnectionTypeEmail = "Email" + ConnectionTypeGCP = "Google Cloud" + ConnectionTypeGenericWebhook = "Generic Webhook" + ConnectionTypeGit = "Git" + ConnectionTypeGithub = "Github" + ConnectionTypeGoogleChat = "Google Chat" + ConnectionTypeHTTP = "HTTP" + ConnectionTypeIFTTT = "IFTTT" + ConnectionTypeJMeter = "JMeter" + ConnectionTypeKubernetes = "Kubernetes" + ConnectionTypeLDAP = "LDAP" + ConnectionTypeMatrix = "Matrix" + ConnectionTypeMattermost = "Mattermost" + ConnectionTypeMongo = "Mongo" + ConnectionTypeMySQL = "MySQL" + ConnectionTypeNtfy = "Ntfy" + ConnectionTypeOpsGenie = "OpsGenie" + ConnectionTypePostgres = "Postgres" + ConnectionTypePrometheus = "Prometheus" + ConnectionTypePushbullet = "Pushbullet" + ConnectionTypePushover = "Pushover" + ConnectionTypeRedis = "Redis" + ConnectionTypeRestic = "Restic" + ConnectionTypeRocketchat = "Rocketchat" + ConnectionTypeSFTP = "SFTP" + ConnectionTypeSlack = "Slack" + ConnectionTypeSlackWebhook = "SlackWebhook" + ConnectionTypeSMB = "SMB" + ConnectionTypeSQLServer = "SQL Server" + ConnectionTypeTeams = "Teams" + ConnectionTypeTelegram = "Telegram" + ConnectionTypeWebhook = "Webhook" + ConnectionTypeWindows = "Windows" + ConnectionTypeZulipChat = "Zulip Chat" +) + type Connection struct { ID uuid.UUID `gorm:"primaryKey;unique_index;not null;column:id" json:"id" faker:"uuid_hyphenated" ` Name string `gorm:"column:name" json:"name" faker:"name" ` @@ -48,3 +93,79 @@ func (c Connection) String() string { func (c Connection) AsMap(removeFields ...string) map[string]any { return asMap(c, removeFields...) } + +// AsGoGetterURL returns the connection as a url that's supported by https://github.com/hashicorp/go-getter +// Connection details are added to the url as query params +func (c Connection) AsGoGetterURL() (string, error) { + parsedURL, err := url.Parse(c.URL) + if err != nil { + return "", err + } + + var output string + switch c.Type { + case ConnectionTypeHTTP: + if c.Username != "" || c.Password != "" { + parsedURL.User = url.UserPassword(c.Username, c.Password) + } + + output = parsedURL.String() + + case ConnectionTypeGit: + q := parsedURL.Query() + q.Set("sshkey", c.Certificate) + + if v, ok := c.Properties["ref"]; ok { + q.Set("ref", v) + } + + if v, ok := c.Properties["depth"]; ok { + q.Set("depth", v) + } + + parsedURL.RawQuery = q.Encode() + output = parsedURL.String() + + case ConnectionTypeAWS: + q := parsedURL.Query() + q.Set("aws_access_key_id", c.Username) + q.Set("aws_access_key_secret", c.Password) + + if v, ok := c.Properties["profile"]; ok { + q.Set("aws_profile", v) + } + + if v, ok := c.Properties["region"]; ok { + q.Set("region", v) + } + + // For S3 + if v, ok := c.Properties["version"]; ok { + q.Set("version", v) + } + + parsedURL.RawQuery = q.Encode() + output = parsedURL.String() + } + + return output, nil +} + +func (c Connection) AsEnv() []string { + var envs []string + switch c.Type { + case ConnectionTypeAWS: + envs = append(envs, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) + envs = append(envs, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) + + if v, ok := c.Properties["profile"]; ok { + envs = append(envs, fmt.Sprintf("AWS_DEFAULT_PROFILE=%s", v)) + } + + if v, ok := c.Properties["region"]; ok { + envs = append(envs, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) + } + } + + return envs +} From 83ad067cd25b41866b9248a2e958699f5b1b26c0 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 15:09:30 +0545 Subject: [PATCH 2/7] feat: return file content on AsEnv() * added tests as well --- models/connections.go | 20 +++++-- models/connections_test.go | 103 +++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 models/connections_test.go diff --git a/models/connections.go b/models/connections.go index 024f4289..c5210305 100644 --- a/models/connections.go +++ b/models/connections.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "regexp" + "strings" "time" "github.com/flanksource/duty/types" @@ -151,21 +152,34 @@ func (c Connection) AsGoGetterURL() (string, error) { return output, nil } -func (c Connection) AsEnv() []string { - var envs []string +// AsEnv generates environment variables and a configuration file content based on the connection type. +func (c Connection) AsEnv() ([]string, string) { + var ( + envs []string + file strings.Builder + ) + switch c.Type { case ConnectionTypeAWS: envs = append(envs, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) envs = append(envs, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) + file.WriteString("[default]\n") + file.WriteString(fmt.Sprintf("aws_access_key_id = %s\n", c.Username)) + file.WriteString(fmt.Sprintf("aws_secret_access_key = %s\n", c.Password)) + if v, ok := c.Properties["profile"]; ok { envs = append(envs, fmt.Sprintf("AWS_DEFAULT_PROFILE=%s", v)) } if v, ok := c.Properties["region"]; ok { envs = append(envs, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) + file.WriteString(fmt.Sprintf("region = %s\n", v)) } + + case ConnectionTypeGCP: + file.WriteString(c.Certificate) } - return envs + return envs, file.String() } diff --git a/models/connections_test.go b/models/connections_test.go new file mode 100644 index 00000000..2d813c67 --- /dev/null +++ b/models/connections_test.go @@ -0,0 +1,103 @@ +package models + +import ( + "testing" +) + +func Test_Connection_AsGoGetterURL(t *testing.T) { + testCases := []struct { + name string + connection Connection + expectedURL string + expectedError error + }{ + { + name: "HTTP Connection", + connection: Connection{ + Type: ConnectionTypeHTTP, + URL: "http://example.com", + Username: "testuser", + Password: "testpassword", + }, + expectedURL: "http://testuser:testpassword@example.com", + expectedError: nil, + }, + { + name: "Git Connection", + connection: Connection{ + Type: ConnectionTypeGit, + URL: "https://github.com/repo.git", + Certificate: "cert123", + Properties: map[string]string{"ref": "main"}, + }, + expectedURL: "https://github.com/repo.git?ref=main&sshkey=cert123", + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resultURL, err := tc.connection.AsGoGetterURL() + + if resultURL != tc.expectedURL { + t.Errorf("Expected URL: %s, but got: %s", tc.expectedURL, resultURL) + } + + if err != tc.expectedError { + t.Errorf("Expected error: %v, but got: %v", tc.expectedError, err) + } + }) + } +} + +func Test_Connection_AsEnv(t *testing.T) { + testCases := []struct { + name string + connection Connection + expectedEnv []string + expectedFile string + }{ + { + name: "AWS Connection", + connection: Connection{ + Type: ConnectionTypeAWS, + Username: "awsuser", + Password: "awssecret", + Properties: map[string]string{"profile": "awsprofile", "region": "us-east-1"}, + }, + expectedEnv: []string{ + "AWS_ACCESS_KEY_ID=awsuser", + "AWS_SECRET_ACCESS_KEY=awssecret", + "AWS_DEFAULT_PROFILE=awsprofile", + "AWS_DEFAULT_REGION=us-east-1", + }, + expectedFile: "[default]\naws_access_key_id = awsuser\naws_secret_access_key = awssecret\nregion = us-east-1\n", + }, + { + name: "GCP Connection", + connection: Connection{ + Type: ConnectionTypeGCP, + Username: "gcpuser", + Certificate: `{"account": "gcpuser"}`, + }, + expectedEnv: []string{}, + expectedFile: `{"account": "gcpuser"}`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + env, file := tc.connection.AsEnv() + + for i, expected := range tc.expectedEnv { + if env[i] != expected { + t.Errorf("Expected environment variable: %s, but got: %s", expected, env[i]) + } + } + + if file != tc.expectedFile { + t.Errorf("Expected file content:\n%s\nBut got:\n%s", tc.expectedFile, file) + } + }) + } +} From 02d2b04a924d9a4957f6f716d797ddb975552892 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 16:48:46 +0545 Subject: [PATCH 3/7] feat: env prep --- models/connections.go | 102 +++++++++++++++++++++++++++++++------ models/connections_test.go | 10 ++-- 2 files changed, 91 insertions(+), 21 deletions(-) diff --git a/models/connections.go b/models/connections.go index c5210305..41d6f6dd 100644 --- a/models/connections.go +++ b/models/connections.go @@ -1,10 +1,13 @@ package models import ( + "bytes" + "context" "fmt" "net/url" + "os" + "os/exec" "regexp" - "strings" "time" "github.com/flanksource/duty/types" @@ -153,33 +156,100 @@ func (c Connection) AsGoGetterURL() (string, error) { } // AsEnv generates environment variables and a configuration file content based on the connection type. -func (c Connection) AsEnv() ([]string, string) { - var ( - envs []string - file strings.Builder - ) +func (c Connection) AsEnv() EnvPrep { + envPrep := EnvPrep{ + Conn: c, + } switch c.Type { case ConnectionTypeAWS: - envs = append(envs, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) - envs = append(envs, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) + envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) + envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) - file.WriteString("[default]\n") - file.WriteString(fmt.Sprintf("aws_access_key_id = %s\n", c.Username)) - file.WriteString(fmt.Sprintf("aws_secret_access_key = %s\n", c.Password)) + envPrep.File.WriteString("[default]\n") + envPrep.File.WriteString(fmt.Sprintf("aws_access_key_id = %s\n", c.Username)) + envPrep.File.WriteString(fmt.Sprintf("aws_secret_access_key = %s\n", c.Password)) if v, ok := c.Properties["profile"]; ok { - envs = append(envs, fmt.Sprintf("AWS_DEFAULT_PROFILE=%s", v)) + envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_DEFAULT_PROFILE=%s", v)) } if v, ok := c.Properties["region"]; ok { - envs = append(envs, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) - file.WriteString(fmt.Sprintf("region = %s\n", v)) + envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) + envPrep.File.WriteString(fmt.Sprintf("region = %s\n", v)) } + case ConnectionTypeAzure: + // Do nothing + case ConnectionTypeGCP: - file.WriteString(c.Certificate) + envPrep.File.WriteString(c.Certificate) + } + + return envPrep +} + +type EnvPrep struct { + Conn Connection + + // Env is the connection credentials in environment variables + Env []string + + // File contains the content of the configuration file based on the connection + File bytes.Buffer +} + +func (c *EnvPrep) Apply(ctx context.Context, cmd *exec.Cmd, configAbsPath string) error { + switch c.Conn.Type { + case ConnectionTypeAWS: + if err := saveConfig(c.File.Bytes(), configAbsPath); err != nil { + return err + } + + cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 + cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configAbsPath)) + if v, ok := c.Conn.Properties["region"]; ok { + cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) + } + + case ConnectionTypeGCP: + if err := saveConfig(c.File.Bytes(), configAbsPath); err != nil { + return err + } + + // to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS, + // we need to explicitly activate it + runCmd := exec.Command("gcloud", "auth", "activate-service-account", "--key-file", configAbsPath) + if err := runCmd.Run(); err != nil { + return fmt.Errorf("failed to activate GCP service account: %w", err) + } + + cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configAbsPath)) + + case ConnectionTypeAzure: + args := []string{"login", "--service-principal", "--username", c.Conn.Username, "--password", c.Conn.Password} + if v, ok := c.Conn.Properties["tenant"]; ok { + args = append(args, "--tenant") + args = append(args, v) + } + + // login with service principal + runCmd := exec.CommandContext(ctx, "az", args...) + if err := runCmd.Run(); err != nil { + return err + } + } + + return nil +} + +func saveConfig(content []byte, absPath string) error { + file, err := os.Create(absPath) + if err != nil { + return err } + defer file.Close() - return envs, file.String() + _, err = file.Write(content) + return err } diff --git a/models/connections_test.go b/models/connections_test.go index 2d813c67..40b5b975 100644 --- a/models/connections_test.go +++ b/models/connections_test.go @@ -87,16 +87,16 @@ func Test_Connection_AsEnv(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - env, file := tc.connection.AsEnv() + envPrep := tc.connection.AsEnv() for i, expected := range tc.expectedEnv { - if env[i] != expected { - t.Errorf("Expected environment variable: %s, but got: %s", expected, env[i]) + if envPrep.Env[i] != expected { + t.Errorf("Expected environment variable: %s, but got: %s", expected, envPrep.Env[i]) } } - if file != tc.expectedFile { - t.Errorf("Expected file content:\n%s\nBut got:\n%s", tc.expectedFile, file) + if envPrep.File.String() != tc.expectedFile { + t.Errorf("Expected file content:\n%s\nBut got:\n%s", tc.expectedFile, envPrep.File.String()) } }) } From 2ea1cc1474adb0566e00156020e856e3d38a4324 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 18:55:57 +0545 Subject: [PATCH 4/7] make envprep context aware --- models/connections.go | 95 ++++++++++++++++++-------------------- models/connections_test.go | 28 +++++++---- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/models/connections.go b/models/connections.go index 41d6f6dd..0fe9ce8b 100644 --- a/models/connections.go +++ b/models/connections.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "fmt" + "math/rand" "net/url" "os" "os/exec" + "path/filepath" "regexp" "time" @@ -156,9 +158,9 @@ func (c Connection) AsGoGetterURL() (string, error) { } // AsEnv generates environment variables and a configuration file content based on the connection type. -func (c Connection) AsEnv() EnvPrep { - envPrep := EnvPrep{ - Conn: c, +func (c Connection) AsEnv(ctx context.Context) EnvPrep { + var envPrep = EnvPrep{ + Files: make(map[string]bytes.Buffer), } switch c.Type { @@ -166,9 +168,13 @@ func (c Connection) AsEnv() EnvPrep { envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) - envPrep.File.WriteString("[default]\n") - envPrep.File.WriteString(fmt.Sprintf("aws_access_key_id = %s\n", c.Username)) - envPrep.File.WriteString(fmt.Sprintf("aws_secret_access_key = %s\n", c.Password)) + // credentialFilePath :="$HOME/.aws/credentials" + credentialFilePath := filepath.Join(".creds", "aws", fmt.Sprintf("cred-%d", rand.Intn(100000000))) + + var credentialFile bytes.Buffer + credentialFile.WriteString("[default]\n") + credentialFile.WriteString(fmt.Sprintf("aws_access_key_id = %s\n", c.Username)) + credentialFile.WriteString(fmt.Sprintf("aws_secret_access_key = %s\n", c.Password)) if v, ok := c.Properties["profile"]; ok { envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_DEFAULT_PROFILE=%s", v)) @@ -176,71 +182,58 @@ func (c Connection) AsEnv() EnvPrep { if v, ok := c.Properties["region"]; ok { envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) - envPrep.File.WriteString(fmt.Sprintf("region = %s\n", v)) + credentialFile.WriteString(fmt.Sprintf("region = %s\n", v)) } + envPrep.Files[credentialFilePath] = credentialFile + case ConnectionTypeAzure: - // Do nothing + args := []string{"login", "--service-principal", "--username", c.Username, "--password", c.Password} + if v, ok := c.Properties["tenant"]; ok { + args = append(args, "--tenant") + args = append(args, v) + } + + // login with service principal + envPrep.PreRuns = append(envPrep.PreRuns, exec.CommandContext(ctx, "az", args...)) case ConnectionTypeGCP: - envPrep.File.WriteString(c.Certificate) + var credentialFile bytes.Buffer + credentialFile.WriteString(c.Certificate) + + // credentialFilePath := "$HOME/.config/gcloud/credentials" + credentialFilePath := filepath.Join(".creds", "gcp", fmt.Sprintf("cred-%d", rand.Intn(100000000))) + + // to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS, + // we need to explicitly activate it + envPrep.PreRuns = append(envPrep.PreRuns, exec.CommandContext(ctx, "gcloud", "auth", "activate-service-account", "--key-file", credentialFilePath)) + envPrep.Files[credentialFilePath] = credentialFile } return envPrep } type EnvPrep struct { - Conn Connection - // Env is the connection credentials in environment variables Env []string + PreRuns []*exec.Cmd + // File contains the content of the configuration file based on the connection - File bytes.Buffer + Files map[string]bytes.Buffer } -func (c *EnvPrep) Apply(ctx context.Context, cmd *exec.Cmd, configAbsPath string) error { - switch c.Conn.Type { - case ConnectionTypeAWS: - if err := saveConfig(c.File.Bytes(), configAbsPath); err != nil { - return err - } - - cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 - cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configAbsPath)) - if v, ok := c.Conn.Properties["region"]; ok { - cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) - } - - case ConnectionTypeGCP: - if err := saveConfig(c.File.Bytes(), configAbsPath); err != nil { - return err - } - - // to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS, - // we need to explicitly activate it - runCmd := exec.Command("gcloud", "auth", "activate-service-account", "--key-file", configAbsPath) - if err := runCmd.Run(); err != nil { - return fmt.Errorf("failed to activate GCP service account: %w", err) - } - - cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configAbsPath)) - - case ConnectionTypeAzure: - args := []string{"login", "--service-principal", "--username", c.Conn.Username, "--password", c.Conn.Password} - if v, ok := c.Conn.Properties["tenant"]; ok { - args = append(args, "--tenant") - args = append(args, v) - } - - // login with service principal - runCmd := exec.CommandContext(ctx, "az", args...) - if err := runCmd.Run(); err != nil { - return err +// Inject creates the config file & injects the necessary environment variable into the command +func (c *EnvPrep) Inject(ctx context.Context, cmd *exec.Cmd) ([]*exec.Cmd, error) { + for path, file := range c.Files { + if err := saveConfig(file.Bytes(), path); err != nil { + return nil, fmt.Errorf("error saving config to %s: %w", path, err) } } - return nil + cmd.Env = append(cmd.Env, c.Env...) + + return c.PreRuns, nil } func saveConfig(content []byte, absPath string) error { diff --git a/models/connections_test.go b/models/connections_test.go index 40b5b975..347a0afd 100644 --- a/models/connections_test.go +++ b/models/connections_test.go @@ -1,6 +1,7 @@ package models import ( + "context" "testing" ) @@ -52,10 +53,10 @@ func Test_Connection_AsGoGetterURL(t *testing.T) { func Test_Connection_AsEnv(t *testing.T) { testCases := []struct { - name string - connection Connection - expectedEnv []string - expectedFile string + name string + connection Connection + expectedEnv []string + expectedFiles map[string]string }{ { name: "AWS Connection", @@ -71,7 +72,9 @@ func Test_Connection_AsEnv(t *testing.T) { "AWS_DEFAULT_PROFILE=awsprofile", "AWS_DEFAULT_REGION=us-east-1", }, - expectedFile: "[default]\naws_access_key_id = awsuser\naws_secret_access_key = awssecret\nregion = us-east-1\n", + expectedFiles: map[string]string{ + "$HOME/.aws/credentials": "[default]\naws_access_key_id = awsuser\naws_secret_access_key = awssecret\nregion = us-east-1\n", + }, }, { name: "GCP Connection", @@ -80,14 +83,16 @@ func Test_Connection_AsEnv(t *testing.T) { Username: "gcpuser", Certificate: `{"account": "gcpuser"}`, }, - expectedEnv: []string{}, - expectedFile: `{"account": "gcpuser"}`, + expectedEnv: []string{}, + expectedFiles: map[string]string{ + "$HOME/.config/gcloud/credentials": `{"account": "gcpuser"}`, + }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - envPrep := tc.connection.AsEnv() + envPrep := tc.connection.AsEnv(context.Background()) for i, expected := range tc.expectedEnv { if envPrep.Env[i] != expected { @@ -95,8 +100,11 @@ func Test_Connection_AsEnv(t *testing.T) { } } - if envPrep.File.String() != tc.expectedFile { - t.Errorf("Expected file content:\n%s\nBut got:\n%s", tc.expectedFile, envPrep.File.String()) + for path, expected := range tc.expectedFiles { + got := envPrep.Files[path] + if got.String() != expected { + t.Errorf("Expected file content: %s, but got: %s", expected, got.String()) + } } }) } From 55e833d581e6337e1c4343258927a86c74996201 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 19:08:51 +0545 Subject: [PATCH 5/7] chore: use lowercase for connection types --- models/connections.go | 94 +++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/models/connections.go b/models/connections.go index 0fe9ce8b..a8883358 100644 --- a/models/connections.go +++ b/models/connections.go @@ -3,6 +3,7 @@ package models import ( "bytes" "context" + "encoding/base64" "fmt" "math/rand" "net/url" @@ -10,6 +11,7 @@ import ( "os/exec" "path/filepath" "regexp" + "strings" "time" "github.com/flanksource/duty/types" @@ -18,46 +20,46 @@ import ( // List of all connection types const ( - ConnectionTypeAWS = "AWS" - ConnectionTypeAzure = "Azure" - ConnectionTypeAzureDevops = "Azure Devops" - ConnectionTypeDiscord = "Discord" - ConnectionTypeDynatrace = "Dynatrace" - ConnectionTypeElasticSearch = "ElasticSearch" - ConnectionTypeEmail = "Email" - ConnectionTypeGCP = "Google Cloud" - ConnectionTypeGenericWebhook = "Generic Webhook" - ConnectionTypeGit = "Git" - ConnectionTypeGithub = "Github" - ConnectionTypeGoogleChat = "Google Chat" - ConnectionTypeHTTP = "HTTP" - ConnectionTypeIFTTT = "IFTTT" - ConnectionTypeJMeter = "JMeter" - ConnectionTypeKubernetes = "Kubernetes" - ConnectionTypeLDAP = "LDAP" - ConnectionTypeMatrix = "Matrix" - ConnectionTypeMattermost = "Mattermost" - ConnectionTypeMongo = "Mongo" - ConnectionTypeMySQL = "MySQL" - ConnectionTypeNtfy = "Ntfy" - ConnectionTypeOpsGenie = "OpsGenie" - ConnectionTypePostgres = "Postgres" - ConnectionTypePrometheus = "Prometheus" - ConnectionTypePushbullet = "Pushbullet" - ConnectionTypePushover = "Pushover" - ConnectionTypeRedis = "Redis" - ConnectionTypeRestic = "Restic" - ConnectionTypeRocketchat = "Rocketchat" - ConnectionTypeSFTP = "SFTP" - ConnectionTypeSlack = "Slack" - ConnectionTypeSlackWebhook = "SlackWebhook" - ConnectionTypeSMB = "SMB" - ConnectionTypeSQLServer = "SQL Server" - ConnectionTypeTeams = "Teams" - ConnectionTypeTelegram = "Telegram" - ConnectionTypeWebhook = "Webhook" - ConnectionTypeWindows = "Windows" - ConnectionTypeZulipChat = "Zulip Chat" + ConnectionTypeAWS = "aws" + ConnectionTypeAzure = "azure" + ConnectionTypeAzureDevops = "azure_devops" + ConnectionTypeDiscord = "discord" + ConnectionTypeDynatrace = "dynatrace" + ConnectionTypeElasticSearch = "elasticsearch" + ConnectionTypeEmail = "email" + ConnectionTypeGCP = "google_cloud" + ConnectionTypeGenericWebhook = "generic_webhook" + ConnectionTypeGit = "git" + ConnectionTypeGithub = "github" + ConnectionTypeGoogleChat = "google_chat" + ConnectionTypeHTTP = "http" + ConnectionTypeIFTTT = "ifttt" + ConnectionTypeJMeter = "jmeter" + ConnectionTypeKubernetes = "kubernetes" + ConnectionTypeLDAP = "ldap" + ConnectionTypeMatrix = "matrix" + ConnectionTypeMattermost = "mattermost" + ConnectionTypeMongo = "mongo" + ConnectionTypeMySQL = "mysql" + ConnectionTypeNtfy = "ntfy" + ConnectionTypeOpsGenie = "opsgenie" + ConnectionTypePostgres = "postgres" + ConnectionTypePrometheus = "prometheus" + ConnectionTypePushbullet = "pushbullet" + ConnectionTypePushover = "pushover" + ConnectionTypeRedis = "redis" + ConnectionTypeRestic = "restic" + ConnectionTypeRocketchat = "rocketchat" + ConnectionTypeSFTP = "sftp" + ConnectionTypeSlack = "slack" + ConnectionTypeSlackWebhook = "slackwebhook" + ConnectionTypeSMB = "smb" + ConnectionTypeSQLServer = "sql_server" + ConnectionTypeTeams = "teams" + ConnectionTypeTelegram = "telegram" + ConnectionTypeWebhook = "webhook" + ConnectionTypeWindows = "windows" + ConnectionTypeZulipChat = "zulip_chat" ) type Connection struct { @@ -76,9 +78,10 @@ type Connection struct { } func (c Connection) String() string { - if c.Type == "aws" { + if strings.ToLower(c.Type) == ConnectionTypeAWS { return "AWS::" + c.Username } + var connection string // Obfuscate passwords of the form ' password=xxxxx ' from connectionString since // connectionStrings are used as metric labels and we don't want to leak passwords @@ -109,7 +112,7 @@ func (c Connection) AsGoGetterURL() (string, error) { } var output string - switch c.Type { + switch strings.ReplaceAll(strings.ToLower(c.Type), " ", "_") { case ConnectionTypeHTTP: if c.Username != "" || c.Password != "" { parsedURL.User = url.UserPassword(c.Username, c.Password) @@ -119,7 +122,10 @@ func (c Connection) AsGoGetterURL() (string, error) { case ConnectionTypeGit: q := parsedURL.Query() - q.Set("sshkey", c.Certificate) + + if c.Certificate != "" { + q.Set("sshkey", base64.URLEncoding.EncodeToString([]byte(c.Certificate))) + } if v, ok := c.Properties["ref"]; ok { q.Set("ref", v) @@ -163,7 +169,7 @@ func (c Connection) AsEnv(ctx context.Context) EnvPrep { Files: make(map[string]bytes.Buffer), } - switch c.Type { + switch strings.ReplaceAll(strings.ToLower(c.Type), " ", "_") { case ConnectionTypeAWS: envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", c.Username)) envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", c.Password)) From 8d695985dff95036757898b134bea82e2b593a23 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 19:18:09 +0545 Subject: [PATCH 6/7] fix: use new CmdEnv --- models/connections.go | 15 ++++++++++++++- models/connections_test.go | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/models/connections.go b/models/connections.go index a8883358..d4a6f1ea 100644 --- a/models/connections.go +++ b/models/connections.go @@ -188,11 +188,17 @@ func (c Connection) AsEnv(ctx context.Context) EnvPrep { if v, ok := c.Properties["region"]; ok { envPrep.Env = append(envPrep.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) + credentialFile.WriteString(fmt.Sprintf("region = %s\n", v)) + + envPrep.CmdEnvs = append(envPrep.CmdEnvs, fmt.Sprintf("AWS_DEFAULT_REGION=%s", v)) } envPrep.Files[credentialFilePath] = credentialFile + envPrep.CmdEnvs = append(envPrep.CmdEnvs, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 + envPrep.CmdEnvs = append(envPrep.CmdEnvs, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", credentialFilePath)) + case ConnectionTypeAzure: args := []string{"login", "--service-principal", "--username", c.Username, "--password", c.Password} if v, ok := c.Properties["tenant"]; ok { @@ -214,6 +220,8 @@ func (c Connection) AsEnv(ctx context.Context) EnvPrep { // we need to explicitly activate it envPrep.PreRuns = append(envPrep.PreRuns, exec.CommandContext(ctx, "gcloud", "auth", "activate-service-account", "--key-file", credentialFilePath)) envPrep.Files[credentialFilePath] = credentialFile + + envPrep.CmdEnvs = append(envPrep.CmdEnvs, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", credentialFilePath)) } return envPrep @@ -223,6 +231,11 @@ type EnvPrep struct { // Env is the connection credentials in environment variables Env []string + // CmdEnvs is a list of env vars that will be passed to the command + CmdEnvs []string + + // List of commands that need to be run before the actual command. + // These commands will setup the connection. PreRuns []*exec.Cmd // File contains the content of the configuration file based on the connection @@ -237,7 +250,7 @@ func (c *EnvPrep) Inject(ctx context.Context, cmd *exec.Cmd) ([]*exec.Cmd, error } } - cmd.Env = append(cmd.Env, c.Env...) + cmd.Env = append(cmd.Env, c.CmdEnvs...) return c.PreRuns, nil } diff --git a/models/connections_test.go b/models/connections_test.go index 347a0afd..29252b5a 100644 --- a/models/connections_test.go +++ b/models/connections_test.go @@ -31,7 +31,7 @@ func Test_Connection_AsGoGetterURL(t *testing.T) { Certificate: "cert123", Properties: map[string]string{"ref": "main"}, }, - expectedURL: "https://github.com/repo.git?ref=main&sshkey=cert123", + expectedURL: "https://github.com/repo.git?ref=main&sshkey=Y2VydDEyMw%3D%3D", expectedError: nil, }, } From 52d0bf08cd37bde32d5e0848958a04866400aff3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 18 Oct 2023 19:29:09 +0545 Subject: [PATCH 7/7] chore: fix test --- models/connections_test.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/models/connections_test.go b/models/connections_test.go index 29252b5a..39269688 100644 --- a/models/connections_test.go +++ b/models/connections_test.go @@ -53,10 +53,10 @@ func Test_Connection_AsGoGetterURL(t *testing.T) { func Test_Connection_AsEnv(t *testing.T) { testCases := []struct { - name string - connection Connection - expectedEnv []string - expectedFiles map[string]string + name string + connection Connection + expectedEnv []string + expectedFileContent string }{ { name: "AWS Connection", @@ -72,9 +72,7 @@ func Test_Connection_AsEnv(t *testing.T) { "AWS_DEFAULT_PROFILE=awsprofile", "AWS_DEFAULT_REGION=us-east-1", }, - expectedFiles: map[string]string{ - "$HOME/.aws/credentials": "[default]\naws_access_key_id = awsuser\naws_secret_access_key = awssecret\nregion = us-east-1\n", - }, + expectedFileContent: "[default]\naws_access_key_id = awsuser\naws_secret_access_key = awssecret\nregion = us-east-1\n", }, { name: "GCP Connection", @@ -83,10 +81,8 @@ func Test_Connection_AsEnv(t *testing.T) { Username: "gcpuser", Certificate: `{"account": "gcpuser"}`, }, - expectedEnv: []string{}, - expectedFiles: map[string]string{ - "$HOME/.config/gcloud/credentials": `{"account": "gcpuser"}`, - }, + expectedEnv: []string{}, + expectedFileContent: `{"account": "gcpuser"}`, }, } @@ -100,10 +96,9 @@ func Test_Connection_AsEnv(t *testing.T) { } } - for path, expected := range tc.expectedFiles { - got := envPrep.Files[path] - if got.String() != expected { - t.Errorf("Expected file content: %s, but got: %s", expected, got.String()) + for _, content := range envPrep.Files { + if content.String() != tc.expectedFileContent { + t.Errorf("Expected file content: %s, but got: %s", tc.expectedFileContent, content.String()) } } })