From c2bffce101bd9f0fa1da56473c0ec64259fb6285 Mon Sep 17 00:00:00 2001 From: adranwit Date: Sun, 17 Mar 2024 13:05:38 -0700 Subject: [PATCH] updated secret with scy --- .gitignore | 5 +++ connection.go | 84 ++++++++++++++++------------------------------ go.mod | 53 +++++++++++++++++++++++++++++ manager_factory.go | 15 +-------- manager_test.go | 3 +- test/config.yaml | 2 +- 6 files changed, 90 insertions(+), 72 deletions(-) create mode 100644 .gitignore create mode 100644 go.mod diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9035052 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +local_test.go +.idea/ +*.iml + + diff --git a/connection.go b/connection.go index 67f71a8..3240fd9 100644 --- a/connection.go +++ b/connection.go @@ -2,19 +2,17 @@ package bgc import ( "fmt" + "github.com/viant/scy/auth/gcp" + "github.com/viant/scy/auth/gcp/client" + "github.com/viant/scy/cred/secret" "net/http" "github.com/viant/dsc" - "github.com/viant/toolbox/secret" + "golang.org/x/net/context" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - "golang.org/x/oauth2/jwt" "google.golang.org/api/bigquery/v2" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" - "io/ioutil" - "os" "reflect" ) @@ -97,66 +95,40 @@ type connectionProvider struct { *dsc.AbstractConnectionProvider } -func (cp *connectionProvider) newAuthConfigWithCredentialsFile() (*jwt.Config, error) { - config, err := secret.New("", false).GetCredentials(cp.Config().Credentials) - if err != nil { - return nil, err - } - return config.NewJWTConfig(bigQueryScope, bigQueryInsertScope, googleDriveReadOnlyScope) -} - -func (cp *connectionProvider) newAuthConfig() (*jwt.Config, error) { - config := cp.Config() - serviceAccountID := config.Get(ServiceAccountIdKey) - var privateKey []byte - if config.Has(PrivateKey) { - privateKey = []byte(config.Get(PrivateKey)) - } else { - var err error - privateKeyPath := config.Get(PrivateKeyPathKey) - privateKey, err = ioutil.ReadFile(privateKeyPath) - if err != nil { - hostname, _ := os.Hostname() - return nil, fmt.Errorf("failed to create bigquery connection - unable to read private key from path %v:%v, %v", hostname, privateKeyPath, err) - } - } - authConfig := &jwt.Config{ - Email: serviceAccountID, - PrivateKey: privateKey, - Subject: serviceAccountID, - Scopes: []string{bigQueryScope, bigQueryInsertScope, googleDriveReadOnlyScope}, - TokenURL: google.JWTTokenURL, - } - return authConfig, nil -} - func (cp *connectionProvider) NewConnection() (dsc.Connection, error) { config := cp.ConnectionProvider.Config() var err error - var authConfig *jwt.Config + ctx := context.Background() var result = &connection{context: &ctx} - if config.CredConfig != nil { - authConfig, _, err = config.CredConfig.JWTConfig(bigQueryScope, bigQueryInsertScope, googleDriveReadOnlyScope) - } else if config.Credentials != "" { - authConfig, err = cp.newAuthConfigWithCredentialsFile() - } else if hasPrivateKey(config) { - authConfig, err = cp.newAuthConfig() - } - if err != nil { - return nil, err + sec := secret.New() + var options = make([]option.ClientOption, 0) + options = append(options, option.WithScopes(bigQueryScope, bigQueryInsertScope, googleDriveReadOnlyScope)) + usesAuth := false + if config.Credentials != "" { + aSecret, err := sec.Lookup(context.Background(), secret.Resource(config.Credentials)) + if err != nil { + return nil, err + } + data := aSecret.String() + options = append(options, option.WithCredentialsJSON([]byte(data))) + options = append(options, option.WithUserAgent(userAgent)) + + usesAuth = true } - var httpClient *http.Client - if authConfig != nil { - httpClient = oauth2.NewClient(ctx, authConfig.TokenSource(ctx)) - } else { - if httpClient, err = getDefaultClient(ctx); err != nil { - return nil, err + if !usesAuth { + gcpService := gcp.New(client.NewScy()) + httpClient, err := gcpService.AuthClient(context.Background(), append(gcp.Scopes, bigQueryScope, bigQueryInsertScope, googleDriveReadOnlyScope)...) + if err == nil && httpClient != nil { + options = append(options, option.WithHTTPClient(httpClient)) } + } - result.service, err = bigquery.New(httpClient) + + service, err := bigquery.NewService(ctx, options...) + result.service = service if err != nil { return nil, fmt.Errorf("failed to create bigquery connection - unable to create client:%v", err) } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6dbc42a --- /dev/null +++ b/go.mod @@ -0,0 +1,53 @@ +module github.com/viant/bgc + +go 1.21.5 + +require ( + github.com/stretchr/testify v1.9.0 + github.com/viant/dsc v0.19.0 + github.com/viant/dsunit v0.11.0 + github.com/viant/scy v0.8.0 + github.com/viant/toolbox v0.34.6-0.20221112031702-3e7cdde7f888 + golang.org/x/net v0.22.0 + google.golang.org/api v0.153.0 +) + +require ( + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/go-errors/errors v1.5.1 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect + github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/iter v1.0.2 // indirect + github.com/lestrrat-go/jwx v1.2.29 // indirect + github.com/lestrrat-go/option v1.0.1 // indirect + github.com/mattn/go-sqlite3 v1.14.18 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/viant/afs v1.25.1-0.20231110184132-877ed98abca1 // indirect + github.com/viant/assertly v0.9.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/oauth2 v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/manager_factory.go b/manager_factory.go index fe4354b..f82d85a 100644 --- a/manager_factory.go +++ b/manager_factory.go @@ -3,7 +3,6 @@ package bgc import ( "fmt" "github.com/viant/dsc" - "github.com/viant/toolbox/secret" ) const defaultTimeFormat = "yyyy-MM-dd HH:mm:ss z" @@ -11,21 +10,9 @@ const defaultTimeFormat = "yyyy-MM-dd HH:mm:ss z" type managerFactory struct{} func (f *managerFactory) configInit(config *dsc.Config) error { - if config.CredConfig != nil { - _, projectID, _ := config.CredConfig.JWTConfig() - if !config.Has(ProjectIDKey) { - config.Parameters[ProjectIDKey] = projectID - } - } else if config.Credentials != "" { - secrets := secret.New("", false) - credConfig, err := secrets.GetCredentials(config.Credentials) - if err != nil { - return err - } - config.CredConfig = credConfig if !config.Has(ProjectIDKey) { - config.Parameters[ProjectIDKey] = credConfig.ProjectID + config.Parameters[ProjectIDKey] = config.CredConfig.ProjectID } } if !config.Has(DateFormatKey) { diff --git a/manager_test.go b/manager_test.go index fa7b3bb..68b0723 100644 --- a/manager_test.go +++ b/manager_test.go @@ -18,7 +18,7 @@ var inited int32 = 0 func initDb(t *testing.T) bool { - if !toolbox.FileExists(path.Join(os.Getenv("HOME"), ".secret/bq.json")) { + if !toolbox.FileExists(path.Join(os.Getenv("HOME"), ".secret/viant-e2e.json")) { return false } @@ -74,6 +74,7 @@ func TestReadSingle(t *testing.T) { } manager := GetManager(t) traveler := Traveler{} + success, err := manager.ReadSingle(&traveler, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity FROM travelers1 WHERE id = ?", []interface{}{4}, nil) assert.Nil(t, err) assert.True(t, success) diff --git a/test/config.yaml b/test/config.yaml index ae5eab2..c9e837b 100644 --- a/test/config.yaml +++ b/test/config.yaml @@ -1,4 +1,4 @@ driverName: bigquery -credentials: bq # place your big query secret json to ~/.secret/bg.json +#credentials: viant-e2e # place your big query secret json to ~/.secret/bg.json parameters: datasetId: myDataset \ No newline at end of file