From afc0d15d391786cf32d57b5d19a7671c9dc67f1c Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 13 Jun 2024 13:21:18 -0700 Subject: [PATCH 01/17] start adding tls --- cmd/createIndex.go | 2 +- cmd/dropIndex.go | 4 +- cmd/listIndex.go | 4 +- cmd/utils.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++ e2e_test.go | 4 +- go.mod | 3 +- go.sum | 2 - 7 files changed, 105 insertions(+), 8 deletions(-) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 150086a..6ce6b41 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -138,7 +138,7 @@ func newCreateIndexCmd() *cobra.Command { defer cancel() adminClient, err := avs.NewAdminClient( - ctx, hosts, createIndexFlags.listenerName.Val, isLoadBalancer, logger, + ctx, hosts, createIndexFlags.listenerName.Val, isLoadBalancer, nil, logger, ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 65b699c..28fc395 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -84,7 +84,9 @@ func newDropIndexCommand() *cobra.Command { ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() - adminClient, err := avs.NewAdminClient(ctx, hosts, nil, isLoadBalancer, logger) + adminClient, err := avs.NewAdminClient( + ctx, hosts, nil, isLoadBalancer, nil, logger, + ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) return err diff --git a/cmd/listIndex.go b/cmd/listIndex.go index dcd8ccc..d9f4008 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -76,7 +76,9 @@ func newListIndexCmd() *cobra.Command { ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) defer cancel() - adminClient, err := avs.NewAdminClient(ctx, hosts, listIndexFlags.listenerName.Val, isLoadBalancer, logger) + adminClient, err := avs.NewAdminClient( + ctx, hosts, listIndexFlags.listenerName.Val, isLoadBalancer, nil, logger, + ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) return err diff --git a/cmd/utils.go b/cmd/utils.go index 7e973d6..e33b22c 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -2,10 +2,104 @@ package cmd import ( "asvec/cmd/flags" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" avs "github.com/aerospike/aerospike-proximus-client-go" ) +func newTLSConfig(rootCA [][]byte, cert []byte, key []byte, keyPass []byte, tlsProtoMin int, tlsProtoMax int) (*tls.Config, error) { + if len(rootCA) == 0 && len(cert) == 0 && len(key) == 0 { + return nil, nil + } + + var ( + clientPool []tls.Certificate + serverPool *x509.CertPool + err error + ) + + serverPool = loadCACerts(rootCA) + + if len(cert) > 0 || len(key) > 0 { + clientPool, err = loadServerCertAndKey(cert, key, keyPass) + if err != nil { + return nil, fmt.Errorf("failed to load client authentication certificate and key `%s`", err) + } + } + + tlsConfig := &tls.Config{ //nolint:gosec // aerospike default tls version is TLSv1.2 + Certificates: clientPool, + RootCAs: serverPool, + InsecureSkipVerify: false, + PreferServerCipherSuites: true, + MinVersion: uint16(tlsProtoMin), + MaxVersion: uint16(tlsProtoMax), + } + + return tlsConfig, nil +} + +// loadCACerts returns CA set of certificates (cert pool) +// reads CA certificate based on the certConfig and adds it to the pool +func loadCACerts(certsBytes [][]byte) *x509.CertPool { + certificates, err := x509.SystemCertPool() + if certificates == nil || err != nil { + certificates = x509.NewCertPool() + } + + for _, cert := range certsBytes { + if len(cert) > 0 { + certificates.AppendCertsFromPEM(cert) + } + } + + return certificates +} + +// loadServerCertAndKey reads server certificate and associated key file based on certConfig and keyConfig +// returns parsed server certificate +// if the private key is encrypted, it will be decrypted using key file passphrase +func loadServerCertAndKey(certFileBytes, keyFileBytes, keyPassBytes []byte) ([]tls.Certificate, error) { + var certificates []tls.Certificate + + // Decode PEM data + keyBlock, _ := pem.Decode(keyFileBytes) + + if keyBlock == nil { + return nil, fmt.Errorf("failed to decode PEM data for key or certificate") + } + + // Check and Decrypt the Key Block using passphrase + if x509.IsEncryptedPEMBlock(keyBlock) { //nolint:staticcheck,lll // This needs to be addressed by aerospike as multiple projects require this functionality + decryptedDERBytes, err := x509.DecryptPEMBlock(keyBlock, keyPassBytes) //nolint:staticcheck,lll // This needs to be addressed by aerospike as multiple projects require this functionality + if err != nil { + return nil, fmt.Errorf("failed to decrypt PEM Block: `%s`", err) + } + + keyBlock.Bytes = decryptedDERBytes + keyBlock.Headers = nil + } + + // Encode PEM data + keyPEM := pem.EncodeToMemory(keyBlock) + + if keyPEM == nil { + return nil, fmt.Errorf("failed to encode PEM data for key or certificate") + } + + cert, err := tls.X509KeyPair(certFileBytes, keyPEM) + if err != nil { + return nil, fmt.Errorf("failed to add certificate and key to the pool: `%s`", err) + } + + certificates = append(certificates, cert) + + return certificates, nil +} + func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFlag) (avs.HostPortSlice, bool) { isLoadBalancer := false hosts := avs.HostPortSlice{} diff --git a/e2e_test.go b/e2e_test.go index cf8a23b..76a2fab 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -74,7 +74,9 @@ func (suite *CmdTestSuite) SetupSuite() { defer cancel() for { - suite.avsClient, err = avs.NewAdminClient(ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, logger) + suite.avsClient, err = avs.NewAdminClient( + ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, nil, logger, + ) if err != nil { fmt.Printf("unable to create avs client %v", err) diff --git a/go.mod b/go.mod index 5f42fff..76d251b 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,7 @@ module asvec go 1.21.7 -//replace github.com/aerospike/aerospike-proximus-client-go => github.com/aerospike/aerospike-proximus-client-go VEC-155-admin-client -//replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go require ( github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 diff --git a/go.sum b/go.sum index a31c1ab..16c4edc 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= From 56dd4b19b20f4ea9ce7a72a60f567586a5203222 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 13 Jun 2024 15:39:11 -0700 Subject: [PATCH 02/17] add tls flags --- cmd/constants.go | 6 +++++ cmd/createIndex.go | 3 +++ cmd/dropIndex.go | 3 +++ cmd/flags/tls.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/listIndex.go | 3 +++ 5 files changed, 76 insertions(+) create mode 100644 cmd/flags/tls.go diff --git a/cmd/constants.go b/cmd/constants.go index 0b6ee36..f4cce6c 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -22,4 +22,10 @@ const ( flagNameBatchMaxRecords = "hnsw-batch-max-records" flagNameBatchInterval = "hnsw-batch-interval" flagNameBatchEnabled = "hnsw-batch-enabled" + flagNameTLSProtocols = "tls-protocols" + flagNameTLSCaFile = "tls-cafile" + flagNameTLSCaPath = "tls-capath" + flagNameTLSCertFile = "tls-certfile" + flagNameTLSKeyFile = "tls-keyfile" + flagNameTLSKeyFilePass = "tls-keyfile-password" ) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 6ce6b41..81b4fb7 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -40,6 +40,7 @@ var createIndexFlags = &struct { hnswBatchInterval flags.Uint32OptionalFlag hnswBatchEnabled flags.BoolOptionalFlag timeout time.Duration + tls *flags.TLSFlags }{ host: flags.NewDefaultHostPortFlag(), seeds: &flags.SeedsSliceFlag{}, @@ -51,6 +52,7 @@ var createIndexFlags = &struct { hnswBatchMaxRecords: flags.Uint32OptionalFlag{}, hnswBatchInterval: flags.Uint32OptionalFlag{}, hnswBatchEnabled: flags.BoolOptionalFlag{}, + tls: &flags.TLSFlags{}, } func newCreateIndexFlagSet() *pflag.FlagSet { @@ -74,6 +76,7 @@ func newCreateIndexFlagSet() *pflag.FlagSet { flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability + flagSet.AddFlagSet(createIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) return flagSet } diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 28fc395..d8deb5f 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -26,9 +26,11 @@ var dropIndexFlags = &struct { sets []string indexName string timeout time.Duration + tls *flags.TLSFlags }{ host: flags.NewDefaultHostPortFlag(), seeds: &flags.SeedsSliceFlag{}, + tls: &flags.TLSFlags{}, } func newDropIndexFlagSet() *pflag.FlagSet { @@ -40,6 +42,7 @@ func newDropIndexFlagSet() *pflag.FlagSet { flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(dropIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) return flagSet } diff --git a/cmd/flags/tls.go b/cmd/flags/tls.go new file mode 100644 index 0000000..1dd6f91 --- /dev/null +++ b/cmd/flags/tls.go @@ -0,0 +1,61 @@ +package flags + +import ( + "crypto/tls" + + commonClient "github.com/aerospike/tools-common-go/client" + commonFlags "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/pflag" +) + +type TLSFlags struct { + TLSProtocols commonFlags.TLSProtocolsFlag + TLSRootCAFile commonFlags.CertFlag + TLSRootCAPath commonFlags.CertPathFlag + TLSCertFile commonFlags.CertFlag + TLSKeyFile commonFlags.CertFlag + TLSKeyFilePass commonFlags.PasswordFlag +} + +func NewTLSFlags() *TLSFlags { + return &TLSFlags{ + TLSProtocols: commonFlags.NewDefaultTLSProtocolsFlag(), + } +} + +// NewAerospikeFlagSet returns a new pflag.FlagSet with Aerospike flags defined. +// Values set in the returned FlagSet will be stored in the AerospikeFlags argument. +func (tf *TLSFlags) NewFlagSet(fmtUsage commonFlags.UsageFormatter) *pflag.FlagSet { + f := &pflag.FlagSet{} + + f.Var(&tf.TLSRootCAFile, "tls-cafile", fmtUsage("The CA used when connecting to AVS.")) + f.Var(&tf.TLSRootCAPath, "tls-capath", fmtUsage("A path containing CAs for connecting to AVS.")) + f.Var(&tf.TLSCertFile, "tls-certfile", fmtUsage("The certificate file for mutual TLS authentication with AVS.")) + f.Var(&tf.TLSKeyFile, "tls-keyfile", fmtUsage("The key file used for mutual TLS authentication with AVS.")) + f.Var(&tf.TLSKeyFilePass, "tls-keyfile-password", fmtUsage("The password used to decrypt the key-file if encrypted.")) + f.Var(&tf.TLSProtocols, "tls-protocols", fmtUsage( + "Set the TLS protocol selection criteria. This format is the same as"+ + " Apache's SSLProtocol documented at https://httpd.apache.org/docs/current/mod/mod_ssl.html#ssl protocol.", + )) + + return f +} + +func (tf *TLSFlags) NewTLSConfig() (*tls.Config, error) { + rootCA := [][]byte{} + + if len(tf.TLSRootCAFile) != 0 { + rootCA = append(rootCA, tf.TLSRootCAFile) + } + + rootCA = append(rootCA, tf.TLSRootCAPath...) + + return commonClient.NewTLSConfig( + rootCA, + tf.TLSCertFile, + tf.TLSKeyFile, + tf.TLSKeyFilePass, + tf.TLSProtocols.Min, + tf.TLSProtocols.Max, + ).NewGoTLSConfig() +} diff --git a/cmd/listIndex.go b/cmd/listIndex.go index d9f4008..0ab7e6a 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -25,9 +25,11 @@ var listIndexFlags = &struct { listenerName flags.StringOptionalFlag verbose bool timeout time.Duration + tls *flags.TLSFlags }{ host: flags.NewDefaultHostPortFlag(), seeds: &flags.SeedsSliceFlag{}, + tls: &flags.TLSFlags{}, } func newListIndexFlagSet() *pflag.FlagSet { @@ -37,6 +39,7 @@ func newListIndexFlagSet() *pflag.FlagSet { flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(listIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) return flagSet } From 50c732655452233ba6ae925c19a91555c0339b43 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 18 Jun 2024 11:35:51 -0700 Subject: [PATCH 03/17] add tls tests and re-org flags --- .github/workflows/tests.yml | 1 + .gitignore | 2 +- cmd/constants.go | 31 -- cmd/createIndex.go | 115 ++++---- cmd/dropIndex.go | 67 ++--- cmd/flags/client.go | 34 +++ cmd/flags/constants.go | 31 ++ cmd/flags/tls.go | 6 +- cmd/listIndex.go | 56 ++-- cmd/root.go | 4 +- docker/tls/config/aerospike-proximus.yml | 86 ++++++ docker/tls/config/aerospike.conf | 82 ++++++ .../config/tls/connector.aerospike.com.crt | 22 ++ .../tls/connector.aerospike.com.keystore.jks | Bin 0 -> 2285 bytes docker/tls/config/tls/keypass | 1 + docker/tls/config/tls/storepass | 1 + docker/tls/docker-compose.yml | 23 ++ e2e_test.go | 269 ++++++------------ go.mod | 8 +- go.sum | 6 + test_utils.go | 160 +++++++++++ 21 files changed, 666 insertions(+), 339 deletions(-) delete mode 100644 cmd/constants.go create mode 100644 cmd/flags/client.go create mode 100644 cmd/flags/constants.go create mode 100644 docker/tls/config/aerospike-proximus.yml create mode 100644 docker/tls/config/aerospike.conf create mode 100644 docker/tls/config/tls/connector.aerospike.com.crt create mode 100644 docker/tls/config/tls/connector.aerospike.com.keystore.jks create mode 100644 docker/tls/config/tls/keypass create mode 100644 docker/tls/config/tls/storepass create mode 100644 docker/tls/docker-compose.yml create mode 100644 test_utils.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a82f7a..6530556 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,7 @@ jobs: echo "$FEATURES_CONF" > docker/config/features.conf - name: Run tests run: | + echo '127.0.0.1 connector.aerospike.com' >> /etc/hosts make coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/.gitignore b/.gitignore index d87c693..00ec609 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/docker/config/features.conf +features.conf /bin/* embed_*.go /tmp diff --git a/cmd/constants.go b/cmd/constants.go deleted file mode 100644 index f4cce6c..0000000 --- a/cmd/constants.go +++ /dev/null @@ -1,31 +0,0 @@ -package cmd - -const ( - logLevelFlagName = "log-level" - flagNameSeeds = "seeds" - flagNameHost = "host" - flagNameListenerName = "listener-name" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVectorField = "vector-field" - flagNameDimension = "dimension" - flagNameDistanceMetric = "distance-metric" - flagNameIndexMeta = "index-meta" - flagNameTimeout = "timeout" - flagNameVerbose = "verbose" - flagNameStorageNamespace = "storage-namespace" - flagNameStorageSet = "storage-set" - flagNameMaxEdges = "hnsw-max-edges" - flagNameConstructionEf = "hnsw-ef-construction" - flagNameEf = "hnsw-ef" - flagNameBatchMaxRecords = "hnsw-batch-max-records" - flagNameBatchInterval = "hnsw-batch-interval" - flagNameBatchEnabled = "hnsw-batch-enabled" - flagNameTLSProtocols = "tls-protocols" - flagNameTLSCaFile = "tls-cafile" - flagNameTLSCaPath = "tls-capath" - flagNameTLSCertFile = "tls-certfile" - flagNameTLSKeyFile = "tls-keyfile" - flagNameTLSKeyFilePass = "tls-keyfile-password" -) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 81b4fb7..cb4a381 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -21,9 +21,7 @@ import ( //nolint:govet // Padding not a concern for a CLI var createIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag + flags.ClientFlags namespace string sets []string indexName string @@ -40,10 +38,8 @@ var createIndexFlags = &struct { hnswBatchInterval flags.Uint32OptionalFlag hnswBatchEnabled flags.BoolOptionalFlag timeout time.Duration - tls *flags.TLSFlags }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, + ClientFlags: *flags.NewClientFlags(), storageNamespace: flags.StringOptionalFlag{}, storageSet: flags.StringOptionalFlag{}, hnswMaxEdges: flags.Uint32OptionalFlag{}, @@ -52,41 +48,37 @@ var createIndexFlags = &struct { hnswBatchMaxRecords: flags.Uint32OptionalFlag{}, hnswBatchInterval: flags.Uint32OptionalFlag{}, hnswBatchEnabled: flags.BoolOptionalFlag{}, - tls: &flags.TLSFlags{}, } func newCreateIndexFlagSet() *pflag.FlagSet { - flagSet := &pflag.FlagSet{} - flagSet.VarP(createIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(createIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&createIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability - flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability - flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageSet, flagNameStorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswMaxEdges, flagNameMaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswConstructionEf, flagNameConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswEf, flagNameEf, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability - flagSet.AddFlagSet(createIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) + flagSet := &pflag.FlagSet{} //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringArrayVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.vectorField, flags.VectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability + flagSet.Uint32VarP(&createIndexFlags.dimensions, flags.Dimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.distanceMetric, flags.DistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability + flagSet.StringToStringVar(&createIndexFlags.indexMeta, flags.IndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.DurationVar(&createIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageNamespace, flags.StorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageSet, flags.StorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswMaxEdges, flags.MaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswConstructionEf, flags.ConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswEf, flags.Ef, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flags.BatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchInterval, flags.BatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchEnabled, flags.BatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability + flagSet.AddFlagSet(createIndexFlags.NewClientFlagSet()) return flagSet } var createIndexRequiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, - flagNameVectorField, - flagNameDimension, - flagNameDistanceMetric, + flags.Namespace, + flags.IndexName, + flags.VectorField, + flags.Dimension, + flags.DistanceMetric, } // createIndexCmd represents the createIndex command @@ -106,42 +98,53 @@ func newCreateIndexCmd() *cobra.Command { --storage-namespace test --hnsw-batch-enabled false `, PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { - hosts, isLoadBalancer := parseBothHostSeedsFlag(createIndexFlags.seeds, createIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(createIndexFlags.Seeds, createIndexFlags.Host) logger.Debug("parsed flags", - slog.String(flagNameHost, createIndexFlags.host.String()), - slog.String(flagNameSeeds, createIndexFlags.seeds.String()), - slog.String(flagNameListenerName, createIndexFlags.listenerName.String()), - slog.String(flagNameNamespace, createIndexFlags.namespace), - slog.Any(flagNameSets, createIndexFlags.sets), - slog.String(flagNameIndexName, createIndexFlags.indexName), - slog.String(flagNameVectorField, createIndexFlags.vectorField), - slog.Uint64(flagNameDimension, uint64(createIndexFlags.dimensions)), - slog.Any(flagNameIndexMeta, createIndexFlags.indexMeta), - slog.String(flagNameDistanceMetric, createIndexFlags.distanceMetric.String()), - slog.Duration(flagNameTimeout, createIndexFlags.timeout), - slog.Any(flagNameStorageNamespace, createIndexFlags.storageNamespace.String()), - slog.Any(flagNameStorageSet, createIndexFlags.storageSet.String()), - slog.Any(flagNameMaxEdges, createIndexFlags.hnswMaxEdges.String()), - slog.Any(flagNameEf, createIndexFlags.hnswEf), - slog.Any(flagNameConstructionEf, createIndexFlags.hnswConstructionEf.String()), - slog.Any(flagNameBatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), - slog.Any(flagNameBatchInterval, createIndexFlags.hnswBatchInterval.String()), - slog.Any(flagNameBatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + slog.String(flags.Host, createIndexFlags.Host.String()), + slog.String(flags.Seeds, createIndexFlags.Seeds.String()), + slog.String(flags.ListenerName, createIndexFlags.ListenerName.String()), + slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), + slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), + slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), + slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), + slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), + slog.String(flags.Namespace, createIndexFlags.namespace), + slog.Any(flags.Sets, createIndexFlags.sets), + slog.String(flags.IndexName, createIndexFlags.indexName), + slog.String(flags.VectorField, createIndexFlags.vectorField), + slog.Uint64(flags.Dimension, uint64(createIndexFlags.dimensions)), + slog.Any(flags.IndexMeta, createIndexFlags.indexMeta), + slog.String(flags.DistanceMetric, createIndexFlags.distanceMetric.String()), + slog.Duration(flags.Timeout, createIndexFlags.timeout), + slog.Any(flags.StorageNamespace, createIndexFlags.storageNamespace.String()), + slog.Any(flags.StorageSet, createIndexFlags.storageSet.String()), + slog.Any(flags.MaxEdges, createIndexFlags.hnswMaxEdges.String()), + slog.Any(flags.Ef, createIndexFlags.hnswEf), + slog.Any(flags.ConstructionEf, createIndexFlags.hnswConstructionEf.String()), + slog.Any(flags.BatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), + slog.Any(flags.BatchInterval, createIndexFlags.hnswBatchInterval.String()), + slog.Any(flags.BatchEnabled, createIndexFlags.hnswBatchEnabled.String()), ) ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) defer cancel() + tlsConfig, err := createIndexFlags.NewTLSConfig() + if err != nil { + logger.Error("failed to create TLS config", slog.Any("error", err)) + return err + } + adminClient, err := avs.NewAdminClient( - ctx, hosts, createIndexFlags.listenerName.Val, isLoadBalancer, nil, logger, + ctx, hosts, createIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index d8deb5f..6ddfcd1 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -19,37 +19,29 @@ import ( //nolint:govet // Padding not a concern for a CLI var dropIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag - namespace string - sets []string - indexName string - timeout time.Duration - tls *flags.TLSFlags + flags.ClientFlags + namespace string + sets []string + indexName string + timeout time.Duration }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, - tls: &flags.TLSFlags{}, + ClientFlags: *flags.NewClientFlags(), } func newDropIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(dropIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(dropIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&dropIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.StringVarP(&dropIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability - flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability - flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.AddFlagSet(dropIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) + flagSet.StringVarP(&dropIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringArrayVarP(&dropIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&dropIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.DurationVar(&dropIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(dropIndexFlags.NewClientFlagSet()) return flagSet } var dropIndexRequiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, + flags.Namespace, + flags.IndexName, } // dropIndexCmd represents the dropIndex command @@ -65,30 +57,41 @@ func newDropIndexCommand() *cobra.Command { asvec drop index -i myindex -n test `, PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flagNameHost, dropIndexFlags.host.String()), - slog.String(flagNameSeeds, dropIndexFlags.seeds.String()), - slog.String(flagNameListenerName, dropIndexFlags.listenerName.String()), - slog.String(flagNameNamespace, dropIndexFlags.namespace), - slog.Any(flagNameSets, dropIndexFlags.sets), - slog.String(flagNameIndexName, dropIndexFlags.indexName), - slog.Duration(flagNameTimeout, dropIndexFlags.timeout), + slog.String(flags.Host, dropIndexFlags.Host.String()), + slog.String(flags.Seeds, dropIndexFlags.Seeds.String()), + slog.String(flags.ListenerName, dropIndexFlags.ListenerName.String()), + slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), + slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), + slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), + slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), + slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), + slog.String(flags.Namespace, dropIndexFlags.namespace), + slog.Any(flags.Sets, dropIndexFlags.sets), + slog.String(flags.IndexName, dropIndexFlags.indexName), + slog.Duration(flags.Timeout, dropIndexFlags.timeout), ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(dropIndexFlags.seeds, dropIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(dropIndexFlags.Seeds, dropIndexFlags.Host) ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() + tlsConfig, err := dropIndexFlags.NewTLSConfig() + if err != nil { + logger.Error("failed to create TLS config", slog.Any("error", err)) + return err + } + adminClient, err := avs.NewAdminClient( - ctx, hosts, nil, isLoadBalancer, nil, logger, + ctx, hosts, dropIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) diff --git a/cmd/flags/client.go b/cmd/flags/client.go new file mode 100644 index 0000000..95dc670 --- /dev/null +++ b/cmd/flags/client.go @@ -0,0 +1,34 @@ +package flags + +import ( + "fmt" + + commonFlags "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/pflag" +) + +type ClientFlags struct { + Host *HostPortFlag + Seeds *SeedsSliceFlag + ListenerName StringOptionalFlag + TLSFlags +} + +func NewClientFlags() *ClientFlags { + return &ClientFlags{ + Host: NewDefaultHostPortFlag(), + Seeds: &SeedsSliceFlag{}, + TLSFlags: *NewTLSFlags(), + } +} + +func (cf *ClientFlags) NewClientFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.VarP(cf.Host, Host, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", Seeds))) //nolint:lll // For readability + flagSet.Var(cf.Seeds, Seeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", Host))) //nolint:lll // For readability + flagSet.VarP(&cf.ListenerName, ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) + + flagSet.AddFlagSet(cf.NewTLSFlagSet(commonFlags.DefaultWrapHelpString)) + + return flagSet +} diff --git a/cmd/flags/constants.go b/cmd/flags/constants.go new file mode 100644 index 0000000..1e06a87 --- /dev/null +++ b/cmd/flags/constants.go @@ -0,0 +1,31 @@ +package flags + +const ( + LogLevel = "log-level" + Seeds = "seeds" + Host = "host" + ListenerName = "listener-name" + Namespace = "namespace" + Sets = "sets" + IndexName = "index-name" + VectorField = "vector-field" + Dimension = "dimension" + DistanceMetric = "distance-metric" + IndexMeta = "index-meta" + Timeout = "timeout" + Verbose = "verbose" + StorageNamespace = "storage-namespace" + StorageSet = "storage-set" + MaxEdges = "hnsw-max-edges" + ConstructionEf = "hnsw-ef-construction" + Ef = "hnsw-ef" + BatchMaxRecords = "hnsw-batch-max-records" + BatchInterval = "hnsw-batch-interval" + BatchEnabled = "hnsw-batch-enabled" + TLSProtocols = "tls-protocols" + TLSCaFile = "tls-cafile" + TLSCaPath = "tls-capath" + TLSCertFile = "tls-certfile" + TLSKeyFile = "tls-keyfile" + TLSKeyFilePass = "tls-keyfile-password" +) diff --git a/cmd/flags/tls.go b/cmd/flags/tls.go index 1dd6f91..30927c4 100644 --- a/cmd/flags/tls.go +++ b/cmd/flags/tls.go @@ -25,7 +25,7 @@ func NewTLSFlags() *TLSFlags { // NewAerospikeFlagSet returns a new pflag.FlagSet with Aerospike flags defined. // Values set in the returned FlagSet will be stored in the AerospikeFlags argument. -func (tf *TLSFlags) NewFlagSet(fmtUsage commonFlags.UsageFormatter) *pflag.FlagSet { +func (tf *TLSFlags) NewTLSFlagSet(fmtUsage commonFlags.UsageFormatter) *pflag.FlagSet { f := &pflag.FlagSet{} f.Var(&tf.TLSRootCAFile, "tls-cafile", fmtUsage("The CA used when connecting to AVS.")) @@ -55,7 +55,7 @@ func (tf *TLSFlags) NewTLSConfig() (*tls.Config, error) { tf.TLSCertFile, tf.TLSKeyFile, tf.TLSKeyFilePass, - tf.TLSProtocols.Min, - tf.TLSProtocols.Max, + 0, + 0, ).NewGoTLSConfig() } diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 0ab7e6a..d294caf 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -20,26 +20,21 @@ import ( ) var listIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag - verbose bool - timeout time.Duration - tls *flags.TLSFlags + flags.ClientFlags + verbose bool + timeout time.Duration }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, - tls: &flags.TLSFlags{}, + ClientFlags: *flags.NewClientFlags(), } func newListIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(listIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(listIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability - flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.AddFlagSet(listIndexFlags.tls.NewFlagSet(commonFlags.DefaultWrapHelpString)) + flagSet.VarP(listIndexFlags.Host, flags.Host, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flags.Seeds))) //nolint:lll // For readability + flagSet.Var(listIndexFlags.Seeds, flags.Seeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flags.Host))) //nolint:lll // For readability + flagSet.VarP(&listIndexFlags.ListenerName, flags.ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.BoolVarP(&listIndexFlags.verbose, flags.Verbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability + flagSet.DurationVar(&listIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(listIndexFlags.NewClientFlagSet()) return flagSet } @@ -57,30 +52,41 @@ func newListIndexCmd() *cobra.Command { For example: export ASVEC_HOST=:5000 asvec list index - `, flagNameVerbose), + `, flags.Verbose), PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flagNameHost, listIndexFlags.host.String()), - slog.String(flagNameSeeds, listIndexFlags.seeds.String()), - slog.String(flagNameListenerName, listIndexFlags.listenerName.String()), - slog.Bool(flagNameVerbose, listIndexFlags.verbose), - slog.Duration(flagNameTimeout, listIndexFlags.timeout), + slog.String(flags.Host, listIndexFlags.Host.String()), + slog.String(flags.Seeds, listIndexFlags.Seeds.String()), + slog.String(flags.ListenerName, listIndexFlags.ListenerName.String()), + slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), + slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), + slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), + slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), + slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), + slog.Bool(flags.Verbose, listIndexFlags.verbose), + slog.Duration(flags.Timeout, listIndexFlags.timeout), ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(listIndexFlags.seeds, listIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(listIndexFlags.Seeds, listIndexFlags.Host) ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) defer cancel() + tlsConfig, err := listIndexFlags.NewTLSConfig() + if err != nil { + logger.Error("failed to create TLS config", slog.Any("error", err)) + return err + } + adminClient, err := avs.NewAdminClient( - ctx, hosts, listIndexFlags.listenerName.Val, isLoadBalancer, nil, logger, + ctx, hosts, listIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, ) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) diff --git a/cmd/root.go b/cmd/root.go index 28bb576..db16927 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -94,11 +94,11 @@ func init() { common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") - if err := viper.BindEnv(flagNameHost); err != nil { + if err := viper.BindEnv(flags.Host); err != nil { logger.Error("failed to bind environment variable", slog.Any("error", err)) } - if err := viper.BindEnv(flagNameSeeds); err != nil { + if err := viper.BindEnv(flags.Seeds); err != nil { logger.Error("failed to bind environment variable", slog.Any("error", err)) } } diff --git a/docker/tls/config/aerospike-proximus.yml b/docker/tls/config/aerospike-proximus.yml new file mode 100644 index 0000000..740d40f --- /dev/null +++ b/docker/tls/config/aerospike-proximus.yml @@ -0,0 +1,86 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/tls/config/aerospike.conf b/docker/tls/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/tls/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/tls/config/tls/connector.aerospike.com.crt b/docker/tls/config/tls/connector.aerospike.com.crt new file mode 100644 index 0000000..3e53361 --- /dev/null +++ b/docker/tls/config/tls/connector.aerospike.com.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDpjCCAo6gAwIBAgIJAJuWztiFvTcCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgx +MDMyMjFaFw0yOTA3MDUxMDMyMjFaMFYxCzAJBgNVBAYTAklOMQswCQYDVQQIDAJL +QTEYMBYGA1UECgwPQWVyb3NwaWtlLCBJbmMuMSAwHgYDVQQDDBdjb25uZWN0b3Iu +YWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAO70 +L+G2TP6s1jPPMvIWB2LSdiVKkfKAuihTf5lJEuF1jyCt6c8q7j4gqAEA5PoAaTpM +ukKt3EvmCyRZTjBr1WqSOuwU4mw+rsQ+gWdFiaEOLXS6kHcpunb5/wLeYAC6rGLk +Di9FBSwX3sNke5M1v/P+Vcd3ozA3bsv5JTFCbZiwarG7i8ZatRXFDHYwS0t36pLA +JcyVUyehT7SSctkQQym0goMjLpnEKgimC0RKgw9yhmPmK5oasIzythr6MSv2SkiO +BJfJDBJZcqvNFn66ghuofJ7vyxxW59DSwrRAlXPfAaxQ4mcgECuoe1xMUtrMya64 +4Hz7ZijnbMrFKTqgy3sCAwEAAaNBMD8wPQYDVR0RBDYwNIIXY29ubmVjdG9yLmFl +cm9zcGlrZS5jb22CGSouY29ubmVjdG9yLmFlcm9zcGlrZS5jb20wDQYJKoZIhvcN +AQELBQADggEBAFTbTRmftwDZlC/60I9w6u5/vRRJRCQvp5mBhrbsfA1OcY7HOBOT +G3hFtDHl14s5cpZAgecdgACtvGvnex6q/gun2JzdAmJjslpRAQgZA0L182X+Sc5z +5QIhMhS6iMCae9KV4GaEq0VMfZ/KlKPfESj1DZE3VQNWUPTwBbamnzpEqSzoFdrp +duFah12yIwnUoTi+fNfhhm76Y5GkG5Cx4HJvF6qiJ1uOIieLYZ07e1nFxIVqnEz/ +zZCkDkKqXWb+ii8hlBl8kiu4vVwKEjrlruSaKb+3UTJxB16zDESjn5i58A9TgtsZ +VvECl552NbEBT+VOgRPsNJRBpPbPfYqUVfk= +-----END CERTIFICATE----- diff --git a/docker/tls/config/tls/connector.aerospike.com.keystore.jks b/docker/tls/config/tls/connector.aerospike.com.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..29490f53bdc89036e1e5bb37638e5e1c43e28f7e GIT binary patch literal 2285 zcmZ{lc{J4f8^`B6gQ1yWFi1(3v5xr82u0aSg-KZp=~t$)??jl$Oo(KO#H4I<#e|Rs zlOjtlLP)ajS<(Nc+K{dLc;bMEht=bY#LocB4O*LiO8x#2I~+AF91?`zJr2%+i-jU51Qe?r#}k8X>9HovSD09VZ3uKML#UW)wM{t82Ju;{U|7!OAA~l+(GS=rBb(p+a(e zcjIx!sN$>gs(~)mNWN5Dld@RU?dP8IqZSrL-n-6ijzLO8iBo~q_;^d|>vE^~i1ZH4 z4%oVk+tn61_oB##nL+DDrF%95I!iQkXSmWs*2&f9&Fv{ieMSts<<4J>cC`OusKMeE zFwUR}1s+|DPV1brB>_(qtp?|^7rS0k7gff!bQyCUw-wl3RkJETZogo<`$`LGh5o*Bp-vV{eO6`tq*0Yve^gl@=3yaym1#s&y9$)q zWZ(0jJ>75IP0R4NIv@-+dQ* zS*Cpf?6Up91aHuSr}Vr?!OITKYHk;qQ(3J~H4`G>y;pZs=Nda_Vw)m8t?;gQa{MCQ zlr+c4ek;S93VlJ51_2W?Ef@vhduHx?)CK?Qp%?6HrnHe0VbfY|a@YyA7b6KDUF!7g zlJDdNH)ty);Amme`;_1CBHAVlwqyOt`nmW5JD&Dh*~0Xu()N4g(Ho-iQTK)uHVvz5 zm4qT65v;o0S*wP~Ce;f^{dp%T4{R$5kGuacIB0o>GJi=IHRAA=KP4N%AKm6~5-ugY ze4zY=31KGEFX9n(0QF-4szUuZhafGdiv8l(LdDbfaE1vuNUtRPWr$=KBpL9x3_ zhbDARlhf+>Gb0UiBl!dR2vZ>g`f5c@zYRXfMYPmVz3rVB^)I{hQsF+I2~g{@PScv< z6^)VD&n&^GPM_-6q#{C#*lrRb4e*ts>3N

6n7JQhQ&j)8?-K<%@7sFx1q&~KB!swwn>+`idzbyAc0fp#|S)ovMo=* zG=CJ1hwX^k)q#R2J^|I3V=!RgUH@^Nl@a44~9gO&a3}p zRVVp+<9CC*{yAL$5Z^Kw{$c>7B~bYNpboeX4{8z!cn1j-;fK`vuLSLWBK-^Oc2N9p zJ|Ux@d}=bLJ|uO#3@H6Kkp#)t$A?S`@(uVE45R{b{~OHK-vOfnLXfTbHYgPUAe>Jc z6D%UP_>E>4VMQG7+!L%|ma=k%t!y2cX(}>tF-fLmzDt#(FH;CWroTemb%|`_l40|A z2&}yY=-KCSQ+E+F>7`%Vq#xyK5|=BuCy1RKqQVZ|*o2OrgRqO8rv)`kc-16E8!v>X zYSpcBZCgU}K^>p=4F$ZhcSe~<`IGA}93G1`BZEP6^N@v`^$IU$d+r=wDyoR>%OvX6U$Nb`@kGP2=mUB)4s6WyRJY=YCUJerI}pB{kK^oP-jwwgC)Q( z2{T?>C38(X181PJ1PuFHJw3eV*0{^n2PVYJ*==e0W5UX7s1zMrnBB=w%e<_DY~8~} zYI9;YcETTjg_of*!*1Impr`c{idl;1>m6CutVd%0>%U}09tvOWvC+T$BVE_OC literal 0 HcmV?d00001 diff --git a/docker/tls/config/tls/keypass b/docker/tls/config/tls/keypass new file mode 100644 index 0000000..b1f833e --- /dev/null +++ b/docker/tls/config/tls/keypass @@ -0,0 +1 @@ +citrusstore diff --git a/docker/tls/config/tls/storepass b/docker/tls/config/tls/storepass new file mode 100644 index 0000000..b1f833e --- /dev/null +++ b/docker/tls/config/tls/storepass @@ -0,0 +1 @@ +citrusstore diff --git a/docker/tls/docker-compose.yml b/docker/tls/docker-compose.yml new file mode 100644 index 0000000..30a3af1 --- /dev/null +++ b/docker/tls/docker-compose.yml @@ -0,0 +1,23 @@ +services: + aerospike: + image: aerospike/aerospike-server-enterprise:7.0.0.2 + ports: + - "3000:3000" + networks: + - avs-demo + volumes: + - ./config:/opt/aerospike/etc/aerospike + command: + - "--config-file" + - "/opt/aerospike/etc/aerospike/aerospike.conf" + avs: + image: aerospike.jfrog.io/docker/aerospike/aerospike-proximus-private:0.5.0-SNAPSHOT + ports: + - "10000:10000" + networks: + - avs-demo + volumes: + - ./config:/etc/aerospike-proximus + +networks: + avs-demo: {} diff --git a/e2e_test.go b/e2e_test.go index 76a2fab..eb9b9ac 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -1,10 +1,15 @@ //go:build integration -package main_test +package main import ( + "asvec/cmd/flags" "context" + "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" + "log" "log/slog" "os" "os/exec" @@ -28,31 +33,68 @@ var ( barNamespace = "bar" ) +func GetCACert(cert string) (*x509.CertPool, error) { + // read in file + certBytes, err := ioutil.ReadFile(cert) + if err != nil { + log.Fatalf("unable to read cert file %v", err) + return nil, err + } + + certificates := x509.NewCertPool() + certificates.AppendCertsFromPEM(certBytes) + + return certificates, nil +} + type CmdTestSuite struct { suite.Suite - app string - coverFile string - coverFileCounter int - avsIP string - avsPort int - avsHostPort *avs.HostPort - avsClient *avs.AdminClient + app string + composeFile string + suiteName string + suiteArgs string + coverFile string + avsIP string + avsPort int + avsHostPort *avs.HostPort + avsTLSConfig *tls.Config + avsClient *avs.AdminClient } -func TestDistanceMetricFlagSuite(t *testing.T) { - suite.Run(t, new(CmdTestSuite)) +func TestCmdSuite(t *testing.T) { + logger = logger.WithGroup("test-logger") + rootCA, err := GetCACert("docker/tls/config/tls/ca.aerospike.com.crt") + if err != nil { + t.Fatalf("unable to read root ca %v", err) + t.FailNow() + logger.Error("Failed to read cert") + } + + logger.Info("%v", slog.Any("cert", rootCA)) + + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/docker-compose.yml", + suiteArgs: "--log-level debug", + avsIP: "localhost", + }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/tls/docker-compose.yml", + suiteArgs: fmt.Sprintf("--%s %s --log-level debug", flags.TLSCaFile, "docker/tls/config/tls/connector.aerospike.com.crt"), + avsTLSConfig: &tls.Config{ + Certificates: nil, + RootCAs: rootCA, + }, + avsIP: "connector.aerospike.com", + }) } func (suite *CmdTestSuite) SetupSuite() { suite.app = path.Join(wd, "app.test") suite.coverFile = path.Join(wd, "../coverage/cmd-coverage.cov") - suite.coverFileCounter = 0 - suite.avsIP = "127.0.0.1" suite.avsPort = 10000 suite.avsHostPort = avs.NewHostPort(suite.avsIP, suite.avsPort, false) - // var err error - err := docker_compose_up() + err := docker_compose_up(suite.composeFile) if err != nil { suite.FailNowf("unable to start docker compose up", "%v", err) } @@ -75,7 +117,7 @@ func (suite *CmdTestSuite) SetupSuite() { for { suite.avsClient, err = avs.NewAdminClient( - ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, nil, logger, + ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, suite.avsTLSConfig, logger, ) if err != nil { @@ -100,17 +142,18 @@ func (suite *CmdTestSuite) TearDownSuite() { suite.Assert().NoError(err) suite.avsClient.Close() - err = docker_compose_down() + err = docker_compose_down(suite.composeFile) if err != nil { fmt.Println("unable to stop docker compose down") } } func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { + asvecCmd = append(strings.Split(suite.suiteArgs, " "), asvecCmd...) + logger.Info("running command", slog.String("cmd", strings.Join(asvecCmd, " "))) cmd := exec.Command(suite.app, asvecCmd...) cmd.Env = []string{"GOCOVERDIR=" + os.Getenv("COVERAGE_DIR")} stdout, err := cmd.Output() - // fmt.Printf("stdout: %v", string(stdout)) if err != nil { if ee, ok := err.(*exec.ExitError); ok { @@ -124,161 +167,6 @@ func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { return lines, nil } -func getStrPtr(str string) *string { - ptr := str - return &ptr -} - -func getUint32Ptr(i int) *uint32 { - ptr := uint32(i) - return &ptr -} - -func getBoolPtr(b bool) *bool { - ptr := b - return &ptr -} - -type IndexDefinitionBuilder struct { - indexName string - namespace string - set *string - dimension int - vectorDistanceMetric protos.VectorDistanceMetric - vectorField string - storageNamespace *string - storageSet *string - hnsfM *uint32 - hnsfEfC *uint32 - hnsfEf *uint32 - hnsfBatchingMaxRecord *uint32 - hnsfBatchingInterval *uint32 - hnsfBatchingDisabled *bool -} - -func NewIndexDefinitionBuilder( - indexName, - namespace string, - dimension int, - distanceMetric protos.VectorDistanceMetric, - vectorField string, -) *IndexDefinitionBuilder { - return &IndexDefinitionBuilder{ - indexName: indexName, - namespace: namespace, - dimension: dimension, - vectorDistanceMetric: distanceMetric, - vectorField: vectorField, - } -} - -func (idb *IndexDefinitionBuilder) WithSet(set string) *IndexDefinitionBuilder { - idb.set = &set - return idb -} - -func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { - idb.storageNamespace = &storageNamespace - return idb -} - -func (idb *IndexDefinitionBuilder) WithStorageSet(storageSet string) *IndexDefinitionBuilder { - idb.storageSet = &storageSet - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswM(m uint32) *IndexDefinitionBuilder { - idb.hnsfM = &m - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswEf(ef uint32) *IndexDefinitionBuilder { - idb.hnsfEf = &ef - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswEfConstruction(efConstruction uint32) *IndexDefinitionBuilder { - idb.hnsfEfC = &efConstruction - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingMaxRecord(maxRecord uint32) *IndexDefinitionBuilder { - idb.hnsfBatchingMaxRecord = &maxRecord - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingInterval(interval uint32) *IndexDefinitionBuilder { - idb.hnsfBatchingInterval = &interval - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingDisabled(disabled bool) *IndexDefinitionBuilder { - idb.hnsfBatchingDisabled = &disabled - return idb -} - -func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { - indexDef := &protos.IndexDefinition{ - Id: &protos.IndexId{ - Name: idb.indexName, - Namespace: idb.namespace, - }, - Dimensions: uint32(idb.dimension), - VectorDistanceMetric: idb.vectorDistanceMetric, - Field: idb.vectorField, - Type: protos.IndexType_HNSW, - Storage: &protos.IndexStorage{ - Namespace: &idb.namespace, - Set: &idb.indexName, - }, - Params: &protos.IndexDefinition_HnswParams{ - HnswParams: &protos.HnswParams{ - M: getUint32Ptr(16), - EfConstruction: getUint32Ptr(100), - Ef: getUint32Ptr(100), - BatchingParams: &protos.HnswBatchingParams{ - MaxRecords: getUint32Ptr(100000), - Interval: getUint32Ptr(30000), - Disabled: getBoolPtr(false), - }, - }, - }, - } - - if idb.set != nil { - indexDef.SetFilter = idb.set - } - - if idb.storageNamespace != nil { - indexDef.Storage.Namespace = idb.storageNamespace - } - - if idb.storageSet != nil { - indexDef.Storage.Set = idb.storageSet - } - - if idb.hnsfM != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.M = idb.hnsfM - } - if idb.hnsfEf != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.Ef = idb.hnsfEf - } - if idb.hnsfEfC != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC - } - if idb.hnsfBatchingMaxRecord != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord - } - if idb.hnsfBatchingInterval != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Interval = idb.hnsfBatchingInterval - } - if idb.hnsfBatchingDisabled != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Disabled = idb.hnsfBatchingDisabled - } - - return indexDef -} - func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { testCases := []struct { name string @@ -291,14 +179,14 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with storage config", "index1", "test", - fmt.Sprintf("create index --seeds %s -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), + fmt.Sprintf("create index --host %s -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). WithStorageNamespace("bar"). WithStorageSet("testbar"). Build(), }, { - "test with hnsw params", + "test with hnsw params and seeds", "index2", "test", fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-max-edges 10 --hnsw-ef 11 --hnsw-ef-construction 12", suite.avsHostPort.String()), @@ -312,7 +200,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw batch params", "index3", "test", - fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-enabled false --hnsw-batch-interval 50 --hnsw-batch-max-records 100", suite.avsHostPort.String()), + fmt.Sprintf("create index --timeout 10s --host %s -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-enabled false --hnsw-batch-interval 50 --hnsw-batch-max-records 100", suite.avsHostPort.String()), NewIndexDefinitionBuilder("index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3"). WithHnswBatchingMaxRecord(100). WithHnswBatchingInterval(50). @@ -324,7 +212,11 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { for _, tc := range testCases { suite.Run(tc.name, func() { lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) - suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + + if err != nil { + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + suite.FailNow("unable to create index") + } actual, err := suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) @@ -338,10 +230,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { } func (suite *CmdTestSuite) TestCreateIndexFailsAlreadyExistsCmd() { - lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --host %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) suite.Assert().NoError(err, "index should have NOT existed on first call. error: %s, stdout/err: %s", err, lines) - lines, err = suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + lines, err = suite.runCmd(strings.Split(fmt.Sprintf("create index --host %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) suite.Assert().Error(err, "index should HAVE existed on first call. error: %s, stdout/err: %s", err, lines) suite.Assert().Contains(lines[0], "AlreadyExists") @@ -356,7 +248,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { cmd string }{ { - "test with just namespace", + "test with just namespace and seeds", "indexdrop1", "test", nil, @@ -369,7 +261,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { []string{ "testset", }, - fmt.Sprintf("drop index --seeds %s -n test -s testset -i indexdrop2 --timeout 10s", suite.avsHostPort.String()), + fmt.Sprintf("drop index --host %s -n test -s testset -i indexdrop2 --timeout 10s", suite.avsHostPort.String()), }, } @@ -380,12 +272,18 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { suite.FailNowf("unable to create index", "%v", err) } + time.Sleep(time.Second * 3) + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + if err != nil { + suite.FailNow("unable to drop index") + } + _, err = suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) - time.Sleep(time.Second) + time.Sleep(time.Second * 3) if err == nil { suite.FailNow("err is nil, that means the index still exists") @@ -626,13 +524,12 @@ func (suite *CmdTestSuite) TestFailInvalidArg() { } } -func docker_compose_up() error { +func docker_compose_up(composeFile string) error { fmt.Println("Starting docker containers") ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - // docker/docker-compose.yml - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "up", "-d") + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f%s", composeFile), "up", "-d") output, err := cmd.CombinedOutput() fmt.Printf("docker compose up output: %s\n", string(output)) @@ -647,11 +544,11 @@ func docker_compose_up() error { return nil } -func docker_compose_down() error { +func docker_compose_down(composeFile string) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "down") + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f%s", composeFile), "down") _, err := cmd.Output() if err != nil { diff --git a/go.mod b/go.mod index 76d251b..a9f8493 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,13 @@ module asvec go 1.21.7 -replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +// replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go + +// replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go require ( - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 - github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 + github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5 + github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 16c4edc..a0cec2d 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5 h1:OfqJsUs8T8DaYqDLyZwXDY55FvWtyVybbm2mc3Pi+1s= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= +github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 h1:5rYc5QsaQeAnSzUm30gOUANEIEsMS8knbnjouenRV7E= +github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/test_utils.go b/test_utils.go new file mode 100644 index 0000000..9bcb1cc --- /dev/null +++ b/test_utils.go @@ -0,0 +1,160 @@ +//go:build integration + +package main + +import "github.com/aerospike/aerospike-proximus-client-go/protos" + +func getStrPtr(str string) *string { + ptr := str + return &ptr +} + +func getUint32Ptr(i int) *uint32 { + ptr := uint32(i) + return &ptr +} + +func getBoolPtr(b bool) *bool { + ptr := b + return &ptr +} + +type IndexDefinitionBuilder struct { + indexName string + namespace string + set *string + dimension int + vectorDistanceMetric protos.VectorDistanceMetric + vectorField string + storageNamespace *string + storageSet *string + hnsfM *uint32 + hnsfEfC *uint32 + hnsfEf *uint32 + hnsfBatchingMaxRecord *uint32 + hnsfBatchingInterval *uint32 + hnsfBatchingDisabled *bool +} + +func NewIndexDefinitionBuilder( + indexName, + namespace string, + dimension int, + distanceMetric protos.VectorDistanceMetric, + vectorField string, +) *IndexDefinitionBuilder { + return &IndexDefinitionBuilder{ + indexName: indexName, + namespace: namespace, + dimension: dimension, + vectorDistanceMetric: distanceMetric, + vectorField: vectorField, + } +} + +func (idb *IndexDefinitionBuilder) WithSet(set string) *IndexDefinitionBuilder { + idb.set = &set + return idb +} + +func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { + idb.storageNamespace = &storageNamespace + return idb +} + +func (idb *IndexDefinitionBuilder) WithStorageSet(storageSet string) *IndexDefinitionBuilder { + idb.storageSet = &storageSet + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswM(m uint32) *IndexDefinitionBuilder { + idb.hnsfM = &m + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEf(ef uint32) *IndexDefinitionBuilder { + idb.hnsfEf = &ef + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEfConstruction(efConstruction uint32) *IndexDefinitionBuilder { + idb.hnsfEfC = &efConstruction + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingMaxRecord(maxRecord uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingMaxRecord = &maxRecord + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingInterval(interval uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingInterval = &interval + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingDisabled(disabled bool) *IndexDefinitionBuilder { + idb.hnsfBatchingDisabled = &disabled + return idb +} + +func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { + indexDef := &protos.IndexDefinition{ + Id: &protos.IndexId{ + Name: idb.indexName, + Namespace: idb.namespace, + }, + Dimensions: uint32(idb.dimension), + VectorDistanceMetric: idb.vectorDistanceMetric, + Field: idb.vectorField, + Type: protos.IndexType_HNSW, + Storage: &protos.IndexStorage{ + Namespace: &idb.namespace, + Set: &idb.indexName, + }, + Params: &protos.IndexDefinition_HnswParams{ + HnswParams: &protos.HnswParams{ + M: getUint32Ptr(16), + EfConstruction: getUint32Ptr(100), + Ef: getUint32Ptr(100), + BatchingParams: &protos.HnswBatchingParams{ + MaxRecords: getUint32Ptr(100000), + Interval: getUint32Ptr(30000), + Disabled: getBoolPtr(false), + }, + }, + }, + } + + if idb.set != nil { + indexDef.SetFilter = idb.set + } + + if idb.storageNamespace != nil { + indexDef.Storage.Namespace = idb.storageNamespace + } + + if idb.storageSet != nil { + indexDef.Storage.Set = idb.storageSet + } + + if idb.hnsfM != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.M = idb.hnsfM + } + if idb.hnsfEf != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.Ef = idb.hnsfEf + } + if idb.hnsfEfC != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC + } + if idb.hnsfBatchingMaxRecord != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord + } + if idb.hnsfBatchingInterval != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Interval = idb.hnsfBatchingInterval + } + if idb.hnsfBatchingDisabled != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Disabled = idb.hnsfBatchingDisabled + } + + return indexDef +} From 7082c9278edecd8ff7cee53d018807778d73daab Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 18 Jun 2024 11:37:03 -0700 Subject: [PATCH 04/17] add missing files --- docker/tls/config/tls/ca.aerospike.com.crt | 24 ++++++++++++++++++ .../tls/ca.aerospike.com.truststore.jks | Bin 0 -> 1099 bytes docker/tls/config/tls/localhost.keystore.jks | Bin 0 -> 2581 bytes 3 files changed, 24 insertions(+) create mode 100644 docker/tls/config/tls/ca.aerospike.com.crt create mode 100644 docker/tls/config/tls/ca.aerospike.com.truststore.jks create mode 100644 docker/tls/config/tls/localhost.keystore.jks diff --git a/docker/tls/config/tls/ca.aerospike.com.crt b/docker/tls/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/tls/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/tls/config/tls/ca.aerospike.com.truststore.jks b/docker/tls/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8d72ac67e8199cf74e1aecd70587ac3acac082c6 GIT binary patch literal 1099 zcmezO_TO6u1_mY|W(3m$$%%T2sYUt41)15Yddc~@K#A;&+U|V}tPy&q29^vAEKCMX z%>N9Un4T?QW@2Pw;$+y-((Y)W)Wc-J%f_kI=F#?@mywa1mBFB~+mPFUlZ`o)g-w{r z(+|es;9>H1gmKt;n4J6!g$x8h%D8wq;daBsxOq5JlkKv*?@q=-a_l*;bEp;PZ{$kg1g+;aND z8kfEo@0p)p4D~%`H#>KQ38TZjXJ;d|4i}2a9+|Z4>m9zMQkf#seBjF+8`XKy>3{4cUTqpk4sX`!4YmoI-> z!NRI(@rS>8a>sv5{Wlu#Y$9IzUq2tt#LURRxH#D$(SRSAB4mXb8UM4e8ZZMX13r)d zKS+QDm_XZr=>o{-V-aH!@%giX|EOE^N_XM=YpkA&&f$H;-)ZCKCl{v9^?c6;vev@}Wk~YVb^`5%hSQsrX zzg!C1xZt}ZvX#awu*wm(1cX~|iEMvnS#*L!cM=l!cz5^|Zc zr~RDrZGL~wCa+b_{JYMueYC#Zy=%{|uTKRQ=&GM84gcx(SE}{WPTr-`I;$2vnd$6o mnK?CJ->E4l1=-jRu(E48Ts#u5Eo{?tLq7M!rIn^GL5~3;Cxf8? literal 0 HcmV?d00001 diff --git a/docker/tls/config/tls/localhost.keystore.jks b/docker/tls/config/tls/localhost.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8062bebe308ed12fbfb527ae72cf615cdf153807 GIT binary patch literal 2581 zcmY+EX*3&%8itdIC9zBFiM1GOV^`AFt_UU6UbOa**mo_WVkU~JR;Qx0qE-8p+P5N% zHK@I+u4Ss!Huklyd(XWyXMTL=JLh@7@BR6ph)`}2kO4)6^f5u?NH<6a>_BGVOCqER zOoY^3;A#|+(d{1*qZOFQh`GQwF2<0F^?zS%P$1|f5qyFof)7x#Owj-1i*jKwJWhL| zm(ROcRWEVe7=H$JTPqvP0xJNS@92v_0JrCe$$)ZBT z1+FT2pvT7TR~`Ki5Fj&A8Q$W8Aecpb?Y|C@K>`{fUDvyU^l5y%D_}o3f5B;N+IXH- zwVO=O(^9D2lCjl9y$KBu6(#AV2*oa)bjkf`5x{VCS&l2sChfj^pZ*|LUHKfbUtI4P^h8uIW+T9&eK zq5=OlaT*pBckN&6(nPd??kRsBtXVLVQ_bIn!oQuEaBUgeGNp zW=~F^`lPqNpA8ZWn?M*l2CGtH^A!(PmrjAjPkfSPAlA}!Bf2|-D|>5j#KcwBNV;}z zi)7q3l3!eB)njqSJ2pcdpr42%(GqnI+g@Hw+IBC zHrT|%@s;yKVuxwR_^p`wW?Ge~GiP5x?(?5?ZuJC9j)X?2*M@1?Ja(ZDXja2U^yVfc zH-5P267GE^GP<^TPYf@;VZ)&f^4*@k?OGl=K!=2h8l%^tioJ*H2X{^?KfnFFy0$MK z!)!t0;L2$Tq7t&IJPF5Fcs}ZfEdb7oJp4Exfg7W7Lj`PQfunNU=gMbT6(h7Hg+F7? zVnAUet6?-nkH@TGHY^z)e(OVi2=L}+Mc?j;WS@ur3rxDfDsro5r$S&AhR|dfTlvo4 zwxH+&%NwuK5W+VHn2WQXzSmaU(a|ooEtp)~Y!*m5-Sg&3n5T{ex8eJI`(_v8hCX)G z@Q}9OFi}pTWLE2O-pE9A>OL?bW2ecN`s^(6R>g9WPRVP2jn4MTD#f-+@*Ak;WR5{S zPHwWbpC6EaKgQ;$!7dVMkUYHZDb5TVP#23*+MA9JdU@HVeA-u0%S-X3A-sUMEpSC3 z@!fTV7*z>#5cO)@vVR4@*=qhT%#c`;t!5seIaH4|C83cXYs-Js;4=~V{SG*2%hDii z%8-l|n)&4&6}y9DwR(~%_LjV$UWS1t=>H=9$MutpJreez_;z=dSVC)oa74}M&0))@w*1K+&JTm(p~$)t+jQZ<7e_}lTsatM?a-C5iD7D zRSn;H)addgSdM$abKVirOY2Yvn(e+JStr>MZGGLKP3bffy)%*NOZ>MLUX>lnDQkar zog{A|dWuUE0z?2@()N*MNezWY(r0-dlA7|}alH1u9s{iI`*k}ru?7*d=n)vj;vL>Z z!Auss*ZV=$W6I`1-;!atWCll2J#bQq!~oh|fAp;SW`7}0+(+E?hYhWdBCzVgBKoaT zZ^54tzndWLGMei+mE+KmGk#zwb~>gcKtfcju-6RHVnxBLHI;L^s$DK6OEJdE8Yu3+ zR0)AAgGE39Wt8~eh*ud7=CSp|dxJ75i`?;MXz3 zelfjk-~mRqHl83Tww}#!*Ts!DBbAatC~3iMb!&VZ4u@66GdUzDVaUQBS6opRdm*aV z^EMaM_Uhhx)ztLa`ol-aB?{hrY!KrHU;Shl+jD?J#pBn0FXcx~wkDRas>-t7ZlNaM zedgLidi!M^6ZvGZfNY7A96ce01ZL7afqtJ&>FJGJHt~o{YQ+rc&bcuw#;;YZx7W1l zzndaJV#_Nfg})QH#oqapt8668c$&VMzsnm;Gr%XjK!gX_;w7I;rRI8IMr-o;-EqE9 zU5n47M}!-CLYzv|TlBq1XkT6)}}GRsaBbPw37NV~U(E1ssVZ87b_B z*w1L~BGEbXQ5$O2gAW+j4%%K%aY|@4t3>^#P+VcYl>mIbE7&9-oL8}(6Y?eeh2M(9 z0aPzp*GBZFfT{80Zd6xXdZi>ZJmym+?{XI(lG~L(O-lV|eRfk6lFQ9oX^N!25NkDz z1#(TFinzWYcN2=nHr3>I0c@kfF<-RB6!bF=Nfb0dfi6cYlTNNo$!U7s?Tc)%X! z;y!$NK7~6ejv5+hA{_uzv0qTLu!xcenSA&Sw*(hl93-Rwr5oA~&TQg!kkdJ4*n-SB z9lu~J3dJHMFp88CXM)An-`=F;8uQC{wGS&M$o^y{H7!~PY?s&q45@FF+tIPQpd)|s zDuJ>4n^s85B(&@cI5K)1Xw$lLm*Q2eUZ&NTv@R6ltY>UEwPccdn8=pfyc8hA7mosm z(Gs6Yd zd;JfaFA-qb=`GS@(%fT*b1}``UEwM_@m=yaqhAl;%SYUWzk}B%ZcrEv(_4Ta~UEB&nrm>}Z!- zy&RD~&bTasx`q-*L6{ijIT%1fi~uma(wy_v{8OE9YDWwb9cPG;ee9qDhU+`Y Date: Tue, 18 Jun 2024 11:38:07 -0700 Subject: [PATCH 05/17] rm edit of /etc/hosts --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6530556..9a82f7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,6 @@ jobs: echo "$FEATURES_CONF" > docker/config/features.conf - name: Run tests run: | - echo '127.0.0.1 connector.aerospike.com' >> /etc/hosts make coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 54da4c2d1a0c82ad1e3a8f4e386bfcd8931908fc Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 18 Jun 2024 11:42:51 -0700 Subject: [PATCH 06/17] remove unused files --- .gitignore | 2 +- .../config/tls/connector.aerospike.com.crt | 22 ------------------ .../tls/connector.aerospike.com.keystore.jks | Bin 2285 -> 0 bytes docker/tls/config/tls/keypass | 2 +- 4 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 docker/tls/config/tls/connector.aerospike.com.crt delete mode 100644 docker/tls/config/tls/connector.aerospike.com.keystore.jks diff --git a/.gitignore b/.gitignore index 00ec609..f7114ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ features.conf /bin/* embed_*.go -/tmp +tmp /vendor /coverage \ No newline at end of file diff --git a/docker/tls/config/tls/connector.aerospike.com.crt b/docker/tls/config/tls/connector.aerospike.com.crt deleted file mode 100644 index 3e53361..0000000 --- a/docker/tls/config/tls/connector.aerospike.com.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDpjCCAo6gAwIBAgIJAJuWztiFvTcCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD -VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv -c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl -LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgx -MDMyMjFaFw0yOTA3MDUxMDMyMjFaMFYxCzAJBgNVBAYTAklOMQswCQYDVQQIDAJL -QTEYMBYGA1UECgwPQWVyb3NwaWtlLCBJbmMuMSAwHgYDVQQDDBdjb25uZWN0b3Iu -YWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAO70 -L+G2TP6s1jPPMvIWB2LSdiVKkfKAuihTf5lJEuF1jyCt6c8q7j4gqAEA5PoAaTpM -ukKt3EvmCyRZTjBr1WqSOuwU4mw+rsQ+gWdFiaEOLXS6kHcpunb5/wLeYAC6rGLk -Di9FBSwX3sNke5M1v/P+Vcd3ozA3bsv5JTFCbZiwarG7i8ZatRXFDHYwS0t36pLA -JcyVUyehT7SSctkQQym0goMjLpnEKgimC0RKgw9yhmPmK5oasIzythr6MSv2SkiO -BJfJDBJZcqvNFn66ghuofJ7vyxxW59DSwrRAlXPfAaxQ4mcgECuoe1xMUtrMya64 -4Hz7ZijnbMrFKTqgy3sCAwEAAaNBMD8wPQYDVR0RBDYwNIIXY29ubmVjdG9yLmFl -cm9zcGlrZS5jb22CGSouY29ubmVjdG9yLmFlcm9zcGlrZS5jb20wDQYJKoZIhvcN -AQELBQADggEBAFTbTRmftwDZlC/60I9w6u5/vRRJRCQvp5mBhrbsfA1OcY7HOBOT -G3hFtDHl14s5cpZAgecdgACtvGvnex6q/gun2JzdAmJjslpRAQgZA0L182X+Sc5z -5QIhMhS6iMCae9KV4GaEq0VMfZ/KlKPfESj1DZE3VQNWUPTwBbamnzpEqSzoFdrp -duFah12yIwnUoTi+fNfhhm76Y5GkG5Cx4HJvF6qiJ1uOIieLYZ07e1nFxIVqnEz/ -zZCkDkKqXWb+ii8hlBl8kiu4vVwKEjrlruSaKb+3UTJxB16zDESjn5i58A9TgtsZ -VvECl552NbEBT+VOgRPsNJRBpPbPfYqUVfk= ------END CERTIFICATE----- diff --git a/docker/tls/config/tls/connector.aerospike.com.keystore.jks b/docker/tls/config/tls/connector.aerospike.com.keystore.jks deleted file mode 100644 index 29490f53bdc89036e1e5bb37638e5e1c43e28f7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2285 zcmZ{lc{J4f8^`B6gQ1yWFi1(3v5xr82u0aSg-KZp=~t$)??jl$Oo(KO#H4I<#e|Rs zlOjtlLP)ajS<(Nc+K{dLc;bMEht=bY#LocB4O*LiO8x#2I~+AF91?`zJr2%+i-jU51Qe?r#}k8X>9HovSD09VZ3uKML#UW)wM{t82Ju;{U|7!OAA~l+(GS=rBb(p+a(e zcjIx!sN$>gs(~)mNWN5Dld@RU?dP8IqZSrL-n-6ijzLO8iBo~q_;^d|>vE^~i1ZH4 z4%oVk+tn61_oB##nL+DDrF%95I!iQkXSmWs*2&f9&Fv{ieMSts<<4J>cC`OusKMeE zFwUR}1s+|DPV1brB>_(qtp?|^7rS0k7gff!bQyCUw-wl3RkJETZogo<`$`LGh5o*Bp-vV{eO6`tq*0Yve^gl@=3yaym1#s&y9$)q zWZ(0jJ>75IP0R4NIv@-+dQ* zS*Cpf?6Up91aHuSr}Vr?!OITKYHk;qQ(3J~H4`G>y;pZs=Nda_Vw)m8t?;gQa{MCQ zlr+c4ek;S93VlJ51_2W?Ef@vhduHx?)CK?Qp%?6HrnHe0VbfY|a@YyA7b6KDUF!7g zlJDdNH)ty);Amme`;_1CBHAVlwqyOt`nmW5JD&Dh*~0Xu()N4g(Ho-iQTK)uHVvz5 zm4qT65v;o0S*wP~Ce;f^{dp%T4{R$5kGuacIB0o>GJi=IHRAA=KP4N%AKm6~5-ugY ze4zY=31KGEFX9n(0QF-4szUuZhafGdiv8l(LdDbfaE1vuNUtRPWr$=KBpL9x3_ zhbDARlhf+>Gb0UiBl!dR2vZ>g`f5c@zYRXfMYPmVz3rVB^)I{hQsF+I2~g{@PScv< z6^)VD&n&^GPM_-6q#{C#*lrRb4e*ts>3N

6n7JQhQ&j)8?-K<%@7sFx1q&~KB!swwn>+`idzbyAc0fp#|S)ovMo=* zG=CJ1hwX^k)q#R2J^|I3V=!RgUH@^Nl@a44~9gO&a3}p zRVVp+<9CC*{yAL$5Z^Kw{$c>7B~bYNpboeX4{8z!cn1j-;fK`vuLSLWBK-^Oc2N9p zJ|Ux@d}=bLJ|uO#3@H6Kkp#)t$A?S`@(uVE45R{b{~OHK-vOfnLXfTbHYgPUAe>Jc z6D%UP_>E>4VMQG7+!L%|ma=k%t!y2cX(}>tF-fLmzDt#(FH;CWroTemb%|`_l40|A z2&}yY=-KCSQ+E+F>7`%Vq#xyK5|=BuCy1RKqQVZ|*o2OrgRqO8rv)`kc-16E8!v>X zYSpcBZCgU}K^>p=4F$ZhcSe~<`IGA}93G1`BZEP6^N@v`^$IU$d+r=wDyoR>%OvX6U$Nb`@kGP2=mUB)4s6WyRJY=YCUJerI}pB{kK^oP-jwwgC)Q( z2{T?>C38(X181PJ1PuFHJw3eV*0{^n2PVYJ*==e0W5UX7s1zMrnBB=w%e<_DY~8~} zYI9;YcETTjg_of*!*1Impr`c{idl;1>m6CutVd%0>%U}09tvOWvC+T$BVE_OC diff --git a/docker/tls/config/tls/keypass b/docker/tls/config/tls/keypass index b1f833e..0f673cc 100644 --- a/docker/tls/config/tls/keypass +++ b/docker/tls/config/tls/keypass @@ -1 +1 @@ -citrusstore +citrusstore \ No newline at end of file From c22228f68b51169d0d1c80a892361858257c4cd4 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 19 Jun 2024 12:30:05 -0700 Subject: [PATCH 07/17] rename --- .golangci.yml | 2 +- cmd/createIndex.go | 4 ++-- cmd/dropIndex.go | 2 +- cmd/flags/distanceMetric.go | 2 +- cmd/flags/hostPort.go | 2 +- cmd/flags/hostPort_test.go | 2 +- cmd/listIndex.go | 4 ++-- cmd/utils.go | 2 +- cmd/utils_test.go | 2 +- cmd/view.go | 2 +- cmd/writers/indexList.go | 2 +- e2e_test.go | 4 ++-- go.mod | 6 +++--- go.sum | 8 -------- test_utils.go | 2 +- 15 files changed, 19 insertions(+), 27 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 32c6591..9099fa8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,7 +23,7 @@ linters-settings: allow: - $gostd - github.com/aerospike/tools-common-go - - github.com/aerospike/aerospike-proximus-client-go + - github.com/aerospike/avs-client-go - asvec/cmd - github.com/spf13/cobra - github.com/spf13/viper diff --git a/cmd/createIndex.go b/cmd/createIndex.go index cb4a381..453a3bf 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -11,8 +11,8 @@ import ( "strings" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + avs "github.com/aerospike/avs-client-go" + "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 6ddfcd1..f638a10 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -10,7 +10,7 @@ import ( "log/slog" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/cmd/flags/distanceMetric.go b/cmd/flags/distanceMetric.go index 78d332e..f19b5d1 100644 --- a/cmd/flags/distanceMetric.go +++ b/cmd/flags/distanceMetric.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" ) type DistanceMetricFlag string diff --git a/cmd/flags/hostPort.go b/cmd/flags/hostPort.go index 36d7578..69b0ff3 100644 --- a/cmd/flags/hostPort.go +++ b/cmd/flags/hostPort.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" ) const ( diff --git a/cmd/flags/hostPort_test.go b/cmd/flags/hostPort_test.go index 1b14406..50d787a 100644 --- a/cmd/flags/hostPort_test.go +++ b/cmd/flags/hostPort_test.go @@ -5,7 +5,7 @@ package flags import ( "testing" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" "github.com/stretchr/testify/suite" ) diff --git a/cmd/listIndex.go b/cmd/listIndex.go index d294caf..c714ddd 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -11,8 +11,8 @@ import ( "sync" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + avs "github.com/aerospike/avs-client-go" + "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/cmd/utils.go b/cmd/utils.go index e33b22c..a0d0e9e 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -7,7 +7,7 @@ import ( "encoding/pem" "fmt" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" ) func newTLSConfig(rootCA [][]byte, cert []byte, key []byte, keyPass []byte, tlsProtoMin int, tlsProtoMax int) (*tls.Config, error) { diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 53c8a84..cbf7039 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -6,7 +6,7 @@ import ( "asvec/cmd/flags" "testing" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" "github.com/stretchr/testify/assert" ) diff --git a/cmd/view.go b/cmd/view.go index c3cbb14..c3ccc59 100644 --- a/cmd/view.go +++ b/cmd/view.go @@ -6,7 +6,7 @@ import ( "io" "log/slog" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" ) type View struct { diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index da07d78..b854a6c 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -4,7 +4,7 @@ import ( "io" "log/slog" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/e2e_test.go b/e2e_test.go index eb9b9ac..5e5bada 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + avs "github.com/aerospike/avs-client-go" + "github.com/aerospike/avs-client-go/protos" "github.com/stretchr/testify/suite" ) diff --git a/go.mod b/go.mod index a9f8493..7d81a6c 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,12 @@ module asvec go 1.21.7 -// replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +replace github.com/aerospike/avs-client-go => /Users/jesseschmidt/Developer/avs-client-go -// replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go +replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go require ( - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5 + github.com/aerospike/avs-client-go v0.0.0-20240509004238-aa3172c86d65 github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index a0cec2d..4f7b347 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5 h1:OfqJsUs8T8DaYqDLyZwXDY55FvWtyVybbm2mc3Pi+1s= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240618165139-d1f0bb1968a5/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= -github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= -github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= -github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 h1:5rYc5QsaQeAnSzUm30gOUANEIEsMS8knbnjouenRV7E= -github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/test_utils.go b/test_utils.go index 9bcb1cc..9f45b21 100644 --- a/test_utils.go +++ b/test_utils.go @@ -2,7 +2,7 @@ package main -import "github.com/aerospike/aerospike-proximus-client-go/protos" +import "github.com/aerospike/avs-client-go/protos" func getStrPtr(str string) *string { ptr := str From 01c465d0f48883cdc639be9ff97a98402829866e Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 24 Jun 2024 17:19:50 -0700 Subject: [PATCH 08/17] remove code duplication, add user/pass flags and auth tests --- cmd/createIndex.go | 69 +++++-------- cmd/dropIndex.go | 53 +++------- cmd/flags/client.go | 22 ++++- cmd/flags/constants.go | 2 + cmd/listIndex.go | 52 +++------- cmd/root.go | 11 ++- cmd/utils.go | 32 ++++++ cmd/utils_test.go | 4 +- docker/auth/config/aerospike-proximus.yml | 93 ++++++++++++++++++ docker/auth/config/aerospike.conf | 82 +++++++++++++++ docker/auth/config/jwt/private_key.pem | 28 ++++++ docker/auth/config/jwt/public_key.pem | 8 ++ docker/auth/config/tls/ca.aerospike.com.crt | 24 +++++ .../tls/ca.aerospike.com.truststore.jks | Bin 0 -> 1099 bytes docker/auth/config/tls/keypass | 1 + docker/auth/config/tls/localhost.keystore.jks | Bin 0 -> 2581 bytes docker/auth/config/tls/storepass | 1 + docker/auth/docker-compose.yml | 23 +++++ docker/tls/config/aerospike-proximus.yml | 1 + e2e_test.go | 53 +++++++--- go.mod | 6 +- go.sum | 4 + test_utils.go | 10 +- 23 files changed, 431 insertions(+), 148 deletions(-) create mode 100644 docker/auth/config/aerospike-proximus.yml create mode 100644 docker/auth/config/aerospike.conf create mode 100644 docker/auth/config/jwt/private_key.pem create mode 100644 docker/auth/config/jwt/public_key.pem create mode 100644 docker/auth/config/tls/ca.aerospike.com.crt create mode 100644 docker/auth/config/tls/ca.aerospike.com.truststore.jks create mode 100644 docker/auth/config/tls/keypass create mode 100644 docker/auth/config/tls/localhost.keystore.jks create mode 100644 docker/auth/config/tls/storepass create mode 100644 docker/auth/docker-compose.yml diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 453a3bf..dfcce2c 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -11,7 +11,6 @@ import ( "strings" "time" - avs "github.com/aerospike/avs-client-go" "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" @@ -21,7 +20,7 @@ import ( //nolint:govet // Padding not a concern for a CLI var createIndexFlags = &struct { - flags.ClientFlags + clientFlags flags.ClientFlags namespace string sets []string indexName string @@ -39,7 +38,7 @@ var createIndexFlags = &struct { hnswBatchEnabled flags.BoolOptionalFlag timeout time.Duration }{ - ClientFlags: *flags.NewClientFlags(), + clientFlags: *flags.NewClientFlags(), storageNamespace: flags.StringOptionalFlag{}, storageSet: flags.StringOptionalFlag{}, hnswMaxEdges: flags.Uint32OptionalFlag{}, @@ -68,7 +67,7 @@ func newCreateIndexFlagSet() *pflag.FlagSet { flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flags.BatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability flagSet.Var(&createIndexFlags.hnswBatchInterval, flags.BatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability flagSet.Var(&createIndexFlags.hnswBatchEnabled, flags.BatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability - flagSet.AddFlagSet(createIndexFlags.NewClientFlagSet()) + flagSet.AddFlagSet(createIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } @@ -105,56 +104,34 @@ func newCreateIndexCmd() *cobra.Command { return nil }, RunE: func(_ *cobra.Command, _ []string) error { - hosts, isLoadBalancer := parseBothHostSeedsFlag(createIndexFlags.Seeds, createIndexFlags.Host) - logger.Debug("parsed flags", - slog.String(flags.Host, createIndexFlags.Host.String()), - slog.String(flags.Seeds, createIndexFlags.Seeds.String()), - slog.String(flags.ListenerName, createIndexFlags.ListenerName.String()), - slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), - slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), - slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), - slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), - slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), - slog.String(flags.Namespace, createIndexFlags.namespace), - slog.Any(flags.Sets, createIndexFlags.sets), - slog.String(flags.IndexName, createIndexFlags.indexName), - slog.String(flags.VectorField, createIndexFlags.vectorField), - slog.Uint64(flags.Dimension, uint64(createIndexFlags.dimensions)), - slog.Any(flags.IndexMeta, createIndexFlags.indexMeta), - slog.String(flags.DistanceMetric, createIndexFlags.distanceMetric.String()), - slog.Duration(flags.Timeout, createIndexFlags.timeout), - slog.Any(flags.StorageNamespace, createIndexFlags.storageNamespace.String()), - slog.Any(flags.StorageSet, createIndexFlags.storageSet.String()), - slog.Any(flags.MaxEdges, createIndexFlags.hnswMaxEdges.String()), - slog.Any(flags.Ef, createIndexFlags.hnswEf), - slog.Any(flags.ConstructionEf, createIndexFlags.hnswConstructionEf.String()), - slog.Any(flags.BatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), - slog.Any(flags.BatchInterval, createIndexFlags.hnswBatchInterval.String()), - slog.Any(flags.BatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + append(createIndexFlags.clientFlags.NewSLogAttr(), + slog.String(flags.Namespace, createIndexFlags.namespace), + slog.Any(flags.Sets, createIndexFlags.sets), + slog.String(flags.IndexName, createIndexFlags.indexName), + slog.String(flags.VectorField, createIndexFlags.vectorField), + slog.Uint64(flags.Dimension, uint64(createIndexFlags.dimensions)), + slog.Any(flags.IndexMeta, createIndexFlags.indexMeta), + slog.String(flags.DistanceMetric, createIndexFlags.distanceMetric.String()), + slog.Duration(flags.Timeout, createIndexFlags.timeout), + slog.Any(flags.StorageNamespace, createIndexFlags.storageNamespace.String()), + slog.Any(flags.StorageSet, createIndexFlags.storageSet.String()), + slog.Any(flags.MaxEdges, createIndexFlags.hnswMaxEdges.String()), + slog.Any(flags.Ef, createIndexFlags.hnswEf), + slog.Any(flags.ConstructionEf, createIndexFlags.hnswConstructionEf.String()), + slog.Any(flags.BatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), + slog.Any(flags.BatchInterval, createIndexFlags.hnswBatchInterval.String()), + slog.Any(flags.BatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + )..., ) - ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) - defer cancel() - - tlsConfig, err := createIndexFlags.NewTLSConfig() + adminClient, err := createClientFromFlags(&createIndexFlags.clientFlags, createIndexFlags.timeout) if err != nil { - logger.Error("failed to create TLS config", slog.Any("error", err)) return err } - - adminClient, err := avs.NewAdminClient( - ctx, hosts, createIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, - ) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), createIndexFlags.timeout) + ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) defer cancel() // Inverted to make it easier to understand diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index f638a10..63c53d5 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -10,7 +10,6 @@ import ( "log/slog" "time" - avs "github.com/aerospike/avs-client-go" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -19,13 +18,13 @@ import ( //nolint:govet // Padding not a concern for a CLI var dropIndexFlags = &struct { - flags.ClientFlags - namespace string - sets []string - indexName string - timeout time.Duration + clientFlags flags.ClientFlags + namespace string + sets []string + indexName string + timeout time.Duration }{ - ClientFlags: *flags.NewClientFlags(), + clientFlags: *flags.NewClientFlags(), } func newDropIndexFlagSet() *pflag.FlagSet { @@ -34,7 +33,7 @@ func newDropIndexFlagSet() *pflag.FlagSet { flagSet.StringArrayVarP(&dropIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability flagSet.StringVarP(&dropIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability flagSet.DurationVar(&dropIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.AddFlagSet(dropIndexFlags.NewClientFlagSet()) + flagSet.AddFlagSet(dropIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } @@ -65,43 +64,21 @@ func newDropIndexCommand() *cobra.Command { }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flags.Host, dropIndexFlags.Host.String()), - slog.String(flags.Seeds, dropIndexFlags.Seeds.String()), - slog.String(flags.ListenerName, dropIndexFlags.ListenerName.String()), - slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), - slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), - slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), - slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), - slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), - slog.String(flags.Namespace, dropIndexFlags.namespace), - slog.Any(flags.Sets, dropIndexFlags.sets), - slog.String(flags.IndexName, dropIndexFlags.indexName), - slog.Duration(flags.Timeout, dropIndexFlags.timeout), + append(dropIndexFlags.clientFlags.NewSLogAttr(), + slog.String(flags.Namespace, dropIndexFlags.namespace), + slog.Any(flags.Sets, dropIndexFlags.sets), + slog.String(flags.IndexName, dropIndexFlags.indexName), + slog.Duration(flags.Timeout, dropIndexFlags.timeout), + )..., ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(dropIndexFlags.Seeds, dropIndexFlags.Host) - - ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) - defer cancel() - - tlsConfig, err := dropIndexFlags.NewTLSConfig() + adminClient, err := createClientFromFlags(&dropIndexFlags.clientFlags, dropIndexFlags.timeout) if err != nil { - logger.Error("failed to create TLS config", slog.Any("error", err)) return err } - - adminClient, err := avs.NewAdminClient( - ctx, hosts, dropIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, - ) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), dropIndexFlags.timeout) + ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() err = adminClient.IndexDrop(ctx, dropIndexFlags.namespace, dropIndexFlags.indexName) diff --git a/cmd/flags/client.go b/cmd/flags/client.go index 95dc670..45d410e 100644 --- a/cmd/flags/client.go +++ b/cmd/flags/client.go @@ -2,6 +2,7 @@ package flags import ( "fmt" + "log/slog" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/pflag" @@ -11,6 +12,8 @@ type ClientFlags struct { Host *HostPortFlag Seeds *SeedsSliceFlag ListenerName StringOptionalFlag + User StringOptionalFlag + Password commonFlags.PasswordFlag TLSFlags } @@ -26,9 +29,24 @@ func (cf *ClientFlags) NewClientFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} flagSet.VarP(cf.Host, Host, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", Seeds))) //nolint:lll // For readability flagSet.Var(cf.Seeds, Seeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", Host))) //nolint:lll // For readability - flagSet.VarP(&cf.ListenerName, ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) - + flagSet.VarP(&cf.ListenerName, ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.VarP(&cf.User, User, "U", commonFlags.DefaultWrapHelpString("The AVS user to authenticate with.")) //nolint:lll // For readability + flagSet.VarP(&cf.Password, Password, "P", commonFlags.DefaultWrapHelpString("The AVS password for the specified user.")) //nolint:lll // For readability flagSet.AddFlagSet(cf.NewTLSFlagSet(commonFlags.DefaultWrapHelpString)) return flagSet } + +func (cf *ClientFlags) NewSLogAttr() []any { + return []any{slog.String(Host, cf.Host.String()), + slog.String(Seeds, cf.Seeds.String()), + slog.String(ListenerName, cf.ListenerName.String()), + slog.String(User, cf.User.String()), + slog.String(Password, cf.Password.String()), + slog.Bool(TLSCaFile, cf.TLSRootCAFile != nil), + slog.Bool(TLSCaPath, cf.TLSRootCAPath != nil), + slog.Bool(TLSCertFile, cf.TLSCertFile != nil), + slog.Bool(TLSKeyFile, cf.TLSKeyFile != nil), + slog.Bool(TLSKeyFilePass, cf.TLSKeyFilePass != nil), + } +} diff --git a/cmd/flags/constants.go b/cmd/flags/constants.go index 1e06a87..888f67b 100644 --- a/cmd/flags/constants.go +++ b/cmd/flags/constants.go @@ -5,6 +5,8 @@ const ( Seeds = "seeds" Host = "host" ListenerName = "listener-name" + User = "user" + Password = "password" Namespace = "namespace" Sets = "sets" IndexName = "index-name" diff --git a/cmd/listIndex.go b/cmd/listIndex.go index c714ddd..2e81f5b 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -11,7 +11,6 @@ import ( "sync" "time" - avs "github.com/aerospike/avs-client-go" "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" @@ -20,21 +19,18 @@ import ( ) var listIndexFlags = &struct { - flags.ClientFlags - verbose bool - timeout time.Duration + clientFlags flags.ClientFlags + verbose bool + timeout time.Duration }{ - ClientFlags: *flags.NewClientFlags(), + clientFlags: *flags.NewClientFlags(), } func newListIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(listIndexFlags.Host, flags.Host, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flags.Seeds))) //nolint:lll // For readability - flagSet.Var(listIndexFlags.Seeds, flags.Seeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flags.Host))) //nolint:lll // For readability - flagSet.VarP(&listIndexFlags.ListenerName, flags.ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.BoolVarP(&listIndexFlags.verbose, flags.Verbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability - flagSet.DurationVar(&listIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.AddFlagSet(listIndexFlags.NewClientFlagSet()) + flagSet.BoolVarP(&listIndexFlags.verbose, flags.Verbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability + flagSet.DurationVar(&listIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(listIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } @@ -62,41 +58,19 @@ func newListIndexCmd() *cobra.Command { }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flags.Host, listIndexFlags.Host.String()), - slog.String(flags.Seeds, listIndexFlags.Seeds.String()), - slog.String(flags.ListenerName, listIndexFlags.ListenerName.String()), - slog.Bool(flags.TLSCaFile, createIndexFlags.TLSRootCAFile != nil), - slog.Bool(flags.TLSCaPath, createIndexFlags.TLSRootCAPath != nil), - slog.Bool(flags.TLSCertFile, createIndexFlags.TLSCertFile != nil), - slog.Bool(flags.TLSKeyFile, createIndexFlags.TLSKeyFile != nil), - slog.Bool(flags.TLSKeyFilePass, createIndexFlags.TLSKeyFilePass != nil), - slog.Bool(flags.Verbose, listIndexFlags.verbose), - slog.Duration(flags.Timeout, listIndexFlags.timeout), + append(listIndexFlags.clientFlags.NewSLogAttr(), + slog.Bool(flags.Verbose, listIndexFlags.verbose), + slog.Duration(flags.Timeout, listIndexFlags.timeout), + )..., ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(listIndexFlags.Seeds, listIndexFlags.Host) - - ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) - defer cancel() - - tlsConfig, err := listIndexFlags.NewTLSConfig() + adminClient, err := createClientFromFlags(&listIndexFlags.clientFlags, listIndexFlags.timeout) if err != nil { - logger.Error("failed to create TLS config", slog.Any("error", err)) return err } - - adminClient, err := avs.NewAdminClient( - ctx, hosts, listIndexFlags.ListenerName.Val, isLoadBalancer, tlsConfig, logger, - ) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) defer cancel() indexList, err := adminClient.IndexList(ctx) diff --git a/cmd/root.go b/cmd/root.go index db16927..1ff5304 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -94,11 +94,12 @@ func init() { common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") - if err := viper.BindEnv(flags.Host); err != nil { - logger.Error("failed to bind environment variable", slog.Any("error", err)) - } + bindEnvs := []string{flags.Host, flags.Seeds, flags.User, flags.Password} - if err := viper.BindEnv(flags.Seeds); err != nil { - logger.Error("failed to bind environment variable", slog.Any("error", err)) + // Bind specified flags to ASVEC_* + for _, env := range bindEnvs { + if err := viper.BindEnv(env); err != nil { + panic(fmt.Sprintf("failed to bind environment variable: %s", err)) + } } } diff --git a/cmd/utils.go b/cmd/utils.go index a0d0e9e..add5701 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -2,14 +2,46 @@ package cmd import ( "asvec/cmd/flags" + "context" "crypto/tls" "crypto/x509" "encoding/pem" "fmt" + "log/slog" + "time" avs "github.com/aerospike/avs-client-go" ) +func createClientFromFlags(clientFlags *flags.ClientFlags, connectTimeout time.Duration) (*avs.AdminClient, error) { + hosts, isLoadBalancer := parseBothHostSeedsFlag(clientFlags.Seeds, clientFlags.Host) + + ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) + defer cancel() + + tlsConfig, err := clientFlags.NewTLSConfig() + if err != nil { + logger.Error("failed to create TLS config", slog.Any("error", err)) + return nil, err + } + + var password *string + if len(clientFlags.Password) != 0 { + strPass := clientFlags.Password.String() + password = &strPass + } + + adminClient, err := avs.NewAdminClient( + ctx, hosts, clientFlags.ListenerName.Val, isLoadBalancer, clientFlags.User.Val, password, tlsConfig, logger, + ) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return nil, err + } + + return adminClient, nil +} + func newTLSConfig(rootCA [][]byte, cert []byte, key []byte, keyPass []byte, tlsProtoMin int, tlsProtoMax int) (*tls.Config, error) { if len(rootCA) == 0 && len(cert) == 0 && len(key) == 0 { return nil, nil diff --git a/cmd/utils_test.go b/cmd/utils_test.go index cbf7039..ba633d6 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -20,12 +20,12 @@ func TestParseBothHostSeedsFlag(t *testing.T) { { &flags.SeedsSliceFlag{ Seeds: avs.HostPortSlice{ - avs.NewHostPort("1.1.1.1", 5000, false), + avs.NewHostPort("1.1.1.1", 5000), }, }, flags.NewDefaultHostPortFlag(), avs.HostPortSlice{ - avs.NewHostPort("1.1.1.1", 5000, false), + avs.NewHostPort("1.1.1.1", 5000), }, false, }, diff --git a/docker/auth/config/aerospike-proximus.yml b/docker/auth/config/aerospike-proximus.yml new file mode 100644 index 0000000..fa7b8a1 --- /dev/null +++ b/docker/auth/config/aerospike-proximus.yml @@ -0,0 +1,93 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + mutual-auth: false + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +security: + auth-token: + private-key: /etc/aerospike-proximus/jwt/private_key.pem + public-key: /etc/aerospike-proximus/jwt/public_key.pem + token-expiry: 30_000 + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/auth/config/aerospike.conf b/docker/auth/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/auth/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/auth/config/jwt/private_key.pem b/docker/auth/config/jwt/private_key.pem new file mode 100644 index 0000000..70c57aa --- /dev/null +++ b/docker/auth/config/jwt/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDBU9XcqJzRuqTQ +d58/zotQN/vUFjS2AhgXIzw18RULCvXo/1mQD6Tej7Gd7Vsm3j/o4/jZrZ10d+iR +yrGSfMw9w7G+SD4UDKcp4Epu2sLme2AYbU/T4Pz2CgD8qkWLeDtfX/Gb5Eoglfiq +PU0cCfNAdATwQZzC5qZ9RCMNQb8WVtQ/fIhEJCltslBLHAdXqOsJnzPUUlFE4Abw +1/HosVvD2GOoOE0yZqrJvpCxH7zoL8TCTuMqhDyg3BfFLIzD0pePceLf30Om9oVU +v0vOhmy5wgjYnSzyHYaPY9WjAaFVTNCw4upGXveqON1qG12ZtNwUBsY6besNAamf +I6bHoLrrAgMBAAECggEADKzbSzXaM51Um/8TzQJU5k6yr0jrbiQnVzEDYW1WhLGp +xiZ5GqBj81xMIr9tUq2PVszHmTda/kgJqAa4QL33mB8XWBNIJVa1CQ0AZFHp0GRg +vF6XtXEStMDwJ/YJTD0hoL2wy23wIZXORr8b1KL0X2fxQPKpp/+pBR0tXSZP3yjM +dhYEKj7/p9qH5pP92ZQftzeWQqK0MOSs+fedUASi1I0ZXWL7Hcsl7e66ZrySevL2 +ad/xdKHShvT2dxlBHpkBkvfAOrssgqmY4d9CZj4qmOXgsQcqUPZiP5bZ0naM6Wdm +B/q/PwPKxWDM8VRybxDaOVesG2358qcB+za3Rk0EXQKBgQD+DXA7qg6J68KMjJud +4nHrPxzw287mJgeNgvdJsAukz4dPfA3Kb95+YLCnhcy4co2lWmJf8f12nJBCcF6p +Oqyw8jUGosaXTXM0ZBeSx8CwFmeYQASmxz5bNofROSIuDXriEYn+RBlRCrvAP9DX +Pbbd2HoR7RlHHdeFUDPGE6AOTQKBgQDCzzpjZ6lBUoVIQDDtoRD9gbmr03epdsCI +DBp4uBv1LuMIKxKy7s2UUiMNIxWKAQf4h9OwMsKzQztawRXi/Q0Q7TZ5P+b//PX7 +ldvO5tMITyj5ZP/LFSIpuuuQCZxxDI93SJWHpI1qk8wOeKTM/TwlcZV0mKxEpnOO +BIV1+go6FwKBgEt9jsYL6DoPdkXxWiR2L0ep/12K01YMIt5n9jdNOoiEqj5yLF7l +EwYWkSeWWJjOYUyrKOQ9sgVWzH8RiO9cYghKUHtiTwSSnRMKUv7ooFcysSHKlBdC +yUKtV3pSMdfhZYxbjdeiQKuWNugPjm0HB5mwQ8Wj4IkDUxFoHZpUJpFZAoGBALGG +OoUfj+PRGVgv9uZ1YpBByTtF58PsTaMstGrSC9gws/9bkRyx9XF2MC7mi9hI/ESF +MmlcuIS3fb1EO3ewdlm4cW2oyA77LAXqGHbBbgoiaQpr+ZXNWmaAye1mdMOoWIyw +/mKvl5dJvChr1HJhSpAso6+u+T1OGfuPLzfFddBdAoGAGdfggY1bR5RWDvJe/ovT +32DnnMDAkD/wQszNraWg+Jr71RoqtBdBo4mkwnLZRulheARJiF62z15CJXYqzmSX +2MxGajz4PW33m+CPwogspNzvkfIZnZyQKJEyYNRYz1J4LCJYKigi7pf9dL1geF5u +9Xq2O8tnSF5u/E5mEZrweFs= +-----END PRIVATE KEY----- diff --git a/docker/auth/config/jwt/public_key.pem b/docker/auth/config/jwt/public_key.pem new file mode 100644 index 0000000..e867ef6 --- /dev/null +++ b/docker/auth/config/jwt/public_key.pem @@ -0,0 +1,8 @@ +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEAwVPV3Kic0bqk0HefP86LUDf71BY0tgIYFyM8NfEVCwr16P9ZkA+k +3o+xne1bJt4/6OP42a2ddHfokcqxknzMPcOxvkg+FAynKeBKbtrC5ntgGG1P0+D8 +9goA/KpFi3g7X1/xm+RKIJX4qj1NHAnzQHQE8EGcwuamfUQjDUG/FlbUP3yIRCQp +bbJQSxwHV6jrCZ8z1FJRROAG8Nfx6LFbw9hjqDhNMmaqyb6QsR+86C/Ewk7jKoQ8 +oNwXxSyMw9KXj3Hi399DpvaFVL9LzoZsucII2J0s8h2Gj2PVowGhVUzQsOLqRl73 +qjjdahtdmbTcFAbGOm3rDQGpnyOmx6C66wIDAQAB +-----END RSA PUBLIC KEY----- diff --git a/docker/auth/config/tls/ca.aerospike.com.crt b/docker/auth/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/auth/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/auth/config/tls/ca.aerospike.com.truststore.jks b/docker/auth/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8d72ac67e8199cf74e1aecd70587ac3acac082c6 GIT binary patch literal 1099 zcmezO_TO6u1_mY|W(3m$$%%T2sYUt41)15Yddc~@K#A;&+U|V}tPy&q29^vAEKCMX z%>N9Un4T?QW@2Pw;$+y-((Y)W)Wc-J%f_kI=F#?@mywa1mBFB~+mPFUlZ`o)g-w{r z(+|es;9>H1gmKt;n4J6!g$x8h%D8wq;daBsxOq5JlkKv*?@q=-a_l*;bEp;PZ{$kg1g+;aND z8kfEo@0p)p4D~%`H#>KQ38TZjXJ;d|4i}2a9+|Z4>m9zMQkf#seBjF+8`XKy>3{4cUTqpk4sX`!4YmoI-> z!NRI(@rS>8a>sv5{Wlu#Y$9IzUq2tt#LURRxH#D$(SRSAB4mXb8UM4e8ZZMX13r)d zKS+QDm_XZr=>o{-V-aH!@%giX|EOE^N_XM=YpkA&&f$H;-)ZCKCl{v9^?c6;vev@}Wk~YVb^`5%hSQsrX zzg!C1xZt}ZvX#awu*wm(1cX~|iEMvnS#*L!cM=l!cz5^|Zc zr~RDrZGL~wCa+b_{JYMueYC#Zy=%{|uTKRQ=&GM84gcx(SE}{WPTr-`I;$2vnd$6o mnK?CJ->E4l1=-jRu(E48Ts#u5Eo{?tLq7M!rIn^GL5~3;Cxf8? literal 0 HcmV?d00001 diff --git a/docker/auth/config/tls/keypass b/docker/auth/config/tls/keypass new file mode 100644 index 0000000..0f673cc --- /dev/null +++ b/docker/auth/config/tls/keypass @@ -0,0 +1 @@ +citrusstore \ No newline at end of file diff --git a/docker/auth/config/tls/localhost.keystore.jks b/docker/auth/config/tls/localhost.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8062bebe308ed12fbfb527ae72cf615cdf153807 GIT binary patch literal 2581 zcmY+EX*3&%8itdIC9zBFiM1GOV^`AFt_UU6UbOa**mo_WVkU~JR;Qx0qE-8p+P5N% zHK@I+u4Ss!Huklyd(XWyXMTL=JLh@7@BR6ph)`}2kO4)6^f5u?NH<6a>_BGVOCqER zOoY^3;A#|+(d{1*qZOFQh`GQwF2<0F^?zS%P$1|f5qyFof)7x#Owj-1i*jKwJWhL| zm(ROcRWEVe7=H$JTPqvP0xJNS@92v_0JrCe$$)ZBT z1+FT2pvT7TR~`Ki5Fj&A8Q$W8Aecpb?Y|C@K>`{fUDvyU^l5y%D_}o3f5B;N+IXH- zwVO=O(^9D2lCjl9y$KBu6(#AV2*oa)bjkf`5x{VCS&l2sChfj^pZ*|LUHKfbUtI4P^h8uIW+T9&eK zq5=OlaT*pBckN&6(nPd??kRsBtXVLVQ_bIn!oQuEaBUgeGNp zW=~F^`lPqNpA8ZWn?M*l2CGtH^A!(PmrjAjPkfSPAlA}!Bf2|-D|>5j#KcwBNV;}z zi)7q3l3!eB)njqSJ2pcdpr42%(GqnI+g@Hw+IBC zHrT|%@s;yKVuxwR_^p`wW?Ge~GiP5x?(?5?ZuJC9j)X?2*M@1?Ja(ZDXja2U^yVfc zH-5P267GE^GP<^TPYf@;VZ)&f^4*@k?OGl=K!=2h8l%^tioJ*H2X{^?KfnFFy0$MK z!)!t0;L2$Tq7t&IJPF5Fcs}ZfEdb7oJp4Exfg7W7Lj`PQfunNU=gMbT6(h7Hg+F7? zVnAUet6?-nkH@TGHY^z)e(OVi2=L}+Mc?j;WS@ur3rxDfDsro5r$S&AhR|dfTlvo4 zwxH+&%NwuK5W+VHn2WQXzSmaU(a|ooEtp)~Y!*m5-Sg&3n5T{ex8eJI`(_v8hCX)G z@Q}9OFi}pTWLE2O-pE9A>OL?bW2ecN`s^(6R>g9WPRVP2jn4MTD#f-+@*Ak;WR5{S zPHwWbpC6EaKgQ;$!7dVMkUYHZDb5TVP#23*+MA9JdU@HVeA-u0%S-X3A-sUMEpSC3 z@!fTV7*z>#5cO)@vVR4@*=qhT%#c`;t!5seIaH4|C83cXYs-Js;4=~V{SG*2%hDii z%8-l|n)&4&6}y9DwR(~%_LjV$UWS1t=>H=9$MutpJreez_;z=dSVC)oa74}M&0))@w*1K+&JTm(p~$)t+jQZ<7e_}lTsatM?a-C5iD7D zRSn;H)addgSdM$abKVirOY2Yvn(e+JStr>MZGGLKP3bffy)%*NOZ>MLUX>lnDQkar zog{A|dWuUE0z?2@()N*MNezWY(r0-dlA7|}alH1u9s{iI`*k}ru?7*d=n)vj;vL>Z z!Auss*ZV=$W6I`1-;!atWCll2J#bQq!~oh|fAp;SW`7}0+(+E?hYhWdBCzVgBKoaT zZ^54tzndWLGMei+mE+KmGk#zwb~>gcKtfcju-6RHVnxBLHI;L^s$DK6OEJdE8Yu3+ zR0)AAgGE39Wt8~eh*ud7=CSp|dxJ75i`?;MXz3 zelfjk-~mRqHl83Tww}#!*Ts!DBbAatC~3iMb!&VZ4u@66GdUzDVaUQBS6opRdm*aV z^EMaM_Uhhx)ztLa`ol-aB?{hrY!KrHU;Shl+jD?J#pBn0FXcx~wkDRas>-t7ZlNaM zedgLidi!M^6ZvGZfNY7A96ce01ZL7afqtJ&>FJGJHt~o{YQ+rc&bcuw#;;YZx7W1l zzndaJV#_Nfg})QH#oqapt8668c$&VMzsnm;Gr%XjK!gX_;w7I;rRI8IMr-o;-EqE9 zU5n47M}!-CLYzv|TlBq1XkT6)}}GRsaBbPw37NV~U(E1ssVZ87b_B z*w1L~BGEbXQ5$O2gAW+j4%%K%aY|@4t3>^#P+VcYl>mIbE7&9-oL8}(6Y?eeh2M(9 z0aPzp*GBZFfT{80Zd6xXdZi>ZJmym+?{XI(lG~L(O-lV|eRfk6lFQ9oX^N!25NkDz z1#(TFinzWYcN2=nHr3>I0c@kfF<-RB6!bF=Nfb0dfi6cYlTNNo$!U7s?Tc)%X! z;y!$NK7~6ejv5+hA{_uzv0qTLu!xcenSA&Sw*(hl93-Rwr5oA~&TQg!kkdJ4*n-SB z9lu~J3dJHMFp88CXM)An-`=F;8uQC{wGS&M$o^y{H7!~PY?s&q45@FF+tIPQpd)|s zDuJ>4n^s85B(&@cI5K)1Xw$lLm*Q2eUZ&NTv@R6ltY>UEwPccdn8=pfyc8hA7mosm z(Gs6Yd zd;JfaFA-qb=`GS@(%fT*b1}``UEwM_@m=yaqhAl;%SYUWzk}B%ZcrEv(_4Ta~UEB&nrm>}Z!- zy&RD~&bTasx`q-*L6{ijIT%1fi~uma(wy_v{8OE9YDWwb9cPG;ee9qDhU+`Y /Users/jesseschmidt/Developer/avs-client-go +// replace github.com/aerospike/avs-client-go => /Users/jesseschmidt/Developer/avs-client-go -replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go +// replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go require ( - github.com/aerospike/avs-client-go v0.0.0-20240509004238-aa3172c86d65 + github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0 github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 4f7b347..8b40062 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,10 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= +github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0 h1:ptyMV3i84ErLHyXIs3aTqIxTgJnJInfa6Nn1+L8GqO0= +github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0/go.mod h1:brxlQNig1y4tU7EP/jX+kXVgmwLERvtP/xhRoNw8Ark= +github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 h1:5rYc5QsaQeAnSzUm30gOUANEIEsMS8knbnjouenRV7E= +github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/test_utils.go b/test_utils.go index 9f45b21..9628bc8 100644 --- a/test_utils.go +++ b/test_utils.go @@ -2,7 +2,11 @@ package main -import "github.com/aerospike/avs-client-go/protos" +import ( + "fmt" + + "github.com/aerospike/avs-client-go/protos" +) func getStrPtr(str string) *string { ptr := str @@ -19,6 +23,10 @@ func getBoolPtr(b bool) *bool { return &ptr } +func createFlagStr(name, value string) string { + return fmt.Sprintf("--%s %s", name, value) +} + type IndexDefinitionBuilder struct { indexName string namespace string From 3c4862bf9a3bc2c0feb848ef03aa59d0b392db8c Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 24 Jun 2024 17:23:05 -0700 Subject: [PATCH 09/17] uncomment tests --- e2e_test.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/e2e_test.go b/e2e_test.go index cc1d711..2207f5b 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -74,23 +74,23 @@ func TestCmdSuite(t *testing.T) { logger.Info("%v", slog.Any("cert", rootCA)) - // suite.Run(t, &CmdTestSuite{ - // composeFile: "docker/docker-compose.yml", - // suiteFlags: []string{"--log-level debug"}, - // avsIP: "localhost", - // }) - // suite.Run(t, &CmdTestSuite{ - // composeFile: "docker/tls/docker-compose.yml", - // suiteFlags: []string{ - // "--log-level debug", - // createFlagStr(flags.TLSCaFile, "docker/tls/config/tls/ca.aerospike.com.crt"), - // }, - // avsTLSConfig: &tls.Config{ - // Certificates: nil, - // RootCAs: rootCA, - // }, - // avsIP: "localhost", - // }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/docker-compose.yml", + suiteFlags: []string{"--log-level debug"}, + avsIP: "localhost", + }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/tls/docker-compose.yml", + suiteFlags: []string{ + "--log-level debug", + createFlagStr(flags.TLSCaFile, "docker/tls/config/tls/ca.aerospike.com.crt"), + }, + avsTLSConfig: &tls.Config{ + Certificates: nil, + RootCAs: rootCA, + }, + avsIP: "localhost", + }) suite.Run(t, &CmdTestSuite{ composeFile: "docker/auth/docker-compose.yml", suiteFlags: []string{ From 2b984f4a1695bf1bdc1b14fc36d0df8622d40cf3 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 24 Jun 2024 17:25:02 -0700 Subject: [PATCH 10/17] create pre-release --- .github/workflows/create-prerelease.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index 556ad73..3bfe314 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -11,6 +11,10 @@ on: description: 'Delete existing pre-releases?' required: false type: boolean + + push: + branches: + - "VEC-189-tls" # remove before merge into main jobs: build-and-release: runs-on: macos-13 From 2364a02bd9f11b45872e04bb3d4cd50e0b57f78f Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 25 Jun 2024 10:35:12 -0700 Subject: [PATCH 11/17] add mTLS tests --- cmd/utils.go | 21 ++++- docker/mtls/config/aerospike-proximus.yml | 87 ++++++++++++++++++ docker/mtls/config/aerospike.conf | 82 +++++++++++++++++ docker/mtls/config/tls/ca.aerospike.com.crt | 24 +++++ .../tls/ca.aerospike.com.truststore.jks | Bin 0 -> 1099 bytes docker/mtls/config/tls/keypass | 1 + docker/mtls/config/tls/localhost.crt | 21 +++++ docker/mtls/config/tls/localhost.key | 27 ++++++ docker/mtls/config/tls/localhost.keystore.jks | Bin 0 -> 2581 bytes docker/mtls/config/tls/storepass | 1 + docker/mtls/docker-compose.yml | 23 +++++ e2e_test.go | 53 +++++++++-- 12 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 docker/mtls/config/aerospike-proximus.yml create mode 100644 docker/mtls/config/aerospike.conf create mode 100644 docker/mtls/config/tls/ca.aerospike.com.crt create mode 100644 docker/mtls/config/tls/ca.aerospike.com.truststore.jks create mode 100644 docker/mtls/config/tls/keypass create mode 100644 docker/mtls/config/tls/localhost.crt create mode 100644 docker/mtls/config/tls/localhost.key create mode 100644 docker/mtls/config/tls/localhost.keystore.jks create mode 100644 docker/mtls/config/tls/storepass create mode 100644 docker/mtls/docker-compose.yml diff --git a/cmd/utils.go b/cmd/utils.go index add5701..0160f9f 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -8,8 +8,11 @@ import ( "encoding/pem" "fmt" "log/slog" + "os" "time" + "golang.org/x/term" + avs "github.com/aerospike/avs-client-go" ) @@ -26,9 +29,21 @@ func createClientFromFlags(clientFlags *flags.ClientFlags, connectTimeout time.D } var password *string - if len(clientFlags.Password) != 0 { - strPass := clientFlags.Password.String() - password = &strPass + if clientFlags.User.Val != nil { + if len(clientFlags.Password) != 0 { + strPass := clientFlags.Password.String() + password = &strPass + } else { + fmt.Print("Enter Password: ") + bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + logger.Error("failed to read password", slog.Any("error", err)) + return nil, err + } + fmt.Println() // Print a newline after the password input + strPass := string(bytePassword) + password = &strPass + } } adminClient, err := avs.NewAdminClient( diff --git a/docker/mtls/config/aerospike-proximus.yml b/docker/mtls/config/aerospike-proximus.yml new file mode 100644 index 0000000..60473cf --- /dev/null +++ b/docker/mtls/config/aerospike-proximus.yml @@ -0,0 +1,87 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + mutual-auth: true + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/mtls/config/aerospike.conf b/docker/mtls/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/mtls/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/mtls/config/tls/ca.aerospike.com.crt b/docker/mtls/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/mtls/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/mtls/config/tls/ca.aerospike.com.truststore.jks b/docker/mtls/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8d72ac67e8199cf74e1aecd70587ac3acac082c6 GIT binary patch literal 1099 zcmezO_TO6u1_mY|W(3m$$%%T2sYUt41)15Yddc~@K#A;&+U|V}tPy&q29^vAEKCMX z%>N9Un4T?QW@2Pw;$+y-((Y)W)Wc-J%f_kI=F#?@mywa1mBFB~+mPFUlZ`o)g-w{r z(+|es;9>H1gmKt;n4J6!g$x8h%D8wq;daBsxOq5JlkKv*?@q=-a_l*;bEp;PZ{$kg1g+;aND z8kfEo@0p)p4D~%`H#>KQ38TZjXJ;d|4i}2a9+|Z4>m9zMQkf#seBjF+8`XKy>3{4cUTqpk4sX`!4YmoI-> z!NRI(@rS>8a>sv5{Wlu#Y$9IzUq2tt#LURRxH#D$(SRSAB4mXb8UM4e8ZZMX13r)d zKS+QDm_XZr=>o{-V-aH!@%giX|EOE^N_XM=YpkA&&f$H;-)ZCKCl{v9^?c6;vev@}Wk~YVb^`5%hSQsrX zzg!C1xZt}ZvX#awu*wm(1cX~|iEMvnS#*L!cM=l!cz5^|Zc zr~RDrZGL~wCa+b_{JYMueYC#Zy=%{|uTKRQ=&GM84gcx(SE}{WPTr-`I;$2vnd$6o mnK?CJ->E4l1=-jRu(E48Ts#u5Eo{?tLq7M!rIn^GL5~3;Cxf8? literal 0 HcmV?d00001 diff --git a/docker/mtls/config/tls/keypass b/docker/mtls/config/tls/keypass new file mode 100644 index 0000000..0f673cc --- /dev/null +++ b/docker/mtls/config/tls/keypass @@ -0,0 +1 @@ +citrusstore \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.crt b/docker/mtls/config/tls/localhost.crt new file mode 100644 index 0000000..e713708 --- /dev/null +++ b/docker/mtls/config/tls/localhost.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhzCCAm+gAwIBAgIUES99at1e+OgIcnfEwS6+uBLdLlkwDQYJKoZIhvcNAQEL +BQAwgYsxCzAJBgNVBAYTAklOMQswCQYDVQQIDAJLQTELMAkGA1UEBwwCQk4xEjAQ +BgNVBAoMCWFlcm9zcGlrZTESMBAGA1UECwwJZWNvc3lzdGVtMRkwFwYDVQQDDBBj +YS5hZXJvc3Bpa2UuY29tMR8wHQYJKoZIhvcNAQkBFhBjYUBhZXJvc3Bpa2UuY29t +MB4XDTE5MDcxNDE3NTI1M1oXDTI5MDcxMTE3NTI1M1owSDELMAkGA1UEBhMCSU4x +CzAJBgNVBAgMAktBMRgwFgYDVQQKDA9BZXJvc3Bpa2UsIEluYy4xEjAQBgNVBAMM +CWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANOOLzx9 +a8XnBLrTQn2RBJfUUi7qZZTBAEraQ0Tsh04wqEycwxsh4HfpqP4QDF8JpAs5mvMj +fotnamVCNncXJgvOIaPQVRoDdGAAfCcOvT9B0Y7xBH3jj+e9YBiOEfee7cb7mjnX +2XYa9UGepNIktHUR0RgsuNooxDcT8wIbt+QMK0ZqF5GBQtgX24646ow5VOzfAGZy +SZfc9J0mWIZwHc0Gkb+V15iJBzdwJ/15FrmHSiJqGIsesqThrdAh5SYZCG/1+RL8 +FuwtZuoKm3V4OoQzQ0TIz6fCaiLI1PgrKJnMiGa3PIPX5DyPEU+5opXJCClGW2iM +pGmDQ5W+GmZwickCAwEAAaMlMCMwIQYDVR0RBBowGIIJbG9jYWxob3N0ggsqLmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAdVCQWrRC3bCI6nQ1GuEfbEa7Fn2x +88EvMoJb7PLCLnumDd68AJ38qHUrmvrTbFFJemrCJ2cI0vGQPf5bMm+L2KyxVD9D +zSyy/mbKmAvJl05gv+Si0N9Ikhnqyj9NuF2hnMVJp7IowCBWU/JTfmXJFJU+bJDr +NntdjuUGHYq+zlTPowGS1p1Bx5QC+6YpelAcYiV7JiUXPQXd0SpJMcp0iYhJLjHv +qoM/jPl0gIHrn8WMrIKmf81Uv3mNd2nkKhiNuqqrHs5tVarYHfY2IJVnf8nMKlh9 +LVUXkze20oB2uc6IKxrb7nbXk0Q2XKb9xmpaFb+Fut3musY5sYrE9l+bkw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.key b/docker/mtls/config/tls/localhost.key new file mode 100644 index 0000000..23b929e --- /dev/null +++ b/docker/mtls/config/tls/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA044vPH1rxecEutNCfZEEl9RSLupllMEAStpDROyHTjCoTJzD +GyHgd+mo/hAMXwmkCzma8yN+i2dqZUI2dxcmC84ho9BVGgN0YAB8Jw69P0HRjvEE +feOP571gGI4R957txvuaOdfZdhr1QZ6k0iS0dRHRGCy42ijENxPzAhu35AwrRmoX +kYFC2BfbjrjqjDlU7N8AZnJJl9z0nSZYhnAdzQaRv5XXmIkHN3An/XkWuYdKImoY +ix6ypOGt0CHlJhkIb/X5EvwW7C1m6gqbdXg6hDNDRMjPp8JqIsjU+CsomcyIZrc8 +g9fkPI8RT7milckIKUZbaIykaYNDlb4aZnCJyQIDAQABAoIBAQCntYX43Cy93KBB +Qwzo4jfT7TuheaxBuqbysAi38RJqh+RDp9p7/eUm6pNPpYVJKiljxKzzpuXAuaD8 +2Pq4eh9tKGI+rP9p+ecd3ASQKf0Y0qLAQI0hB2+jdNtjW+0ecl1pazgeNuFr4X8g +IBXlibeNPyyVj46TU9IJH8V7nGGxZArm5hOWbLxfygVzulkeEsozjeDcVaUZPA1o +gN9QgqSC850jjqddV7a0sZuHyd3KECkfU1uRoSHikTKHJLxaV6BIHzliQijRutLT +hTFpk4QzM+7xixY66hh56YAoTB44A0sXHNrH5Zt3YkUv4mT/P9ue9DQ1mhz2Eh4h +DAzWFq4BAoGBAPOCOyar6K0mogxUl1Q1LBH34HvyrZ4uVN7ICeE12neh/fS04fnV +lM5e6bxKj2HEzqqJ8JUxjQypurz6aQBSI1o0hc13eZqykoNa7c9TREY8swM4Vwl9 +NOdyBuysJIqG9+tPp+vReP4ISyLG8fyeJNDl1OEaaz2GcxPGDbf7hR1hAoGBAN5o +V0tXp5wG/RnUZn0FU2g8faw5tfB4iO38igYJPEo5YbiC7FcJHYiPUlEfjlWmWwGT +RNggYK6Q7lQ4NKJU80DTG2k5AK5yLHtmtRmPPhYMZAt5txCXF7zpuoQS7OTKnc4l +R9pnsEYn+kYN/YA3EC56j/uSj6VEdB/AgHNvs51pAoGBAN4DlrqjcfisiIKFfZOh +BxU60skvcWwPAgI8kAVtfEomv8wkPwPx30Jo9uJdeGzDa0nBij/8dYVeGovCI4nP +WbwctwGmNJD+zuZEOR4V5OHE5dHBxFk6dsmuBPIz4P0MIW3BqnAvBAlYtmh2yppv +9VEguv6hf7UQqEsW/9sGz08BAoGBAKMnripCMl3+rnvdWhYK6yYDgjnu2C6BbgoQ +Afztl4Hn2G0v9krfEABXC48hdBwW/poIPC/EiMhm379+v/X6Fb0PYQNu4rYWYdVh +Aieu8l/gVSAp+Qa9oJdgawhqjchFb0CEDtMEz8aXmzz7FGWTf1ZpaOinmqMltX55 +jIGihwRJAoGATH9eCCCv1rbdnM8+qY3R8PZcZAOugICN55woXHJBCRRT+CpY4kiG +LNXyLZILWs8LXbzkKtynCXmVHMGbehtY/PnGXoSpsgLLz15CuGrbyQLoLnTr8NTB +atV5a96f8fOPhFl9EIRzV8u3DIF4CoYvNAFws7VwPDw01FCw5MncX2g= +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.keystore.jks b/docker/mtls/config/tls/localhost.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8062bebe308ed12fbfb527ae72cf615cdf153807 GIT binary patch literal 2581 zcmY+EX*3&%8itdIC9zBFiM1GOV^`AFt_UU6UbOa**mo_WVkU~JR;Qx0qE-8p+P5N% zHK@I+u4Ss!Huklyd(XWyXMTL=JLh@7@BR6ph)`}2kO4)6^f5u?NH<6a>_BGVOCqER zOoY^3;A#|+(d{1*qZOFQh`GQwF2<0F^?zS%P$1|f5qyFof)7x#Owj-1i*jKwJWhL| zm(ROcRWEVe7=H$JTPqvP0xJNS@92v_0JrCe$$)ZBT z1+FT2pvT7TR~`Ki5Fj&A8Q$W8Aecpb?Y|C@K>`{fUDvyU^l5y%D_}o3f5B;N+IXH- zwVO=O(^9D2lCjl9y$KBu6(#AV2*oa)bjkf`5x{VCS&l2sChfj^pZ*|LUHKfbUtI4P^h8uIW+T9&eK zq5=OlaT*pBckN&6(nPd??kRsBtXVLVQ_bIn!oQuEaBUgeGNp zW=~F^`lPqNpA8ZWn?M*l2CGtH^A!(PmrjAjPkfSPAlA}!Bf2|-D|>5j#KcwBNV;}z zi)7q3l3!eB)njqSJ2pcdpr42%(GqnI+g@Hw+IBC zHrT|%@s;yKVuxwR_^p`wW?Ge~GiP5x?(?5?ZuJC9j)X?2*M@1?Ja(ZDXja2U^yVfc zH-5P267GE^GP<^TPYf@;VZ)&f^4*@k?OGl=K!=2h8l%^tioJ*H2X{^?KfnFFy0$MK z!)!t0;L2$Tq7t&IJPF5Fcs}ZfEdb7oJp4Exfg7W7Lj`PQfunNU=gMbT6(h7Hg+F7? zVnAUet6?-nkH@TGHY^z)e(OVi2=L}+Mc?j;WS@ur3rxDfDsro5r$S&AhR|dfTlvo4 zwxH+&%NwuK5W+VHn2WQXzSmaU(a|ooEtp)~Y!*m5-Sg&3n5T{ex8eJI`(_v8hCX)G z@Q}9OFi}pTWLE2O-pE9A>OL?bW2ecN`s^(6R>g9WPRVP2jn4MTD#f-+@*Ak;WR5{S zPHwWbpC6EaKgQ;$!7dVMkUYHZDb5TVP#23*+MA9JdU@HVeA-u0%S-X3A-sUMEpSC3 z@!fTV7*z>#5cO)@vVR4@*=qhT%#c`;t!5seIaH4|C83cXYs-Js;4=~V{SG*2%hDii z%8-l|n)&4&6}y9DwR(~%_LjV$UWS1t=>H=9$MutpJreez_;z=dSVC)oa74}M&0))@w*1K+&JTm(p~$)t+jQZ<7e_}lTsatM?a-C5iD7D zRSn;H)addgSdM$abKVirOY2Yvn(e+JStr>MZGGLKP3bffy)%*NOZ>MLUX>lnDQkar zog{A|dWuUE0z?2@()N*MNezWY(r0-dlA7|}alH1u9s{iI`*k}ru?7*d=n)vj;vL>Z z!Auss*ZV=$W6I`1-;!atWCll2J#bQq!~oh|fAp;SW`7}0+(+E?hYhWdBCzVgBKoaT zZ^54tzndWLGMei+mE+KmGk#zwb~>gcKtfcju-6RHVnxBLHI;L^s$DK6OEJdE8Yu3+ zR0)AAgGE39Wt8~eh*ud7=CSp|dxJ75i`?;MXz3 zelfjk-~mRqHl83Tww}#!*Ts!DBbAatC~3iMb!&VZ4u@66GdUzDVaUQBS6opRdm*aV z^EMaM_Uhhx)ztLa`ol-aB?{hrY!KrHU;Shl+jD?J#pBn0FXcx~wkDRas>-t7ZlNaM zedgLidi!M^6ZvGZfNY7A96ce01ZL7afqtJ&>FJGJHt~o{YQ+rc&bcuw#;;YZx7W1l zzndaJV#_Nfg})QH#oqapt8668c$&VMzsnm;Gr%XjK!gX_;w7I;rRI8IMr-o;-EqE9 zU5n47M}!-CLYzv|TlBq1XkT6)}}GRsaBbPw37NV~U(E1ssVZ87b_B z*w1L~BGEbXQ5$O2gAW+j4%%K%aY|@4t3>^#P+VcYl>mIbE7&9-oL8}(6Y?eeh2M(9 z0aPzp*GBZFfT{80Zd6xXdZi>ZJmym+?{XI(lG~L(O-lV|eRfk6lFQ9oX^N!25NkDz z1#(TFinzWYcN2=nHr3>I0c@kfF<-RB6!bF=Nfb0dfi6cYlTNNo$!U7s?Tc)%X! z;y!$NK7~6ejv5+hA{_uzv0qTLu!xcenSA&Sw*(hl93-Rwr5oA~&TQg!kkdJ4*n-SB z9lu~J3dJHMFp88CXM)An-`=F;8uQC{wGS&M$o^y{H7!~PY?s&q45@FF+tIPQpd)|s zDuJ>4n^s85B(&@cI5K)1Xw$lLm*Q2eUZ&NTv@R6ltY>UEwPccdn8=pfyc8hA7mosm z(Gs6Yd zd;JfaFA-qb=`GS@(%fT*b1}``UEwM_@m=yaqhAl;%SYUWzk}B%ZcrEv(_4Ta~UEB&nrm>}Z!- zy&RD~&bTasx`q-*L6{ijIT%1fi~uma(wy_v{8OE9YDWwb9cPG;ee9qDhU+`Y Date: Tue, 25 Jun 2024 12:51:39 -0700 Subject: [PATCH 12/17] prompt when creating and dropping index --- cmd/createIndex.go | 19 +++++++++++++++---- cmd/dropIndex.go | 13 ++++++++++++- cmd/utils.go | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index dfcce2c..e05f4a7 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -52,7 +52,7 @@ var createIndexFlags = &struct { func newCreateIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} //nolint:lll // For readability flagSet.StringVarP(&createIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringSliceVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability flagSet.StringVarP(&createIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability flagSet.StringVarP(&createIndexFlags.vectorField, flags.VectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability flagSet.Uint32VarP(&createIndexFlags.dimensions, flags.Dimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability @@ -131,9 +131,6 @@ func newCreateIndexCmd() *cobra.Command { } defer adminClient.Close() - ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) - defer cancel() - // Inverted to make it easier to understand var hnswBatchDisabled *bool if createIndexFlags.hnswBatchEnabled.Val != nil { @@ -157,6 +154,20 @@ func newCreateIndexCmd() *cobra.Command { }, } + if !confirm(fmt.Sprintf( + "Are you sure you want to create the index %s field %s?", + nsAndSetString( + createIndexFlags.namespace, + createIndexFlags.sets, + ), + createIndexFlags.vectorField, + )) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) + defer cancel() + err = adminClient.IndexCreate( ctx, createIndexFlags.namespace, diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 63c53d5..d0c6306 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -30,7 +30,7 @@ var dropIndexFlags = &struct { func newDropIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} flagSet.StringVarP(&dropIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&dropIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringSliceVarP(&dropIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability flagSet.StringVarP(&dropIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability flagSet.DurationVar(&dropIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability flagSet.AddFlagSet(dropIndexFlags.clientFlags.NewClientFlagSet()) @@ -78,6 +78,17 @@ func newDropIndexCommand() *cobra.Command { } defer adminClient.Close() + if !confirm(fmt.Sprintf( + "Are you sure you want to drop the index %s on field %s?", + nsAndSetString( + createIndexFlags.namespace, + createIndexFlags.sets, + ), + createIndexFlags.vectorField, + )) { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() diff --git a/cmd/utils.go b/cmd/utils.go index 0160f9f..46ce4d1 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "os" + "strings" "time" "golang.org/x/term" @@ -165,3 +166,26 @@ func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFla return hosts, isLoadBalancer } + +func nsAndSetString(namespace string, sets []string) string { + var setStr string + + if len(sets) == 0 { + setStr = "*" + } else if len(sets) == 1 { + setStr = sets[0] + } else { + setStr = fmt.Sprintf("%v", sets) + } + + return fmt.Sprintf("%s.%s", namespace, setStr) +} + +func confirm(prompt string) bool { + var confirm string + + fmt.Print(prompt + " (y/n): ") + fmt.Scanln(&confirm) + + return strings.ToLower(confirm) == "y" +} From 6cd4ff54b99125b163090d6b2d43787787a4d0ec Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 25 Jun 2024 15:31:24 -0700 Subject: [PATCH 13/17] fix help text --- cmd/createIndex.go | 38 +++++++++++++++++++------------------- cmd/root.go | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index e05f4a7..d178b20 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -50,23 +50,23 @@ var createIndexFlags = &struct { } func newCreateIndexFlagSet() *pflag.FlagSet { - flagSet := &pflag.FlagSet{} //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringSliceVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.vectorField, flags.VectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability - flagSet.Uint32VarP(&createIndexFlags.dimensions, flags.Dimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.distanceMetric, flags.DistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability - flagSet.StringToStringVar(&createIndexFlags.indexMeta, flags.IndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.DurationVar(&createIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageNamespace, flags.StorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageSet, flags.StorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswMaxEdges, flags.MaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswConstructionEf, flags.ConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswEf, flags.Ef, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flags.BatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchInterval, flags.BatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchEnabled, flags.BatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability + flagSet := &pflag.FlagSet{} //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringSliceVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.vectorField, flags.VectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability + flagSet.Uint32VarP(&createIndexFlags.dimensions, flags.Dimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.distanceMetric, flags.DistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability + flagSet.StringToStringVar(&createIndexFlags.indexMeta, flags.IndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.DurationVar(&createIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageNamespace, flags.StorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageSet, flags.StorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswMaxEdges, flags.MaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswConstructionEf, flags.ConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswEf, flags.Ef, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flags.BatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchInterval, flags.BatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchEnabled, flags.BatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability flagSet.AddFlagSet(createIndexFlags.clientFlags.NewClientFlagSet()) return flagSet @@ -88,8 +88,8 @@ func newCreateIndexCmd() *cobra.Command { Long: `A command for creating indexes. An index is required to enable vector search on your data. The index tells AVS where your data is located, what your vectors look like, and how vectors should be compared to each other. - You can optionally tweak where your index is stored, and how the HNSW algorithm - behaves. + Optionally, you can tweak where your index is stored and how the HNSW algorithm + behaves. For more information see: https://aerospike.com/docs/vector For example: export ASVEC_HOST=:5000 diff --git a/cmd/root.go b/cmd/root.go index 1ff5304..f3b3fd4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -31,7 +31,7 @@ var rootCmd = &cobra.Command{ Short: "Aerospike Vector Search CLI", Long: `Welcome to the AVS Deployment Manager CLI Tool! To start using this tool, please consult the detailed documentation available at https://aerospike.com/docs/vector. - Should you encounter any issues or have questions, feel free to report them via GitHub issues. + Should you encounter any issues or have questions, feel free to report them by creating a GitHub issue. Enterprise customers requiring support should contact Aerospike Support directly at https://aerospike.com/support.`, PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { if rootFlags.logLevel.NotSet() { From 40fdf4ce9eb13df727e603cef07a2fcf62a7b303 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 26 Jun 2024 10:08:50 -0700 Subject: [PATCH 14/17] fix verbose requirement for unmerged record count --- cmd/listIndex.go | 52 +++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 2e81f5b..5bd334e 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -81,35 +81,33 @@ func newListIndexCmd() *cobra.Command { indexStatusList := make([]*protos.IndexStatusResponse, len(indexList.GetIndices())) - if listIndexFlags.verbose { - cancel() - - ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) - defer cancel() - - wg := sync.WaitGroup{} - for i, index := range indexList.GetIndices() { - wg.Add(1) - go func(i int, index *protos.IndexDefinition) { - defer wg.Done() - indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) - if err != nil { - logger.ErrorContext(ctx, - "failed to get index status", - slog.Any("error", err), - slog.String("index", index.Id.String()), - ) - return - } - - indexStatusList[i] = indexStatus - logger.Debug("server index status", slog.Int("index", i), slog.Any("response", indexStatus)) - }(i, index) - } - - wg.Wait() + cancel() + + ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + defer cancel() + + wg := sync.WaitGroup{} + for i, index := range indexList.GetIndices() { + wg.Add(1) + go func(i int, index *protos.IndexDefinition) { + defer wg.Done() + indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) + if err != nil { + logger.ErrorContext(ctx, + "failed to get index status", + slog.Any("error", err), + slog.String("index", index.Id.String()), + ) + return + } + + indexStatusList[i] = indexStatus + logger.Debug("server index status", slog.Int("index", i), slog.Any("response", indexStatus)) + }(i, index) } + wg.Wait() + logger.Debug("server index list", slog.String("response", indexList.String())) view.PrintIndexes(indexList, indexStatusList, listIndexFlags.verbose) From edb32069a4336bb21630d166191e9105e9dddc2f Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 2 Jul 2024 15:10:59 -0700 Subject: [PATCH 15/17] fix --- cmd/root.go | 2 +- go.mod | 1 + go.sum | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index f3b3fd4..4b38a74 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -88,7 +88,7 @@ func Execute() { func init() { rootCmd.PersistentFlags().Var( &rootFlags.logLevel, - logLevelFlagName, + flags.LogLevel, common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", "))), //nolint:lll // For readability ) common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version diff --git a/go.mod b/go.mod index 9adea42..47fb4a6 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/term v0.21.0 ) require ( diff --git a/go.sum b/go.sum index 8b40062..0c4b52f 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= From 1548ac32b7d95f556ae7c8606704f1edaa215e5a Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 3 Jul 2024 09:08:43 -0700 Subject: [PATCH 16/17] review changes --- cmd/flags/tls.go | 4 +-- cmd/utils.go | 94 ------------------------------------------------ 2 files changed, 2 insertions(+), 96 deletions(-) diff --git a/cmd/flags/tls.go b/cmd/flags/tls.go index 30927c4..837f2c7 100644 --- a/cmd/flags/tls.go +++ b/cmd/flags/tls.go @@ -23,8 +23,8 @@ func NewTLSFlags() *TLSFlags { } } -// NewAerospikeFlagSet returns a new pflag.FlagSet with Aerospike flags defined. -// Values set in the returned FlagSet will be stored in the AerospikeFlags argument. +// NewTLSFlagSet returns a new pflag.FlagSet with TLS flags defined. Values +// are stored in the TLSFlags struct. func (tf *TLSFlags) NewTLSFlagSet(fmtUsage commonFlags.UsageFormatter) *pflag.FlagSet { f := &pflag.FlagSet{} diff --git a/cmd/utils.go b/cmd/utils.go index 46ce4d1..71a062b 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -3,9 +3,6 @@ package cmd import ( "asvec/cmd/flags" "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" "fmt" "log/slog" "os" @@ -57,97 +54,6 @@ func createClientFromFlags(clientFlags *flags.ClientFlags, connectTimeout time.D return adminClient, nil } - -func newTLSConfig(rootCA [][]byte, cert []byte, key []byte, keyPass []byte, tlsProtoMin int, tlsProtoMax int) (*tls.Config, error) { - if len(rootCA) == 0 && len(cert) == 0 && len(key) == 0 { - return nil, nil - } - - var ( - clientPool []tls.Certificate - serverPool *x509.CertPool - err error - ) - - serverPool = loadCACerts(rootCA) - - if len(cert) > 0 || len(key) > 0 { - clientPool, err = loadServerCertAndKey(cert, key, keyPass) - if err != nil { - return nil, fmt.Errorf("failed to load client authentication certificate and key `%s`", err) - } - } - - tlsConfig := &tls.Config{ //nolint:gosec // aerospike default tls version is TLSv1.2 - Certificates: clientPool, - RootCAs: serverPool, - InsecureSkipVerify: false, - PreferServerCipherSuites: true, - MinVersion: uint16(tlsProtoMin), - MaxVersion: uint16(tlsProtoMax), - } - - return tlsConfig, nil -} - -// loadCACerts returns CA set of certificates (cert pool) -// reads CA certificate based on the certConfig and adds it to the pool -func loadCACerts(certsBytes [][]byte) *x509.CertPool { - certificates, err := x509.SystemCertPool() - if certificates == nil || err != nil { - certificates = x509.NewCertPool() - } - - for _, cert := range certsBytes { - if len(cert) > 0 { - certificates.AppendCertsFromPEM(cert) - } - } - - return certificates -} - -// loadServerCertAndKey reads server certificate and associated key file based on certConfig and keyConfig -// returns parsed server certificate -// if the private key is encrypted, it will be decrypted using key file passphrase -func loadServerCertAndKey(certFileBytes, keyFileBytes, keyPassBytes []byte) ([]tls.Certificate, error) { - var certificates []tls.Certificate - - // Decode PEM data - keyBlock, _ := pem.Decode(keyFileBytes) - - if keyBlock == nil { - return nil, fmt.Errorf("failed to decode PEM data for key or certificate") - } - - // Check and Decrypt the Key Block using passphrase - if x509.IsEncryptedPEMBlock(keyBlock) { //nolint:staticcheck,lll // This needs to be addressed by aerospike as multiple projects require this functionality - decryptedDERBytes, err := x509.DecryptPEMBlock(keyBlock, keyPassBytes) //nolint:staticcheck,lll // This needs to be addressed by aerospike as multiple projects require this functionality - if err != nil { - return nil, fmt.Errorf("failed to decrypt PEM Block: `%s`", err) - } - - keyBlock.Bytes = decryptedDERBytes - keyBlock.Headers = nil - } - - // Encode PEM data - keyPEM := pem.EncodeToMemory(keyBlock) - - if keyPEM == nil { - return nil, fmt.Errorf("failed to encode PEM data for key or certificate") - } - - cert, err := tls.X509KeyPair(certFileBytes, keyPEM) - if err != nil { - return nil, fmt.Errorf("failed to add certificate and key to the pool: `%s`", err) - } - - certificates = append(certificates, cert) - - return certificates, nil -} - func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFlag) (avs.HostPortSlice, bool) { isLoadBalancer := false hosts := avs.HostPortSlice{} From 9ef1a0a7a3c9eb50bc5daff90e23b175c64e3697 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 3 Jul 2024 09:09:59 -0700 Subject: [PATCH 17/17] update deps --- go.mod | 8 ++------ go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 47fb4a6..e39cfcb 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,9 @@ module asvec go 1.21.7 -// replace github.com/aerospike/avs-client-go => /Users/jesseschmidt/Developer/avs-client-go - -// replace github.com/aerospike/tools-common-go => /Users/jesseschmidt/Developer/tools-common-go - require ( - github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0 - github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 + github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05 + github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 0c4b52f..3896d72 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= -github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0 h1:ptyMV3i84ErLHyXIs3aTqIxTgJnJInfa6Nn1+L8GqO0= -github.com/aerospike/avs-client-go v0.0.0-20240625001515-2ad1adca5de0/go.mod h1:brxlQNig1y4tU7EP/jX+kXVgmwLERvtP/xhRoNw8Ark= -github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89 h1:5rYc5QsaQeAnSzUm30gOUANEIEsMS8knbnjouenRV7E= -github.com/aerospike/tools-common-go v0.0.0-20240618165632-595098741f89/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= +github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05 h1:Egu9UMm2SIRP871wuwp5PoqWKjPrc9XNDMg/4REOuaQ= +github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05/go.mod h1:brxlQNig1y4tU7EP/jX+kXVgmwLERvtP/xhRoNw8Ark= +github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 h1:tOLqcGsc6A656WNEZhYZYxDiX7d6wCkEN6+jLDSDeUU= +github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=