diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index 556ad73..3bfe314 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -11,6 +11,10 @@ on: description: 'Delete existing pre-releases?' required: false type: boolean + + push: + branches: + - "VEC-189-tls" # remove before merge into main jobs: build-and-release: runs-on: macos-13 diff --git a/.gitignore b/.gitignore index d87c693..f7114ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -/docker/config/features.conf +features.conf /bin/* embed_*.go -/tmp +tmp /vendor /coverage \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 32c6591..9099fa8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,7 +23,7 @@ linters-settings: allow: - $gostd - github.com/aerospike/tools-common-go - - github.com/aerospike/aerospike-proximus-client-go + - github.com/aerospike/avs-client-go - asvec/cmd - github.com/spf13/cobra - github.com/spf13/viper diff --git a/cmd/constants.go b/cmd/constants.go deleted file mode 100644 index 0b6ee36..0000000 --- a/cmd/constants.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -const ( - logLevelFlagName = "log-level" - flagNameSeeds = "seeds" - flagNameHost = "host" - flagNameListenerName = "listener-name" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVectorField = "vector-field" - flagNameDimension = "dimension" - flagNameDistanceMetric = "distance-metric" - flagNameIndexMeta = "index-meta" - flagNameTimeout = "timeout" - flagNameVerbose = "verbose" - flagNameStorageNamespace = "storage-namespace" - flagNameStorageSet = "storage-set" - flagNameMaxEdges = "hnsw-max-edges" - flagNameConstructionEf = "hnsw-ef-construction" - flagNameEf = "hnsw-ef" - flagNameBatchMaxRecords = "hnsw-batch-max-records" - flagNameBatchInterval = "hnsw-batch-interval" - flagNameBatchEnabled = "hnsw-batch-enabled" -) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 150086a..d178b20 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -11,8 +11,7 @@ import ( "strings" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -21,9 +20,7 @@ import ( //nolint:govet // Padding not a concern for a CLI var createIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag + clientFlags flags.ClientFlags namespace string sets []string indexName string @@ -41,8 +38,7 @@ var createIndexFlags = &struct { hnswBatchEnabled flags.BoolOptionalFlag timeout time.Duration }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, + clientFlags: *flags.NewClientFlags(), storageNamespace: flags.StringOptionalFlag{}, storageSet: flags.StringOptionalFlag{}, hnswMaxEdges: flags.Uint32OptionalFlag{}, @@ -54,36 +50,34 @@ var createIndexFlags = &struct { } func newCreateIndexFlagSet() *pflag.FlagSet { - flagSet := &pflag.FlagSet{} - flagSet.VarP(createIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(createIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&createIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability - flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability - flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability - flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.storageSet, flagNameStorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswMaxEdges, flagNameMaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswConstructionEf, flagNameConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswEf, flagNameEf, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability - flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability + flagSet := &pflag.FlagSet{} //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringSliceVarP(&createIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.vectorField, flags.VectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability + flagSet.Uint32VarP(&createIndexFlags.dimensions, flags.Dimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.distanceMetric, flags.DistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability + flagSet.StringToStringVar(&createIndexFlags.indexMeta, flags.IndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.DurationVar(&createIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageNamespace, flags.StorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageSet, flags.StorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswMaxEdges, flags.MaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswConstructionEf, flags.ConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswEf, flags.Ef, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flags.BatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchInterval, flags.BatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchEnabled, flags.BatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability + flagSet.AddFlagSet(createIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } var createIndexRequiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, - flagNameVectorField, - flagNameDimension, - flagNameDistanceMetric, + flags.Namespace, + flags.IndexName, + flags.VectorField, + flags.Dimension, + flags.DistanceMetric, } // createIndexCmd represents the createIndex command @@ -94,8 +88,8 @@ func newCreateIndexCmd() *cobra.Command { Long: `A command for creating indexes. An index is required to enable vector search on your data. The index tells AVS where your data is located, what your vectors look like, and how vectors should be compared to each other. - You can optionally tweak where your index is stored, and how the HNSW algorithm - behaves. + Optionally, you can tweak where your index is stored and how the HNSW algorithm + behaves. For more information see: https://aerospike.com/docs/vector For example: export ASVEC_HOST=:5000 @@ -103,54 +97,40 @@ func newCreateIndexCmd() *cobra.Command { --storage-namespace test --hnsw-batch-enabled false `, PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { - hosts, isLoadBalancer := parseBothHostSeedsFlag(createIndexFlags.seeds, createIndexFlags.host) - logger.Debug("parsed flags", - slog.String(flagNameHost, createIndexFlags.host.String()), - slog.String(flagNameSeeds, createIndexFlags.seeds.String()), - slog.String(flagNameListenerName, createIndexFlags.listenerName.String()), - slog.String(flagNameNamespace, createIndexFlags.namespace), - slog.Any(flagNameSets, createIndexFlags.sets), - slog.String(flagNameIndexName, createIndexFlags.indexName), - slog.String(flagNameVectorField, createIndexFlags.vectorField), - slog.Uint64(flagNameDimension, uint64(createIndexFlags.dimensions)), - slog.Any(flagNameIndexMeta, createIndexFlags.indexMeta), - slog.String(flagNameDistanceMetric, createIndexFlags.distanceMetric.String()), - slog.Duration(flagNameTimeout, createIndexFlags.timeout), - slog.Any(flagNameStorageNamespace, createIndexFlags.storageNamespace.String()), - slog.Any(flagNameStorageSet, createIndexFlags.storageSet.String()), - slog.Any(flagNameMaxEdges, createIndexFlags.hnswMaxEdges.String()), - slog.Any(flagNameEf, createIndexFlags.hnswEf), - slog.Any(flagNameConstructionEf, createIndexFlags.hnswConstructionEf.String()), - slog.Any(flagNameBatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), - slog.Any(flagNameBatchInterval, createIndexFlags.hnswBatchInterval.String()), - slog.Any(flagNameBatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + append(createIndexFlags.clientFlags.NewSLogAttr(), + slog.String(flags.Namespace, createIndexFlags.namespace), + slog.Any(flags.Sets, createIndexFlags.sets), + slog.String(flags.IndexName, createIndexFlags.indexName), + slog.String(flags.VectorField, createIndexFlags.vectorField), + slog.Uint64(flags.Dimension, uint64(createIndexFlags.dimensions)), + slog.Any(flags.IndexMeta, createIndexFlags.indexMeta), + slog.String(flags.DistanceMetric, createIndexFlags.distanceMetric.String()), + slog.Duration(flags.Timeout, createIndexFlags.timeout), + slog.Any(flags.StorageNamespace, createIndexFlags.storageNamespace.String()), + slog.Any(flags.StorageSet, createIndexFlags.storageSet.String()), + slog.Any(flags.MaxEdges, createIndexFlags.hnswMaxEdges.String()), + slog.Any(flags.Ef, createIndexFlags.hnswEf), + slog.Any(flags.ConstructionEf, createIndexFlags.hnswConstructionEf.String()), + slog.Any(flags.BatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), + slog.Any(flags.BatchInterval, createIndexFlags.hnswBatchInterval.String()), + slog.Any(flags.BatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + )..., ) - ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) - defer cancel() - - adminClient, err := avs.NewAdminClient( - ctx, hosts, createIndexFlags.listenerName.Val, isLoadBalancer, logger, - ) + adminClient, err := createClientFromFlags(&createIndexFlags.clientFlags, createIndexFlags.timeout) if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) return err } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), createIndexFlags.timeout) - defer cancel() - // Inverted to make it easier to understand var hnswBatchDisabled *bool if createIndexFlags.hnswBatchEnabled.Val != nil { @@ -174,6 +154,20 @@ func newCreateIndexCmd() *cobra.Command { }, } + if !confirm(fmt.Sprintf( + "Are you sure you want to create the index %s field %s?", + nsAndSetString( + createIndexFlags.namespace, + createIndexFlags.sets, + ), + createIndexFlags.vectorField, + )) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) + defer cancel() + err = adminClient.IndexCreate( ctx, createIndexFlags.namespace, diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 65b699c..d0c6306 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -10,7 +10,6 @@ import ( "log/slog" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -19,34 +18,29 @@ import ( //nolint:govet // Padding not a concern for a CLI var dropIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag - namespace string - sets []string - indexName string - timeout time.Duration + clientFlags flags.ClientFlags + namespace string + sets []string + indexName string + timeout time.Duration }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, + clientFlags: *flags.NewClientFlags(), } func newDropIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(dropIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(dropIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&dropIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.StringVarP(&dropIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability - flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability - flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability - flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&dropIndexFlags.namespace, flags.Namespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringSliceVarP(&dropIndexFlags.sets, flags.Sets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&dropIndexFlags.indexName, flags.IndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.DurationVar(&dropIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(dropIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } var dropIndexRequiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, + flags.Namespace, + flags.IndexName, } // dropIndexCmd represents the dropIndex command @@ -62,38 +56,40 @@ func newDropIndexCommand() *cobra.Command { asvec drop index -i myindex -n test `, PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flagNameHost, dropIndexFlags.host.String()), - slog.String(flagNameSeeds, dropIndexFlags.seeds.String()), - slog.String(flagNameListenerName, dropIndexFlags.listenerName.String()), - slog.String(flagNameNamespace, dropIndexFlags.namespace), - slog.Any(flagNameSets, dropIndexFlags.sets), - slog.String(flagNameIndexName, dropIndexFlags.indexName), - slog.Duration(flagNameTimeout, dropIndexFlags.timeout), + append(dropIndexFlags.clientFlags.NewSLogAttr(), + slog.String(flags.Namespace, dropIndexFlags.namespace), + slog.Any(flags.Sets, dropIndexFlags.sets), + slog.String(flags.IndexName, dropIndexFlags.indexName), + slog.Duration(flags.Timeout, dropIndexFlags.timeout), + )..., ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(dropIndexFlags.seeds, dropIndexFlags.host) - - ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) - defer cancel() - - adminClient, err := avs.NewAdminClient(ctx, hosts, nil, isLoadBalancer, logger) + adminClient, err := createClientFromFlags(&dropIndexFlags.clientFlags, dropIndexFlags.timeout) if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) return err } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), dropIndexFlags.timeout) + if !confirm(fmt.Sprintf( + "Are you sure you want to drop the index %s on field %s?", + nsAndSetString( + createIndexFlags.namespace, + createIndexFlags.sets, + ), + createIndexFlags.vectorField, + )) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() err = adminClient.IndexDrop(ctx, dropIndexFlags.namespace, dropIndexFlags.indexName) diff --git a/cmd/flags/client.go b/cmd/flags/client.go new file mode 100644 index 0000000..45d410e --- /dev/null +++ b/cmd/flags/client.go @@ -0,0 +1,52 @@ +package flags + +import ( + "fmt" + "log/slog" + + commonFlags "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/pflag" +) + +type ClientFlags struct { + Host *HostPortFlag + Seeds *SeedsSliceFlag + ListenerName StringOptionalFlag + User StringOptionalFlag + Password commonFlags.PasswordFlag + TLSFlags +} + +func NewClientFlags() *ClientFlags { + return &ClientFlags{ + Host: NewDefaultHostPortFlag(), + Seeds: &SeedsSliceFlag{}, + TLSFlags: *NewTLSFlags(), + } +} + +func (cf *ClientFlags) NewClientFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.VarP(cf.Host, Host, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", Seeds))) //nolint:lll // For readability + flagSet.Var(cf.Seeds, Seeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", Host))) //nolint:lll // For readability + flagSet.VarP(&cf.ListenerName, ListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.VarP(&cf.User, User, "U", commonFlags.DefaultWrapHelpString("The AVS user to authenticate with.")) //nolint:lll // For readability + flagSet.VarP(&cf.Password, Password, "P", commonFlags.DefaultWrapHelpString("The AVS password for the specified user.")) //nolint:lll // For readability + flagSet.AddFlagSet(cf.NewTLSFlagSet(commonFlags.DefaultWrapHelpString)) + + return flagSet +} + +func (cf *ClientFlags) NewSLogAttr() []any { + return []any{slog.String(Host, cf.Host.String()), + slog.String(Seeds, cf.Seeds.String()), + slog.String(ListenerName, cf.ListenerName.String()), + slog.String(User, cf.User.String()), + slog.String(Password, cf.Password.String()), + slog.Bool(TLSCaFile, cf.TLSRootCAFile != nil), + slog.Bool(TLSCaPath, cf.TLSRootCAPath != nil), + slog.Bool(TLSCertFile, cf.TLSCertFile != nil), + slog.Bool(TLSKeyFile, cf.TLSKeyFile != nil), + slog.Bool(TLSKeyFilePass, cf.TLSKeyFilePass != nil), + } +} diff --git a/cmd/flags/constants.go b/cmd/flags/constants.go new file mode 100644 index 0000000..888f67b --- /dev/null +++ b/cmd/flags/constants.go @@ -0,0 +1,33 @@ +package flags + +const ( + LogLevel = "log-level" + Seeds = "seeds" + Host = "host" + ListenerName = "listener-name" + User = "user" + Password = "password" + Namespace = "namespace" + Sets = "sets" + IndexName = "index-name" + VectorField = "vector-field" + Dimension = "dimension" + DistanceMetric = "distance-metric" + IndexMeta = "index-meta" + Timeout = "timeout" + Verbose = "verbose" + StorageNamespace = "storage-namespace" + StorageSet = "storage-set" + MaxEdges = "hnsw-max-edges" + ConstructionEf = "hnsw-ef-construction" + Ef = "hnsw-ef" + BatchMaxRecords = "hnsw-batch-max-records" + BatchInterval = "hnsw-batch-interval" + BatchEnabled = "hnsw-batch-enabled" + TLSProtocols = "tls-protocols" + TLSCaFile = "tls-cafile" + TLSCaPath = "tls-capath" + TLSCertFile = "tls-certfile" + TLSKeyFile = "tls-keyfile" + TLSKeyFilePass = "tls-keyfile-password" +) diff --git a/cmd/flags/distanceMetric.go b/cmd/flags/distanceMetric.go index 78d332e..f19b5d1 100644 --- a/cmd/flags/distanceMetric.go +++ b/cmd/flags/distanceMetric.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" ) type DistanceMetricFlag string diff --git a/cmd/flags/hostPort.go b/cmd/flags/hostPort.go index 36d7578..69b0ff3 100644 --- a/cmd/flags/hostPort.go +++ b/cmd/flags/hostPort.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" ) const ( diff --git a/cmd/flags/hostPort_test.go b/cmd/flags/hostPort_test.go index 1b14406..50d787a 100644 --- a/cmd/flags/hostPort_test.go +++ b/cmd/flags/hostPort_test.go @@ -5,7 +5,7 @@ package flags import ( "testing" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" "github.com/stretchr/testify/suite" ) diff --git a/cmd/flags/tls.go b/cmd/flags/tls.go new file mode 100644 index 0000000..837f2c7 --- /dev/null +++ b/cmd/flags/tls.go @@ -0,0 +1,61 @@ +package flags + +import ( + "crypto/tls" + + commonClient "github.com/aerospike/tools-common-go/client" + commonFlags "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/pflag" +) + +type TLSFlags struct { + TLSProtocols commonFlags.TLSProtocolsFlag + TLSRootCAFile commonFlags.CertFlag + TLSRootCAPath commonFlags.CertPathFlag + TLSCertFile commonFlags.CertFlag + TLSKeyFile commonFlags.CertFlag + TLSKeyFilePass commonFlags.PasswordFlag +} + +func NewTLSFlags() *TLSFlags { + return &TLSFlags{ + TLSProtocols: commonFlags.NewDefaultTLSProtocolsFlag(), + } +} + +// NewTLSFlagSet returns a new pflag.FlagSet with TLS flags defined. Values +// are stored in the TLSFlags struct. +func (tf *TLSFlags) NewTLSFlagSet(fmtUsage commonFlags.UsageFormatter) *pflag.FlagSet { + f := &pflag.FlagSet{} + + f.Var(&tf.TLSRootCAFile, "tls-cafile", fmtUsage("The CA used when connecting to AVS.")) + f.Var(&tf.TLSRootCAPath, "tls-capath", fmtUsage("A path containing CAs for connecting to AVS.")) + f.Var(&tf.TLSCertFile, "tls-certfile", fmtUsage("The certificate file for mutual TLS authentication with AVS.")) + f.Var(&tf.TLSKeyFile, "tls-keyfile", fmtUsage("The key file used for mutual TLS authentication with AVS.")) + f.Var(&tf.TLSKeyFilePass, "tls-keyfile-password", fmtUsage("The password used to decrypt the key-file if encrypted.")) + f.Var(&tf.TLSProtocols, "tls-protocols", fmtUsage( + "Set the TLS protocol selection criteria. This format is the same as"+ + " Apache's SSLProtocol documented at https://httpd.apache.org/docs/current/mod/mod_ssl.html#ssl protocol.", + )) + + return f +} + +func (tf *TLSFlags) NewTLSConfig() (*tls.Config, error) { + rootCA := [][]byte{} + + if len(tf.TLSRootCAFile) != 0 { + rootCA = append(rootCA, tf.TLSRootCAFile) + } + + rootCA = append(rootCA, tf.TLSRootCAPath...) + + return commonClient.NewTLSConfig( + rootCA, + tf.TLSCertFile, + tf.TLSKeyFile, + tf.TLSKeyFilePass, + 0, + 0, + ).NewGoTLSConfig() +} diff --git a/cmd/listIndex.go b/cmd/listIndex.go index dcd8ccc..5bd334e 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -11,8 +11,7 @@ import ( "sync" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -20,23 +19,18 @@ import ( ) var listIndexFlags = &struct { - host *flags.HostPortFlag - seeds *flags.SeedsSliceFlag - listenerName flags.StringOptionalFlag - verbose bool - timeout time.Duration + clientFlags flags.ClientFlags + verbose bool + timeout time.Duration }{ - host: flags.NewDefaultHostPortFlag(), - seeds: &flags.SeedsSliceFlag{}, + clientFlags: *flags.NewClientFlags(), } func newListIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(listIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability - flagSet.Var(listIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability - flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability - flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability - flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.BoolVarP(&listIndexFlags.verbose, flags.Verbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability + flagSet.DurationVar(&listIndexFlags.timeout, flags.Timeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.AddFlagSet(listIndexFlags.clientFlags.NewClientFlagSet()) return flagSet } @@ -54,38 +48,29 @@ func newListIndexCmd() *cobra.Command { For example: export ASVEC_HOST=:5000 asvec list index - `, flagNameVerbose), + `, flags.Verbose), PreRunE: func(_ *cobra.Command, _ []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) + if viper.IsSet(flags.Seeds) && viper.IsSet(flags.Host) { + return fmt.Errorf("only --%s or --%s allowed", flags.Seeds, flags.Host) } return nil }, RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", - slog.String(flagNameHost, listIndexFlags.host.String()), - slog.String(flagNameSeeds, listIndexFlags.seeds.String()), - slog.String(flagNameListenerName, listIndexFlags.listenerName.String()), - slog.Bool(flagNameVerbose, listIndexFlags.verbose), - slog.Duration(flagNameTimeout, listIndexFlags.timeout), + append(listIndexFlags.clientFlags.NewSLogAttr(), + slog.Bool(flags.Verbose, listIndexFlags.verbose), + slog.Duration(flags.Timeout, listIndexFlags.timeout), + )..., ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(listIndexFlags.seeds, listIndexFlags.host) - - ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) - defer cancel() - - adminClient, err := avs.NewAdminClient(ctx, hosts, listIndexFlags.listenerName.Val, isLoadBalancer, logger) + adminClient, err := createClientFromFlags(&listIndexFlags.clientFlags, listIndexFlags.timeout) if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) return err } - - cancel() defer adminClient.Close() - ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) defer cancel() indexList, err := adminClient.IndexList(ctx) @@ -96,35 +81,33 @@ func newListIndexCmd() *cobra.Command { indexStatusList := make([]*protos.IndexStatusResponse, len(indexList.GetIndices())) - if listIndexFlags.verbose { - cancel() - - ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) - defer cancel() - - wg := sync.WaitGroup{} - for i, index := range indexList.GetIndices() { - wg.Add(1) - go func(i int, index *protos.IndexDefinition) { - defer wg.Done() - indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) - if err != nil { - logger.ErrorContext(ctx, - "failed to get index status", - slog.Any("error", err), - slog.String("index", index.Id.String()), - ) - return - } - - indexStatusList[i] = indexStatus - logger.Debug("server index status", slog.Int("index", i), slog.Any("response", indexStatus)) - }(i, index) - } - - wg.Wait() + cancel() + + ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + defer cancel() + + wg := sync.WaitGroup{} + for i, index := range indexList.GetIndices() { + wg.Add(1) + go func(i int, index *protos.IndexDefinition) { + defer wg.Done() + indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) + if err != nil { + logger.ErrorContext(ctx, + "failed to get index status", + slog.Any("error", err), + slog.String("index", index.Id.String()), + ) + return + } + + indexStatusList[i] = indexStatus + logger.Debug("server index status", slog.Int("index", i), slog.Any("response", indexStatus)) + }(i, index) } + wg.Wait() + logger.Debug("server index list", slog.String("response", indexList.String())) view.PrintIndexes(indexList, indexStatusList, listIndexFlags.verbose) diff --git a/cmd/root.go b/cmd/root.go index 28bb576..4b38a74 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -31,7 +31,7 @@ var rootCmd = &cobra.Command{ Short: "Aerospike Vector Search CLI", Long: `Welcome to the AVS Deployment Manager CLI Tool! To start using this tool, please consult the detailed documentation available at https://aerospike.com/docs/vector. - Should you encounter any issues or have questions, feel free to report them via GitHub issues. + Should you encounter any issues or have questions, feel free to report them by creating a GitHub issue. Enterprise customers requiring support should contact Aerospike Support directly at https://aerospike.com/support.`, PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { if rootFlags.logLevel.NotSet() { @@ -88,17 +88,18 @@ func Execute() { func init() { rootCmd.PersistentFlags().Var( &rootFlags.logLevel, - logLevelFlagName, + flags.LogLevel, common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", "))), //nolint:lll // For readability ) common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") - if err := viper.BindEnv(flagNameHost); err != nil { - logger.Error("failed to bind environment variable", slog.Any("error", err)) - } + bindEnvs := []string{flags.Host, flags.Seeds, flags.User, flags.Password} - if err := viper.BindEnv(flagNameSeeds); err != nil { - logger.Error("failed to bind environment variable", slog.Any("error", err)) + // Bind specified flags to ASVEC_* + for _, env := range bindEnvs { + if err := viper.BindEnv(env); err != nil { + panic(fmt.Sprintf("failed to bind environment variable: %s", err)) + } } } diff --git a/cmd/utils.go b/cmd/utils.go index 7e973d6..71a062b 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -2,10 +2,58 @@ package cmd import ( "asvec/cmd/flags" + "context" + "fmt" + "log/slog" + "os" + "strings" + "time" - avs "github.com/aerospike/aerospike-proximus-client-go" + "golang.org/x/term" + + avs "github.com/aerospike/avs-client-go" ) +func createClientFromFlags(clientFlags *flags.ClientFlags, connectTimeout time.Duration) (*avs.AdminClient, error) { + hosts, isLoadBalancer := parseBothHostSeedsFlag(clientFlags.Seeds, clientFlags.Host) + + ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) + defer cancel() + + tlsConfig, err := clientFlags.NewTLSConfig() + if err != nil { + logger.Error("failed to create TLS config", slog.Any("error", err)) + return nil, err + } + + var password *string + if clientFlags.User.Val != nil { + if len(clientFlags.Password) != 0 { + strPass := clientFlags.Password.String() + password = &strPass + } else { + fmt.Print("Enter Password: ") + bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + logger.Error("failed to read password", slog.Any("error", err)) + return nil, err + } + fmt.Println() // Print a newline after the password input + strPass := string(bytePassword) + password = &strPass + } + } + + adminClient, err := avs.NewAdminClient( + ctx, hosts, clientFlags.ListenerName.Val, isLoadBalancer, clientFlags.User.Val, password, tlsConfig, logger, + ) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return nil, err + } + + return adminClient, nil +} func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFlag) (avs.HostPortSlice, bool) { isLoadBalancer := false hosts := avs.HostPortSlice{} @@ -24,3 +72,26 @@ func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFla return hosts, isLoadBalancer } + +func nsAndSetString(namespace string, sets []string) string { + var setStr string + + if len(sets) == 0 { + setStr = "*" + } else if len(sets) == 1 { + setStr = sets[0] + } else { + setStr = fmt.Sprintf("%v", sets) + } + + return fmt.Sprintf("%s.%s", namespace, setStr) +} + +func confirm(prompt string) bool { + var confirm string + + fmt.Print(prompt + " (y/n): ") + fmt.Scanln(&confirm) + + return strings.ToLower(confirm) == "y" +} diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 53c8a84..ba633d6 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -6,7 +6,7 @@ import ( "asvec/cmd/flags" "testing" - avs "github.com/aerospike/aerospike-proximus-client-go" + avs "github.com/aerospike/avs-client-go" "github.com/stretchr/testify/assert" ) @@ -20,12 +20,12 @@ func TestParseBothHostSeedsFlag(t *testing.T) { { &flags.SeedsSliceFlag{ Seeds: avs.HostPortSlice{ - avs.NewHostPort("1.1.1.1", 5000, false), + avs.NewHostPort("1.1.1.1", 5000), }, }, flags.NewDefaultHostPortFlag(), avs.HostPortSlice{ - avs.NewHostPort("1.1.1.1", 5000, false), + avs.NewHostPort("1.1.1.1", 5000), }, false, }, diff --git a/cmd/view.go b/cmd/view.go index c3cbb14..c3ccc59 100644 --- a/cmd/view.go +++ b/cmd/view.go @@ -6,7 +6,7 @@ import ( "io" "log/slog" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" ) type View struct { diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index da07d78..b854a6c 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -4,7 +4,7 @@ import ( "io" "log/slog" - "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/avs-client-go/protos" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/docker/auth/config/aerospike-proximus.yml b/docker/auth/config/aerospike-proximus.yml new file mode 100644 index 0000000..fa7b8a1 --- /dev/null +++ b/docker/auth/config/aerospike-proximus.yml @@ -0,0 +1,93 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + mutual-auth: false + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +security: + auth-token: + private-key: /etc/aerospike-proximus/jwt/private_key.pem + public-key: /etc/aerospike-proximus/jwt/public_key.pem + token-expiry: 30_000 + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/auth/config/aerospike.conf b/docker/auth/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/auth/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/auth/config/jwt/private_key.pem b/docker/auth/config/jwt/private_key.pem new file mode 100644 index 0000000..70c57aa --- /dev/null +++ b/docker/auth/config/jwt/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDBU9XcqJzRuqTQ +d58/zotQN/vUFjS2AhgXIzw18RULCvXo/1mQD6Tej7Gd7Vsm3j/o4/jZrZ10d+iR +yrGSfMw9w7G+SD4UDKcp4Epu2sLme2AYbU/T4Pz2CgD8qkWLeDtfX/Gb5Eoglfiq +PU0cCfNAdATwQZzC5qZ9RCMNQb8WVtQ/fIhEJCltslBLHAdXqOsJnzPUUlFE4Abw +1/HosVvD2GOoOE0yZqrJvpCxH7zoL8TCTuMqhDyg3BfFLIzD0pePceLf30Om9oVU +v0vOhmy5wgjYnSzyHYaPY9WjAaFVTNCw4upGXveqON1qG12ZtNwUBsY6besNAamf +I6bHoLrrAgMBAAECggEADKzbSzXaM51Um/8TzQJU5k6yr0jrbiQnVzEDYW1WhLGp +xiZ5GqBj81xMIr9tUq2PVszHmTda/kgJqAa4QL33mB8XWBNIJVa1CQ0AZFHp0GRg +vF6XtXEStMDwJ/YJTD0hoL2wy23wIZXORr8b1KL0X2fxQPKpp/+pBR0tXSZP3yjM +dhYEKj7/p9qH5pP92ZQftzeWQqK0MOSs+fedUASi1I0ZXWL7Hcsl7e66ZrySevL2 +ad/xdKHShvT2dxlBHpkBkvfAOrssgqmY4d9CZj4qmOXgsQcqUPZiP5bZ0naM6Wdm +B/q/PwPKxWDM8VRybxDaOVesG2358qcB+za3Rk0EXQKBgQD+DXA7qg6J68KMjJud +4nHrPxzw287mJgeNgvdJsAukz4dPfA3Kb95+YLCnhcy4co2lWmJf8f12nJBCcF6p +Oqyw8jUGosaXTXM0ZBeSx8CwFmeYQASmxz5bNofROSIuDXriEYn+RBlRCrvAP9DX +Pbbd2HoR7RlHHdeFUDPGE6AOTQKBgQDCzzpjZ6lBUoVIQDDtoRD9gbmr03epdsCI +DBp4uBv1LuMIKxKy7s2UUiMNIxWKAQf4h9OwMsKzQztawRXi/Q0Q7TZ5P+b//PX7 +ldvO5tMITyj5ZP/LFSIpuuuQCZxxDI93SJWHpI1qk8wOeKTM/TwlcZV0mKxEpnOO +BIV1+go6FwKBgEt9jsYL6DoPdkXxWiR2L0ep/12K01YMIt5n9jdNOoiEqj5yLF7l +EwYWkSeWWJjOYUyrKOQ9sgVWzH8RiO9cYghKUHtiTwSSnRMKUv7ooFcysSHKlBdC +yUKtV3pSMdfhZYxbjdeiQKuWNugPjm0HB5mwQ8Wj4IkDUxFoHZpUJpFZAoGBALGG +OoUfj+PRGVgv9uZ1YpBByTtF58PsTaMstGrSC9gws/9bkRyx9XF2MC7mi9hI/ESF +MmlcuIS3fb1EO3ewdlm4cW2oyA77LAXqGHbBbgoiaQpr+ZXNWmaAye1mdMOoWIyw +/mKvl5dJvChr1HJhSpAso6+u+T1OGfuPLzfFddBdAoGAGdfggY1bR5RWDvJe/ovT +32DnnMDAkD/wQszNraWg+Jr71RoqtBdBo4mkwnLZRulheARJiF62z15CJXYqzmSX +2MxGajz4PW33m+CPwogspNzvkfIZnZyQKJEyYNRYz1J4LCJYKigi7pf9dL1geF5u +9Xq2O8tnSF5u/E5mEZrweFs= +-----END PRIVATE KEY----- diff --git a/docker/auth/config/jwt/public_key.pem b/docker/auth/config/jwt/public_key.pem new file mode 100644 index 0000000..e867ef6 --- /dev/null +++ b/docker/auth/config/jwt/public_key.pem @@ -0,0 +1,8 @@ +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEAwVPV3Kic0bqk0HefP86LUDf71BY0tgIYFyM8NfEVCwr16P9ZkA+k +3o+xne1bJt4/6OP42a2ddHfokcqxknzMPcOxvkg+FAynKeBKbtrC5ntgGG1P0+D8 +9goA/KpFi3g7X1/xm+RKIJX4qj1NHAnzQHQE8EGcwuamfUQjDUG/FlbUP3yIRCQp +bbJQSxwHV6jrCZ8z1FJRROAG8Nfx6LFbw9hjqDhNMmaqyb6QsR+86C/Ewk7jKoQ8 +oNwXxSyMw9KXj3Hi399DpvaFVL9LzoZsucII2J0s8h2Gj2PVowGhVUzQsOLqRl73 +qjjdahtdmbTcFAbGOm3rDQGpnyOmx6C66wIDAQAB +-----END RSA PUBLIC KEY----- diff --git a/docker/auth/config/tls/ca.aerospike.com.crt b/docker/auth/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/auth/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/auth/config/tls/ca.aerospike.com.truststore.jks b/docker/auth/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000..8d72ac6 Binary files /dev/null and b/docker/auth/config/tls/ca.aerospike.com.truststore.jks differ diff --git a/docker/auth/config/tls/keypass b/docker/auth/config/tls/keypass new file mode 100644 index 0000000..0f673cc --- /dev/null +++ b/docker/auth/config/tls/keypass @@ -0,0 +1 @@ +citrusstore \ No newline at end of file diff --git a/docker/auth/config/tls/localhost.keystore.jks b/docker/auth/config/tls/localhost.keystore.jks new file mode 100644 index 0000000..8062beb Binary files /dev/null and b/docker/auth/config/tls/localhost.keystore.jks differ diff --git a/docker/auth/config/tls/storepass b/docker/auth/config/tls/storepass new file mode 100644 index 0000000..b1f833e --- /dev/null +++ b/docker/auth/config/tls/storepass @@ -0,0 +1 @@ +citrusstore diff --git a/docker/auth/docker-compose.yml b/docker/auth/docker-compose.yml new file mode 100644 index 0000000..30a3af1 --- /dev/null +++ b/docker/auth/docker-compose.yml @@ -0,0 +1,23 @@ +services: + aerospike: + image: aerospike/aerospike-server-enterprise:7.0.0.2 + ports: + - "3000:3000" + networks: + - avs-demo + volumes: + - ./config:/opt/aerospike/etc/aerospike + command: + - "--config-file" + - "/opt/aerospike/etc/aerospike/aerospike.conf" + avs: + image: aerospike.jfrog.io/docker/aerospike/aerospike-proximus-private:0.5.0-SNAPSHOT + ports: + - "10000:10000" + networks: + - avs-demo + volumes: + - ./config:/etc/aerospike-proximus + +networks: + avs-demo: {} diff --git a/docker/mtls/config/aerospike-proximus.yml b/docker/mtls/config/aerospike-proximus.yml new file mode 100644 index 0000000..60473cf --- /dev/null +++ b/docker/mtls/config/aerospike-proximus.yml @@ -0,0 +1,87 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + mutual-auth: true + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/mtls/config/aerospike.conf b/docker/mtls/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/mtls/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/mtls/config/tls/ca.aerospike.com.crt b/docker/mtls/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/mtls/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/mtls/config/tls/ca.aerospike.com.truststore.jks b/docker/mtls/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000..8d72ac6 Binary files /dev/null and b/docker/mtls/config/tls/ca.aerospike.com.truststore.jks differ diff --git a/docker/mtls/config/tls/keypass b/docker/mtls/config/tls/keypass new file mode 100644 index 0000000..0f673cc --- /dev/null +++ b/docker/mtls/config/tls/keypass @@ -0,0 +1 @@ +citrusstore \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.crt b/docker/mtls/config/tls/localhost.crt new file mode 100644 index 0000000..e713708 --- /dev/null +++ b/docker/mtls/config/tls/localhost.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhzCCAm+gAwIBAgIUES99at1e+OgIcnfEwS6+uBLdLlkwDQYJKoZIhvcNAQEL +BQAwgYsxCzAJBgNVBAYTAklOMQswCQYDVQQIDAJLQTELMAkGA1UEBwwCQk4xEjAQ +BgNVBAoMCWFlcm9zcGlrZTESMBAGA1UECwwJZWNvc3lzdGVtMRkwFwYDVQQDDBBj +YS5hZXJvc3Bpa2UuY29tMR8wHQYJKoZIhvcNAQkBFhBjYUBhZXJvc3Bpa2UuY29t +MB4XDTE5MDcxNDE3NTI1M1oXDTI5MDcxMTE3NTI1M1owSDELMAkGA1UEBhMCSU4x +CzAJBgNVBAgMAktBMRgwFgYDVQQKDA9BZXJvc3Bpa2UsIEluYy4xEjAQBgNVBAMM +CWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANOOLzx9 +a8XnBLrTQn2RBJfUUi7qZZTBAEraQ0Tsh04wqEycwxsh4HfpqP4QDF8JpAs5mvMj +fotnamVCNncXJgvOIaPQVRoDdGAAfCcOvT9B0Y7xBH3jj+e9YBiOEfee7cb7mjnX +2XYa9UGepNIktHUR0RgsuNooxDcT8wIbt+QMK0ZqF5GBQtgX24646ow5VOzfAGZy +SZfc9J0mWIZwHc0Gkb+V15iJBzdwJ/15FrmHSiJqGIsesqThrdAh5SYZCG/1+RL8 +FuwtZuoKm3V4OoQzQ0TIz6fCaiLI1PgrKJnMiGa3PIPX5DyPEU+5opXJCClGW2iM +pGmDQ5W+GmZwickCAwEAAaMlMCMwIQYDVR0RBBowGIIJbG9jYWxob3N0ggsqLmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAdVCQWrRC3bCI6nQ1GuEfbEa7Fn2x +88EvMoJb7PLCLnumDd68AJ38qHUrmvrTbFFJemrCJ2cI0vGQPf5bMm+L2KyxVD9D +zSyy/mbKmAvJl05gv+Si0N9Ikhnqyj9NuF2hnMVJp7IowCBWU/JTfmXJFJU+bJDr +NntdjuUGHYq+zlTPowGS1p1Bx5QC+6YpelAcYiV7JiUXPQXd0SpJMcp0iYhJLjHv +qoM/jPl0gIHrn8WMrIKmf81Uv3mNd2nkKhiNuqqrHs5tVarYHfY2IJVnf8nMKlh9 +LVUXkze20oB2uc6IKxrb7nbXk0Q2XKb9xmpaFb+Fut3musY5sYrE9l+bkw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.key b/docker/mtls/config/tls/localhost.key new file mode 100644 index 0000000..23b929e --- /dev/null +++ b/docker/mtls/config/tls/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA044vPH1rxecEutNCfZEEl9RSLupllMEAStpDROyHTjCoTJzD +GyHgd+mo/hAMXwmkCzma8yN+i2dqZUI2dxcmC84ho9BVGgN0YAB8Jw69P0HRjvEE +feOP571gGI4R957txvuaOdfZdhr1QZ6k0iS0dRHRGCy42ijENxPzAhu35AwrRmoX +kYFC2BfbjrjqjDlU7N8AZnJJl9z0nSZYhnAdzQaRv5XXmIkHN3An/XkWuYdKImoY +ix6ypOGt0CHlJhkIb/X5EvwW7C1m6gqbdXg6hDNDRMjPp8JqIsjU+CsomcyIZrc8 +g9fkPI8RT7milckIKUZbaIykaYNDlb4aZnCJyQIDAQABAoIBAQCntYX43Cy93KBB +Qwzo4jfT7TuheaxBuqbysAi38RJqh+RDp9p7/eUm6pNPpYVJKiljxKzzpuXAuaD8 +2Pq4eh9tKGI+rP9p+ecd3ASQKf0Y0qLAQI0hB2+jdNtjW+0ecl1pazgeNuFr4X8g +IBXlibeNPyyVj46TU9IJH8V7nGGxZArm5hOWbLxfygVzulkeEsozjeDcVaUZPA1o +gN9QgqSC850jjqddV7a0sZuHyd3KECkfU1uRoSHikTKHJLxaV6BIHzliQijRutLT +hTFpk4QzM+7xixY66hh56YAoTB44A0sXHNrH5Zt3YkUv4mT/P9ue9DQ1mhz2Eh4h +DAzWFq4BAoGBAPOCOyar6K0mogxUl1Q1LBH34HvyrZ4uVN7ICeE12neh/fS04fnV +lM5e6bxKj2HEzqqJ8JUxjQypurz6aQBSI1o0hc13eZqykoNa7c9TREY8swM4Vwl9 +NOdyBuysJIqG9+tPp+vReP4ISyLG8fyeJNDl1OEaaz2GcxPGDbf7hR1hAoGBAN5o +V0tXp5wG/RnUZn0FU2g8faw5tfB4iO38igYJPEo5YbiC7FcJHYiPUlEfjlWmWwGT +RNggYK6Q7lQ4NKJU80DTG2k5AK5yLHtmtRmPPhYMZAt5txCXF7zpuoQS7OTKnc4l +R9pnsEYn+kYN/YA3EC56j/uSj6VEdB/AgHNvs51pAoGBAN4DlrqjcfisiIKFfZOh +BxU60skvcWwPAgI8kAVtfEomv8wkPwPx30Jo9uJdeGzDa0nBij/8dYVeGovCI4nP +WbwctwGmNJD+zuZEOR4V5OHE5dHBxFk6dsmuBPIz4P0MIW3BqnAvBAlYtmh2yppv +9VEguv6hf7UQqEsW/9sGz08BAoGBAKMnripCMl3+rnvdWhYK6yYDgjnu2C6BbgoQ +Afztl4Hn2G0v9krfEABXC48hdBwW/poIPC/EiMhm379+v/X6Fb0PYQNu4rYWYdVh +Aieu8l/gVSAp+Qa9oJdgawhqjchFb0CEDtMEz8aXmzz7FGWTf1ZpaOinmqMltX55 +jIGihwRJAoGATH9eCCCv1rbdnM8+qY3R8PZcZAOugICN55woXHJBCRRT+CpY4kiG +LNXyLZILWs8LXbzkKtynCXmVHMGbehtY/PnGXoSpsgLLz15CuGrbyQLoLnTr8NTB +atV5a96f8fOPhFl9EIRzV8u3DIF4CoYvNAFws7VwPDw01FCw5MncX2g= +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/docker/mtls/config/tls/localhost.keystore.jks b/docker/mtls/config/tls/localhost.keystore.jks new file mode 100644 index 0000000..8062beb Binary files /dev/null and b/docker/mtls/config/tls/localhost.keystore.jks differ diff --git a/docker/mtls/config/tls/storepass b/docker/mtls/config/tls/storepass new file mode 100644 index 0000000..b1f833e --- /dev/null +++ b/docker/mtls/config/tls/storepass @@ -0,0 +1 @@ +citrusstore diff --git a/docker/mtls/docker-compose.yml b/docker/mtls/docker-compose.yml new file mode 100644 index 0000000..30a3af1 --- /dev/null +++ b/docker/mtls/docker-compose.yml @@ -0,0 +1,23 @@ +services: + aerospike: + image: aerospike/aerospike-server-enterprise:7.0.0.2 + ports: + - "3000:3000" + networks: + - avs-demo + volumes: + - ./config:/opt/aerospike/etc/aerospike + command: + - "--config-file" + - "/opt/aerospike/etc/aerospike/aerospike.conf" + avs: + image: aerospike.jfrog.io/docker/aerospike/aerospike-proximus-private:0.5.0-SNAPSHOT + ports: + - "10000:10000" + networks: + - avs-demo + volumes: + - ./config:/etc/aerospike-proximus + +networks: + avs-demo: {} diff --git a/docker/tls/config/aerospike-proximus.yml b/docker/tls/config/aerospike-proximus.yml new file mode 100644 index 0000000..aa58cb6 --- /dev/null +++ b/docker/tls/config/aerospike-proximus.yml @@ -0,0 +1,87 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +tls: + service-tls: + mutual-auth: false + trust-store: + store-file: /etc/aerospike-proximus/tls/ca.aerospike.com.truststore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-store: + store-file: /etc/aerospike-proximus/tls/localhost.keystore.jks + store-password-file: /etc/aerospike-proximus/tls/storepass + key-password-file: /etc/aerospike-proximus/tls/keypass + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: + # If TLS needs to be enabled, tls configuration id. + tls-id: service-tls + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: + tls-id: service-tls + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/tls/config/aerospike.conf b/docker/tls/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/tls/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/tls/config/tls/ca.aerospike.com.crt b/docker/tls/config/tls/ca.aerospike.com.crt new file mode 100644 index 0000000..522410a --- /dev/null +++ b/docker/tls/config/tls/ca.aerospike.com.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIJALiEh0EwIowCMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD +VQQGEwJJTjELMAkGA1UECAwCS0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJv +c3Bpa2UxEjAQBgNVBAsMCWVjb3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtl +LmNvbTEfMB0GCSqGSIb3DQEJARYQY2FAYWVyb3NwaWtlLmNvbTAeFw0xOTA3MDgw +OTA2NTZaFw0zOTA3MDMwOTA2NTZaMIGLMQswCQYDVQQGEwJJTjELMAkGA1UECAwC +S0ExCzAJBgNVBAcMAkJOMRIwEAYDVQQKDAlhZXJvc3Bpa2UxEjAQBgNVBAsMCWVj +b3N5c3RlbTEZMBcGA1UEAwwQY2EuYWVyb3NwaWtlLmNvbTEfMB0GCSqGSIb3DQEJ +ARYQY2FAYWVyb3NwaWtlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAOkpqtQIiWhUouKqsyoHs7+mXL4fahOKZBG1asUaNpY/rhR500OpczWfHXK1 ++W5WA2yizhIFFpgzNJOtXW2Sai0Dqk5MO7hPLoKzlA/pSYnVFyM34kECWiqFo9PZ +6FQxOBfRJjE4sLuLJGh+Pr/bii8Cb8GwuckUGFQaJLv9VZXcsGkpvyOEy9CsRI7o +7wPn0VVNxj6bbag0AUCe5s1ZKsNxFh3Ekqbx+pFA/7KxxVSmOCVhX+W87K4qJz0+ +XfGtBJjM4YqGun3ul4JCzcPHoqA5vKAB00LHa7bNY/5Zf2iGcfPLEmyk09PyqAQF +KTj8D4OTiP85L+wo7jxY6U/Xz1cCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFEz8sA/FRlupRxPfrDrnFZwN4g+HMB8G +A1UdIwQYMBaAFEz8sA/FRlupRxPfrDrnFZwN4g+HMA0GCSqGSIb3DQEBCwUAA4IB +AQDTDCjMVX11S9GZGNExnqtGVzOEAlKyWnx0g1pULTffN3vc8DG1gynY6n7/9vPI +V2pheuyd/aKoR0Ig8CvjOnj90DcMePwh3Zk6eG7SlUK41x4yrkw04VEqvyw02Dw7 +SPZRgEs5/AHVLscOaDeJxW6Nzm5XYS5mfhto5nZCBEq5u5FfsktwYisIlK9JLbYE +ATjQbkwoNeg2Ubdtddn9HgnCEV0ht0VE2bZc0OUmv29R5XTNEIEIf/bXjdgnbv57 +IhJElLyHziPbD08JgkqqQw+6zAbxO9OLury69eUQoC0nynVX+Ub9GoXSuQ2lGyyq +ouSZQ0M5aZVQvsqUyREGBsAF +-----END CERTIFICATE----- diff --git a/docker/tls/config/tls/ca.aerospike.com.truststore.jks b/docker/tls/config/tls/ca.aerospike.com.truststore.jks new file mode 100644 index 0000000..8d72ac6 Binary files /dev/null and b/docker/tls/config/tls/ca.aerospike.com.truststore.jks differ diff --git a/docker/tls/config/tls/keypass b/docker/tls/config/tls/keypass new file mode 100644 index 0000000..0f673cc --- /dev/null +++ b/docker/tls/config/tls/keypass @@ -0,0 +1 @@ +citrusstore \ No newline at end of file diff --git a/docker/tls/config/tls/localhost.keystore.jks b/docker/tls/config/tls/localhost.keystore.jks new file mode 100644 index 0000000..8062beb Binary files /dev/null and b/docker/tls/config/tls/localhost.keystore.jks differ diff --git a/docker/tls/config/tls/storepass b/docker/tls/config/tls/storepass new file mode 100644 index 0000000..b1f833e --- /dev/null +++ b/docker/tls/config/tls/storepass @@ -0,0 +1 @@ +citrusstore diff --git a/docker/tls/docker-compose.yml b/docker/tls/docker-compose.yml new file mode 100644 index 0000000..30a3af1 --- /dev/null +++ b/docker/tls/docker-compose.yml @@ -0,0 +1,23 @@ +services: + aerospike: + image: aerospike/aerospike-server-enterprise:7.0.0.2 + ports: + - "3000:3000" + networks: + - avs-demo + volumes: + - ./config:/opt/aerospike/etc/aerospike + command: + - "--config-file" + - "/opt/aerospike/etc/aerospike/aerospike.conf" + avs: + image: aerospike.jfrog.io/docker/aerospike/aerospike-proximus-private:0.5.0-SNAPSHOT + ports: + - "10000:10000" + networks: + - avs-demo + volumes: + - ./config:/etc/aerospike-proximus + +networks: + avs-demo: {} diff --git a/e2e_test.go b/e2e_test.go index cf8a23b..3b636e2 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -1,10 +1,14 @@ //go:build integration -package main_test +package main import ( + "asvec/cmd/flags" "context" + "crypto/tls" + "crypto/x509" "fmt" + "log" "log/slog" "os" "os/exec" @@ -14,8 +18,9 @@ import ( "testing" "time" - avs "github.com/aerospike/aerospike-proximus-client-go" - "github.com/aerospike/aerospike-proximus-client-go/protos" + avs "github.com/aerospike/avs-client-go" + "github.com/aerospike/avs-client-go/protos" + "github.com/aerospike/tools-common-go/client" "github.com/stretchr/testify/suite" ) @@ -28,31 +33,122 @@ var ( barNamespace = "bar" ) +func GetCACert(cert string) (*x509.CertPool, error) { + // read in file + certBytes, err := os.ReadFile(cert) + if err != nil { + log.Fatalf("unable to read cert file %v", err) + return nil, err + } + + return client.LoadCACerts([][]byte{certBytes}), nil +} + +func GetCertificates(certFile string, keyFile string) ([]tls.Certificate, error) { + cert, err := os.ReadFile(certFile) + if err != nil { + log.Fatalf("unable to read cert file %v", err) + return nil, err + } + + key, err := os.ReadFile(keyFile) + if err != nil { + log.Fatalf("unable to read key file %v", err) + return nil, err + } + + return client.LoadServerCertAndKey([]byte(cert), []byte(key), nil) +} + type CmdTestSuite struct { suite.Suite - app string - coverFile string - coverFileCounter int - avsIP string - avsPort int - avsHostPort *avs.HostPort - avsClient *avs.AdminClient + app string + composeFile string + suiteName string + suiteFlags []string + coverFile string + avsIP string + avsPort int + avsHostPort *avs.HostPort + avsTLSConfig *tls.Config + avsUser *string + avsPassword *string + avsClient *avs.AdminClient } -func TestDistanceMetricFlagSuite(t *testing.T) { - suite.Run(t, new(CmdTestSuite)) +func TestCmdSuite(t *testing.T) { + logger = logger.With(slog.Bool("test-logger", true)) // makes it easy to see which logger is which + rootCA, err := GetCACert("docker/tls/config/tls/ca.aerospike.com.crt") + if err != nil { + t.Fatalf("unable to read root ca %v", err) + t.FailNow() + logger.Error("Failed to read cert") + } + + certificates, err := GetCertificates("docker/mtls/config/tls/localhost.crt", "docker/mtls/config/tls/localhost.key") + if err != nil { + t.Fatalf("unable to read certificates %v", err) + t.FailNow() + logger.Error("Failed to read cert") + } + + logger.Info("%v", slog.Any("cert", rootCA)) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/docker-compose.yml", // vanilla + suiteFlags: []string{"--log-level debug"}, + avsIP: "localhost", + }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/tls/docker-compose.yml", // tls + suiteFlags: []string{ + "--log-level debug", + createFlagStr(flags.TLSCaFile, "docker/tls/config/tls/ca.aerospike.com.crt"), + }, + avsTLSConfig: &tls.Config{ + Certificates: nil, + RootCAs: rootCA, + }, + avsIP: "localhost", + }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/mtls/docker-compose.yml", // mutual tls + suiteFlags: []string{ + "--log-level debug", + createFlagStr(flags.TLSCaFile, "docker/mtls/config/tls/ca.aerospike.com.crt"), + createFlagStr(flags.TLSCertFile, "docker/mtls/config/tls/localhost.crt"), + createFlagStr(flags.TLSKeyFile, "docker/mtls/config/tls/localhost.key"), + }, + avsTLSConfig: &tls.Config{ + Certificates: certificates, + RootCAs: rootCA, + }, + avsIP: "localhost", + }) + suite.Run(t, &CmdTestSuite{ + composeFile: "docker/auth/docker-compose.yml", // tls + auth (auth requires tls) + suiteFlags: []string{ + "--log-level debug", + createFlagStr(flags.TLSCaFile, "docker/auth/config/tls/ca.aerospike.com.crt"), + createFlagStr(flags.User, "admin"), + createFlagStr(flags.Password, "admin"), + }, + avsUser: getStrPtr("admin"), + avsPassword: getStrPtr("admin"), + avsTLSConfig: &tls.Config{ + Certificates: nil, + RootCAs: rootCA, + }, + avsIP: "localhost", + }) } func (suite *CmdTestSuite) SetupSuite() { suite.app = path.Join(wd, "app.test") suite.coverFile = path.Join(wd, "../coverage/cmd-coverage.cov") - suite.coverFileCounter = 0 - suite.avsIP = "127.0.0.1" suite.avsPort = 10000 - suite.avsHostPort = avs.NewHostPort(suite.avsIP, suite.avsPort, false) - // var err error + suite.avsHostPort = avs.NewHostPort(suite.avsIP, suite.avsPort) - err := docker_compose_up() + err := docker_compose_up(suite.composeFile) if err != nil { suite.FailNowf("unable to start docker compose up", "%v", err) } @@ -74,7 +170,16 @@ func (suite *CmdTestSuite) SetupSuite() { defer cancel() for { - suite.avsClient, err = avs.NewAdminClient(ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, logger) + suite.avsClient, err = avs.NewAdminClient( + ctx, + avs.HostPortSlice{suite.avsHostPort}, + nil, + true, + suite.avsUser, + suite.avsPassword, + suite.avsTLSConfig, + logger, + ) if err != nil { fmt.Printf("unable to create avs client %v", err) @@ -98,17 +203,19 @@ func (suite *CmdTestSuite) TearDownSuite() { suite.Assert().NoError(err) suite.avsClient.Close() - err = docker_compose_down() + err = docker_compose_down(suite.composeFile) if err != nil { fmt.Println("unable to stop docker compose down") } } func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { + suiteFlags := strings.Split(strings.Join(suite.suiteFlags, " "), " ") + asvecCmd = append(suiteFlags, asvecCmd...) + logger.Info("running command", slog.String("cmd", strings.Join(asvecCmd, " "))) cmd := exec.Command(suite.app, asvecCmd...) cmd.Env = []string{"GOCOVERDIR=" + os.Getenv("COVERAGE_DIR")} stdout, err := cmd.Output() - // fmt.Printf("stdout: %v", string(stdout)) if err != nil { if ee, ok := err.(*exec.ExitError); ok { @@ -122,161 +229,6 @@ func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { return lines, nil } -func getStrPtr(str string) *string { - ptr := str - return &ptr -} - -func getUint32Ptr(i int) *uint32 { - ptr := uint32(i) - return &ptr -} - -func getBoolPtr(b bool) *bool { - ptr := b - return &ptr -} - -type IndexDefinitionBuilder struct { - indexName string - namespace string - set *string - dimension int - vectorDistanceMetric protos.VectorDistanceMetric - vectorField string - storageNamespace *string - storageSet *string - hnsfM *uint32 - hnsfEfC *uint32 - hnsfEf *uint32 - hnsfBatchingMaxRecord *uint32 - hnsfBatchingInterval *uint32 - hnsfBatchingDisabled *bool -} - -func NewIndexDefinitionBuilder( - indexName, - namespace string, - dimension int, - distanceMetric protos.VectorDistanceMetric, - vectorField string, -) *IndexDefinitionBuilder { - return &IndexDefinitionBuilder{ - indexName: indexName, - namespace: namespace, - dimension: dimension, - vectorDistanceMetric: distanceMetric, - vectorField: vectorField, - } -} - -func (idb *IndexDefinitionBuilder) WithSet(set string) *IndexDefinitionBuilder { - idb.set = &set - return idb -} - -func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { - idb.storageNamespace = &storageNamespace - return idb -} - -func (idb *IndexDefinitionBuilder) WithStorageSet(storageSet string) *IndexDefinitionBuilder { - idb.storageSet = &storageSet - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswM(m uint32) *IndexDefinitionBuilder { - idb.hnsfM = &m - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswEf(ef uint32) *IndexDefinitionBuilder { - idb.hnsfEf = &ef - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswEfConstruction(efConstruction uint32) *IndexDefinitionBuilder { - idb.hnsfEfC = &efConstruction - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingMaxRecord(maxRecord uint32) *IndexDefinitionBuilder { - idb.hnsfBatchingMaxRecord = &maxRecord - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingInterval(interval uint32) *IndexDefinitionBuilder { - idb.hnsfBatchingInterval = &interval - return idb -} - -func (idb *IndexDefinitionBuilder) WithHnswBatchingDisabled(disabled bool) *IndexDefinitionBuilder { - idb.hnsfBatchingDisabled = &disabled - return idb -} - -func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { - indexDef := &protos.IndexDefinition{ - Id: &protos.IndexId{ - Name: idb.indexName, - Namespace: idb.namespace, - }, - Dimensions: uint32(idb.dimension), - VectorDistanceMetric: idb.vectorDistanceMetric, - Field: idb.vectorField, - Type: protos.IndexType_HNSW, - Storage: &protos.IndexStorage{ - Namespace: &idb.namespace, - Set: &idb.indexName, - }, - Params: &protos.IndexDefinition_HnswParams{ - HnswParams: &protos.HnswParams{ - M: getUint32Ptr(16), - EfConstruction: getUint32Ptr(100), - Ef: getUint32Ptr(100), - BatchingParams: &protos.HnswBatchingParams{ - MaxRecords: getUint32Ptr(100000), - Interval: getUint32Ptr(30000), - Disabled: getBoolPtr(false), - }, - }, - }, - } - - if idb.set != nil { - indexDef.SetFilter = idb.set - } - - if idb.storageNamespace != nil { - indexDef.Storage.Namespace = idb.storageNamespace - } - - if idb.storageSet != nil { - indexDef.Storage.Set = idb.storageSet - } - - if idb.hnsfM != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.M = idb.hnsfM - } - if idb.hnsfEf != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.Ef = idb.hnsfEf - } - if idb.hnsfEfC != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC - } - if idb.hnsfBatchingMaxRecord != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord - } - if idb.hnsfBatchingInterval != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Interval = idb.hnsfBatchingInterval - } - if idb.hnsfBatchingDisabled != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Disabled = idb.hnsfBatchingDisabled - } - - return indexDef -} - func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { testCases := []struct { name string @@ -289,14 +241,14 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with storage config", "index1", "test", - fmt.Sprintf("create index --seeds %s -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), + fmt.Sprintf("create index --host %s -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). WithStorageNamespace("bar"). WithStorageSet("testbar"). Build(), }, { - "test with hnsw params", + "test with hnsw params and seeds", "index2", "test", fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-max-edges 10 --hnsw-ef 11 --hnsw-ef-construction 12", suite.avsHostPort.String()), @@ -310,7 +262,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw batch params", "index3", "test", - fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-enabled false --hnsw-batch-interval 50 --hnsw-batch-max-records 100", suite.avsHostPort.String()), + fmt.Sprintf("create index --timeout 10s --host %s -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-enabled false --hnsw-batch-interval 50 --hnsw-batch-max-records 100", suite.avsHostPort.String()), NewIndexDefinitionBuilder("index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3"). WithHnswBatchingMaxRecord(100). WithHnswBatchingInterval(50). @@ -322,7 +274,11 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { for _, tc := range testCases { suite.Run(tc.name, func() { lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) - suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + + if err != nil { + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + suite.FailNow("unable to create index") + } actual, err := suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) @@ -336,10 +292,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { } func (suite *CmdTestSuite) TestCreateIndexFailsAlreadyExistsCmd() { - lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --host %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) suite.Assert().NoError(err, "index should have NOT existed on first call. error: %s, stdout/err: %s", err, lines) - lines, err = suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + lines, err = suite.runCmd(strings.Split(fmt.Sprintf("create index --host %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) suite.Assert().Error(err, "index should HAVE existed on first call. error: %s, stdout/err: %s", err, lines) suite.Assert().Contains(lines[0], "AlreadyExists") @@ -354,7 +310,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { cmd string }{ { - "test with just namespace", + "test with just namespace and seeds", "indexdrop1", "test", nil, @@ -367,7 +323,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { []string{ "testset", }, - fmt.Sprintf("drop index --seeds %s -n test -s testset -i indexdrop2 --timeout 10s", suite.avsHostPort.String()), + fmt.Sprintf("drop index --host %s -n test -s testset -i indexdrop2 --timeout 10s", suite.avsHostPort.String()), }, } @@ -378,12 +334,18 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { suite.FailNowf("unable to create index", "%v", err) } + time.Sleep(time.Second * 3) + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + if err != nil { + suite.FailNow("unable to drop index") + } + _, err = suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) - time.Sleep(time.Second) + time.Sleep(time.Second * 3) if err == nil { suite.FailNow("err is nil, that means the index still exists") @@ -624,13 +586,12 @@ func (suite *CmdTestSuite) TestFailInvalidArg() { } } -func docker_compose_up() error { +func docker_compose_up(composeFile string) error { fmt.Println("Starting docker containers") ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - // docker/docker-compose.yml - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "up", "-d") + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f%s", composeFile), "up", "-d") output, err := cmd.CombinedOutput() fmt.Printf("docker compose up output: %s\n", string(output)) @@ -645,11 +606,11 @@ func docker_compose_up() error { return nil } -func docker_compose_down() error { +func docker_compose_down(composeFile string) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "down") + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f%s", composeFile), "down") _, err := cmd.Output() if err != nil { diff --git a/go.mod b/go.mod index 5f42fff..e39cfcb 100644 --- a/go.mod +++ b/go.mod @@ -2,17 +2,15 @@ module asvec go 1.21.7 -//replace github.com/aerospike/aerospike-proximus-client-go => github.com/aerospike/aerospike-proximus-client-go VEC-155-admin-client -//replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go - require ( - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 - github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 + github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05 + github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/term v0.21.0 ) require ( diff --git a/go.sum b/go.sum index a31c1ab..3896d72 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= -github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= -github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= +github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05 h1:Egu9UMm2SIRP871wuwp5PoqWKjPrc9XNDMg/4REOuaQ= +github.com/aerospike/avs-client-go v0.0.0-20240702180233-787c93a41b05/go.mod h1:brxlQNig1y4tU7EP/jX+kXVgmwLERvtP/xhRoNw8Ark= +github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 h1:tOLqcGsc6A656WNEZhYZYxDiX7d6wCkEN6+jLDSDeUU= +github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -123,6 +123,8 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= diff --git a/test_utils.go b/test_utils.go new file mode 100644 index 0000000..9628bc8 --- /dev/null +++ b/test_utils.go @@ -0,0 +1,168 @@ +//go:build integration + +package main + +import ( + "fmt" + + "github.com/aerospike/avs-client-go/protos" +) + +func getStrPtr(str string) *string { + ptr := str + return &ptr +} + +func getUint32Ptr(i int) *uint32 { + ptr := uint32(i) + return &ptr +} + +func getBoolPtr(b bool) *bool { + ptr := b + return &ptr +} + +func createFlagStr(name, value string) string { + return fmt.Sprintf("--%s %s", name, value) +} + +type IndexDefinitionBuilder struct { + indexName string + namespace string + set *string + dimension int + vectorDistanceMetric protos.VectorDistanceMetric + vectorField string + storageNamespace *string + storageSet *string + hnsfM *uint32 + hnsfEfC *uint32 + hnsfEf *uint32 + hnsfBatchingMaxRecord *uint32 + hnsfBatchingInterval *uint32 + hnsfBatchingDisabled *bool +} + +func NewIndexDefinitionBuilder( + indexName, + namespace string, + dimension int, + distanceMetric protos.VectorDistanceMetric, + vectorField string, +) *IndexDefinitionBuilder { + return &IndexDefinitionBuilder{ + indexName: indexName, + namespace: namespace, + dimension: dimension, + vectorDistanceMetric: distanceMetric, + vectorField: vectorField, + } +} + +func (idb *IndexDefinitionBuilder) WithSet(set string) *IndexDefinitionBuilder { + idb.set = &set + return idb +} + +func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { + idb.storageNamespace = &storageNamespace + return idb +} + +func (idb *IndexDefinitionBuilder) WithStorageSet(storageSet string) *IndexDefinitionBuilder { + idb.storageSet = &storageSet + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswM(m uint32) *IndexDefinitionBuilder { + idb.hnsfM = &m + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEf(ef uint32) *IndexDefinitionBuilder { + idb.hnsfEf = &ef + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEfConstruction(efConstruction uint32) *IndexDefinitionBuilder { + idb.hnsfEfC = &efConstruction + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingMaxRecord(maxRecord uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingMaxRecord = &maxRecord + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingInterval(interval uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingInterval = &interval + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingDisabled(disabled bool) *IndexDefinitionBuilder { + idb.hnsfBatchingDisabled = &disabled + return idb +} + +func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { + indexDef := &protos.IndexDefinition{ + Id: &protos.IndexId{ + Name: idb.indexName, + Namespace: idb.namespace, + }, + Dimensions: uint32(idb.dimension), + VectorDistanceMetric: idb.vectorDistanceMetric, + Field: idb.vectorField, + Type: protos.IndexType_HNSW, + Storage: &protos.IndexStorage{ + Namespace: &idb.namespace, + Set: &idb.indexName, + }, + Params: &protos.IndexDefinition_HnswParams{ + HnswParams: &protos.HnswParams{ + M: getUint32Ptr(16), + EfConstruction: getUint32Ptr(100), + Ef: getUint32Ptr(100), + BatchingParams: &protos.HnswBatchingParams{ + MaxRecords: getUint32Ptr(100000), + Interval: getUint32Ptr(30000), + Disabled: getBoolPtr(false), + }, + }, + }, + } + + if idb.set != nil { + indexDef.SetFilter = idb.set + } + + if idb.storageNamespace != nil { + indexDef.Storage.Namespace = idb.storageNamespace + } + + if idb.storageSet != nil { + indexDef.Storage.Set = idb.storageSet + } + + if idb.hnsfM != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.M = idb.hnsfM + } + if idb.hnsfEf != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.Ef = idb.hnsfEf + } + if idb.hnsfEfC != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC + } + if idb.hnsfBatchingMaxRecord != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord + } + if idb.hnsfBatchingInterval != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Interval = idb.hnsfBatchingInterval + } + if idb.hnsfBatchingDisabled != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Disabled = idb.hnsfBatchingDisabled + } + + return indexDef +}