From 0d60aed678952c6fd84f7c13e4f1dd4928eda8c8 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 12:25:26 +0530 Subject: [PATCH 01/16] feat: config rebase --- config/config.go | 475 ++++++------------ config/parser.go | 226 +++++++++ config/validator.go | 97 ++++ dice.toml | 53 -- dicedb.conf | 54 ++ go.mod | 30 +- go.sum | 62 +-- .../commands/async/command_default_test.go | 4 +- integration_tests/commands/http/setup.go | 4 +- .../commands/resp/abort/server_abort_test.go | 6 + integration_tests/commands/resp/dump_test.go | 3 +- .../commands/websocket/json_test.go | 2 + integration_tests/config/config_test.go | 260 +++++----- integration_tests/server/server_abort_test.go | 5 + internal/cli/cli.go | 203 ++++++++ internal/eval/eval_amd64.go | 2 +- internal/server/httpServer.go | 2 +- internal/server/resp/server.go | 2 +- main.go | 2 - 19 files changed, 926 insertions(+), 566 deletions(-) create mode 100644 config/parser.go create mode 100644 config/validator.go delete mode 100644 dice.toml create mode 100644 dicedb.conf create mode 100644 internal/cli/cli.go diff --git a/config/config.go b/config/config.go index e23ad1f71..a9d5e882e 100644 --- a/config/config.go +++ b/config/config.go @@ -1,36 +1,28 @@ package config import ( - "errors" + "log" "log/slog" "os" "path/filepath" - "runtime" - "strings" "time" "github.com/dicedb/dice/internal/server/utils" - "github.com/pelletier/go-toml/v2" - "github.com/spf13/viper" ) const ( - DiceDBVersion string = "0.0.5" + DiceDBVersion string = "0.0.5" + DefaultConfigName string = "dicedb.conf" - DefaultHost string = "0.0.0.0" - DefaultPort int = 7379 - DefaultConfigName string = "dice.toml" - DefaultConfigFilePath string = "./" - - EvictBatchKeysLRU string = "batch_keys_lru" - - DefaultKeysLimit int = 200000000 - DefaultEvictionRatio float64 = 0.1 + EvictSimpleFirst string = "simple-first" + EvictAllKeysRandom string = "allkeys-random" + EvictAllKeysLRU string = "allkeys-lru" + EvictAllKeysLFU string = "allkeys-lfu" ) var ( - Host = DefaultHost - Port = DefaultPort + Host = "0.0.0.0" + Port = 7379 EnableMultiThreading = false EnableHTTP = true @@ -48,8 +40,7 @@ var ( InitConfigCmd = false - KeysLimit = DefaultKeysLimit - EvictionRatio = DefaultEvictionRatio + KeysLimit = 200000000 EnableProfiling = false @@ -62,213 +53,96 @@ var ( ) type Config struct { - Version string `mapstructure:"version"` - InstanceID string `mapstructure:"instance_id"` - AsyncServer struct { - Addr string `mapstructure:"addr"` - Port int `mapstructure:"port"` - KeepAlive int32 `mapstructure:"keepalive"` - Timeout int32 `mapstructure:"timeout"` - MaxConn int32 `mapstructure:"max-conn"` - } `mapstructure:"asyncserver"` - - HTTP struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - } `mapstructure:"http"` - - WebSocket struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` - WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` - } `mapstructure:"websocket"` - - Performance struct { - WatchChanBufSize int `mapstructure:"watchchanbufsize"` - ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` - MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` - MaxClients int32 `mapstructure:"maxclients"` - EnableMultiThreading bool `mapstructure:"enablemultithreading"` - StoreMapInitSize int `mapstructure:"storemapinitsize"` - AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` - EnableProfiling bool `mapstructure:"profiling"` - } `mapstructure:"performance"` - - Memory struct { - MaxMemory int64 `mapstructure:"maxmemory"` - EvictionStrategy string `mapstructure:"evictionstrategy"` - EvictionRatio float64 `mapstructure:"evictionratio"` - KeysLimit int `mapstructure:"keyslimit"` - LFULogFactor int `mapstructure:"lfulogfactor"` - } `mapstructure:"memory"` - - Persistence struct { - AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` - WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` - } `mapstructure:"persistence"` - - Logging struct { - LogLevel string `mapstructure:"loglevel"` - PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` - } `mapstructure:"logging"` - - Auth struct { - UserName string `mapstructure:"username"` - Password string `mapstructure:"password"` - } `mapstructure:"auth"` - - Network struct { - IOBufferLength int `mapstructure:"iobufferlength"` - IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` - } `mapstructure:"network"` - - NumShards int `mapstructure:"num_shards"` + Version string `config:"version" default:"0.0.5"` + InstanceID string `config:"instance_id"` + Auth auth `config:"auth"` + AsyncServer asyncServer `config:"async_server"` + HTTP http `config:"http"` + WebSocket websocket `config:"websocket"` + Performance performance `config:"performance"` + Memory memory `config:"memory"` + Persistence persistence `config:"persistence"` + Logging logging `config:"logging"` + Network network `config:"network"` + NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` } -// Default configurations for internal use -var baseConfig = Config{ - Version: DiceDBVersion, - AsyncServer: struct { - Addr string `mapstructure:"addr"` - Port int `mapstructure:"port"` - KeepAlive int32 `mapstructure:"keepalive"` - Timeout int32 `mapstructure:"timeout"` - MaxConn int32 `mapstructure:"max-conn"` - }{ - Addr: DefaultHost, - Port: DefaultPort, - KeepAlive: int32(300), - Timeout: int32(300), - MaxConn: int32(0), - }, - HTTP: struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - }{ - Enabled: EnableHTTP, - Port: HTTPPort, - }, - WebSocket: struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` - WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` - }{ - Enabled: EnableWebsocket, - Port: WebsocketPort, - MaxWriteResponseRetries: 3, - WriteResponseTimeout: 10 * time.Second, - }, - Performance: struct { - WatchChanBufSize int `mapstructure:"watchchanbufsize"` - ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` - MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` - MaxClients int32 `mapstructure:"maxclients"` - EnableMultiThreading bool `mapstructure:"enablemultithreading"` - StoreMapInitSize int `mapstructure:"storemapinitsize"` - AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` - EnableProfiling bool `mapstructure:"profiling"` - }{ - WatchChanBufSize: 20000, - ShardCronFrequency: 1 * time.Second, - MultiplexerPollTimeout: 100 * time.Millisecond, - MaxClients: int32(20000), - EnableMultiThreading: false, - StoreMapInitSize: 1024000, - AdhocReqChanBufSize: 20, // assuming we wouldn't have more than 20 adhoc requests being sent at a time. - }, - Memory: struct { - MaxMemory int64 `mapstructure:"maxmemory"` - EvictionStrategy string `mapstructure:"evictionstrategy"` - EvictionRatio float64 `mapstructure:"evictionratio"` - KeysLimit int `mapstructure:"keyslimit"` - LFULogFactor int `mapstructure:"lfulogfactor"` - }{ - MaxMemory: 0, - EvictionStrategy: EvictBatchKeysLRU, - EvictionRatio: DefaultEvictionRatio, - KeysLimit: DefaultKeysLimit, - LFULogFactor: 10, - }, - Persistence: struct { - AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` - WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` - }{ - PersistenceEnabled: true, - AOFFile: "./dice-master.aof", - WriteAOFOnCleanup: false, - }, - Logging: struct { - LogLevel string `mapstructure:"loglevel"` - PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` - }{ - LogLevel: "info", - PrettyPrintLogs: true, - }, - Auth: struct { - UserName string `mapstructure:"username"` - Password string `mapstructure:"password"` - }{ - UserName: "dice", - Password: RequirePass, - }, - Network: struct { - IOBufferLength int `mapstructure:"iobufferlength"` - IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` - }{ - IOBufferLength: 512, - IOBufferLengthMAX: 50 * 1024, - }, +type auth struct { + UserName string `config:"username" default:"dice"` + Password string `validate:"min=8"` } -var defaultConfig Config +type asyncServer struct { + Addr string `config:"addr" default:"0.0.0.0"` + Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` + KeepAlive int32 `config:"keepalive" default:"300"` + Timeout int32 `config:"timeout" default:"300"` + MaxConn int32 `config:"max_conn" default:"0"` +} -func init() { - config := baseConfig - config.Logging.PrettyPrintLogs = false - config.Logging.LogLevel = "info" - defaultConfig = config +type http struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` } -// DiceConfig is the global configuration object for dice -var DiceConfig = &defaultConfig +type websocket struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` + MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` + WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` +} -func SetupConfig() { - if InitConfigCmd { - FileLocation = getConfigPath() - createConfigFile(FileLocation) - return - } +type performance struct { + WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` + ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` + MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` + MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` + EnableMultiThreading bool `config:"enable_multithreading" default:"false"` + StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` + AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` + EnableProfiling bool `config:"profiling" default:"false"` + EnableWatch bool `config:"enable_watch" default:"false"` +} - // Check if both -o and -c flags are set - if areBothFlagsSet() { - slog.Error("Both -o and -c flags are set. Please use only one flag.") - return - } +type memory struct { + MaxMemory int64 `config:"max_memory" default:"0"` + EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` + EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` + KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` + LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` +} - // Check if -o flag is set - if CustomConfigFilePath != utils.EmptyStr && isValidDirPath() { - createConfigFile(filepath.Join(CustomConfigFilePath, DefaultConfigName)) - return - } +type persistence struct { + AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` + PersistenceEnabled bool `config:"persistence_enabled" default:"true"` + WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` +} - // Check if -c flag is set - if FileLocation != utils.EmptyStr || isConfigFilePresent() { - setUpViperConfig(FileLocation) - return - } +type logging struct { + LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` +} + +type network struct { + IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' + IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` +} - // If no flags are set, use default configurations with prioritizing command line flags - mergeFlagsWithConfig() +func init() { + configFilePath := filepath.Join(".", DefaultConfigName) + if err := loadDiceConfig(configFilePath); err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } } -func createConfigFile(configFilePath string) { +// DiceConfig is the global configuration object for dice +var DiceConfig = &Config{} + +func CreateConfigFile(configFilePath string) { if _, err := os.Stat(configFilePath); err == nil { slog.Warn("config file already exists", slog.String("path", configFilePath)) - setUpViperConfig(configFilePath) + if err := loadDiceConfig(configFilePath); err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } return } @@ -277,11 +151,71 @@ func createConfigFile(configFilePath string) { return } - setUpViperConfig(configFilePath) + if err := loadDiceConfig(configFilePath); err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } slog.Info("config file created at", slog.Any("path", configFilePath)) } +// writeConfigFile writes the default configuration to the specified file path func writeConfigFile(configFilePath string) error { + content := `# Configuration file for Dicedb + +# Version +version = "0.0.5" + +# Async Server Configuration +async_server.addr = "0.0.0.0" +async_server.port = 7379 +async_server.keepalive = 300 +async_server.timeout = 300 +async_server.max_conn = 0 + +# HTTP Configuration +http.enabled = false +http.port = 8082 + +# WebSocket Configuration +websocket.enabled = false +websocket.port = 8379 +websocket.max_write_response_retries = 3 +websocket.write_response_timeout = 10s + +# Performance Configuration +performance.watch_chan_buf_size = 20000 +performance.shard_cron_frequency = 1s +performance.multiplexer_poll_timeout = 100ms +performance.max_clients = 20000 +performance.enable_multithreading = false +performance.store_map_init_size = 1024000 +performance.adhoc_req_chan_buf_size = 20 +performance.enable_profiling = false + +# Memory Configuration +memory.max_memory = 0 +memory.eviction_policy = "allkeys-lfu" +memory.eviction_ratio = 0.9 +memory.keys_limit = 200000000 +memory.lfu_log_factor = 10 + +# Persistence Configuration +persistence.aof_file = "./dice-master.aof" +persistence.persistence_enabled = true +persistence.write_aof_on_cleanup = false + +# Logging Configuration +logging.log_level = "info" + +# Authentication Configuration +auth.username = "dice" +auth.password = "vinit" + +# Network Configuration +network.io_buffer_length = 512 +network.io_buffer_length_max = 51200 +` + + // Check if the directory exists or not dir := filepath.Dir(configFilePath) if _, err := os.Stat(dir); err != nil { return err @@ -294,112 +228,35 @@ func writeConfigFile(configFilePath string) error { } defer file.Close() - encoder := toml.NewEncoder(file) - err = encoder.Encode(defaultConfig) - return err -} - -func isValidDirPath() bool { - info, err := os.Stat(CustomConfigFilePath) - if os.IsNotExist(err) || err != nil { - return false - } - - if !info.IsDir() { - return false - } - return true -} - -// This function checks if both -o and -c flags are set or not -func areBothFlagsSet() bool { - return FileLocation != utils.EmptyStr && CustomConfigFilePath != utils.EmptyStr -} - -func setUpViperConfig(configFilePath string) { - if configFilePath != filepath.Join(DefaultConfigFilePath, DefaultConfigName) { - viper.SetConfigName(strings.Split(filepath.Base(configFilePath), ".")[0]) - } else { - viper.SetConfigName("dice") - } - - if configFilePath == utils.EmptyStr { - viper.AddConfigPath(DefaultConfigFilePath) - } else { - viper.AddConfigPath(filepath.Dir(configFilePath)) - } - - viper.SetConfigType("toml") - if err := viper.ReadInConfig(); err != nil { - var configFileNotFoundError viper.ConfigFileNotFoundError - if errors.As(err, &configFileNotFoundError) { - slog.Warn("config file not found. Using default configurations.") - return - } - slog.Error("Error reading config file", slog.Any("error", err)) - } - - if err := viper.Unmarshal(&DiceConfig); err != nil { - slog.Error("Error unmarshalling config file", slog.Any("error", err)) - slog.Warn("starting DiceDB with default configurations.") - return - } - - // override default configurations with command line flags - mergeFlagsWithConfig() -} - -func mergeFlagsWithConfig() { - if RequirePass != utils.EmptyStr { - DiceConfig.Auth.Password = RequirePass - } - - if Host != DefaultHost { - DiceConfig.AsyncServer.Addr = Host - } - - if Port != DefaultPort { - DiceConfig.AsyncServer.Port = Port - } - - if KeysLimit != DefaultKeysLimit { - DiceConfig.Memory.KeysLimit = KeysLimit + if _, err := file.WriteString(content); err != nil { + return err } - if EvictionRatio != DefaultEvictionRatio { - DiceConfig.Memory.EvictionRatio = EvictionRatio - } + return nil } -// This function checks if the config file is present or not at default location or at -c flag location -func isConfigFilePresent() bool { - // If -c flag is not set then look for config file in current directory use it - if _, err := os.Stat(filepath.Join(".", DefaultConfigName)); FileLocation == utils.EmptyStr && err == nil { - FileLocation = filepath.Join(".", DefaultConfigName) - return true +func loadDiceConfig(configFilePath string) error { + parser := NewConfigParser() + if err := parser.ParseFromFile(configFilePath); err != nil { + slog.Warn("Failed to parse config file", slog.String("error", err.Error()), slog.String("message", "Loading default configurations")) + return parser.ParseDefaults(DiceConfig) } - // will be executed if -c flag is used - _, err := os.Stat(FileLocation) - - return err == nil + return parser.Loadconfig(DiceConfig) } // This function returns the config file path based on the OS -func getConfigPath() string { - switch runtime.GOOS { - case "windows": - FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) - case "darwin", "linux": - FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) - default: - // Default to current directory if OS is unknown - FileLocation = filepath.Join(".", DefaultConfigName) - } - return FileLocation -} +// func getConfigPath() string { +// switch runtime.GOOS { +// case "windows": +// FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) +// case "darwin", "linux": +// FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) +// default: +// // Default to current directory if OS is unknown +// FileLocation = filepath.Join(".", DefaultConfigName) +// } +// return FileLocation +// } // ResetConfig resets the DiceConfig to default configurations. This function is only used for testing purposes -func ResetConfig() { - DiceConfig = &defaultConfig -} diff --git a/config/parser.go b/config/parser.go new file mode 100644 index 000000000..84e141990 --- /dev/null +++ b/config/parser.go @@ -0,0 +1,226 @@ +package config + +import ( + "bufio" + "fmt" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +// ConfigParser handles the parsing of configuration files +type ConfigParser struct { + // store holds the raw key-value pairs from the config file + store map[string]string +} + +// NewConfigParser creates a new instance of ConfigParser +func NewConfigParser() *ConfigParser { + return &ConfigParser{ + store: make(map[string]string), + } +} + +// ParseFromFile reads the configuration data from a file +func (p *ConfigParser) ParseFromFile(filename string) error { + file, err := os.Open(filename) + if err != nil { + return fmt.Errorf("error opening config file: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + return processConfigData(scanner, p) +} + +// processConfigData reads the configuration data line by line and stores it in the ConfigParser +func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), "\"") + p.store[key] = value + } + + return scanner.Err() +} + +// Loadconfig populates a struct with configuration values based on struct tags +func (p *ConfigParser) Loadconfig(cfg interface{}) error { + val := reflect.ValueOf(cfg) + if val.Kind() != reflect.Ptr || val.IsNil() { + return fmt.Errorf("config must be a non-nil pointer to a struct") + } + + val = val.Elem() + if val.Kind() != reflect.Struct { + return fmt.Errorf("config must be a pointer to a struct") + } + + if err := p.unmarshalStruct(val, ""); err != nil { + return fmt.Errorf("failed to unmarshal config: %w", err) + } + + if err := validateConfig(DiceConfig); err != nil { + return fmt.Errorf("failed to validate config: %w", err) + } + + return nil +} + +// unmarshalStruct handles the recursive struct parsing. +func (p *ConfigParser) unmarshalStruct(val reflect.Value, prefix string) error { + typ := val.Type() + + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldType := typ.Field(i) + + // Skip unexported fields just like how encoding/json does + if !field.CanSet() { + continue + } + + // Get config key or field name + key := fieldType.Tag.Get("config") + + // Use field name as key if not specified in tag + if key == "" { + key = strings.ToLower(fieldType.Name) + } + + // Skip fields with "-" tag + if key == "-" { + continue + } + + // Apply nested struct's tag as prefix + fullKey := key + if prefix != "" { + fullKey = fmt.Sprintf("%s.%s", prefix, key) + } + + // Recursively process nested structs with their prefix + if field.Kind() == reflect.Struct { + if err := p.unmarshalStruct(field, fullKey); err != nil { + return err + } + continue + } + + // Fetch and set value for non-struct fields + value, exists := p.store[fullKey] + if !exists { + // Use default value from tag if available + if defaultValue := fieldType.Tag.Get("default"); defaultValue != "" { + value = defaultValue + } else { + continue + } + } + + if err := setField(field, value); err != nil { + return fmt.Errorf("error setting field %s: %w", fullKey, err) + } + } + + return nil +} + +// setField sets the appropriate field value based on its type +func setField(field reflect.Value, value string) error { + switch field.Kind() { + case reflect.String: + field.SetString(value) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if field.Type() == reflect.TypeOf(time.Duration(0)) { + // Handle time.Duration type + duration, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("failed to parse duration: %w", err) + } + field.Set(reflect.ValueOf(duration)) + } else { + // Handle other integer types + val, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse integer: %w", err) + } + field.SetInt(val) + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + val, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return err + } + field.SetUint(val) + + case reflect.Float32, reflect.Float64: + val, err := strconv.ParseFloat(value, 64) + if err != nil { + return err + } + field.SetFloat(val) + + case reflect.Bool: + val, err := strconv.ParseBool(value) + if err != nil { + return err + } + field.SetBool(val) + + case reflect.Slice: + // Get the type of the elements in the slice + elemType := field.Type().Elem() + values := strings.Split(value, ",") + slice := reflect.MakeSlice(field.Type(), len(values), len(values)) + for i, v := range values { + elem := slice.Index(i) + elemVal := reflect.New(elemType).Elem() + if err := setField(elemVal, strings.TrimSpace(v)); err != nil { + return err + } + elem.Set(elemVal) + } + field.Set(slice) + + default: + return fmt.Errorf("unsupported type: %s", field.Type()) + } + + return nil +} + +// perse from stdin reads the configuration data from stdin +func (p *ConfigParser) ParseFromStdin() error { + scanner := bufio.NewScanner(os.Stdin) + return processConfigData(scanner, p) +} + +// ParseDefaults populates a struct with default values, will be used in case the config file is missing +func (p *ConfigParser) ParseDefaults(cfg interface{}) error { + val := reflect.ValueOf(cfg) + if val.Kind() != reflect.Ptr || val.IsNil() { + return fmt.Errorf("config must be a non-nil pointer to a struct") + } + + val = val.Elem() + if val.Kind() != reflect.Struct { + return fmt.Errorf("config must be a pointer to a struct") + } + + return p.unmarshalStruct(val, "") +} diff --git a/config/validator.go b/config/validator.go new file mode 100644 index 000000000..7c9c8ae0a --- /dev/null +++ b/config/validator.go @@ -0,0 +1,97 @@ +package config + +import ( + "fmt" + "log" + "reflect" + "strings" + + "github.com/go-playground/validator/v10" +) + +func validateConfig(config *Config) error { + validate := validator.New() + validate.RegisterStructValidation(validateShardCount, Config{}) + + if err := validate.Struct(config); err != nil { + validationErrors, ok := err.(validator.ValidationErrors) + if !ok { + return fmt.Errorf("unexpected validation error type: %v", err) + } + + processedFields := make(map[string]bool) + + for _, validationErr := range validationErrors { + fieldName := strings.TrimPrefix(validationErr.Namespace(), "Config.") + + if processedFields[fieldName] { + continue + } + processedFields[fieldName] = true + + log.Printf("Field %s failed validation: %s", fieldName, validationErr.Tag()) + + if err := applyDefaultValuesFromTags(config, fieldName); err != nil { + return fmt.Errorf("error setting default for %s: %v", fieldName, err) + } + } + } + return nil +} + +func validateShardCount(sl validator.StructLevel) { + config := sl.Current().Interface().(Config) + if config.NumShards <= 0 && config.NumShards != -1 { + sl.ReportError(config.NumShards, "NumShards", "NumShards", "invalidValue", "must be -1 or greater than 0") + } +} + +func applyDefaultValuesFromTags(config *Config, fieldName string) error { + configType := reflect.TypeOf(config).Elem() + configValue := reflect.ValueOf(config).Elem() + + // Split the field name if it refers to a nested struct + parts := strings.Split(fieldName, ".") + var field reflect.StructField + var fieldValue reflect.Value + var found bool + + // Traverse the struct to find the nested field + for i, part := range parts { + // If it's the first field, just look in the top-level struct + if i == 0 { + field, found = configType.FieldByName(part) + if !found { + log.Printf("Warning: %s field not found", part) + return fmt.Errorf("field %s not found in config struct", part) + } + fieldValue = configValue.FieldByName(part) + } else { + // Otherwise, the struct is nested, so navigate into it + if fieldValue.Kind() == reflect.Struct { + field, found = fieldValue.Type().FieldByName(part) + if !found { + log.Printf("Warning: %s field not found in %s", part, fieldValue.Type()) + return fmt.Errorf("field %s not found in struct %s", part, fieldValue.Type()) + } + fieldValue = fieldValue.FieldByName(part) + } else { + log.Printf("Warning: %s is not a struct", fieldName) + return fmt.Errorf("%s is not a struct", fieldName) + } + } + } + + defaultValue := field.Tag.Get("default") + if defaultValue == "" { + log.Printf("Warning: %s field has no default value to set, leaving empty string", fieldName) + return nil + } + + if err := setField(fieldValue, defaultValue); err != nil { + return fmt.Errorf("error setting default value for %s: %v", fieldName, err) + } + + log.Printf("Setting default value for %s to: %s", fieldName, defaultValue) + return nil +} diff --git a/dice.toml b/dice.toml deleted file mode 100644 index 02f330fc8..000000000 --- a/dice.toml +++ /dev/null @@ -1,53 +0,0 @@ -# Dice configuration file example. -# -# Note that in order to read the configuration file, Dice must be -# started with the c flag and file path as first argument: -# -# go run main.go -c /path/to/dice.toml - -Version = '0.0.5' -InstanceID = '' - -[AsyncServer] -Addr = '0.0.0.0' -Port = 7379 -KeepAlive = 300 -Timeout = 300 -MaxConn = 0 - -[HTTP] -Enabled = true -Port = 8082 - -[WebSocket] -Enabled = true -Port = 8379 - -[Performance] -WatchChanBufSize = 20000 -ShardCronFrequency = 1000000000 -MultiplexerPollTimeout = 100000000 -MaxClients = 20000 -EnableMultiThreading = false -StoreMapInitSize = 1024000 -AdhocReqChanBufSize = 20 - -[Memory] -MaxMemory = 0 -EvictionPolicy = 'allkeys-lfu' -EvictionRatio = 0.1 -KeysLimit = 200000000 -LFULogFactor = 10 - -[Persistence] -AOFFile = './dice-master.aof' -PersistenceEnabled = true -WriteAOFOnCleanup = false - -[Auth] -UserName = 'dice' -Password = '' - -[Network] -IOBufferLength = 512 -IOBufferLengthMAX = 51200 diff --git a/dicedb.conf b/dicedb.conf new file mode 100644 index 000000000..796c9e300 --- /dev/null +++ b/dicedb.conf @@ -0,0 +1,54 @@ +# Configuration file for Dicedb + +# Version +version = "0.0.5" + +# Async Server Configuration +async_server.addr = "0.0.0.0" +async_server.port = 7379 +async_server.keepalive = 300 +async_server.timeout = 300 +async_server.max_conn = 0 + +# HTTP Configuration +http.enabled = false +http.port = 8082 + +# WebSocket Configuration +websocket.enabled = false +websocket.port = 8379 +websocket.max_write_response_retries = 3 +websocket.write_response_timeout = 10s + +# Performance Configuration +performance.watch_chan_buf_size = 20000 +performance.shard_cron_frequency = 1s +performance.multiplexer_poll_timeout = 100ms +performance.max_clients = 20000 +performance.enable_multithreading = false +performance.store_map_init_size = 1024000 +performance.adhoc_req_chan_buf_size = 20 +performance.enable_profiling = false + +# Memory Configuration +memory.max_memory = 0 +memory.eviction_policy = "allkeys-lfu" +memory.eviction_ratio = 0.9 +memory.keys_limit = 200000000 +memory.lfu_log_factor = 10 + +# Persistence Configuration +persistence.aof_file = "./dice-master.aof" +persistence.persistence_enabled = true +persistence.write_aof_on_cleanup = false + +# Logging Configuration +logging.log_level = "info" + +# Authentication Configuration +auth.username = "dice" +auth.password = "" + +# Network Configuration +network.io_buffer_length = 512 +network.io_buffer_length_max = 51200 \ No newline at end of file diff --git a/go.mod b/go.mod index 64bc75043..784dcadf5 100644 --- a/go.mod +++ b/go.mod @@ -11,27 +11,20 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/magiconair/properties v1.8.7 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.12.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -42,6 +35,7 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 + github.com/go-playground/validator/v10 v10.22.1 github.com/gobwas/glob v0.2.3 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 @@ -50,10 +44,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.24 github.com/mmcloughlin/geohash v0.10.0 github.com/ohler55/ojg v1.25.0 - github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.33.0 - github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.8 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 diff --git a/go.sum b/go.sum index 242b25ad3..94cbdc7bb 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -34,12 +35,16 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 h1:JvnAibMNGA0vQH+T47Y/d5/POURIvfJl3fFk0GIEBkQ= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3/go.mod h1:p7x5/3S6wBEmiRMwxavj1I1P1xsSVQS6fcSbeai5ic4= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -51,20 +56,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -73,14 +74,11 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/geohash v0.10.0 h1:9w1HchfDfdeLc+jFEf/04D27KP7E2QmpDu52wPbJWRE= github.com/mmcloughlin/geohash v0.10.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= github.com/ohler55/ojg v1.25.0 h1:sDwc4u4zex65Uz5Nm7O1QwDKTT+YRcpeZQTy1pffRkw= github.com/ohler55/ojg v1.25.0/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -92,20 +90,6 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -115,47 +99,33 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration_tests/commands/async/command_default_test.go b/integration_tests/commands/async/command_default_test.go index f5ca6e542..066fbaa12 100644 --- a/integration_tests/commands/async/command_default_test.go +++ b/integration_tests/commands/async/command_default_test.go @@ -31,9 +31,7 @@ func getCommandDefault(connection net.Conn) []interface{} { return nil } var cmds []interface{} - for _, v := range responseValue.([]interface{}) { - cmds = append(cmds, v) - } + cmds = append(cmds, responseValue.([]interface{})...) return cmds } diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index 26171b2ee..d69f78e3a 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -110,11 +110,11 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.HTTPPort = opt.Port + config.DiceConfig.HTTP.Port = opt.Port // Initialize the HTTPServer testServer := server.NewHTTPServer(shardManager, nil) // Inform the user that the server is starting - fmt.Println("Starting the test server on port", config.HTTPPort) + fmt.Println("Starting the test server on port", config.DiceConfig.HTTP.Port) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) wg.Add(1) go func() { diff --git a/integration_tests/commands/resp/abort/server_abort_test.go b/integration_tests/commands/resp/abort/server_abort_test.go index c3f788e2d..7e13bfd66 100644 --- a/integration_tests/commands/resp/abort/server_abort_test.go +++ b/integration_tests/commands/resp/abort/server_abort_test.go @@ -3,6 +3,7 @@ package abort import ( "context" "fmt" + "log" "net" "sync" "testing" @@ -17,6 +18,11 @@ var testServerOptions = resp.TestServerOptions{ Port: 8740, } +func init() { + config.DiceConfig.AsyncServer.Port = testServerOptions.Port + log.Print("Setting port to ", config.DiceConfig.AsyncServer.Port) +} + func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer ctx.Done() diff --git a/integration_tests/commands/resp/dump_test.go b/integration_tests/commands/resp/dump_test.go index 7e0f04fe8..88f41ace4 100644 --- a/integration_tests/commands/resp/dump_test.go +++ b/integration_tests/commands/resp/dump_test.go @@ -90,8 +90,7 @@ func TestDumpRestore(t *testing.T) { t.Run(tc.name, func(t *testing.T) { FireCommand(conn, "FLUSHALL") for i, cmd := range tc.commands { - var result interface{} - result = FireCommand(conn, cmd) + result := FireCommand(conn, cmd) expected := tc.expected[i] switch exp := expected.(type) { diff --git a/integration_tests/commands/websocket/json_test.go b/integration_tests/commands/websocket/json_test.go index 75ba0b269..71b85a079 100644 --- a/integration_tests/commands/websocket/json_test.go +++ b/integration_tests/commands/websocket/json_test.go @@ -381,6 +381,7 @@ func TestJsonARRTRIM(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") + assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) @@ -472,6 +473,7 @@ func TestJsonARRINSERT(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") + assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) diff --git a/integration_tests/config/config_test.go b/integration_tests/config/config_test.go index acd0de531..a592ae81b 100644 --- a/integration_tests/config/config_test.go +++ b/integration_tests/config/config_test.go @@ -1,130 +1,136 @@ package commands -import ( - "os" - "path/filepath" - "testing" - - "github.com/dicedb/dice/config" -) - -// scenario 1: Create a config file if the directory is provided (-o flag) -func TestSetupConfig_CreateAndLoadDefault(t *testing.T) { - config.ResetConfig() - tempDir := t.TempDir() - - // Simulate the flag: -o= - config.CustomConfigFilePath = tempDir - config.SetupConfig() - - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) - } -} - -// scenario 2: Load default config if no config file or directory is provided -func TestSetupConfig_DefaultConfig(t *testing.T) { - // Simulate no flags being set (default config scenario) - config.ResetConfig() - config.CustomConfigFilePath = "" - config.FileLocation = filepath.Join(config.DefaultConfigFilePath, config.DefaultConfigName) - - // Verify that the configuration was loaded from the default values - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) // 127.0.0.1 - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d, got %d", 8739, config.DiceConfig.AsyncServer.Port) - } -} - -// scenario 3: Config file is present but not well-structured (Malformed) -func TestSetupConfig_InvalidConfigFile(t *testing.T) { - config.DiceConfig = nil - tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = 127.0.0.1 // Missing quotes around string value - port = abc // Invalid integer - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to create invalid test config file: %v", err) - } - - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath - - config.SetupConfig() - - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s' after unmarshal error, got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d after unmarshal error, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) - } -} - -// scenario 4: Config file is present with partial content -func TestSetupConfig_PartialConfigFile(t *testing.T) { - tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = "127.0.0.1" - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to create partial test config file: %v", err) - } - - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath - - config.SetupConfig() - - t.Log(config.DiceConfig.AsyncServer.Port) - - if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { - t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d (default), got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) - } -} - -// scenario 5: Load config from the provided file path -func TestSetupConfig_LoadFromFile(t *testing.T) { - config.ResetConfig() - tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = "127.0.0.1" - port = 8739 - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to write test config file: %v", err) - } - - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath - - config.SetupConfig() - - if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { - t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != 8739 { - t.Fatalf("Expected server port to be 8374, got %d", config.DiceConfig.AsyncServer.Port) - } - +import "testing" + +// import ( +// "os" +// "path/filepath" +// "testing" + +// "github.com/dicedb/dice/config" +// ) + +// // scenario 1: Create a config file if the directory is provided (-o flag) +// func TestSetupConfig_CreateAndLoadDefault(t *testing.T) { +// config.ResetConfig() +// tempDir := t.TempDir() + +// // Simulate the flag: -o= +// config.CustomConfigFilePath = tempDir +// config.SetupConfig() + +// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { +// t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) +// } +// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { +// t.Fatalf("Expected server port to be %d, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) +// } +// } + +// // scenario 2: Load default config if no config file or directory is provided +// func TestSetupConfig_DefaultConfig(t *testing.T) { +// // Simulate no flags being set (default config scenario) +// config.ResetConfig() +// config.CustomConfigFilePath = "" +// config.FileLocation = filepath.Join(config.DefaultConfigFilePath, config.DefaultConfigName) + +// // Verify that the configuration was loaded from the default values +// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { +// t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) // 127.0.0.1 +// } +// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { +// t.Fatalf("Expected server port to be %d, got %d", 8739, config.DiceConfig.AsyncServer.Port) +// } +// } + +// // scenario 3: Config file is present but not well-structured (Malformed) +// func TestSetupConfig_InvalidConfigFile(t *testing.T) { +// config.DiceConfig = nil +// tempDir := t.TempDir() +// configFilePath := filepath.Join(tempDir, "dice.toml") + +// content := ` +// [asyncserver] +// addr = 127.0.0.1 // Missing quotes around string value +// port = abc // Invalid integer +// ` +// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { +// t.Fatalf("Failed to create invalid test config file: %v", err) +// } + +// // Simulate the flag: -c= +// config.CustomConfigFilePath = "" +// config.FileLocation = configFilePath + +// config.SetupConfig() + +// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { +// t.Fatalf("Expected server addr to be '%s' after unmarshal error, got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) +// } +// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { +// t.Fatalf("Expected server port to be %d after unmarshal error, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) +// } +// } + +// // scenario 4: Config file is present with partial content +// func TestSetupConfig_PartialConfigFile(t *testing.T) { +// tempDir := t.TempDir() +// configFilePath := filepath.Join(tempDir, "dice.toml") + +// content := ` +// [asyncserver] +// addr = "127.0.0.1" +// ` +// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { +// t.Fatalf("Failed to create partial test config file: %v", err) +// } + +// // Simulate the flag: -c= +// config.CustomConfigFilePath = "" +// config.FileLocation = configFilePath + +// config.SetupConfig() + +// t.Log(config.DiceConfig.AsyncServer.Port) + +// if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { +// t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) +// } +// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { +// t.Fatalf("Expected server port to be %d (default), got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) +// } +// } + +// // scenario 5: Load config from the provided file path +// func TestSetupConfig_LoadFromFile(t *testing.T) { +// config.ResetConfig() +// tempDir := t.TempDir() +// configFilePath := filepath.Join(tempDir, "dice.toml") + +// content := ` +// [asyncserver] +// addr = "127.0.0.1" +// port = 8739 +// ` +// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { +// t.Fatalf("Failed to write test config file: %v", err) +// } + +// // Simulate the flag: -c= +// config.CustomConfigFilePath = "" +// config.FileLocation = configFilePath + +// config.SetupConfig() + +// if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { +// t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) +// } +// if config.DiceConfig.AsyncServer.Port != 8739 { +// t.Fatalf("Expected server port to be 8374, got %d", config.DiceConfig.AsyncServer.Port) +// } + +// } + +func Test_demo(t *testing.T) { + t.Log("todo implement new tests") } diff --git a/integration_tests/server/server_abort_test.go b/integration_tests/server/server_abort_test.go index f4c98ac4f..fb11e69ae 100644 --- a/integration_tests/server/server_abort_test.go +++ b/integration_tests/server/server_abort_test.go @@ -18,6 +18,11 @@ var testServerOptions = commands.TestServerOptions{ Port: 8740, } +func init() { + parser := config.NewConfigParser() + parser.ParseDefaults(config.DiceConfig) +} + func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 000000000..8ccfad88b --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,203 @@ +package cli + +import ( + "fmt" + "log" + "os" + "runtime" + "strings" + + "github.com/dicedb/dice/config" +) + +type configEntry struct { + Key string + Value interface{} +} + +var configTable = []configEntry{} + +// configuration function used to add configuration values to the print table at the startup. +// add entry to this function to add a new row in the startup configuration table. +func configuration() { + // Add the version of the DiceDB to the configuration table + addEntry("Version", config.DiceDBVersion) + + // Add the port number on which DiceDB is running to the configuration table + addEntry("Port", config.DiceConfig.AsyncServer.Port) + + // Add whether multi-threading is enabled to the configuration table + addEntry("Multi Threading Enabled", config.DiceConfig.Performance.EnableMultiThreading) + + // Add the number of CPU cores available on the machine to the configuration table + addEntry("Cores", runtime.NumCPU()) + + // Conditionally add the number of shards to be used for DiceDB to the configuration table + if config.DiceConfig.Performance.EnableMultiThreading { + if config.DiceConfig.NumShards > 0 { + configTable = append(configTable, configEntry{"Shards", config.DiceConfig.NumShards}) + } else { + configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) + } + } else { + configTable = append(configTable, configEntry{"Shards", 1}) + } + + // Add whether the watch feature is enabled to the configuration table + // addEntry("Watch Enabled", config.EnableWatch) + + // Add whether the watch feature is enabled to the configuration table + addEntry("HTTP Enabled", config.DiceConfig.HTTP.Enabled) + + // Add whether the watch feature is enabled to the configuration table + addEntry("Websocket Enabled", config.DiceConfig.WebSocket.Enabled) +} + +func addEntry(k string, v interface{}) { + configTable = append(configTable, configEntry{k, v}) +} + +// printConfigTable prints key-value pairs in a vertical table format. +func render() { + fmt.Print(` + ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ + ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ + ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ + ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ + ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ + ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ + + `) + configuration() + + // Find the longest key to align the values properly + maxKeyLength := 0 + maxValueLength := 20 // Default value length for alignment + for _, entry := range configTable { + if len(entry.Key) > maxKeyLength { + maxKeyLength = len(entry.Key) + } + if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { + maxValueLength = len(fmt.Sprintf("%v", entry.Value)) + } + } + + // Create the table header and separator line + fmt.Println() + totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") + fmt.Println(strings.Repeat("-", totalWidth)) + + // Print each configuration key-value pair without row lines + for _, entry := range configTable { + fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) + } + + // Final bottom line + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Println() +} + +func Execute() { + if len(os.Args) < 2 { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatal(err) + } + + render() + return + } + + switch os.Args[1] { + case "-v", "--version": + fmt.Println("dicedb version", config.DiceDBVersion) + os.Exit(0) + + case "-h", "--help": + printUsage() + os.Exit(0) + + case "-": + parser := config.NewConfigParser() + if err := parser.ParseFromStdin(); err != nil { + log.Fatal(err) + } + if err := parser.Loadconfig(config.DiceConfig); err != nil { + log.Fatal(err) + } + fmt.Println(config.DiceConfig.Version) + case "-o", "--output": + // TODO: Implement output to file of config + if len(os.Args) < 3 { + log.Fatal("Output file path not provided") + } else { + dirPath := os.Args[2] + if dirPath == "" { + log.Fatal("Output file path not provided") + } + + info, err := os.Stat(dirPath) + switch { + case os.IsNotExist(err): + log.Fatal("Output file path does not exist") + case err != nil: + log.Fatalf("Error checking output file path: %v", err) + case !info.IsDir(): + log.Fatal("Output file path is not a directory") + } + + var filePath string + if strings.HasSuffix(dirPath, "/") { + filePath = dirPath + "dicedb.conf" + } else { + filePath = dirPath + "/dicedb.conf" + } + config.CreateConfigFile(filePath) + render() + } + case "-c", "--config": + if len(os.Args) >= 3 { + filePath := os.Args[2] + if filePath == "" { + log.Fatal("Config file path not provided") + } + + info, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + log.Fatal("Config file does not exist") + } + log.Fatalf("Error checking config file: %v", err) + } + + if info.IsDir() || !strings.HasSuffix(filePath, ".conf") { + log.Fatal("Config file must be a regular file with a .conf extension") + } + + parser := config.NewConfigParser() + if err := parser.ParseFromFile(filePath); err != nil { + log.Fatal(err) + } + if err := parser.Loadconfig(config.DiceConfig); err != nil { + log.Fatal(err) + } + + render() + } else { + log.Fatal("Config file path not provided") + } + + default: + fmt.Printf("Unknown option: %s\n", os.Args[1]) + printUsage() + } +} + +func printUsage() { + fmt.Println(`Usage: ./dicedb [/path/to/dice.conf] [options] [-] + ./dicedb - (read config from stdin) + ./dicedb -v or --version + ./dicedb -h or --help`) +} diff --git a/internal/eval/eval_amd64.go b/internal/eval/eval_amd64.go index d370a64d8..440161b82 100644 --- a/internal/eval/eval_amd64.go +++ b/internal/eval/eval_amd64.go @@ -26,7 +26,7 @@ func EvalBGREWRITEAOF(args []string, store *dstore.Store) []byte { // TODO: Problem at hand: In multi-threaded environment, each shard instance would fork a child process. // TODO: Each child process would now have a copy of the network file descriptor thus resulting in resource leaks. // TODO: We need to find an alternative approach for the multi-threaded environment. - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { return nil } newChild, _, _ := syscall.Syscall(syscall.SYS_FORK, 0, 0, 0) diff --git a/internal/server/httpServer.go b/internal/server/httpServer.go index 2a993c9a0..9cad4e8a6 100644 --- a/internal/server/httpServer.go +++ b/internal/server/httpServer.go @@ -65,7 +65,7 @@ func NewHTTPServer(shardManager *shard.ShardManager, wl wal.AbstractWAL) *HTTPSe mux := http.NewServeMux() caseInsensitiveMux := &CaseInsensitiveMux{mux: mux} srv := &http.Server{ - Addr: fmt.Sprintf(":%d", config.HTTPPort), + Addr: fmt.Sprintf(":%d", config.DiceConfig.HTTP.Port), Handler: caseInsensitiveMux, ReadHeaderTimeout: 5 * time.Second, } diff --git a/internal/server/resp/server.go b/internal/server/resp/server.go index 17244d076..6f857f55d 100644 --- a/internal/server/resp/server.go +++ b/internal/server/resp/server.go @@ -96,7 +96,7 @@ func (s *Server) Run(ctx context.Context) (err error) { } }(wg) - slog.Info("ready to accept and serve requests on", slog.Int("port", config.Port)) + slog.Info("ready to accept and serve requests on", slog.Int("port", config.DiceConfig.AsyncServer.Port)) select { case <-ctx.Done(): diff --git a/main.go b/main.go index a52003d7a..4757df2d3 100644 --- a/main.go +++ b/main.go @@ -76,8 +76,6 @@ func init() { flag.Parse() - config.SetupConfig() - iid := observability.GetOrCreateInstanceID() config.DiceConfig.InstanceID = iid From 2885513837ca6799edd88a3c2c8e7f56053e94db Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 12:25:54 +0530 Subject: [PATCH 02/16] feat: config rebase --- config/config.go | 43 +---- config/validator.go | 4 +- go.mod | 4 +- go.sum | 18 +- integration_tests/commands/websocket/setup.go | 2 +- internal/cli/cli.go | 4 +- internal/shard/shard_manager.go | 2 +- main.go | 181 +++--------------- 8 files changed, 47 insertions(+), 211 deletions(-) diff --git a/config/config.go b/config/config.go index a9d5e882e..6006143fb 100644 --- a/config/config.go +++ b/config/config.go @@ -6,8 +6,6 @@ import ( "os" "path/filepath" "time" - - "github.com/dicedb/dice/internal/server/utils" ) const ( @@ -18,38 +16,9 @@ const ( EvictAllKeysRandom string = "allkeys-random" EvictAllKeysLRU string = "allkeys-lru" EvictAllKeysLFU string = "allkeys-lfu" -) - -var ( - Host = "0.0.0.0" - Port = 7379 - - EnableMultiThreading = false - EnableHTTP = true - HTTPPort = 8082 - - EnableWebsocket = true - WebsocketPort = 8379 - NumShards int = -1 - - // if RequirePass is set to an empty string, no authentication is required - RequirePass = utils.EmptyStr - - CustomConfigFilePath = utils.EmptyStr - FileLocation = utils.EmptyStr - - InitConfigCmd = false - - KeysLimit = 200000000 - - EnableProfiling = false - - EnableWatch = true - LogDir = "" - EnableWAL = true - RestoreFromWAL = false - WALEngine = "sqlite" + DefaultKeysLimit int = 200000000 + DefaultEvictionRatio float64 = 0.1 ) type Config struct { @@ -64,12 +33,11 @@ type Config struct { Persistence persistence `config:"persistence"` Logging logging `config:"logging"` Network network `config:"network"` - NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` } type auth struct { UserName string `config:"username" default:"dice"` - Password string `validate:"min=8"` + Password string `config:"password"` } type asyncServer struct { @@ -102,6 +70,7 @@ type performance struct { AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` EnableProfiling bool `config:"profiling" default:"false"` EnableWatch bool `config:"enable_watch" default:"false"` + NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` } type memory struct { @@ -116,6 +85,10 @@ type persistence struct { AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` PersistenceEnabled bool `config:"persistence_enabled" default:"true"` WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` + EnableWAL bool `config:"enable-wal" default:"false"` + WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` + RestoreFromWAL bool `config:"restore-wal" default:"false"` + WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` } type logging struct { diff --git a/config/validator.go b/config/validator.go index 7c9c8ae0a..66669ff43 100644 --- a/config/validator.go +++ b/config/validator.go @@ -41,8 +41,8 @@ func validateConfig(config *Config) error { func validateShardCount(sl validator.StructLevel) { config := sl.Current().Interface().(Config) - if config.NumShards <= 0 && config.NumShards != -1 { - sl.ReportError(config.NumShards, "NumShards", "NumShards", "invalidValue", "must be -1 or greater than 0") + if config.Performance.NumShards <= 0 && config.Performance.NumShards != -1 { + sl.ReportError(config.Performance.NumShards, "NumShards", "NumShards", "invalidValue", "must be -1 or greater than 0") } } diff --git a/go.mod b/go.mod index 784dcadf5..a193be0cf 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,8 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.11.0 // indirect golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 94cbdc7bb..8a48e609c 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= -github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -107,20 +105,20 @@ github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryB github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index b75b17c4f..b8ea19b8f 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -108,7 +108,7 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.WebsocketPort = opt.Port + config.DiceConfig.WebSocket.Port = opt.Port testServer := server.NewWebSocketServer(shardManager, testPort1, nil) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 8ccfad88b..1c211f264 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -34,8 +34,8 @@ func configuration() { // Conditionally add the number of shards to be used for DiceDB to the configuration table if config.DiceConfig.Performance.EnableMultiThreading { - if config.DiceConfig.NumShards > 0 { - configTable = append(configTable, configEntry{"Shards", config.DiceConfig.NumShards}) + if config.DiceConfig.Performance.NumShards > 0 { + configTable = append(configTable, configEntry{"Shards", config.DiceConfig.Performance.NumShards}) } else { configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) } diff --git a/internal/shard/shard_manager.go b/internal/shard/shard_manager.go index 86bf41fbf..365a6a687 100644 --- a/internal/shard/shard_manager.go +++ b/internal/shard/shard_manager.go @@ -32,7 +32,7 @@ func NewShardManager(shardCount uint8, queryWatchChan chan dstore.QueryWatchEven shardReqMap := make(map[ShardID]chan *ops.StoreOp) shardErrorChan := make(chan *ShardError) - maxKeysPerShard := config.KeysLimit / int(shardCount) + maxKeysPerShard := config.DiceConfig.Memory.KeysLimit / int(shardCount) for i := uint8(0); i < shardCount; i++ { evictionStrategy := dstore.NewBatchEvictionLRU(maxKeysPerShard, config.DiceConfig.Memory.EvictionRatio) // Shards are numbered from 0 to shardCount-1 diff --git a/main.go b/main.go index 4757df2d3..6a5404847 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "flag" "fmt" "log/slog" "net/http" @@ -12,11 +11,11 @@ import ( "runtime" "runtime/pprof" "runtime/trace" - "strings" "sync" "syscall" "time" + "github.com/dicedb/dice/internal/cli" "github.com/dicedb/dice/internal/logger" "github.com/dicedb/dice/internal/server/abstractserver" "github.com/dicedb/dice/internal/wal" @@ -32,145 +31,11 @@ import ( "github.com/dicedb/dice/internal/worker" ) -type configEntry struct { - Key string - Value interface{} -} - -var configTable = []configEntry{} - -func init() { - flag.StringVar(&config.Host, "host", "0.0.0.0", "host for the DiceDB server") - - flag.IntVar(&config.Port, "port", 7379, "port for the DiceDB server") - - flag.IntVar(&config.HTTPPort, "http-port", 7380, "port for accepting requets over HTTP") - flag.BoolVar(&config.EnableHTTP, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") - - flag.IntVar(&config.WebsocketPort, "websocket-port", 7381, "port for accepting requets over WebSocket") - flag.BoolVar(&config.EnableWebsocket, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") - - flag.BoolVar(&config.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") - flag.IntVar(&config.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") - - flag.BoolVar(&config.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") - flag.BoolVar(&config.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") - - flag.StringVar(&config.DiceConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") - flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") - - flag.BoolVar(&config.EnableWAL, "enable-wal", false, "enable write-ahead logging") - flag.BoolVar(&config.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") - flag.StringVar(&config.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") - - flag.StringVar(&config.RequirePass, "requirepass", config.RequirePass, "enable authentication for the default user") - flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the config file") - flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") - flag.BoolVar(&config.InitConfigCmd, "init-config", false, "initialize a new config file") - - flag.IntVar(&config.KeysLimit, "keys-limit", config.KeysLimit, "keys limit for the DiceDB server. "+ - "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ - "total number of shard threads to estimate how much memory will be required at system start up.") - flag.Float64Var(&config.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ - "keys limit is reached") - - flag.Parse() - +func main() { iid := observability.GetOrCreateInstanceID() config.DiceConfig.InstanceID = iid - slog.SetDefault(logger.New()) -} - -func printSplash() { - fmt.Print(` - ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ - ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ - ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ - ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ - ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ - ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ - - `) -} - -// configuration function used to add configuration values to the print table at the startup. -// add entry to this function to add a new row in the startup configuration table. -func configuration() { - // Add the version of the DiceDB to the configuration table - addEntry("Version", config.DiceDBVersion) - - // Add the port number on which DiceDB is running to the configuration table - addEntry("Port", config.Port) - - // Add whether multi-threading is enabled to the configuration table - addEntry("Multi Threading Enabled", config.EnableMultiThreading) - - // Add the number of CPU cores available on the machine to the configuration table - addEntry("Cores", runtime.NumCPU()) - - // Conditionally add the number of shards to be used for DiceDB to the configuration table - if config.EnableMultiThreading { - if config.NumShards > 0 { - configTable = append(configTable, configEntry{"Shards", config.NumShards}) - } else { - configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) - } - } else { - configTable = append(configTable, configEntry{"Shards", 1}) - } - - // Add whether the watch feature is enabled to the configuration table - addEntry("Watch Enabled", config.EnableWatch) - - // Add whether the watch feature is enabled to the configuration table - addEntry("HTTP Enabled", config.EnableHTTP) - - // Add whether the watch feature is enabled to the configuration table - addEntry("Websocket Enabled", config.EnableWebsocket) -} - -func addEntry(k string, v interface{}) { - configTable = append(configTable, configEntry{k, v}) -} - -// printConfigTable prints key-value pairs in a vertical table format. -func printConfigTable() { - configuration() - - // Find the longest key to align the values properly - maxKeyLength := 0 - maxValueLength := 20 // Default value length for alignment - for _, entry := range configTable { - if len(entry.Key) > maxKeyLength { - maxKeyLength = len(entry.Key) - } - if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { - maxValueLength = len(fmt.Sprintf("%v", entry.Value)) - } - } - - // Create the table header and separator line - fmt.Println() - totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") - fmt.Println(strings.Repeat("-", totalWidth)) - - // Print each configuration key-value pair without row lines - for _, entry := range configTable { - fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) - } - - // Final bottom line - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Println() -} - -func main() { - printSplash() - printConfigTable() - + cli.Execute() go observability.Ping() ctx, cancel := context.WithCancel(context.Background()) @@ -188,26 +53,26 @@ func main() { ) wl, _ = wal.NewNullWAL() - slog.Info("running with", slog.Bool("enable-wal", config.EnableWAL)) - if config.EnableWAL { - if config.WALEngine == "sqlite" { - _wl, err := wal.NewSQLiteWAL(config.LogDir) + slog.Info("running with", slog.Bool("enable-wal", config.DiceConfig.Persistence.EnableWAL)) + if config.DiceConfig.Persistence.EnableWAL { + if config.DiceConfig.Persistence.WALEngine == "sqlite" { + _wl, err := wal.NewSQLiteWAL(config.DiceConfig.Persistence.WALDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl - } else if config.WALEngine == "aof" { - _wl, err := wal.NewAOFWAL(config.LogDir) + } else if config.DiceConfig.Persistence.WALEngine == "aof" { + _wl, err := wal.NewAOFWAL(config.DiceConfig.Persistence.WALDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl } else { - slog.Error("unsupported WAL engine", slog.String("engine", config.WALEngine)) + slog.Error("unsupported WAL engine", slog.String("engine", config.DiceConfig.Persistence.WALEngine)) sigs <- syscall.SIGKILL return } @@ -220,14 +85,14 @@ func main() { slog.Debug("WAL initialization complete") - if config.RestoreFromWAL { + if config.DiceConfig.Persistence.RestoreFromWAL { slog.Info("restoring database from WAL") wal.ReplayWAL(wl) slog.Info("database restored from WAL") } } - if config.EnableWatch { + if config.DiceConfig.Performance.EnableWatch { bufSize := config.DiceConfig.Performance.WatchChanBufSize queryWatchChan = make(chan dstore.QueryWatchEvent, bufSize) cmdWatchChan = make(chan dstore.CmdWatchEvent, bufSize) @@ -239,10 +104,10 @@ func main() { // core count ensures the application can make full use of all available hardware. // If multithreading is not enabled, server will run on a single core. var numShards int - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { numShards = runtime.NumCPU() - if config.NumShards > 0 { - numShards = config.NumShards + if config.DiceConfig.Performance.NumShards > 0 { + numShards = config.DiceConfig.Performance.NumShards } } else { numShards = 1 @@ -267,8 +132,8 @@ func main() { var serverWg sync.WaitGroup - if config.EnableMultiThreading { - if config.EnableProfiling { + if config.DiceConfig.Performance.EnableMultiThreading { + if config.DiceConfig.Performance.EnableProfiling { stopProfiling, err := startProfiling() if err != nil { slog.Error("Profiling could not be started", slog.Any("error", err)) @@ -291,15 +156,15 @@ func main() { serverWg.Add(1) go runServer(ctx, &serverWg, asyncServer, serverErrCh) - if config.EnableHTTP { + if config.DiceConfig.HTTP.Enabled { httpServer := server.NewHTTPServer(shardManager, wl) serverWg.Add(1) go runServer(ctx, &serverWg, httpServer, serverErrCh) } } - if config.EnableWebsocket { - websocketServer := server.NewWebSocketServer(shardManager, config.WebsocketPort, wl) + if config.DiceConfig.WebSocket.Enabled { + websocketServer := server.NewWebSocketServer(shardManager, config.DiceConfig.WebSocket.Port, wl) serverWg.Add(1) go runServer(ctx, &serverWg, websocketServer, serverErrCh) } @@ -326,7 +191,7 @@ func main() { close(sigs) - if config.EnableWAL { + if config.DiceConfig.Persistence.EnableWAL { wal.ShutdownBG() } From 1222a127b696515fbdfe9534a34ddfd6db98b9bf Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 13:02:37 +0530 Subject: [PATCH 03/16] feat: test cases of new config management --- config/parser.go | 72 +++---- integration_tests/config/config_test.go | 232 ++++++++++------------- integration_tests/config/parser_test.go | 237 ++++++++++++++++++++++++ 3 files changed, 370 insertions(+), 171 deletions(-) create mode 100644 integration_tests/config/parser_test.go diff --git a/config/parser.go b/config/parser.go index 84e141990..082c589f6 100644 --- a/config/parser.go +++ b/config/parser.go @@ -35,25 +35,25 @@ func (p *ConfigParser) ParseFromFile(filename string) error { return processConfigData(scanner, p) } -// processConfigData reads the configuration data line by line and stores it in the ConfigParser -func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } +// ParseFromStdin reads the configuration data from stdin +func (p *ConfigParser) ParseFromStdin() error { + scanner := bufio.NewScanner(os.Stdin) + return processConfigData(scanner, p) +} - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - continue - } +// ParseDefaults populates a struct with default values based on struct tag `default` +func (p *ConfigParser) ParseDefaults(cfg interface{}) error { + val := reflect.ValueOf(cfg) + if val.Kind() != reflect.Ptr || val.IsNil() { + return fmt.Errorf("config must be a non-nil pointer to a struct") + } - key := strings.TrimSpace(parts[0]) - value := strings.Trim(strings.TrimSpace(parts[1]), "\"") - p.store[key] = value + val = val.Elem() + if val.Kind() != reflect.Struct { + return fmt.Errorf("config must be a pointer to a struct") } - return scanner.Err() + return p.unmarshalStruct(val, "") } // Loadconfig populates a struct with configuration values based on struct tags @@ -79,6 +79,27 @@ func (p *ConfigParser) Loadconfig(cfg interface{}) error { return nil } +// processConfigData reads the configuration data line by line and stores it in the ConfigParser +func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid config line: %s", line) + } + + key := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), "\"") + p.store[key] = value + } + + return scanner.Err() +} + // unmarshalStruct handles the recursive struct parsing. func (p *ConfigParser) unmarshalStruct(val reflect.Value, prefix string) error { typ := val.Type() @@ -203,24 +224,3 @@ func setField(field reflect.Value, value string) error { return nil } - -// perse from stdin reads the configuration data from stdin -func (p *ConfigParser) ParseFromStdin() error { - scanner := bufio.NewScanner(os.Stdin) - return processConfigData(scanner, p) -} - -// ParseDefaults populates a struct with default values, will be used in case the config file is missing -func (p *ConfigParser) ParseDefaults(cfg interface{}) error { - val := reflect.ValueOf(cfg) - if val.Kind() != reflect.Ptr || val.IsNil() { - return fmt.Errorf("config must be a non-nil pointer to a struct") - } - - val = val.Elem() - if val.Kind() != reflect.Struct { - return fmt.Errorf("config must be a pointer to a struct") - } - - return p.unmarshalStruct(val, "") -} diff --git a/integration_tests/config/config_test.go b/integration_tests/config/config_test.go index a592ae81b..c68dae6d0 100644 --- a/integration_tests/config/config_test.go +++ b/integration_tests/config/config_test.go @@ -1,136 +1,98 @@ -package commands - -import "testing" - -// import ( -// "os" -// "path/filepath" -// "testing" - -// "github.com/dicedb/dice/config" -// ) - -// // scenario 1: Create a config file if the directory is provided (-o flag) -// func TestSetupConfig_CreateAndLoadDefault(t *testing.T) { -// config.ResetConfig() -// tempDir := t.TempDir() - -// // Simulate the flag: -o= -// config.CustomConfigFilePath = tempDir -// config.SetupConfig() - -// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { -// t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) -// } -// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { -// t.Fatalf("Expected server port to be %d, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) -// } -// } - -// // scenario 2: Load default config if no config file or directory is provided -// func TestSetupConfig_DefaultConfig(t *testing.T) { -// // Simulate no flags being set (default config scenario) -// config.ResetConfig() -// config.CustomConfigFilePath = "" -// config.FileLocation = filepath.Join(config.DefaultConfigFilePath, config.DefaultConfigName) - -// // Verify that the configuration was loaded from the default values -// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { -// t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) // 127.0.0.1 -// } -// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { -// t.Fatalf("Expected server port to be %d, got %d", 8739, config.DiceConfig.AsyncServer.Port) -// } -// } - -// // scenario 3: Config file is present but not well-structured (Malformed) -// func TestSetupConfig_InvalidConfigFile(t *testing.T) { -// config.DiceConfig = nil -// tempDir := t.TempDir() -// configFilePath := filepath.Join(tempDir, "dice.toml") - -// content := ` -// [asyncserver] -// addr = 127.0.0.1 // Missing quotes around string value -// port = abc // Invalid integer -// ` -// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { -// t.Fatalf("Failed to create invalid test config file: %v", err) -// } - -// // Simulate the flag: -c= -// config.CustomConfigFilePath = "" -// config.FileLocation = configFilePath - -// config.SetupConfig() - -// if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { -// t.Fatalf("Expected server addr to be '%s' after unmarshal error, got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) -// } -// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { -// t.Fatalf("Expected server port to be %d after unmarshal error, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) -// } -// } - -// // scenario 4: Config file is present with partial content -// func TestSetupConfig_PartialConfigFile(t *testing.T) { -// tempDir := t.TempDir() -// configFilePath := filepath.Join(tempDir, "dice.toml") - -// content := ` -// [asyncserver] -// addr = "127.0.0.1" -// ` -// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { -// t.Fatalf("Failed to create partial test config file: %v", err) -// } - -// // Simulate the flag: -c= -// config.CustomConfigFilePath = "" -// config.FileLocation = configFilePath - -// config.SetupConfig() - -// t.Log(config.DiceConfig.AsyncServer.Port) - -// if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { -// t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) -// } -// if config.DiceConfig.AsyncServer.Port != config.DefaultPort { -// t.Fatalf("Expected server port to be %d (default), got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) -// } -// } - -// // scenario 5: Load config from the provided file path -// func TestSetupConfig_LoadFromFile(t *testing.T) { -// config.ResetConfig() -// tempDir := t.TempDir() -// configFilePath := filepath.Join(tempDir, "dice.toml") - -// content := ` -// [asyncserver] -// addr = "127.0.0.1" -// port = 8739 -// ` -// if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { -// t.Fatalf("Failed to write test config file: %v", err) -// } - -// // Simulate the flag: -c= -// config.CustomConfigFilePath = "" -// config.FileLocation = configFilePath - -// config.SetupConfig() - -// if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { -// t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) -// } -// if config.DiceConfig.AsyncServer.Port != 8739 { -// t.Fatalf("Expected server port to be 8374, got %d", config.DiceConfig.AsyncServer.Port) -// } - -// } - -func Test_demo(t *testing.T) { - t.Log("todo implement new tests") +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/dicedb/dice/config" +) + +const configFileName = "dicedb.conf" + +// TestCreateConfigFile_FileExists tests the scenario when config file already exists +func TestCreateConfigFile_FileExists(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, configFileName) + + if err := os.WriteFile(configPath, []byte("test config"), 0644); err != nil { + t.Fatalf("Failed to create test config file: %v", err) + } + + config.CreateConfigFile(configPath) + + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read config file: %v", err) + } + + if string(content) != "test config" { + t.Error("Config file content was modified when it should have been preserved") + } +} + +// TestCreateConfigFile_NewFile tests creating a new config file +func TestCreateConfigFile_NewFile(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, configFileName) + config.CreateConfigFile(configPath) + + if _, err := os.Stat(configPath); os.IsNotExist(err) { + t.Error("Config file was not created") + } + + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read created config file: %v", err) + } + + if len(content) == 0 { + t.Error("Created config file is empty") + } +} + +// TestCreateConfigFile_InvalidPath tests creation with an invalid file path +func TestCreateConfigFile_InvalidPath(t *testing.T) { + configPath := "/nonexistent/directory/dicedb.conf" + config.CreateConfigFile(configPath) + + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Error("Config file should not have been created at invalid path") + } +} + +func TestCreateConfigFile_NoPermission(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("Skipping test when running as root") + } + + tempDir := t.TempDir() + err := os.Chmod(tempDir, 0555) // read + execute only + if err != nil { + t.Fatalf("Failed to change directory permissions: %v", err) + } + defer os.Chmod(tempDir, 0755) // restore permissions + + configPath := filepath.Join(tempDir, configFileName) + config.CreateConfigFile(configPath) + + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Error("Config file should not have been created without permissions") + } +} + +// TestCreateConfigFile_ExistingDirectory tests creation in existing directory +func TestCreateConfigFile_ExistingDirectory(t *testing.T) { + tempDir := t.TempDir() + configDir := filepath.Join(tempDir, "config") + if err := os.MkdirAll(configDir, 0755); err != nil { + t.Fatalf("Failed to create config directory: %v", err) + } + + configPath := filepath.Join(configDir, configFileName) + config.CreateConfigFile(configPath) + + if _, err := os.Stat(configPath); os.IsNotExist(err) { + t.Error("Config file was not created in existing directory") + } } diff --git a/integration_tests/config/parser_test.go b/integration_tests/config/parser_test.go new file mode 100644 index 000000000..60bf6ab96 --- /dev/null +++ b/integration_tests/config/parser_test.go @@ -0,0 +1,237 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/dicedb/dice/config" +) + +// TestConfig is a test struct that mimics your actual config structure +type TestConfig struct { + Host string `default:"localhost"` + Port int `default:"8080"` + LogLevel string `default:"info"` +} + +func TestNewConfigParser(t *testing.T) { + parser := config.NewConfigParser() + if parser == nil { + t.Fatal("NewConfigParser returned nil") + } +} + +func TestParseFromFile(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + setupErr bool + }{ + { + name: "valid config", + content: `host=testhost +port=9090 +log_level=debug`, + wantErr: false, + }, + { + name: "empty file", + content: "", + wantErr: false, + }, + { + name: "malformed config", + content: `host=testhost +invalid-line +port=9090`, + wantErr: true, + }, + { + name: "non-existent file", + setupErr: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Create temporary config file + tempDir := t.TempDir() + filename := filepath.Join(tempDir, "dicedb.conf") + + if !tt.setupErr { + err := os.WriteFile(filename, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + } + + err := parser.ParseFromFile(filename) + if (err != nil) != tt.wantErr { + t.Errorf("ParseFromFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseFromStdin(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + }{ + { + name: "valid input", + input: `host=testhost +port=9090 +log_level=debug`, + wantErr: false, + }, + { + name: "empty input", + input: "", + wantErr: false, + }, + { + name: "malformed input", + input: `host=testhost +invalid-line +port=9090`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Store original stdin + oldStdin := os.Stdin + defer func() { os.Stdin = oldStdin }() + + // Create a pipe and pass the test input + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Failed to create pipe: %v", err) + } + os.Stdin = r + + go func() { + defer w.Close() + w.Write([]byte(tt.input)) + }() + + err = parser.ParseFromStdin() + if (err != nil) != tt.wantErr { + t.Errorf("ParseFromStdin() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseDefaults(t *testing.T) { + tests := []struct { + name string + cfg interface{} + wantErr bool + }{ + { + name: "valid struct", + cfg: &TestConfig{}, + wantErr: false, + }, + { + name: "nil pointer", + cfg: nil, + wantErr: true, + }, + { + name: "non-pointer", + cfg: TestConfig{}, + wantErr: true, + }, + { + name: "pointer to non-struct", + cfg: new(string), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + err := parser.ParseDefaults(tt.cfg) + + if (err != nil) != tt.wantErr { + t.Errorf("ParseDefaults() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr && tt.cfg != nil { + cfg := tt.cfg.(*TestConfig) + if cfg.Host != "localhost" || cfg.Port != 8080 || cfg.LogLevel != "info" { + t.Error("Default values were not properly set") + } + } + }) + } +} + +// TestLoadconfig tests the Loadconfig method +func TestLoadconfig(t *testing.T) { + tests := []struct { + name string + cfg interface{} + content string + wantErr bool + }{ + { + name: "nil pointer", + cfg: nil, + content: "", + wantErr: true, + }, + { + name: "non-pointer", + cfg: TestConfig{}, + content: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Create and populate config file if content is provided + if tt.content != "" { + tempDir := t.TempDir() + filename := filepath.Join(tempDir, "dicedb.conf") + err := os.WriteFile(filename, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + err = parser.ParseFromFile(filename) + if err != nil { + t.Fatalf("Failed to parse test file: %v", err) + } + } + + err := parser.Loadconfig(tt.cfg) + if (err != nil) != tt.wantErr { + t.Errorf("Loadconfig() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr && tt.cfg != nil { + cfg := tt.cfg.(*TestConfig) + if tt.content != "" && (cfg.Host != "customhost" || cfg.Port != 9090 || cfg.LogLevel != "debug") { + t.Error("Config values were not properly loaded") + } + } + }) + } +} From ee1929e0a31a4e03698c9bbe216afefa472411c4 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 18:48:07 +0530 Subject: [PATCH 04/16] refactor: parser logic --- config/parser.go | 4 +- integration_tests/config/config_test.go | 1 + integration_tests/config/parser_test.go | 88 +++++++++++++++++++++++-- 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/config/parser.go b/config/parser.go index 082c589f6..132c5f64b 100644 --- a/config/parser.go +++ b/config/parser.go @@ -3,6 +3,7 @@ package config import ( "bufio" "fmt" + "log/slog" "os" "reflect" "strconv" @@ -89,7 +90,8 @@ func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { - return fmt.Errorf("invalid config line: %s", line) + slog.Warn("invalid config line", slog.String("line", line)) + continue } key := strings.TrimSpace(parts[0]) diff --git a/integration_tests/config/config_test.go b/integration_tests/config/config_test.go index c68dae6d0..c8063c0b0 100644 --- a/integration_tests/config/config_test.go +++ b/integration_tests/config/config_test.go @@ -61,6 +61,7 @@ func TestCreateConfigFile_InvalidPath(t *testing.T) { } } +// TestCreateConfigFile_NoPermission tests creation without write permissions func TestCreateConfigFile_NoPermission(t *testing.T) { if os.Getuid() == 0 { t.Skip("Skipping test when running as root") diff --git a/integration_tests/config/parser_test.go b/integration_tests/config/parser_test.go index 60bf6ab96..be594c34a 100644 --- a/integration_tests/config/parser_test.go +++ b/integration_tests/config/parser_test.go @@ -4,15 +4,89 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/dicedb/dice/config" ) // TestConfig is a test struct that mimics your actual config structure type TestConfig struct { - Host string `default:"localhost"` - Port int `default:"8080"` - LogLevel string `default:"info"` + Version string `config:"version" default:"0.0.5"` + InstanceID string `config:"instance_id"` + Auth auth `config:"auth"` + AsyncServer asyncServer `config:"async_server"` + HTTP http `config:"http"` + WebSocket websocket `config:"websocket"` + Performance performance `config:"performance"` + Memory memory `config:"memory"` + Persistence persistence `config:"persistence"` + Logging logging `config:"logging"` + Network network `config:"network"` +} + +type auth struct { + UserName string `config:"username" default:"dice"` + Password string `config:"password"` +} + +type asyncServer struct { + Addr string `config:"addr" default:"0.0.0.0"` + Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` + KeepAlive int32 `config:"keepalive" default:"300"` + Timeout int32 `config:"timeout" default:"300"` + MaxConn int32 `config:"max_conn" default:"0"` +} + +type http struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` +} + +type websocket struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` + MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` + WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` +} + +type performance struct { + WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` + ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` + MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` + MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` + EnableMultiThreading bool `config:"enable_multithreading" default:"false"` + StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` + AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` + EnableProfiling bool `config:"profiling" default:"false"` + EnableWatch bool `config:"enable_watch" default:"false"` + NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` +} + +type memory struct { + MaxMemory int64 `config:"max_memory" default:"0"` + EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` + EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` + KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` + LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` +} + +type persistence struct { + AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` + PersistenceEnabled bool `config:"persistence_enabled" default:"true"` + WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` + EnableWAL bool `config:"enable-wal" default:"false"` + WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` + RestoreFromWAL bool `config:"restore-wal" default:"false"` + WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` +} + +type logging struct { + LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` +} + +type network struct { + IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' + IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` } func TestNewConfigParser(t *testing.T) { @@ -46,7 +120,7 @@ log_level=debug`, content: `host=testhost invalid-line port=9090`, - wantErr: true, + wantErr: false, }, { name: "non-existent file", @@ -101,7 +175,7 @@ log_level=debug`, input: `host=testhost invalid-line port=9090`, - wantErr: true, + wantErr: false, }, } @@ -172,7 +246,7 @@ func TestParseDefaults(t *testing.T) { if !tt.wantErr && tt.cfg != nil { cfg := tt.cfg.(*TestConfig) - if cfg.Host != "localhost" || cfg.Port != 8080 || cfg.LogLevel != "info" { + if cfg.AsyncServer.Addr != "0.0.0.0" || cfg.AsyncServer.Port != 7379 || cfg.Logging.LogLevel != "info" { t.Error("Default values were not properly set") } } @@ -228,7 +302,7 @@ func TestLoadconfig(t *testing.T) { if !tt.wantErr && tt.cfg != nil { cfg := tt.cfg.(*TestConfig) - if tt.content != "" && (cfg.Host != "customhost" || cfg.Port != 9090 || cfg.LogLevel != "debug") { + if tt.content != "" && (cfg.AsyncServer.Addr != "customhost" || cfg.AsyncServer.Port != 9090 || cfg.Logging.LogLevel != "debug") { t.Error("Config values were not properly loaded") } } From 44ba27d388019859f43bc303e4f721903e5fc949 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 19:01:02 +0530 Subject: [PATCH 05/16] fix: default behaviour --- config/config.go | 3 ++- internal/cli/cli.go | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 6006143fb..aa522f6e4 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ import ( const ( DiceDBVersion string = "0.0.5" DefaultConfigName string = "dicedb.conf" + DefaultConfigDir string = "." EvictSimpleFirst string = "simple-first" EvictAllKeysRandom string = "allkeys-random" @@ -181,7 +182,7 @@ logging.log_level = "info" # Authentication Configuration auth.username = "dice" -auth.password = "vinit" +auth.password = "" # Network Configuration network.io_buffer_length = 512 diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 1c211f264..13b2bfa64 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "path/filepath" "runtime" "strings" @@ -101,10 +102,7 @@ func render() { func Execute() { if len(os.Args) < 2 { - parser := config.NewConfigParser() - if err := parser.ParseDefaults(config.DiceConfig); err != nil { - log.Fatal(err) - } + config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")) render() return From 6dcb41df63237b096916cd1f3b6c316e903e30cd Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Fri, 15 Nov 2024 19:10:39 +0530 Subject: [PATCH 06/16] fixing minor issues --- config/config.go | 36 +++++++++++++++++++++++------------- dicedb.conf | 6 ++++++ internal/cli/cli.go | 45 +++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/config/config.go b/config/config.go index aa522f6e4..5ece4a6fa 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "log" "log/slog" "os" @@ -86,7 +87,7 @@ type persistence struct { AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` PersistenceEnabled bool `config:"persistence_enabled" default:"true"` WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` - EnableWAL bool `config:"enable-wal" default:"false"` + EnableWAL bool `config:"enable-wal" default:"true"` WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` RestoreFromWAL bool `config:"restore-wal" default:"false"` WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` @@ -102,33 +103,37 @@ type network struct { } func init() { - configFilePath := filepath.Join(".", DefaultConfigName) + configFilePath := filepath.Join(DefaultConfigDir, DefaultConfigName) if err := loadDiceConfig(configFilePath); err != nil { - log.Fatalf("Failed to load configuration: %v", err) + log.Fatalf("failed to load configuration: %v", err) } } // DiceConfig is the global configuration object for dice var DiceConfig = &Config{} -func CreateConfigFile(configFilePath string) { +func CreateConfigFile(configFilePath string) error { + // Check if the config file already exists if _, err := os.Stat(configFilePath); err == nil { - slog.Warn("config file already exists", slog.String("path", configFilePath)) if err := loadDiceConfig(configFilePath); err != nil { - log.Fatalf("Failed to load configuration: %v", err) + return fmt.Errorf("failed to load existing configuration: %w", err) } - return + return nil } + // Attempt to write a new config file if err := writeConfigFile(configFilePath); err != nil { - slog.Warn("starting DiceDB with default configurations.", slog.Any("error", err)) - return + slog.Warn("Failed to create config file, starting with defaults.", slog.Any("error", err)) + return nil // Continuing with defaults; may reconsider behavior. } + // Load the new configuration if err := loadDiceConfig(configFilePath); err != nil { - log.Fatalf("Failed to load configuration: %v", err) + return fmt.Errorf("failed to load newly created configuration: %w", err) } - slog.Info("config file created at", slog.Any("path", configFilePath)) + + slog.Info("Config file successfully created.", slog.String("path", configFilePath)) + return nil } // writeConfigFile writes the default configuration to the specified file path @@ -164,6 +169,8 @@ performance.enable_multithreading = false performance.store_map_init_size = 1024000 performance.adhoc_req_chan_buf_size = 20 performance.enable_profiling = false +performance.enable_watch = false +performance.num_shards = -1 # Memory Configuration memory.max_memory = 0 @@ -176,6 +183,10 @@ memory.lfu_log_factor = 10 persistence.aof_file = "./dice-master.aof" persistence.persistence_enabled = true persistence.write_aof_on_cleanup = false +persistence.enable-wal = true +persistence.wal-dir = "./" +persistence.restore-wal = false +persistence.wal-engine = "aof" # Logging Configuration logging.log_level = "info" @@ -186,8 +197,7 @@ auth.password = "" # Network Configuration network.io_buffer_length = 512 -network.io_buffer_length_max = 51200 -` +network.io_buffer_length_max = 51200` // Check if the directory exists or not dir := filepath.Dir(configFilePath) diff --git a/dicedb.conf b/dicedb.conf index 796c9e300..627d19fc9 100644 --- a/dicedb.conf +++ b/dicedb.conf @@ -29,6 +29,8 @@ performance.enable_multithreading = false performance.store_map_init_size = 1024000 performance.adhoc_req_chan_buf_size = 20 performance.enable_profiling = false +performance.enable_watch = false +performance.num_shards = -1 # Memory Configuration memory.max_memory = 0 @@ -41,6 +43,10 @@ memory.lfu_log_factor = 10 persistence.aof_file = "./dice-master.aof" persistence.persistence_enabled = true persistence.write_aof_on_cleanup = false +persistence.enable-wal = true +persistence.wal-dir = "./" +persistence.restore-wal = false +persistence.wal-engine = "aof" # Logging Configuration logging.log_level = "info" diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 13b2bfa64..34c956846 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "log" + "log/slog" "os" "path/filepath" "runtime" @@ -45,7 +46,7 @@ func configuration() { } // Add whether the watch feature is enabled to the configuration table - // addEntry("Watch Enabled", config.EnableWatch) + addEntry("Watch Enabled", config.DiceConfig.Performance.EnableWatch) // Add whether the watch feature is enabled to the configuration table addEntry("HTTP Enabled", config.DiceConfig.HTTP.Enabled) @@ -102,8 +103,9 @@ func render() { func Execute() { if len(os.Args) < 2 { - config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")) - + if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { + log.Fatal(err) + } render() return } @@ -127,7 +129,6 @@ func Execute() { } fmt.Println(config.DiceConfig.Version) case "-o", "--output": - // TODO: Implement output to file of config if len(os.Args) < 3 { log.Fatal("Output file path not provided") } else { @@ -146,32 +147,37 @@ func Execute() { log.Fatal("Output file path is not a directory") } - var filePath string - if strings.HasSuffix(dirPath, "/") { - filePath = dirPath + "dicedb.conf" - } else { - filePath = dirPath + "/dicedb.conf" + filePath := filepath.Join(dirPath, "dicedb.conf") + if _, err := os.Stat(filePath); err == nil { + slog.Warn("Config file already exists at the specified path", slog.String("path", filePath), slog.String("action", "skipping file creation")) + return + } + if err := config.CreateConfigFile(filePath); err != nil { + log.Fatal(err) } - config.CreateConfigFile(filePath) render() } case "-c", "--config": if len(os.Args) >= 3 { filePath := os.Args[2] if filePath == "" { - log.Fatal("Config file path not provided") + log.Fatal("Error: Config file path not provided") } info, err := os.Stat(filePath) - if err != nil { - if os.IsNotExist(err) { - log.Fatal("Config file does not exist") - } - log.Fatalf("Error checking config file: %v", err) + switch { + case os.IsNotExist(err): + log.Fatalf("Config file does not exist: %s", filePath) + case err != nil: + log.Fatalf("Unable to check config file: %v", err) } - if info.IsDir() || !strings.HasSuffix(filePath, ".conf") { - log.Fatal("Config file must be a regular file with a .conf extension") + if info.IsDir() { + log.Fatalf("Config file path points to a directory: %s", filePath) + } + + if !strings.HasSuffix(filePath, ".conf") { + log.Fatalf("Config file must have a .conf extension: %s", filePath) } parser := config.NewConfigParser() @@ -181,7 +187,6 @@ func Execute() { if err := parser.Loadconfig(config.DiceConfig); err != nil { log.Fatal(err) } - render() } else { log.Fatal("Config file path not provided") @@ -195,7 +200,7 @@ func Execute() { func printUsage() { fmt.Println(`Usage: ./dicedb [/path/to/dice.conf] [options] [-] - ./dicedb - (read config from stdin) + ./dicedb - (read config from stdin) e.g. echo "version=1.0" | ./dicedb - ./dicedb -v or --version ./dicedb -h or --help`) } From ea1ba4f87b07911abc2bc513819ffe4ca1f52894 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Sun, 17 Nov 2024 17:16:01 +0530 Subject: [PATCH 07/16] fixing minor issues --- config/config.go | 46 ++++++++++++++++-- go.mod | 1 + go.sum | 2 + internal/cli/cli.go | 112 ++++++++++++++++++++++++++++++++++++-------- 4 files changed, 136 insertions(+), 25 deletions(-) diff --git a/config/config.go b/config/config.go index 5ece4a6fa..14474a26c 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" "time" + + "github.com/dicedb/dice/internal/server/utils" ) const ( @@ -18,11 +20,45 @@ const ( EvictAllKeysRandom string = "allkeys-random" EvictAllKeysLRU string = "allkeys-lru" EvictAllKeysLFU string = "allkeys-lfu" + EvictBatchKeysLRU string = "batch_keys_lru" DefaultKeysLimit int = 200000000 DefaultEvictionRatio float64 = 0.1 ) +var ( + Host string = "0.0.0.0" + Port int = 7379 + + EnableMultiThreading bool = false + EnableHTTP bool = true + HTTPPort int = 8082 + + EnableWebsocket bool = true + WebsocketPort int = 8379 + NumShards int = -1 + + // if RequirePass is set to an empty string, no authentication is required + RequirePass = utils.EmptyStr + + CustomConfigFilePath = utils.EmptyStr + FileLocation = utils.EmptyStr + + InitConfigCmd = false + + KeysLimit = DefaultKeysLimit + EvictionRatio = DefaultEvictionRatio + + EnableProfiling = false + + EnableWatch = true + LogDir = "" + + EnableWAL = true + RestoreFromWAL = false + WALEngine = "sqlite" +) + type Config struct { Version string `config:"version" default:"0.0.5"` InstanceID string `config:"instance_id"` @@ -43,8 +79,8 @@ type auth struct { } type asyncServer struct { - Addr string `config:"addr" default:"0.0.0.0"` - Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` + Addr string `config:"addr" default:"0.0.0.0" validate:"ipv4"` + Port int `config:"port" default:"7379" validate:"number,gte=0,lte=65535"` KeepAlive int32 `config:"keepalive" default:"300"` Timeout int32 `config:"timeout" default:"300"` MaxConn int32 `config:"max_conn" default:"0"` @@ -52,12 +88,12 @@ type asyncServer struct { type http struct { Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` + Port int `config:"port" default:"8082" validate:"number,gte=0,lte=65535"` } type websocket struct { Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` + Port int `config:"port" default:"8379" validate:"number,gte=0,lte=65535"` MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` } @@ -76,7 +112,7 @@ type performance struct { } type memory struct { - MaxMemory int64 `config:"max_memory" default:"0"` + MaxMemory int64 `config:"max_memory" default:"0" validate:"min=0"` EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` diff --git a/go.mod b/go.mod index a193be0cf..834d395d2 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 + github.com/fatih/color v1.18.0 github.com/go-playground/validator/v10 v10.22.1 github.com/gobwas/glob v0.2.3 github.com/google/btree v1.1.3 diff --git a/go.sum b/go.sum index 8a48e609c..9d626bfc8 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 h1:JvnAibMNGA0vQH+T47Y/d5/POURIvfJl3fFk0GIEBkQ= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3/go.mod h1:p7x5/3S6wBEmiRMwxavj1I1P1xsSVQS6fcSbeai5ic4= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 34c956846..7377e75a5 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1,6 +1,7 @@ package cli import ( + "flag" "fmt" "log" "log/slog" @@ -10,6 +11,7 @@ import ( "strings" "github.com/dicedb/dice/config" + "github.com/fatih/color" ) type configEntry struct { @@ -61,6 +63,7 @@ func addEntry(k string, v interface{}) { // printConfigTable prints key-value pairs in a vertical table format. func render() { + color.Set(color.FgHiRed) fmt.Print(` ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ @@ -70,11 +73,21 @@ func render() { ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ `) - configuration() + color.Unset() + renderConfigTable() +} +func renderConfigTable() { + configuration() + color.Set(color.FgGreen) // Find the longest key to align the values properly + // Default value length for alignment + // Create the table header and separator line + // 7 is for spacing and pipes + // Print each configuration key-value pair without row lines + // Final bottom line maxKeyLength := 0 - maxValueLength := 20 // Default value length for alignment + maxValueLength := 20 for _, entry := range configTable { if len(entry.Key) > maxKeyLength { maxKeyLength = len(entry.Key) @@ -84,24 +97,96 @@ func render() { } } - // Create the table header and separator line + color.Set(color.FgGreen) fmt.Println() - totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes + totalWidth := maxKeyLength + maxValueLength + 7 fmt.Println(strings.Repeat("-", totalWidth)) fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") fmt.Println(strings.Repeat("-", totalWidth)) - // Print each configuration key-value pair without row lines for _, entry := range configTable { fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) } - // Final bottom line fmt.Println(strings.Repeat("-", totalWidth)) fmt.Println() + color.Unset() } func Execute() { + flag.StringVar(&config.Host, "host", "0.0.0.0", "host for the DiceDB server") + + flag.IntVar(&config.Port, "port", 7379, "port for the DiceDB server") + + flag.IntVar(&config.HTTPPort, "http-port", 7380, "port for accepting requets over HTTP") + flag.BoolVar(&config.EnableHTTP, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") + + flag.IntVar(&config.WebsocketPort, "websocket-port", 7381, "port for accepting requets over WebSocket") + flag.BoolVar(&config.EnableWebsocket, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") + + flag.BoolVar(&config.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") + flag.IntVar(&config.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") + + flag.BoolVar(&config.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") + flag.BoolVar(&config.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") + + flag.StringVar(&config.DiceConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") + flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") + + flag.BoolVar(&config.EnableWAL, "enable-wal", false, "enable write-ahead logging") + flag.BoolVar(&config.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") + flag.StringVar(&config.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") + + flag.StringVar(&config.RequirePass, "requirepass", config.RequirePass, "enable authentication for the default user") + flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the config file") + flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") + flag.BoolVar(&config.InitConfigCmd, "init-config", false, "initialize a new config file") + + flag.IntVar(&config.KeysLimit, "keys-limit", config.KeysLimit, "keys limit for the DiceDB server. "+ + "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ + "total number of shard threads to estimate how much memory will be required at system start up.") + flag.Float64Var(&config.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ + "keys limit is reached") + + flag.Usage = func() { + color.Set(color.FgYellow) + fmt.Println("Usage: ./dicedb [options] [config-file]") + color.Unset() + + color.Set(color.FgGreen) + fmt.Println("Options:") + color.Unset() + + color.Set(color.FgCyan) + fmt.Println(" -v, --version Show the version of DiceDB") + fmt.Println(" -h, --help Show this help message") + fmt.Println(" -host Host for the DiceDB server (default: \"0.0.0.0\")") + fmt.Println(" -port Port for the DiceDB server (default: 7379)") + fmt.Println(" -http-port Port for accepting requests over HTTP (default: 7380)") + fmt.Println(" -enable-http Enable DiceDB to listen, accept, and process HTTP (default: false)") + fmt.Println(" -websocket-port Port for accepting requests over WebSocket (default: 7381)") + fmt.Println(" -enable-websocket Enable DiceDB to listen, accept, and process WebSocket (default: false)") + fmt.Println(" -enable-multithreading Enable multithreading execution and leverage multiple CPU cores (default: false)") + fmt.Println(" -num-shards Number of shards to create. Defaults to number of cores (default: -1)") + fmt.Println(" -enable-watch Enable support for .WATCH commands and real-time reactivity (default: false)") + fmt.Println(" -enable-profiling Enable profiling and capture critical metrics and traces in .prof files (default: false)") + fmt.Println(" -log-level Log level, values: info, debug (default: \"info\")") + fmt.Println(" -log-dir Log directory path (default: \"/tmp/dicedb\")") + fmt.Println(" -enable-wal Enable write-ahead logging (default: false)") + fmt.Println(" -restore-wal Restore the database from the WAL files (default: false)") + fmt.Println(" -wal-engine WAL engine to use, values: sqlite, aof (default: \"null\")") + fmt.Println(" -requirepass Enable authentication for the default user (default: \"\")") + fmt.Println(" -o Directory path to create the config file (default: \"\")") + fmt.Println(" -c File path of the config file (default: \"\")") + fmt.Println(" -init-config Initialize a new config file (default: false)") + fmt.Println(" -keys-limit Keys limit for the DiceDB server (default: 0)") + fmt.Println(" -eviction-ratio Ratio of keys to evict when the keys limit is reached (default: 0.1)") + color.Unset() + os.Exit(0) + } + + flag.Parse() + if len(os.Args) < 2 { if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { log.Fatal(err) @@ -115,10 +200,6 @@ func Execute() { fmt.Println("dicedb version", config.DiceDBVersion) os.Exit(0) - case "-h", "--help": - printUsage() - os.Exit(0) - case "-": parser := config.NewConfigParser() if err := parser.ParseFromStdin(); err != nil { @@ -191,16 +272,7 @@ func Execute() { } else { log.Fatal("Config file path not provided") } - default: - fmt.Printf("Unknown option: %s\n", os.Args[1]) - printUsage() + render() } } - -func printUsage() { - fmt.Println(`Usage: ./dicedb [/path/to/dice.conf] [options] [-] - ./dicedb - (read config from stdin) e.g. echo "version=1.0" | ./dicedb - - ./dicedb -v or --version - ./dicedb -h or --help`) -} From 0f06f323f35f10139042404ce7b3c0ff3ba7dc86 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 12:00:11 +0530 Subject: [PATCH 08/16] fixing redundent type warning --- integration_tests/commands/http/json_test.go | 52 ++++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/integration_tests/commands/http/json_test.go b/integration_tests/commands/http/json_test.go index f337613ef..8a3b862ec 100644 --- a/integration_tests/commands/http/json_test.go +++ b/integration_tests/commands/http/json_test.go @@ -492,7 +492,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), nil}, assertType: []string{"equal", "equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -505,7 +505,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom","address":{"zip":"10001"}}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -518,7 +518,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"flag":true}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -531,7 +531,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -544,7 +544,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"age":28}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -567,7 +567,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(2), `{"bosses":{"hobby":"swim"}}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -580,7 +580,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -593,7 +593,7 @@ func TestJSONDel(t *testing.T) { expected: []interface{}{float64(1), `{"name":"sugar"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, } @@ -619,7 +619,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), nil}, assertType: []string{"equal", "equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -632,7 +632,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom","address":{"zip":"10001"}}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -645,7 +645,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"flag":true}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -658,7 +658,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -671,7 +671,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"age":28}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -694,7 +694,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(2), `{"bosses":{"hobby":"swim"}}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -707,7 +707,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"name":"Tom"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -720,7 +720,7 @@ func TestJSONForget(t *testing.T) { expected: []interface{}{float64(1), `{"name":"sugar"}`}, assertType: []string{"equal", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, } @@ -748,7 +748,7 @@ func TestJSONTOGGLE(t *testing.T) { expected: []interface{}{[]any{float64(0)}}, assertType: []string{"jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "user"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "user"}}, }, }, { @@ -792,7 +792,7 @@ func TestJSONTOGGLE(t *testing.T) { }, assertType: []string{"jsoneq", "jsoneq", "jsoneq"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "user"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "user"}}, }, }, } @@ -1112,7 +1112,7 @@ func TestJsonNumMultBy(t *testing.T) { expected: []interface{}{"[]"}, assertType: []string{"equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1124,7 +1124,7 @@ func TestJsonNumMultBy(t *testing.T) { expected: []interface{}{"ERR expected value at line 1 column 1"}, assertType: []string{"equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1137,7 +1137,7 @@ func TestJsonNumMultBy(t *testing.T) { expected: []interface{}{"[4,null,10,null]", `{"a":"b","b":[{"a":4},{"a":10},{"a":"c"}]}`}, assertType: []string{"perm_equal", "json_equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1150,7 +1150,7 @@ func TestJsonNumMultBy(t *testing.T) { expected: []interface{}{"[null]", `{"a":"b","b":[{"a":2},{"a":5},{"a":"c"}]}`}, assertType: []string{"perm_equal", "json_equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, } @@ -1355,7 +1355,7 @@ func TestJSONNumIncrBy(t *testing.T) { expected: []interface{}{"ERR expected value at line 1 column 1"}, assertType: []string{"equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1384,7 +1384,7 @@ func TestJSONNumIncrBy(t *testing.T) { expected: []interface{}{"[4.2,7,null,null]", "[null]", "[3.2,6,null,null]", `{"a":"b","b":[{"a":3.2},{"a":6},{"a":"c"}]}`}, assertType: []string{"perm_equal", "perm_equal", "perm_equal", "json_equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1398,7 +1398,7 @@ func TestJSONNumIncrBy(t *testing.T) { expected: []interface{}{"[3]", "[2]", "2"}, assertType: []string{"perm_equal", "perm_equal", "json_equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, { @@ -1412,7 +1412,7 @@ func TestJSONNumIncrBy(t *testing.T) { expected: []interface{}{"[3.5]", "[2.0]", "2.0"}, assertType: []string{"perm_equal", "perm_equal", "json_equal"}, cleanUp: []HTTPCommand{ - HTTPCommand{Command: "DEL", Body: map[string]interface{}{"key": "k"}}, + {Command: "DEL", Body: map[string]interface{}{"key": "k"}}, }, }, } From b5a0e5250ed64d558800deea76c3eb9a5b1015e6 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 12:01:11 +0530 Subject: [PATCH 09/16] fixing flags priority --- config/config.go | 194 ++++++++++-------- integration_tests/commands/async/setup.go | 8 + integration_tests/commands/http/setup.go | 7 + integration_tests/commands/resp/setup.go | 8 + integration_tests/commands/websocket/setup.go | 8 + integration_tests/server/setup.go | 14 ++ internal/cli/cli.go | 42 ++-- 7 files changed, 180 insertions(+), 101 deletions(-) create mode 100644 integration_tests/server/setup.go diff --git a/config/config.go b/config/config.go index 14474a26c..e9cd8b0b0 100644 --- a/config/config.go +++ b/config/config.go @@ -1,8 +1,8 @@ package config import ( + "flag" "fmt" - "log" "log/slog" "os" "path/filepath" @@ -12,18 +12,80 @@ import ( ) const ( - DiceDBVersion string = "0.0.5" - DefaultConfigName string = "dicedb.conf" - DefaultConfigDir string = "." + DiceDBVersion = "0.0.5" + DefaultConfigName = "dicedb.conf" + DefaultConfigDir = "." - EvictSimpleFirst string = "simple-first" - EvictAllKeysRandom string = "allkeys-random" - EvictAllKeysLRU string = "allkeys-lru" - EvictAllKeysLFU string = "allkeys-lfu" - EvictBatchKeysLRU string = "batch_keys_lru" + EvictSimpleFirst = "simple-first" + EvictAllKeysRandom = "allkeys-random" + EvictAllKeysLRU = "allkeys-lru" + EvictAllKeysLFU = "allkeys-lfu" + EvictBatchKeysLRU = "batch_keys_lru" DefaultKeysLimit int = 200000000 DefaultEvictionRatio float64 = 0.1 + + defaultConfigTemplate = `# Configuration file for Dicedb + +# Version +version = "0.0.5" + +# Async Server Configuration +async_server.addr = "0.0.0.0" +async_server.port = 7379 +async_server.keepalive = 300 +async_server.timeout = 300 +async_server.max_conn = 0 + +# HTTP Configuration +http.enabled = false +http.port = 8082 + +# WebSocket Configuration +websocket.enabled = false +websocket.port = 8379 +websocket.max_write_response_retries = 3 +websocket.write_response_timeout = 10s + +# Performance Configuration +performance.watch_chan_buf_size = 20000 +performance.shard_cron_frequency = 1s +performance.multiplexer_poll_timeout = 100ms +performance.max_clients = 20000 +performance.enable_multithreading = false +performance.store_map_init_size = 1024000 +performance.adhoc_req_chan_buf_size = 20 +performance.enable_profiling = false +performance.enable_watch = false +performance.num_shards = -1 + +# Memory Configuration +memory.max_memory = 0 +memory.eviction_policy = "allkeys-lfu" +memory.eviction_ratio = 0.9 +memory.keys_limit = 200000000 +memory.lfu_log_factor = 10 + +# Persistence Configuration +persistence.aof_file = "./dice-master.aof" +persistence.persistence_enabled = true +persistence.write_aof_on_cleanup = false +persistence.enable-wal = true +persistence.wal-dir = "./" +persistence.restore-wal = false +persistence.wal-engine = "aof" + +# Logging Configuration +logging.log_level = "info" +logging.log_dir = "/tmp/dicedb" + +# Authentication Configuration +auth.username = "dice" +auth.password = "" + +# Network Configuration +network.io_buffer_length = 512 +network.io_buffer_length_max = 51200` ) var ( @@ -44,8 +106,6 @@ var ( CustomConfigFilePath = utils.EmptyStr FileLocation = utils.EmptyStr - InitConfigCmd = false - KeysLimit = DefaultKeysLimit EvictionRatio = DefaultEvictionRatio @@ -131,6 +191,7 @@ type persistence struct { type logging struct { LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` + LogDir string `config:"log_dir" default:"/tmp/dicedb" validate:"dirpath"` } type network struct { @@ -138,13 +199,6 @@ type network struct { IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` } -func init() { - configFilePath := filepath.Join(DefaultConfigDir, DefaultConfigName) - if err := loadDiceConfig(configFilePath); err != nil { - log.Fatalf("failed to load configuration: %v", err) - } -} - // DiceConfig is the global configuration object for dice var DiceConfig = &Config{} @@ -174,67 +228,6 @@ func CreateConfigFile(configFilePath string) error { // writeConfigFile writes the default configuration to the specified file path func writeConfigFile(configFilePath string) error { - content := `# Configuration file for Dicedb - -# Version -version = "0.0.5" - -# Async Server Configuration -async_server.addr = "0.0.0.0" -async_server.port = 7379 -async_server.keepalive = 300 -async_server.timeout = 300 -async_server.max_conn = 0 - -# HTTP Configuration -http.enabled = false -http.port = 8082 - -# WebSocket Configuration -websocket.enabled = false -websocket.port = 8379 -websocket.max_write_response_retries = 3 -websocket.write_response_timeout = 10s - -# Performance Configuration -performance.watch_chan_buf_size = 20000 -performance.shard_cron_frequency = 1s -performance.multiplexer_poll_timeout = 100ms -performance.max_clients = 20000 -performance.enable_multithreading = false -performance.store_map_init_size = 1024000 -performance.adhoc_req_chan_buf_size = 20 -performance.enable_profiling = false -performance.enable_watch = false -performance.num_shards = -1 - -# Memory Configuration -memory.max_memory = 0 -memory.eviction_policy = "allkeys-lfu" -memory.eviction_ratio = 0.9 -memory.keys_limit = 200000000 -memory.lfu_log_factor = 10 - -# Persistence Configuration -persistence.aof_file = "./dice-master.aof" -persistence.persistence_enabled = true -persistence.write_aof_on_cleanup = false -persistence.enable-wal = true -persistence.wal-dir = "./" -persistence.restore-wal = false -persistence.wal-engine = "aof" - -# Logging Configuration -logging.log_level = "info" - -# Authentication Configuration -auth.username = "dice" -auth.password = "" - -# Network Configuration -network.io_buffer_length = 512 -network.io_buffer_length_max = 51200` - // Check if the directory exists or not dir := filepath.Dir(configFilePath) if _, err := os.Stat(dir); err != nil { @@ -248,7 +241,7 @@ network.io_buffer_length_max = 51200` } defer file.Close() - if _, err := file.WriteString(content); err != nil { + if _, err := file.WriteString(defaultConfigTemplate); err != nil { return err } @@ -265,6 +258,47 @@ func loadDiceConfig(configFilePath string) error { return parser.Loadconfig(DiceConfig) } +func MergeFlags(flags *Config) { + flagset := flag.CommandLine + flagset.Visit(func(f *flag.Flag) { + // updating values for flags that were explicitly set by the user + switch f.Name { + case "host": + DiceConfig.AsyncServer.Addr = flags.AsyncServer.Addr + case "port": + DiceConfig.AsyncServer.Port = flags.AsyncServer.Port + case "enable-http": + DiceConfig.HTTP.Enabled = flags.HTTP.Enabled + case "http-port": + DiceConfig.HTTP.Port = flags.HTTP.Port + case "enable-websocket": + DiceConfig.WebSocket.Enabled = flags.WebSocket.Enabled + case "websocket-port": + DiceConfig.WebSocket.Port = flags.WebSocket.Port + case "num-shards": + DiceConfig.Performance.NumShards = flags.Performance.NumShards + case "require-pass": + DiceConfig.Auth.Password = flags.Auth.Password + case "keys-limit": + DiceConfig.Memory.KeysLimit = flags.Memory.KeysLimit + case "eviction-ratio": + DiceConfig.Memory.EvictionRatio = flags.Memory.EvictionRatio + case "enable-profiling": + DiceConfig.Performance.EnableProfiling = flags.Performance.EnableProfiling + case "enable-watch": + DiceConfig.Performance.EnableWatch = flags.Performance.EnableWatch + case "log-dir": + DiceConfig.Logging.LogDir = flags.Logging.LogDir + case "enable-wal": + DiceConfig.Persistence.EnableWAL = flags.Persistence.EnableWAL + case "restore-from-wal": + DiceConfig.Persistence.RestoreFromWAL = flags.Persistence.RestoreFromWAL + case "wal-engine": + DiceConfig.Persistence.WALEngine = flags.Persistence.WALEngine + } + }) +} + // This function returns the config file path based on the OS // func getConfigPath() string { // switch runtime.GOOS { diff --git a/integration_tests/commands/async/setup.go b/integration_tests/commands/async/setup.go index b8a3ea684..93df39f22 100644 --- a/integration_tests/commands/async/setup.go +++ b/integration_tests/commands/async/setup.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log" "log/slog" "net" "os" @@ -21,6 +22,13 @@ import ( dicedb "github.com/dicedb/dicedb-go" ) +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} + type TestServerOptions struct { Port int MaxClients int32 diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index d69f78e3a..d27c67fbd 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -36,6 +36,13 @@ type HTTPCommandExecutor struct { baseURL string } +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} + func NewHTTPCommandExecutor() *HTTPCommandExecutor { return &HTTPCommandExecutor{ baseURL: "http://localhost:8083", diff --git a/integration_tests/commands/resp/setup.go b/integration_tests/commands/resp/setup.go index 131bf7bd6..093fbe0d4 100644 --- a/integration_tests/commands/resp/setup.go +++ b/integration_tests/commands/resp/setup.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log" "log/slog" "net" "os" @@ -29,6 +30,13 @@ type TestServerOptions struct { Port int } +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} + // getLocalConnection returns a local TCP connection to the database // //nolint:unused diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index b8ea19b8f..cbfcc286a 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "log/slog" "net/http" "sync" @@ -40,6 +41,13 @@ type WebsocketCommandExecutor struct { upgrader websocket.Upgrader } +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} + func NewWebsocketCommandExecutor() *WebsocketCommandExecutor { return &WebsocketCommandExecutor{ baseURL: URL, diff --git a/integration_tests/server/setup.go b/integration_tests/server/setup.go new file mode 100644 index 000000000..b3afe6a18 --- /dev/null +++ b/integration_tests/server/setup.go @@ -0,0 +1,14 @@ +package server + +import ( + "log" + + "github.com/dicedb/dice/config" +) + +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 7377e75a5..da49a5fa4 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/dicedb/dice/config" + "github.com/dicedb/dice/internal/server/utils" "github.com/fatih/color" ) @@ -114,38 +115,38 @@ func renderConfigTable() { } func Execute() { - flag.StringVar(&config.Host, "host", "0.0.0.0", "host for the DiceDB server") + flagsConfig := config.Config{} + flag.StringVar(&flagsConfig.AsyncServer.Addr, "host", "0.0.0.0", "host for the DiceDB server") - flag.IntVar(&config.Port, "port", 7379, "port for the DiceDB server") + flag.IntVar(&flagsConfig.AsyncServer.Port, "port", 7379, "port for the DiceDB server") - flag.IntVar(&config.HTTPPort, "http-port", 7380, "port for accepting requets over HTTP") - flag.BoolVar(&config.EnableHTTP, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") + flag.IntVar(&flagsConfig.HTTP.Port, "http-port", 7380, "port for accepting requets over HTTP") + flag.BoolVar(&flagsConfig.HTTP.Enabled, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") - flag.IntVar(&config.WebsocketPort, "websocket-port", 7381, "port for accepting requets over WebSocket") - flag.BoolVar(&config.EnableWebsocket, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") + flag.IntVar(&flagsConfig.WebSocket.Port, "websocket-port", 7381, "port for accepting requets over WebSocket") + flag.BoolVar(&flagsConfig.WebSocket.Enabled, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") - flag.BoolVar(&config.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") - flag.IntVar(&config.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") + flag.BoolVar(&flagsConfig.Performance.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") + flag.IntVar(&flagsConfig.Performance.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") - flag.BoolVar(&config.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") - flag.BoolVar(&config.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") + flag.BoolVar(&flagsConfig.Performance.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") + flag.BoolVar(&flagsConfig.Performance.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") - flag.StringVar(&config.DiceConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") + flag.StringVar(&flagsConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") - flag.BoolVar(&config.EnableWAL, "enable-wal", false, "enable write-ahead logging") - flag.BoolVar(&config.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") - flag.StringVar(&config.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") + flag.BoolVar(&flagsConfig.Persistence.EnableWAL, "enable-wal", false, "enable write-ahead logging") + flag.BoolVar(&flagsConfig.Persistence.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") + flag.StringVar(&flagsConfig.Persistence.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") - flag.StringVar(&config.RequirePass, "requirepass", config.RequirePass, "enable authentication for the default user") - flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the config file") + flag.StringVar(&flagsConfig.Auth.Password, "requirepass", utils.EmptyStr, "enable authentication for the default user") + flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the flagsConfig file") flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") - flag.BoolVar(&config.InitConfigCmd, "init-config", false, "initialize a new config file") - flag.IntVar(&config.KeysLimit, "keys-limit", config.KeysLimit, "keys limit for the DiceDB server. "+ + flag.IntVar(&flagsConfig.Memory.KeysLimit, "keys-limit", config.DefaultKeysLimit, "keys limit for the DiceDB server. "+ "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ "total number of shard threads to estimate how much memory will be required at system start up.") - flag.Float64Var(&config.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ + flag.Float64Var(&flagsConfig.Memory.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ "keys limit is reached") flag.Usage = func() { @@ -178,7 +179,6 @@ func Execute() { fmt.Println(" -requirepass Enable authentication for the default user (default: \"\")") fmt.Println(" -o Directory path to create the config file (default: \"\")") fmt.Println(" -c File path of the config file (default: \"\")") - fmt.Println(" -init-config Initialize a new config file (default: false)") fmt.Println(" -keys-limit Keys limit for the DiceDB server (default: 0)") fmt.Println(" -eviction-ratio Ratio of keys to evict when the keys limit is reached (default: 0.1)") color.Unset() @@ -186,7 +186,7 @@ func Execute() { } flag.Parse() - + config.MergeFlags(&flagsConfig) if len(os.Args) < 2 { if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { log.Fatal(err) From 8f6e5ba0fe3eadb0185423e0f323906f0155011d Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 12:08:05 +0530 Subject: [PATCH 10/16] fixing flag name --- config/config.go | 28 +--------------------------- internal/cli/cli.go | 4 ++-- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/config/config.go b/config/config.go index e9cd8b0b0..ff5a17a1c 100644 --- a/config/config.go +++ b/config/config.go @@ -89,34 +89,8 @@ network.io_buffer_length_max = 51200` ) var ( - Host string = "0.0.0.0" - Port int = 7379 - - EnableMultiThreading bool = false - EnableHTTP bool = true - HTTPPort int = 8082 - - EnableWebsocket bool = true - WebsocketPort int = 8379 - NumShards int = -1 - - // if RequirePass is set to an empty string, no authentication is required - RequirePass = utils.EmptyStr - CustomConfigFilePath = utils.EmptyStr - FileLocation = utils.EmptyStr - - KeysLimit = DefaultKeysLimit - EvictionRatio = DefaultEvictionRatio - - EnableProfiling = false - - EnableWatch = true - LogDir = "" - - EnableWAL = true - RestoreFromWAL = false - WALEngine = "sqlite" + CustomConfigDirPath = utils.EmptyStr ) type Config struct { diff --git a/internal/cli/cli.go b/internal/cli/cli.go index da49a5fa4..fd93fb21f 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -133,7 +133,7 @@ func Execute() { flag.BoolVar(&flagsConfig.Performance.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") flag.StringVar(&flagsConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") - flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") + flag.StringVar(&config.DiceConfig.Logging.LogDir, "log-dir", "/tmp/dicedb", "log directory path") flag.BoolVar(&flagsConfig.Persistence.EnableWAL, "enable-wal", false, "enable write-ahead logging") flag.BoolVar(&flagsConfig.Persistence.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") @@ -141,7 +141,7 @@ func Execute() { flag.StringVar(&flagsConfig.Auth.Password, "requirepass", utils.EmptyStr, "enable authentication for the default user") flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the flagsConfig file") - flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") + flag.StringVar(&config.CustomConfigDirPath, "c", config.CustomConfigDirPath, "file path of the config file") flag.IntVar(&flagsConfig.Memory.KeysLimit, "keys-limit", config.DefaultKeysLimit, "keys limit for the DiceDB server. "+ "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ From 126f31845c77358c027c7f27e25b56b396e01083 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 12:12:54 +0530 Subject: [PATCH 11/16] fixing unittest err due to config --- internal/clientio/resp_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/clientio/resp_test.go b/internal/clientio/resp_test.go index 96495e015..2ff0c71cc 100644 --- a/internal/clientio/resp_test.go +++ b/internal/clientio/resp_test.go @@ -3,14 +3,23 @@ package clientio_test import ( "bytes" "fmt" + "log" "math" "testing" + "github.com/dicedb/dice/config" "github.com/dicedb/dice/internal/clientio" "github.com/dicedb/dice/internal/server/utils" "github.com/stretchr/testify/assert" ) +func init() { + parser := config.NewConfigParser() + if err := parser.ParseDefaults(config.DiceConfig); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } +} + func TestSimpleStringDecode(t *testing.T) { cases := map[string]string{ "+OK\r\n": "OK", From d1f309d8627a944a62debcc6dade746de3c49e2f Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 13:02:12 +0530 Subject: [PATCH 12/16] fixing async/hello_test.go --- integration_tests/commands/async/hello_test.go | 4 +++- integration_tests/commands/async/setup.go | 3 +-- internal/eval/eval.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/integration_tests/commands/async/hello_test.go b/integration_tests/commands/async/hello_test.go index 263865f35..bca2c91e0 100644 --- a/integration_tests/commands/async/hello_test.go +++ b/integration_tests/commands/async/hello_test.go @@ -1,8 +1,10 @@ package async import ( + "fmt" "testing" + "github.com/dicedb/dice/config" "github.com/stretchr/testify/assert" ) @@ -12,7 +14,7 @@ func TestHello(t *testing.T) { expected := []interface{}{ "proto", int64(2), - "id", "0.0.0.0:7379", + "id", fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port), "mode", "standalone", "role", "master", "modules", []interface{}{}, diff --git a/integration_tests/commands/async/setup.go b/integration_tests/commands/async/setup.go index 93df39f22..d11ad1463 100644 --- a/integration_tests/commands/async/setup.go +++ b/integration_tests/commands/async/setup.go @@ -34,9 +34,8 @@ type TestServerOptions struct { MaxClients int32 } -//nolint:unused func getLocalConnection() net.Conn { - conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", config.DiceConfig.AsyncServer.Port)) + conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port)) if err != nil { panic(err) } diff --git a/internal/eval/eval.go b/internal/eval/eval.go index 25391578a..26fe9feb7 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -91,7 +91,6 @@ const ( func init() { diceCommandsCount = len(DiceCmds) TxnCommands = map[string]bool{"EXEC": true, "DISCARD": true} - serverID = fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port) } // evalPING returns with an encoded "PONG" @@ -387,6 +386,7 @@ func evalHELLO(args []string, store *dstore.Store) []byte { } var resp []interface{} + serverID = fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port) resp = append(resp, "proto", 2, "id", serverID, From 877b4e8b53f162369bd58fe335169fbef0360aa6 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 13:07:28 +0530 Subject: [PATCH 13/16] fixing minor issues related unittest and config --- internal/eval/eval_darwin_arm64.go | 2 +- internal/eval/eval_linux_arm64.go | 2 +- internal/eval/eval_test.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/eval/eval_darwin_arm64.go b/internal/eval/eval_darwin_arm64.go index 94cd73b5d..af247f052 100644 --- a/internal/eval/eval_darwin_arm64.go +++ b/internal/eval/eval_darwin_arm64.go @@ -25,7 +25,7 @@ func EvalBGREWRITEAOF(args []string, store *dstore.Store) []byte { // TODO: Problem at hand: In multi-threaded environment, each shard instance would fork a child process. // TODO: Each child process would now have a copy of the network file descriptor thus resulting in resource leaks. // TODO: We need to find an alternative approach for the multi-threaded environment. - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { return nil } diff --git a/internal/eval/eval_linux_arm64.go b/internal/eval/eval_linux_arm64.go index cbc58c95c..8b68f3a7a 100644 --- a/internal/eval/eval_linux_arm64.go +++ b/internal/eval/eval_linux_arm64.go @@ -26,7 +26,7 @@ func EvalBGREWRITEAOF(args []string, store *dstore.Store) []byte { // TODO: Problem at hand: In multi-threaded environment, each shard instance would fork a child process. // TODO: Each child process would now have a copy of the network file descriptor thus resulting in resource leaks. // TODO: We need to find an alternative approach for the multi-threaded environment. - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { return nil } childThreadID, _, _ := syscall.Syscall(syscall.SYS_GETTID, 0, 0, 0) diff --git a/internal/eval/eval_test.go b/internal/eval/eval_test.go index bf5dfa76c..513da3f13 100644 --- a/internal/eval/eval_test.go +++ b/internal/eval/eval_test.go @@ -14,6 +14,7 @@ import ( "github.com/axiomhq/hyperloglog" "github.com/bytedance/sonic" + "github.com/dicedb/dice/config" "github.com/dicedb/dice/internal/clientio" diceerrors "github.com/dicedb/dice/internal/errors" "github.com/dicedb/dice/internal/eval/sortedset" @@ -169,6 +170,7 @@ func testEvalECHO(t *testing.T, store *dstore.Store) { } func testEvalHELLO(t *testing.T, store *dstore.Store) { + serverID = fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port) resp := []interface{}{ "proto", 2, "id", serverID, From 7862c1c02a061f0dc7861bbb4360d9aa51967232 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Tue, 19 Nov 2024 13:15:15 +0530 Subject: [PATCH 14/16] linter issue --- integration_tests/commands/async/setup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/commands/async/setup.go b/integration_tests/commands/async/setup.go index d11ad1463..fbfabfeec 100644 --- a/integration_tests/commands/async/setup.go +++ b/integration_tests/commands/async/setup.go @@ -34,6 +34,7 @@ type TestServerOptions struct { MaxClients int32 } +//nolint:unused func getLocalConnection() net.Conn { conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", config.DiceConfig.AsyncServer.Addr, config.DiceConfig.AsyncServer.Port)) if err != nil { From 77b4295a5bd3fbf89cf93789c20a28698b2a8430 Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Wed, 20 Nov 2024 13:54:07 +0530 Subject: [PATCH 15/16] fixing flags related bug --- config/parser.go | 27 +++++++++++++++++++++------ dicedb.conf | 2 +- internal/cli/cli.go | 19 ++++++++++--------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/config/parser.go b/config/parser.go index 132c5f64b..937ff9f5b 100644 --- a/config/parser.go +++ b/config/parser.go @@ -170,7 +170,7 @@ func setField(field reflect.Value, value string) error { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: if field.Type() == reflect.TypeOf(time.Duration(0)) { // Handle time.Duration type - duration, err := time.ParseDuration(value) + duration, err := parseDuration(value) if err != nil { return fmt.Errorf("failed to parse duration: %w", err) } @@ -187,26 +187,26 @@ func setField(field reflect.Value, value string) error { case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: val, err := strconv.ParseUint(value, 10, 64) if err != nil { - return err + return fmt.Errorf("failed to parse unsigned integer: %w", err) } field.SetUint(val) case reflect.Float32, reflect.Float64: val, err := strconv.ParseFloat(value, 64) if err != nil { - return err + return fmt.Errorf("failed to parse float: %w", err) } field.SetFloat(val) case reflect.Bool: val, err := strconv.ParseBool(value) if err != nil { - return err + return fmt.Errorf("failed to parse boolean: %w", err) } field.SetBool(val) case reflect.Slice: - // Get the type of the elements in the slice + // Handle slices of basic types elemType := field.Type().Elem() values := strings.Split(value, ",") slice := reflect.MakeSlice(field.Type(), len(values), len(values)) @@ -214,7 +214,7 @@ func setField(field reflect.Value, value string) error { elem := slice.Index(i) elemVal := reflect.New(elemType).Elem() if err := setField(elemVal, strings.TrimSpace(v)); err != nil { - return err + return fmt.Errorf("failed to parse slice element at index %d: %w", i, err) } elem.Set(elemVal) } @@ -226,3 +226,18 @@ func setField(field reflect.Value, value string) error { return nil } + +// parseDuration handles parsing of time.Duration with proper validation. +func parseDuration(value string) (time.Duration, error) { + if value == "" { + return 0, fmt.Errorf("duration string is empty") + } + duration, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid duration format: %s", value) + } + if duration <= 0 { + return 0, fmt.Errorf("duration must be positive, got: %s", value) + } + return duration, nil +} diff --git a/dicedb.conf b/dicedb.conf index 627d19fc9..ca44ecd6a 100644 --- a/dicedb.conf +++ b/dicedb.conf @@ -29,7 +29,7 @@ performance.enable_multithreading = false performance.store_map_init_size = 1024000 performance.adhoc_req_chan_buf_size = 20 performance.enable_profiling = false -performance.enable_watch = false +performance.enable_watch = true performance.num_shards = -1 # Memory Configuration diff --git a/internal/cli/cli.go b/internal/cli/cli.go index fd93fb21f..f275af954 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -186,14 +186,6 @@ func Execute() { } flag.Parse() - config.MergeFlags(&flagsConfig) - if len(os.Args) < 2 { - if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { - log.Fatal(err) - } - render() - return - } switch os.Args[1] { case "-v", "--version": @@ -228,7 +220,7 @@ func Execute() { log.Fatal("Output file path is not a directory") } - filePath := filepath.Join(dirPath, "dicedb.conf") + filePath := filepath.Join(dirPath, config.DefaultConfigName) if _, err := os.Stat(filePath); err == nil { slog.Warn("Config file already exists at the specified path", slog.String("path", filePath), slog.String("action", "skipping file creation")) return @@ -236,6 +228,8 @@ func Execute() { if err := config.CreateConfigFile(filePath); err != nil { log.Fatal(err) } + + config.MergeFlags(&flagsConfig) render() } case "-c", "--config": @@ -268,11 +262,18 @@ func Execute() { if err := parser.Loadconfig(config.DiceConfig); err != nil { log.Fatal(err) } + + config.MergeFlags(&flagsConfig) render() } else { log.Fatal("Config file path not provided") } default: + if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, config.DefaultConfigName)); err != nil { + log.Fatal(err) + } + + config.MergeFlags(&flagsConfig) render() } } From a9b887933299281f7580f35e84a33667de0a14de Mon Sep 17 00:00:00 2001 From: vinitparekh17 Date: Wed, 20 Nov 2024 14:17:44 +0530 Subject: [PATCH 16/16] refactor: simplifing persistence properties --- config/config.go | 35 +++++++++------------------------- dicedb.conf | 4 ++-- internal/cli/cli.go | 2 +- internal/shard/shard_thread.go | 2 +- main.go | 6 +++--- 5 files changed, 16 insertions(+), 33 deletions(-) diff --git a/config/config.go b/config/config.go index ff5a17a1c..0660c5d84 100644 --- a/config/config.go +++ b/config/config.go @@ -67,10 +67,10 @@ memory.keys_limit = 200000000 memory.lfu_log_factor = 10 # Persistence Configuration +persistence.enabled = true persistence.aof_file = "./dice-master.aof" persistence.persistence_enabled = true persistence.write_aof_on_cleanup = false -persistence.enable-wal = true persistence.wal-dir = "./" persistence.restore-wal = false persistence.wal-engine = "aof" @@ -154,13 +154,12 @@ type memory struct { } type persistence struct { - AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` - PersistenceEnabled bool `config:"persistence_enabled" default:"true"` - WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` - EnableWAL bool `config:"enable-wal" default:"true"` - WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` - RestoreFromWAL bool `config:"restore-wal" default:"false"` - WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` + Enabled bool `config:"enabled" default:"true"` + AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` + WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` + WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` + RestoreFromWAL bool `config:"restore-wal" default:"false"` + WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` } type logging struct { @@ -263,8 +262,8 @@ func MergeFlags(flags *Config) { DiceConfig.Performance.EnableWatch = flags.Performance.EnableWatch case "log-dir": DiceConfig.Logging.LogDir = flags.Logging.LogDir - case "enable-wal": - DiceConfig.Persistence.EnableWAL = flags.Persistence.EnableWAL + case "persistence-enable": + DiceConfig.Persistence.Enabled = flags.Persistence.Enabled case "restore-from-wal": DiceConfig.Persistence.RestoreFromWAL = flags.Persistence.RestoreFromWAL case "wal-engine": @@ -272,19 +271,3 @@ func MergeFlags(flags *Config) { } }) } - -// This function returns the config file path based on the OS -// func getConfigPath() string { -// switch runtime.GOOS { -// case "windows": -// FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) -// case "darwin", "linux": -// FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) -// default: -// // Default to current directory if OS is unknown -// FileLocation = filepath.Join(".", DefaultConfigName) -// } -// return FileLocation -// } - -// ResetConfig resets the DiceConfig to default configurations. This function is only used for testing purposes diff --git a/dicedb.conf b/dicedb.conf index ca44ecd6a..17351b8e5 100644 --- a/dicedb.conf +++ b/dicedb.conf @@ -29,7 +29,7 @@ performance.enable_multithreading = false performance.store_map_init_size = 1024000 performance.adhoc_req_chan_buf_size = 20 performance.enable_profiling = false -performance.enable_watch = true +performance.enable_watch = false performance.num_shards = -1 # Memory Configuration @@ -40,10 +40,10 @@ memory.keys_limit = 200000000 memory.lfu_log_factor = 10 # Persistence Configuration +persistence.enabled = true persistence.aof_file = "./dice-master.aof" persistence.persistence_enabled = true persistence.write_aof_on_cleanup = false -persistence.enable-wal = true persistence.wal-dir = "./" persistence.restore-wal = false persistence.wal-engine = "aof" diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f275af954..48cecbc03 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -135,7 +135,7 @@ func Execute() { flag.StringVar(&flagsConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") flag.StringVar(&config.DiceConfig.Logging.LogDir, "log-dir", "/tmp/dicedb", "log directory path") - flag.BoolVar(&flagsConfig.Persistence.EnableWAL, "enable-wal", false, "enable write-ahead logging") + flag.BoolVar(&flagsConfig.Persistence.Enabled, "persistence-enable", false, "enable write-ahead logging") flag.BoolVar(&flagsConfig.Persistence.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") flag.StringVar(&flagsConfig.Persistence.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") diff --git a/internal/shard/shard_thread.go b/internal/shard/shard_thread.go index 635c9dafe..a9ffecc9d 100644 --- a/internal/shard/shard_thread.go +++ b/internal/shard/shard_thread.go @@ -134,7 +134,7 @@ func (shard *ShardThread) processRequest(op *ops.StoreOp) { // cleanup handles cleanup logic when the shard stops. func (shard *ShardThread) cleanup() { close(shard.ReqChan) - if !config.DiceConfig.Persistence.WriteAOFOnCleanup { + if !config.DiceConfig.Persistence.Enabled || !config.DiceConfig.Persistence.WriteAOFOnCleanup { return } diff --git a/main.go b/main.go index 6a5404847..f0afab731 100644 --- a/main.go +++ b/main.go @@ -53,8 +53,8 @@ func main() { ) wl, _ = wal.NewNullWAL() - slog.Info("running with", slog.Bool("enable-wal", config.DiceConfig.Persistence.EnableWAL)) - if config.DiceConfig.Persistence.EnableWAL { + slog.Info("running with", slog.Bool("persistence-enabled", config.DiceConfig.Persistence.Enabled)) + if config.DiceConfig.Persistence.Enabled { if config.DiceConfig.Persistence.WALEngine == "sqlite" { _wl, err := wal.NewSQLiteWAL(config.DiceConfig.Persistence.WALDir) if err != nil { @@ -191,7 +191,7 @@ func main() { close(sigs) - if config.DiceConfig.Persistence.EnableWAL { + if config.DiceConfig.Persistence.Enabled { wal.ShutdownBG() }