diff --git a/README.md b/README.md index a37f8a7..b5b19dc 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Hord is designed to be a database-agnostic library that provides a common interf | [BoltDB](https://github.com/etcd-io/bbolt) | ✅ | | | | [Cassandra](https://cassandra.apache.org/) | ✅ | | [ScyllaDB](https://www.scylladb.com/), [YugabyteDB](https://www.yugabyte.com/), [Azure Cosmos DB](https://learn.microsoft.com/en-us/azure/cosmos-db/introduction) | | [Couchbase](https://www.couchbase.com/) | Pending ||| -| Hashmap | ✅ ||| +| Hashmap | ✅ | Optionally allows storing to YAML or JSON file || | Mock | ✅ | Mock Database interactions within unit tests || | [NATS](https://nats.io/) | ✅ | Experimental || | [Redis](https://redis.io/) | ✅ || [Dragonfly](https://www.dragonflydb.io/), [KeyDB](https://docs.keydb.dev/) | diff --git a/drivers/hashmap/byte_slice.go b/drivers/hashmap/byte_slice.go new file mode 100644 index 0000000..ed66814 --- /dev/null +++ b/drivers/hashmap/byte_slice.go @@ -0,0 +1,27 @@ +package hashmap + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// ByteSlice is a struct that is used to implement custom YAML Marshal/Unmarshal because +// the default marshal for a []byte will write an array of integers +type ByteSlice []byte + +// MarshalYAML simply casts the ByteSlice to a string +func (bs ByteSlice) MarshalYAML() (interface{}, error) { + return string(bs), nil +} + +// UnmarshalYAML converts the YAML string back into ByteSlice +func (bs *ByteSlice) UnmarshalYAML(value *yaml.Node) error { + switch value.Tag { + case "!!str": + *bs = []byte(value.Value) + default: + return fmt.Errorf("expected string, but got %s", value.Tag) + } + return nil +} diff --git a/drivers/hashmap/byte_slice_test.go b/drivers/hashmap/byte_slice_test.go new file mode 100644 index 0000000..faf67e8 --- /dev/null +++ b/drivers/hashmap/byte_slice_test.go @@ -0,0 +1,43 @@ +package hashmap + +import ( + "testing" + + "gopkg.in/yaml.v3" +) + +func TestByteSliceMarshalYAML(t *testing.T) { + data := map[string]ByteSlice{ + "key": []byte("value"), + } + + dataBytes, err := yaml.Marshal(data) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + expected := "key: value\n" + if string(dataBytes) != expected { + t.Errorf("expected %q but got: %v", expected, string(dataBytes)) + } +} + +func TestByteSliceUnmarshalYAML(t *testing.T) { + var data map[string]ByteSlice + err := yaml.Unmarshal([]byte(`key: value`), &data) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if string(data["key"]) != "value" { + t.Errorf("unexpected value: %v", string(data["key"])) + } +} + +func TestByteSliceUnmarshalYAMLError(t *testing.T) { + var data map[string]ByteSlice + err := yaml.Unmarshal([]byte(`key: 1`), &data) + if err == nil || err.Error() != "expected string, but got !!int" { + t.Errorf("error did not match: %v", err) + } +} diff --git a/drivers/hashmap/common_test.go b/drivers/hashmap/common_test.go index 614f3c2..6e9f973 100644 --- a/drivers/hashmap/common_test.go +++ b/drivers/hashmap/common_test.go @@ -3,6 +3,7 @@ package hashmap import ( "context" "fmt" + "os" "testing" "time" @@ -12,6 +13,14 @@ import ( func TestInterfaceHappyPath(t *testing.T) { cfgs := make(map[string]Config) cfgs["Hashmap"] = Config{} + cfgs["HashmapWithJSONStorage"] = Config{ + Filename: "testdata/common_test.json", + } + cfgs["HashmapWithYAMLStorage"] = Config{ + Filename: "testdata/common_test.yml", + } + defer os.RemoveAll("testdata/common_test.json") + defer os.RemoveAll("testdata/common_test.yml") // Loop through valid Configs and validate the driver adheres to the Hord interface for name, cfg := range cfgs { diff --git a/drivers/hashmap/hashmap.go b/drivers/hashmap/hashmap.go index 8ccb873..39b29de 100644 --- a/drivers/hashmap/hashmap.go +++ b/drivers/hashmap/hashmap.go @@ -57,31 +57,83 @@ Hord provides a simple abstraction for working with the hashmap driver, with eas package hashmap import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" "sync" "github.com/madflojo/hord" + "gopkg.in/yaml.v3" ) // Config represents the configuration for the hashmap database. -type Config struct{} +type Config struct { + // Filename is an optional parameter that accepts the path to a YAML or JSON file to read/write data + Filename string +} // Database is an in-memory hashmap implementation of the hord.Database interface. type Database struct { sync.RWMutex + config Config + // data is used to store data in a simple map - data map[string][]byte + data map[string]ByteSlice } // Dial initializes and returns a new hashmap database instance. -func Dial(_ Config) (*Database, error) { - db := &Database{} - db.data = make(map[string][]byte) +func Dial(conf Config) (*Database, error) { + if conf.Filename != "" { + switch filepath.Ext(conf.Filename) { + case ".yaml", ".yml", ".json": + default: + return nil, errors.New("filename must have yaml, yml, or json extension") + } + } + + db := &Database{config: conf} + db.data = make(map[string]ByteSlice) return db, nil } -// Setup sets up the hashmap database. This function does nothing for the hashmap driver. +// Setup sets up the hashmap database. If file storage is enabled, this will load from the file or create it if it does not exist. func (db *Database) Setup() error { + if db.config.Filename == "" { + return nil + } + + db.Lock() + defer db.Unlock() + + // check file and create if it does not exist + file, err := os.OpenFile(db.config.Filename, os.O_RDONLY|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("error checking file %q: %w", db.config.Filename, err) + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("unable to read local file: %w", err) + } + + switch filepath.Ext(db.config.Filename) { + case ".json": + // json fails to read empty input + if len(data) != 0 { + err = json.Unmarshal(data, &db.data) + } + case ".yaml", ".yml": + err = yaml.Unmarshal(data, &db.data) + } + if err != nil { + return fmt.Errorf("unable to unmarshal data from file: %w", err) + } + return nil } @@ -123,7 +175,7 @@ func (db *Database) Set(key string, data []byte) error { } db.data[key] = data - return nil + return db.saveToLocalFile() } // Delete removes data from the hashmap database based on the provided key. @@ -140,7 +192,7 @@ func (db *Database) Delete(key string) error { } delete(db.data, key) - return nil + return db.saveToLocalFile() } // Keys retrieves a list of keys stored in the hashmap database. @@ -167,12 +219,46 @@ func (db *Database) HealthCheck() error { return hord.ErrNoDial } + if db.config.Filename != "" { + _, err := os.Stat(db.config.Filename) + if err != nil { + return fmt.Errorf("error checking if file exists: %w", err) + } + } + return nil } -// Close closes the hashmap database connection and clears all stored data. +// Close closes the hashmap database connection and clears all stored data from memory (file remains if used). func (db *Database) Close() { db.Lock() defer db.Unlock() db.data = nil } + +// saveToLocalFile is a helper function for methods that change the data (Set, Delete) and should +// only be used after acquiring Write lock +func (db *Database) saveToLocalFile() error { + if db.config.Filename == "" { + return nil + } + + var err error + var content []byte + switch filepath.Ext(db.config.Filename) { + case ".json": + content, err = json.Marshal(db.data) + case ".yaml", ".yml": + content, err = yaml.Marshal(db.data) + } + if err != nil { + return fmt.Errorf("error marshalling data: %w", err) + } + + err = os.WriteFile(db.config.Filename, content, 0755) + if err != nil { + return fmt.Errorf("error writing data to file %q: %w", db.config.Filename, err) + } + + return nil +} diff --git a/drivers/hashmap/hashmap_test.go b/drivers/hashmap/hashmap_test.go new file mode 100644 index 0000000..e259bd8 --- /dev/null +++ b/drivers/hashmap/hashmap_test.go @@ -0,0 +1,269 @@ +package hashmap + +import ( + "encoding/json" + "os" + "testing" + + "gopkg.in/yaml.v3" +) + +var fileTypeCases = []struct { + extension string + unmarshal func([]byte, interface{}) error + marshal func(interface{}) ([]byte, error) +}{ + { + "yaml", + yaml.Unmarshal, + yaml.Marshal, + }, + { + "json", + json.Unmarshal, + json.Marshal, + }, +} + +func TestSetupCreatesFileIfNotExist(t *testing.T) { + tests := []struct { + ext string + }{ + {"yaml"}, + {"yml"}, + {"json"}, + } + + for _, tt := range tests { + t.Run(tt.ext, func(t *testing.T) { + filename := "testdata/empty_file." + tt.ext + defer os.RemoveAll(filename) + + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + // make sure file exists + _, err = os.Stat(filename) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestDial(t *testing.T) { + t.Run("ErrorInvalidFilename", func(t *testing.T) { + _, err := Dial(Config{Filename: "testdata/empty_file.txt"}) + if err == nil || err.Error() != "filename must have yaml, yml, or json extension" { + t.Errorf("error did not match: %v", err) + } + }) +} + +func TestSaveToLocalFileAfterSet(t *testing.T) { + for _, tt := range fileTypeCases { + t.Run(tt.extension, func(t *testing.T) { + filename := "testdata/save_test." + tt.extension + defer os.RemoveAll(filename) + + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Set("key", []byte("value")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + db.Close() + + data, err := readFile(filename, tt.unmarshal) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + result := string(data["key"]) + if result != "value" { + t.Errorf("unexpected value: %v", result) + } + }) + } +} + +func TestLoadingFromExistingFile(t *testing.T) { + for _, tt := range fileTypeCases { + t.Run("ValidFileContents_"+tt.extension, func(t *testing.T) { + filename := "testdata/load_data_test." + tt.extension + + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + value, err := db.Get("key") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if string(value) != "value" { + t.Errorf("unexpected value: %s", string(value)) + } + }) + + t.Run("InvalidFileContents_"+tt.extension, func(t *testing.T) { + filename := "testdata/load_data_invalid_test." + tt.extension + + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err == nil { + t.Error("expected error but was nil") + } + + // different error expectations depending on format + var expectedErr string + switch tt.extension { + case "yaml": + expectedErr = "unable to unmarshal data from file: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `this is...` into map[string]hashmap.ByteSlice" + case "json": + expectedErr = "unable to unmarshal data from file: invalid character 'h' in literal true (expecting 'r')" + } + + if err.Error() != expectedErr { + t.Errorf("error did not match: %v", err) + } + }) + } +} + +func TestComplexObjectSaveToFile(t *testing.T) { + data := map[string]interface{}{ + "string": "string", + "integer": 1, + "float": 1.5, + "string_array": []string{"a", "b", "c"}, + "integer_array": []int{1, 2, 3}, + "object": map[string]interface{}{ + "key": "value", + }, + } + + for _, tt := range fileTypeCases { + t.Run(tt.extension, func(t *testing.T) { + filename := "testdata/complex_data_test." + tt.extension + defer os.RemoveAll(filename) + + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + // Save objects to DB as JSON and YAML + dataJSON, err := json.Marshal(data) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Set("data_json", dataJSON) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + dataYAML, err := yaml.Marshal(data) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Set("data_yaml", dataYAML) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + db.Close() + + t.Run("CompareSavedFilesToExpectations", func(t *testing.T) { + data, err := readFile(filename, tt.unmarshal) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + expectedData, err := readFile("testdata/complex_data_test_expected."+tt.extension, tt.unmarshal) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if string(data["data"]) != string(expectedData["data"]) { + t.Errorf("data did not match: %v", string(data["data"])) + } + }) + + t.Run("RecreateDBAndGetData", func(t *testing.T) { + db, err := Dial(Config{Filename: filename}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + err = db.Setup() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + newDataJSON, err := db.Get("data_json") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(newDataJSON) != string(dataJSON) { + t.Errorf("data did not match after reading from file: %s", string(newDataJSON)) + } + + newDataYAML, err := db.Get("data_yaml") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(newDataYAML) != string(dataYAML) { + t.Errorf("data did not match after reading from file: %s", string(newDataYAML)) + } + }) + }) + } +} + +func readFile(filename string, unmarshal func([]byte, interface{}) error) (map[string]ByteSlice, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + var parsedData map[string]ByteSlice + err = unmarshal(data, &parsedData) + if err != nil { + return nil, err + } + + return parsedData, nil +} diff --git a/drivers/hashmap/testdata/complex_data_test_expected.json b/drivers/hashmap/testdata/complex_data_test_expected.json new file mode 100755 index 0000000..fc5e515 --- /dev/null +++ b/drivers/hashmap/testdata/complex_data_test_expected.json @@ -0,0 +1 @@ +{"data_json":"eyJmbG9hdCI6MS41LCJpbnRlZ2VyIjoxLCJpbnRlZ2VyX2FycmF5IjpbMSwyLDNdLCJvYmplY3QiOnsia2V5IjoidmFsdWUifSwic3RyaW5nIjoic3RyaW5nIiwic3RyaW5nX2FycmF5IjpbImEiLCJiIiwiYyJdfQ==","data_yaml":"ZmxvYXQ6IDEuNQppbnRlZ2VyOiAxCmludGVnZXJfYXJyYXk6CiAgICAtIDEKICAgIC0gMgogICAgLSAzCm9iamVjdDoKICAgIGtleTogdmFsdWUKc3RyaW5nOiBzdHJpbmcKc3RyaW5nX2FycmF5OgogICAgLSBhCiAgICAtIGIKICAgIC0gYwo="} \ No newline at end of file diff --git a/drivers/hashmap/testdata/complex_data_test_expected.yaml b/drivers/hashmap/testdata/complex_data_test_expected.yaml new file mode 100755 index 0000000..e0f591d --- /dev/null +++ b/drivers/hashmap/testdata/complex_data_test_expected.yaml @@ -0,0 +1,15 @@ +data_json: '{"float":1.5,"integer":1,"integer_array":[1,2,3],"object":{"key":"value"},"string":"string","string_array":["a","b","c"]}' +data_yaml: | + float: 1.5 + integer: 1 + integer_array: + - 1 + - 2 + - 3 + object: + key: value + string: string + string_array: + - a + - b + - c diff --git a/drivers/hashmap/testdata/load_data_invalid_test.json b/drivers/hashmap/testdata/load_data_invalid_test.json new file mode 100644 index 0000000..7e31dc3 --- /dev/null +++ b/drivers/hashmap/testdata/load_data_invalid_test.json @@ -0,0 +1 @@ +this is not JSON \ No newline at end of file diff --git a/drivers/hashmap/testdata/load_data_invalid_test.yaml b/drivers/hashmap/testdata/load_data_invalid_test.yaml new file mode 100644 index 0000000..0cc0efa --- /dev/null +++ b/drivers/hashmap/testdata/load_data_invalid_test.yaml @@ -0,0 +1 @@ +this is not YAML diff --git a/drivers/hashmap/testdata/load_data_test.json b/drivers/hashmap/testdata/load_data_test.json new file mode 100755 index 0000000..7216c8e --- /dev/null +++ b/drivers/hashmap/testdata/load_data_test.json @@ -0,0 +1,3 @@ +{ + "key": "dmFsdWU=" +} \ No newline at end of file diff --git a/drivers/hashmap/testdata/load_data_test.yaml b/drivers/hashmap/testdata/load_data_test.yaml new file mode 100755 index 0000000..b4f46b7 --- /dev/null +++ b/drivers/hashmap/testdata/load_data_test.yaml @@ -0,0 +1 @@ +key: value diff --git a/go.mod b/go.mod index c621cea..cc7b761 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gomodule/redigo v1.8.9 github.com/nats-io/nats.go v1.25.0 go.etcd.io/bbolt v1.3.7 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index 11082ed..ce60a58 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=