allow human-readable memory size in FifoCache config (grafana#2527)

* allow human-readable memory size in FifoCache config Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> * update CHANGELOG, usage message Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> * addressed comments Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> * update tests Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> * fixed integration test failure Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> * added unit test Signed-off-by: Dmitry Shmulevich <dmitry.shmulevich@sysdig.com> Co-authored-by: Dmitry Shmulevich <dima@dev.local>
cyriltovena · Apr 29, 2020 · a55fd69 · a55fd69
1 parent a1d518d
commit a55fd69
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 11 deletions.
diff --git a/cache/cache.go b/cache/cache.go
@@ -55,6 +55,10 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, description string, f
 	cfg.Prefix = prefix
 }
 
+func (cfg *Config) Validate() error {
+	return cfg.Fifocache.Validate()
+}
+
 // New creates a new Cache using Config.
 func New(cfg Config) (Cache, error) {
 	if cfg.Cache != nil {

diff --git a/cache/fifo_cache.go b/cache/fifo_cache.go
@@ -8,7 +8,9 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/dustin/go-humanize"
 	"github.com/go-kit/kit/log/level"
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -86,7 +88,7 @@ const (
 
 // FifoCacheConfig holds config for the FifoCache.
 type FifoCacheConfig struct {
-	MaxSizeBytes int           `yaml:"max_size_bytes"`
+	MaxSizeBytes string        `yaml:"max_size_bytes"`
 	MaxSizeItems int           `yaml:"max_size_items"`
 	Validity     time.Duration `yaml:"validity"`
 
@@ -95,20 +97,36 @@ type FifoCacheConfig struct {
 
 // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet
 func (cfg *FifoCacheConfig) RegisterFlagsWithPrefix(prefix, description string, f *flag.FlagSet) {
-	f.IntVar(&cfg.MaxSizeBytes, prefix+"fifocache.max-size-bytes", 0, description+"Maximum memory size of the cache.")
+	f.StringVar(&cfg.MaxSizeBytes, prefix+"fifocache.max-size-bytes", "", description+"Maximum memory size of the cache in bytes. A unit suffix (KB, MB, GB) may be applied.")
 	f.IntVar(&cfg.MaxSizeItems, prefix+"fifocache.max-size-items", 0, description+"Maximum number of entries in the cache.")
 	f.DurationVar(&cfg.Validity, prefix+"fifocache.duration", 0, description+"The expiry duration for the cache.")
 
 	f.IntVar(&cfg.DeprecatedSize, prefix+"fifocache.size", 0, "Deprecated (use max-size-items or max-size-bytes instead): "+description+"The number of entries to cache. ")
 }
 
+func (cfg *FifoCacheConfig) Validate() error {
+	_, err := parsebytes(cfg.MaxSizeBytes)
+	return err
+}
+
+func parsebytes(s string) (uint64, error) {
+	if len(s) == 0 {
+		return 0, nil
+	}
+	bytes, err := humanize.ParseBytes(s)
+	if err != nil {
+		return 0, errors.Wrap(err, "invalid FifoCache config")
+	}
+	return bytes, nil
+}
+
 // FifoCache is a simple string -> interface{} cache which uses a fifo slide to
 // manage evictions.  O(1) inserts and updates, O(1) gets.
 type FifoCache struct {
 	lock          sync.RWMutex
 	maxSizeItems  int
-	maxSizeBytes  int
-	currSizeBytes int
+	maxSizeBytes  uint64
+	currSizeBytes uint64
 	validity      time.Duration
 
 	entries map[string]*list.Element
@@ -140,14 +158,16 @@ func NewFifoCache(name string, cfg FifoCacheConfig) *FifoCache {
 		level.Warn(util.Logger).Log("msg", "running with DEPRECATED flag fifocache.size, use fifocache.max-size-items or fifocache.max-size-bytes instead", "cache", name)
 		cfg.MaxSizeItems = cfg.DeprecatedSize
 	}
-	if cfg.MaxSizeBytes == 0 && cfg.MaxSizeItems == 0 {
+	maxSizeBytes, _ := parsebytes(cfg.MaxSizeBytes)
+
+	if maxSizeBytes == 0 && cfg.MaxSizeItems == 0 {
 		// zero cache capacity - no need to create cache
 		level.Warn(util.Logger).Log("msg", "neither fifocache.max-size-bytes nor fifocache.max-size-items is set", "cache", name)
 		return nil
 	}
 	return &FifoCache{
 		maxSizeItems: cfg.MaxSizeItems,
-		maxSizeBytes: cfg.MaxSizeBytes,
+		maxSizeBytes: maxSizeBytes,
 		validity:     cfg.Validity,
 		entries:      make(map[string]*list.Element),
 		lru:          list.New(),
@@ -281,10 +301,10 @@ func (c *FifoCache) Get(ctx context.Context, key string) ([]byte, bool) {
 	return nil, false
 }
 
-func sizeOf(item *cacheEntry) int {
-	return int(unsafe.Sizeof(*item)) + // size of cacheEntry
+func sizeOf(item *cacheEntry) uint64 {
+	return uint64(int(unsafe.Sizeof(*item)) + // size of cacheEntry
 		len(item.key) + // size of key
 		cap(item.value) + // size of value
 		elementSize + // size of the element in linked list
-		elementPrtSize // size of the pointer to an element in the map
+		elementPrtSize) // size of the pointer to an element in the map
 }
diff --git a/cache/fifo_cache_test.go b/cache/fifo_cache_test.go
@@ -3,6 +3,7 @@ package cache
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"testing"
 	"time"
 
@@ -27,7 +28,7 @@ func TestFifoCacheEviction(t *testing.T) {
 	}{
 		{
 			name: "test-memory-eviction",
-			cfg:  FifoCacheConfig{MaxSizeBytes: cnt * sizeOf(itemTemplate), Validity: 1 * time.Minute},
+			cfg:  FifoCacheConfig{MaxSizeBytes: strconv.FormatInt(int64(cnt*sizeOf(itemTemplate)), 10), Validity: 1 * time.Minute},
 		},
 		{
 			name: "test-items-eviction",
@@ -175,7 +176,7 @@ func TestFifoCacheExpiry(t *testing.T) {
 	}{
 		{
 			name: "test-memory-expiry",
-			cfg:  FifoCacheConfig{MaxSizeBytes: memorySz, Validity: 5 * time.Millisecond},
+			cfg:  FifoCacheConfig{MaxSizeBytes: strconv.FormatInt(int64(memorySz), 10), Validity: 5 * time.Millisecond},
 		},
 		{
 			name: "test-items-expiry",
@@ -236,3 +237,41 @@ func genBytes(n uint8) []byte {
 	}
 	return arr
 }
+
+func TestBytesParsing(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected uint64
+	}{
+		{input: "", expected: 0},
+		{input: "123", expected: 123},
+		{input: "1234567890", expected: 1234567890},
+		{input: "25k", expected: 25000},
+		{input: "25K", expected: 25000},
+		{input: "25kb", expected: 25000},
+		{input: "25kB", expected: 25000},
+		{input: "25Kb", expected: 25000},
+		{input: "25KB", expected: 25000},
+		{input: "25kib", expected: 25600},
+		{input: "25KiB", expected: 25600},
+		{input: "25m", expected: 25000000},
+		{input: "25M", expected: 25000000},
+		{input: "25mB", expected: 25000000},
+		{input: "25MB", expected: 25000000},
+		{input: "2.5MB", expected: 2500000},
+		{input: "25MiB", expected: 26214400},
+		{input: "25mib", expected: 26214400},
+		{input: "2.5mib", expected: 2621440},
+		{input: "25g", expected: 25000000000},
+		{input: "25G", expected: 25000000000},
+		{input: "25gB", expected: 25000000000},
+		{input: "25Gb", expected: 25000000000},
+		{input: "25GiB", expected: 26843545600},
+		{input: "25gib", expected: 26843545600},
+	}
+	for _, test := range tests {
+		output, err := parsebytes(test.input)
+		assert.Nil(t, err)
+		assert.Equal(t, test.expected, output)
+	}
+}
diff --git a/chunk_store.go b/chunk_store.go
@@ -71,6 +71,16 @@ func (cfg *StoreConfig) RegisterFlags(f *flag.FlagSet) {
 	f.Var(&cfg.MaxLookBackPeriod, "store.max-look-back-period", "Limit how long back data can be queried")
 }
 
+func (cfg *StoreConfig) Validate() error {
+	if err := cfg.ChunkCacheConfig.Validate(); err != nil {
+		return err
+	}
+	if err := cfg.WriteDedupeCacheConfig.Validate(); err != nil {
+		return err
+	}
+	return nil
+}
+
 type baseStore struct {
 	cfg StoreConfig
 

diff --git a/storage/factory.go b/storage/factory.go
@@ -103,6 +103,9 @@ func (cfg *Config) Validate() error {
 	if err := cfg.Swift.Validate(); err != nil {
 		return errors.Wrap(err, "invalid Swift Storage config")
 	}
+	if err := cfg.IndexQueriesCacheConfig.Validate(); err != nil {
+		return errors.Wrap(err, "invalid Index Queries Cache config")
+	}
 	return nil
 }