server: Cache PreparedEvalQuery's for data requests

This adds in caching of prepared queries for versioned and unversioned
data queries (POST/GET to `/`, `/data`, and `/v1/data`).

The cache has a max size of 100 and just starts acting as a circular
buffer for queries (FIFO, no smarts for LRU caching or anything). It
seems like it would be unlikely for these API's to hit the cache max
size. Most OPA use-cases have a single query that is re-used over
and over with different inputs.

There is a new metric `counter_server_query_cache_hit` which will
show whether or not a request used the query cache or not. It is there
primarily to help explain away why sometimes a handful of the other
metrics aren't there (the query parse/compile/etc).

In the future this could be added to the query API's too. This change
does not touch anything other than the "data" API's.

Closes: #1567
Signed-off-by: Patrick East <east.patrick@gmail.com>

CHANGELOG.md

@@ -5,6 +5,18 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+### Fixes
+- Rego parse and compile metrics are now correctly tracked and returned
+  when using bundles.
+
+### Compatibility Notes
+- The `Path` string provided to decision log implementations via the
+  `server.Info#Path` parameter is no longer dot-notation rego reference
+  string. It is now a `/` separated path which has had the `data` prefix
+  removed. This change corrects is issue described in
+  [1532](https://github.com/open-policy-agent/opa/issues/1532) and may
+  affect custom decision log plugins that rely on the `Path`.
+
 ## 0.15.1
 
 In this release we reached a milestone for Wasm: any Rego policy can

metrics/metrics.go

@@ -19,17 +19,18 @@ import (
 
 // Well-known metric names.
 const (
-	ServerHandler     = "server_handler"
-	RegoQueryCompile  = "rego_query_compile"
-	RegoQueryEval     = "rego_query_eval"
-	RegoQueryParse    = "rego_query_parse"
-	RegoModuleParse   = "rego_module_parse"
-	RegoDataParse     = "rego_data_parse"
-	RegoModuleCompile = "rego_module_compile"
-	RegoPartialEval   = "rego_partial_eval"
-	RegoInputParse    = "rego_input_parse"
-	RegoLoadFiles     = "rego_load_files"
-	RegoLoadBundles   = "rego_load_bundles"
+	ServerHandler       = "server_handler"
+	ServerQueryCacheHit = "server_query_cache_hit"
+	RegoQueryCompile    = "rego_query_compile"
+	RegoQueryEval       = "rego_query_eval"
+	RegoQueryParse      = "rego_query_parse"
+	RegoModuleParse     = "rego_module_parse"
+	RegoDataParse       = "rego_data_parse"
+	RegoModuleCompile   = "rego_module_compile"
+	RegoPartialEval     = "rego_partial_eval"
+	RegoInputParse      = "rego_input_parse"
+	RegoLoadFiles       = "rego_load_files"
+	RegoLoadBundles     = "rego_load_bundles"
 )
 
 // Info contains attributes describing the underlying metrics provider.

server/cache.go

@@ -0,0 +1,53 @@
+package server
+
+import "sync"
+
+type cache struct {
+	data    map[string]interface{}
+	keylist []string
+	idx     int
+	maxSize int
+	mtx     sync.RWMutex
+}
+
+func newCache(maxSize int) *cache {
+	return &cache{
+		data:    map[string]interface{}{},
+		keylist: []string{},
+		maxSize: maxSize,
+	}
+}
+
+func (c *cache) Get(k string) (interface{}, bool) {
+	c.mtx.RLock()
+	v, ok := c.data[k]
+	c.mtx.RUnlock()
+	return v, ok
+}
+
+func (c *cache) Insert(k string, v interface{}) {
+
+	// Short path if its already in the cache
+	_, ok := c.Get(k)
+	if ok {
+		return
+	}
+
+	// Slow path, grab the write lock and insert
+	c.mtx.Lock()
+	_, ok = c.data[k]
+	if !ok {
+		c.data[k] = v
+		if len(c.keylist) < c.maxSize {
+			// Haven't reached max size yet, keep adding keys.
+			c.keylist = append(c.keylist, k)
+		} else {
+			// Start recycling spots in the key list and
+			// dropping cache entries for them.
+			delete(c.data, c.keylist[c.idx])
+			c.keylist[c.idx] = k
+			c.idx = (c.idx + 1) % c.maxSize
+		}
+	}
+	c.mtx.Unlock()
+}

server/cache_test.go

@@ -0,0 +1,78 @@
+package server
+
+import (
+	"strconv"
+	"testing"
+)
+
+func TestCacheBase(t *testing.T) {
+	c := newCache(5)
+	foo := struct{}{}
+	c.Insert("foo", foo)
+	ensureCacheKey(t, c, "foo", foo)
+}
+
+func TestCacheLimit(t *testing.T) {
+	max := 10
+	c := newCache(max)
+
+	// Fill the cache with values
+	var i int
+	for i = 0; i < max; i++ {
+		c.Insert(strconv.Itoa(i), i)
+	}
+
+	// Ensure its at the max size
+	ensureCacheSize(t, c, max)
+
+	// Ensure they are all stored..
+	for j := 0; j < max; j++ {
+		ensureCacheKey(t, c, strconv.Itoa(j), j)
+	}
+
+	// Continue filling the cache, expect old keys to be dropped
+	c.Insert(strconv.Itoa(i), i)
+	ensureCacheKey(t, c, strconv.Itoa(i), i)
+
+	// Should still be at max size
+	ensureCacheSize(t, c, max)
+
+	// Expect that "0" got dropped
+	_, ok := c.Get("0")
+	if ok {
+		t.Fatal("Expected key '0' to not be found")
+	}
+
+	// Load the cache with many more than max
+	for ; i < max*20; i++ {
+		c.Insert(strconv.Itoa(i), i)
+	}
+	ensureCacheSize(t, c, max)
+
+	// Ensure the last set of "max" number are available, and everything else is not
+	for j := 0; j < i; j++ {
+		k := strconv.Itoa(j)
+		if j >= (i - max) {
+			ensureCacheKey(t, c, k, j)
+		} else {
+			_, ok := c.Get(k)
+			if ok {
+				t.Fatalf("Expected key %s to not be found", k)
+			}
+		}
+	}
+}
+
+func ensureCacheKey(t *testing.T, c *cache, k string, v interface{}) {
+	t.Helper()
+	actual, ok := c.Get(k)
+	if !ok || v != actual {
+		t.Fatalf("expected to retrieve value %v for key %s, got %v ok==%t", v, k, actual, ok)
+	}
+}
+
+func ensureCacheSize(t *testing.T, c *cache, size int) {
+	if len(c.data) != size && len(c.keylist) != size {
+		t.Fatalf("Unexpected cache size len(data)=%d len(keylist)=%d, expected %d", size, len(c.data), len(c.keylist))
+	}
+}