add denylist stuff

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gomodule/redigo v1.8.5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c // indirect
 	github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect
 	github.com/juju/loggo v0.0.0-20200526014432-9ce3a2e09b5e // indirect

go.sum

@@ -140,6 +140,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=

lib/denylist/http.go

@@ -0,0 +1,108 @@
+package denylist
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+)
+
+// CollectionEndpoint serves the endpoints for the whole Denylist at /denylist
+func CollectionEndpoint(denylist *Denylist) func(http.ResponseWriter, *http.Request) {
+	return func(response http.ResponseWriter, request *http.Request) {
+		switch request.Method {
+		case "GET":
+			listDenylistKeys(response, denylist)
+		case "PUT":
+			createDenylistEntry(response, request, denylist)
+		default:
+			http.Error(response, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		}
+	}
+}
+
+// SingleEndpoint serves the endpoints for particular Denylist entries at /denylist/...
+func SingleEndpoint(denylist *Denylist) func(http.ResponseWriter, *http.Request) {
+	return func(response http.ResponseWriter, request *http.Request) {
+		switch request.Method {
+		case "GET":
+			getDenylistEntry(response, request, denylist)
+		case "DELETE":
+			deleteDenylistEntry(response, request, denylist)
+		default:
+			http.Error(response, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		}
+	}
+}
+
+// GET /denylist
+func listDenylistKeys(response http.ResponseWriter, denylist *Denylist) {
+	keys := denylist.GetKeys()
+
+	response.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(response).Encode(keys)
+	response.WriteHeader(http.StatusOK)
+}
+
+// PUT /denylist
+func createDenylistEntry(response http.ResponseWriter, request *http.Request, denylist *Denylist) {
+	if request.Header.Get("Content-Type") != "application/json" {
+		http.Error(response, "request must be JSON", http.StatusBadRequest)
+		return
+	}
+	decoder := json.NewDecoder(request.Body)
+	var payload map[string]string
+	err := decoder.Decode(&payload)
+	if err != nil {
+		http.Error(response, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	unparsedKeys, keysOk := payload["keys"]
+	unparsedRegex, regexOk := payload["regex"]
+	if !keysOk || !regexOk {
+		http.Error(response, "request body must contain `keys` and `regex`", http.StatusBadRequest)
+		return
+	}
+
+	id, err := denylist.AppendEntry(unparsedKeys, unparsedRegex)
+	if err != nil {
+		http.Error(response, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	response.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(response).Encode(id)
+	response.WriteHeader(http.StatusCreated)
+}
+
+// GET /denylist/...
+func getDenylistEntry(response http.ResponseWriter, request *http.Request, denylist *Denylist) {
+	id := request.URL.Path
+	entry := denylist.GetEntry(id)
+	if entry == nil {
+		http.Error(response, "denylist entry not found with that id", http.StatusNotFound)
+		return
+	}
+
+	payload := map[string]string{
+		"keys":  strings.Join(entry.Keys, KeysSeparator),
+		"regex": entry.Regex.String(),
+	}
+
+	response.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(response).Encode(payload)
+	response.WriteHeader(http.StatusOK)
+}
+
+// DELETE /denylist/...
+func deleteDenylistEntry(response http.ResponseWriter, request *http.Request, denylist *Denylist) {
+	id := request.URL.Path
+	deleted := denylist.DeleteEntry(id)
+
+	if !deleted {
+		http.Error(response, "denylist entry not found with that id", http.StatusNotFound)
+		return
+	}
+
+	response.WriteHeader(http.StatusNoContent)
+}

lib/denylist/main.go

@@ -0,0 +1,115 @@
+package denylist
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+// KeysSeparator is the character that separates the object keys when parsing a new denylist entry
+const KeysSeparator = "."
+
+// DenylistEntry is one active rule for when an oplog update would be skipped
+type DenylistEntry struct {
+	// Keys is an array of object keys to index into an oplog update document.
+	// Each key is applied to the object at the previous key, so to index the text in an object
+	// like {"a":{"b":{"c":"text"}}}, the Keys array would be ["a", "b", "c"]
+	Keys []string
+	// Regex is a regular expression to test against the oplog document's data at the specified index.
+	// If the regex is found anywhere in the string, the update document will be skipped.
+	// For instance, to skip the above document if it contained exactly the string text, the regex could be
+	// `^text$`
+	Regex *regexp.Regexp
+}
+
+// Denylist is a list of rules for skipping oplog updates
+type Denylist map[string]*DenylistEntry
+
+// NewDenylist creates a new empty Denylist with no rules
+func NewDenylist() *Denylist {
+	return &Denylist{}
+}
+
+// GetKeys returns a list of identifiers for the active rules of this Denylist
+func (dl *Denylist) GetKeys() []string {
+	keys := make([]string, len(*dl))
+
+	i := 0
+	for k := range *dl {
+		keys[i] = k
+		i++
+	}
+
+	return keys
+}
+
+// GetEntry returns an active Denylist rule corresponding to the provided identifier
+func (dl *Denylist) GetEntry(key string) *DenylistEntry {
+	if dle, ok := (*dl)[key]; ok {
+		return dle
+	}
+	return nil
+}
+
+// DeleteEntry removes a rule from this Denylist, so it will no longer cause oplog updates to be skipped.
+// Returns true if the rule existed (and was removed), or false if it didn't (and thus wasn't).
+func (dl *Denylist) DeleteEntry(key string) bool {
+	if _, ok := (*dl)[key]; ok {
+		delete(*dl, key)
+		return true
+	}
+	return false
+}
+
+// AppendEntry constructs and adds a new rule to this Denylist. The contents of the rule (keys to check and regex)
+// are provided as strings. The unparsed keys array should be delimited by the keysSeparator character.
+// Returns the random identifier for the new rule in the Denylist, or an error if the regex couldn't be compiled.
+func (dl *Denylist) AppendEntry(unparsedKeys string, unparsedRegex string) (string, error) {
+	keys := strings.Split(unparsedKeys, KeysSeparator)
+	regex, err := regexp.Compile(unparsedRegex)
+	if err != nil {
+		return "", errors.Wrap(err, "parsing denylist regex")
+	}
+
+	entryKey := uuid.New().String()
+	(*dl)[entryKey] = &DenylistEntry{
+		Keys:  keys,
+		Regex: regex,
+	}
+
+	return entryKey, nil
+}
+
+// PassFilter tests if a provided object passes this Denylist rule.
+// First, the keys are used to index into the object. If, for any key, the
+// object is not a map that can be indexed, the object automatically _passes_ the filter.
+// Then, if the object at the indexed location is not a string, it automatically _passes_ the filter.
+// Otherwise, it will fail the filter if the regex could be found somewhere in the string, otherwise it passes.
+func (dle *DenylistEntry) PassFilter(obj interface{}) bool {
+	for _, key := range dle.Keys {
+		if mapObj, ok := obj.(map[string]interface{}); ok {
+			obj = mapObj[key]
+		} else {
+			return true
+		}
+	}
+	if str, ok := obj.(string); ok {
+		return !dle.Regex.MatchString(str)
+	} else {
+		return true
+	}
+}
+
+// Filter tests if a provided object passes every Denylist rule.
+// If it fails any rule, it fails the entire Denylist, and returns the ID of that rule.
+// If it passes every route, it passes the list, and returns the empty string.
+func (dl *Denylist) Filter(obj map[string]interface{}) string {
+	for id, dle := range *dl {
+		if !dle.PassFilter(obj) {
+			return id
+		}
+	}
+	return ""
+}