Merge pull request #94 from square/mmc-remove-oldClients

Delete secrets when syncing a removed client

.travis.yml

@@ -27,4 +27,4 @@ install:
 
 script:
  - go test -v ./...
- - docker run -it keysync-test
+ - docker run -it --rm keysync-test

Dockerfile-test

@@ -1,6 +1,6 @@
 FROM openjdk:8-jre-alpine
 
-RUN apk add --update bash go gcc git musl-dev diffutils util-linux coreutils && \
+RUN apk add --update bash go gcc git musl-dev diffutils util-linux coreutils curl && \
     rm -rf /var/cache/apk/*
 
 COPY testing /opt/keysync/testing/

api.go

@@ -50,16 +50,18 @@ type StatusResponse struct {
 
 func writeSuccess(w http.ResponseWriter) {
 	resp := &StatusResponse{Ok: true}
-	out, _ := json.Marshal(resp)
+	out, _ := json.MarshalIndent(resp, "", "")
 	w.WriteHeader(http.StatusOK)
 	_, _ = w.Write(out)
+	_, _ = w.Write([]byte("\n"))
 }
 
 func writeError(w http.ResponseWriter, status int, err error) {
 	resp := &StatusResponse{Ok: false, Message: err.Error()}
-	out, _ := json.Marshal(resp)
+	out, _ := json.MarshalIndent(resp, "", "")
 	w.WriteHeader(status)
 	_, _ = w.Write(out)
+	_, _ = w.Write([]byte("\n"))
 }
 
 func (a *APIServer) syncAll(w http.ResponseWriter, r *http.Request) {
@@ -88,24 +90,29 @@ func (a *APIServer) syncOne(w http.ResponseWriter, r *http.Request) {
 	a.syncer.syncMutex.Lock()
 	defer a.syncer.syncMutex.Unlock()
 
-	err := a.syncer.LoadClients()
+	pendingCleanup, err := a.syncer.LoadClients()
 	if err != nil {
 		logger.WithError(err).Warn("Failed while loading clients")
 		writeError(w, http.StatusInternalServerError, fmt.Errorf("failed while loading clients: %s", err))
 		return
 	}
-
-	syncerEntry, ok := a.syncer.clients[client]
-	if !ok {
-		logger.Infof("Unknown client: %s", client)
-		writeError(w, http.StatusNotFound, fmt.Errorf("unknown client: %s", client))
-		return
-	}
-	err = syncerEntry.Sync()
-	if err != nil {
-		logger.WithError(err).Warnf("Error syncing %s", client)
-		writeError(w, http.StatusInternalServerError, fmt.Errorf("error syncing %s: %s", client, err))
-		return
+	// We do this in a defer because we want it to run regardless of which of the
+	// below cases we end up in.
+	defer pendingCleanup.cleanup(a.logger)
+
+	if syncerEntry, ok := a.syncer.clients[client]; ok {
+		if err = syncerEntry.Sync(); err != nil {
+			logger.WithError(err).Warnf("Error syncing %s", client)
+			writeError(w, http.StatusInternalServerError, fmt.Errorf("error syncing %s: %s", client, err))
+			return
+		}
+	} else {
+		if _, pending := pendingCleanup.Outputs[client]; !pending {
+			// If it's not a current client, or one pending cleanup, return an error
+			logger.Infof("Unknown client: %s", client)
+			writeError(w, http.StatusNotFound, fmt.Errorf("unknown client: %s", client))
+			return
+		}
 	}
 
 	writeSuccess(w)

api_test.go

@@ -134,7 +134,7 @@ func TestApiSyncOneError(t *testing.T) {
 	syncer, err := NewSyncer(config, OutputDirCollection{}, logrus.NewEntry(logrus.New()), metricsForTest())
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	assert.NotNil(t, err)
 
 	NewAPIServer(syncer, port, logrus.NewEntry(logrus.New()), metricsForTest())
@@ -170,7 +170,7 @@ func TestHealthCheck(t *testing.T) {
 	syncer, err := NewSyncer(config, OutputDirCollection{}, logrus.NewEntry(logrus.New()), metricsForTest())
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	assert.NotNil(t, err)
 
 	NewAPIServer(syncer, port, logrus.NewEntry(logrus.New()), metricsForTest())
@@ -208,7 +208,7 @@ func TestMetricsReporting(t *testing.T) {
 	syncer, err := NewSyncer(config, OutputDirCollection{}, logrus.NewEntry(logrus.New()), metricsForTest())
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	assert.NotNil(t, err)
 
 	NewAPIServer(syncer, port, logrus.NewEntry(logrus.New()), metricsForTest())

syncer.go

@@ -25,7 +25,7 @@ import (
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
-	"github.com/square/go-sq-metrics"
+	sqmetrics "github.com/square/go-sq-metrics"
 )
 
 var (
@@ -59,7 +59,6 @@ type Syncer struct {
 	config                 *Config
 	server                 *url.URL
 	clients                map[string]syncerEntry
-	oldClients             map[string]syncerEntry
 	logger                 *logrus.Entry
 	metricsHandle          *sqmetrics.SquareMetrics
 	syncMutex              sync.Mutex
@@ -87,7 +86,6 @@ func NewSyncer(config *Config, outputCollection OutputCollection, logger *logrus
 	syncer := Syncer{
 		config:           config,
 		clients:          map[string]syncerEntry{},
-		oldClients:       map[string]syncerEntry{},
 		logger:           logger,
 		metricsHandle:    metricsHandle,
 		pollInterval:     pollInterval,
@@ -115,7 +113,6 @@ func NewSyncerFromFile(config *Config, clientConfig ClientConfig, bundle string,
 	syncer := Syncer{
 		config:                 config,
 		clients:                map[string]syncerEntry{},
-		oldClients:             map[string]syncerEntry{},
 		logger:                 logger,
 		metricsHandle:          metricsHandle,
 		disableClientReloading: true,
@@ -173,15 +170,38 @@ func (s *Syncer) mostRecentError() (err error) {
 	return *((*error)(atomic.LoadPointer(&s.lastError)))
 }
 
+type pendingCleanup struct {
+	Outputs map[string]Output
+}
+
+func (p *pendingCleanup) cleanup(logger *logrus.Entry) []error {
+	var errors []error
+	if p == nil {
+		return errors
+	}
+
+	for name, output := range p.Outputs {
+		if err := output.RemoveAll(); err != nil {
+			errors = append(errors, err)
+			logger.WithError(err).WithField("name", name).Warn("Failed to remove old client")
+		} else {
+			logger.WithField("name", name).Info("Removed old client")
+		}
+	}
+
+	return errors
+}
+
 // LoadClients gets configured clients,
-func (s *Syncer) LoadClients() error {
+// This function returns clients that have been deconfigured, which are expected to be cleaned up
+func (s *Syncer) LoadClients() (*pendingCleanup, error) {
 	if s.disableClientReloading {
-		return nil
+		return nil, nil
 	}
 
 	newConfigs, err := s.config.LoadClients()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	s.logger.WithField("count", len(newConfigs)).Info("Loaded configs")
 
@@ -207,15 +227,17 @@ func (s *Syncer) LoadClients() error {
 		}
 		s.clients[name] = *client
 	}
+
+	pending := &pendingCleanup{Outputs: map[string]Output{}}
 	for name, client := range s.clients {
 		// Record which clients have gone away, for later cleanup.
 		_, ok := newConfigs[name]
 		if !ok {
-			s.oldClients[name] = client
+			pending.Outputs[name] = client.output
 			delete(s.clients, name)
 		}
 	}
-	return nil
+	return pending, nil
 }
 
 // buildClient collects the configuration and builds a client.  Most of this code should probably be refactored ito NewClient
@@ -275,7 +297,7 @@ func (s *Syncer) RunOnce() []error {
 	s.syncMutex.Lock()
 	defer s.syncMutex.Unlock()
 	var errors []error
-	err := s.LoadClients()
+	pendingCleanup, err := s.LoadClients()
 	if err != nil {
 		return []error{err}
 	}
@@ -292,18 +314,8 @@ func (s *Syncer) RunOnce() []error {
 	}
 
 	// Remove clients that we noticed the configs disappear for.
-	// While the loop below would take care of it too, we don't warn in the expected case.
-	for name, entry := range s.oldClients {
-		if err := entry.output.RemoveAll(); err != nil {
-			errors = append(errors, err)
-			s.logger.WithError(err).WithField("name", name).Warn("Failed to remove old client")
-		} else {
-			s.logger.WithField("name", name).Info("Removed old client")
-		}
-		// This is outside the error check, because we should only try to clean up
-		// once.  If it failed and still exists, the sweep below will catch it.
-		delete(s.oldClients, name)
-	}
+	// While the function below would take care of it too, we don't warn in the expected case.
+	errors = append(errors, pendingCleanup.cleanup(s.logger)...)
 
 	// Clean up any old content in the secrets directory
 	errors = append(errors, s.outputCollection.Cleanup(clientDirs, s.logger)...)

syncer_test.go

@@ -34,11 +34,11 @@ func TestSyncerLoadClients(t *testing.T) {
 	syncer, err := NewSyncer(config, NewInMemoryOutputCollection(), logrus.NewEntry(logrus.New()), metricsForTest())
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 
 	// The clients should reload without error
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 }
 
@@ -49,7 +49,7 @@ func TestSyncerLoadClientsError(t *testing.T) {
 	syncer, err := NewSyncer(config, NewInMemoryOutputCollection(), logrus.NewEntry(logrus.New()), metricsForTest())
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.NotNil(t, err)
 }
 
@@ -156,7 +156,7 @@ func TestSyncerRunLoadClientsFails(t *testing.T) {
 }
 
 func TestNewSyncerFails(t *testing.T) {
-	// Load a test config which fails on LoadCLients
+	// Load a test config which fails on LoadClients
 	config, err := LoadConfig("fixtures/configs/errorconfigs/nonexistent-client-dir-config.yaml")
 	require.Nil(t, err)
 
@@ -176,7 +176,7 @@ func TestSyncerEntrySyncKeywhizFails(t *testing.T) {
 	syncer, err := createNewSyncer("fixtures/configs/test-config.yaml", server)
 	require.Nil(t, err)
 
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 
 	for name, entry := range syncer.clients {
@@ -209,7 +209,7 @@ func TestSyncerEntrySyncKeywhizFails(t *testing.T) {
 
 	// Clear and reload the clients to force them to pick up the new server
 	syncer.clients = make(map[string]syncerEntry)
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 
 	for name, entry := range syncer.clients {
@@ -242,7 +242,7 @@ func TestSyncerEntrySyncKeywhizFails(t *testing.T) {
 
 	// Clear and reload the clients to force them to pick up the new server
 	syncer.clients = make(map[string]syncerEntry)
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 
 	for name, entry := range syncer.clients {
@@ -273,7 +273,7 @@ func TestSyncerEntrySyncKeywhizFails(t *testing.T) {
 
 	// Clear and reload the clients to force them to pick up the new server
 	syncer.clients = make(map[string]syncerEntry)
-	err = syncer.LoadClients()
+	_, err = syncer.LoadClients()
 	require.Nil(t, err)
 
 	for _, entry := range syncer.clients {

testing/run-tests.sh

@@ -2,32 +2,63 @@
 
 set -eu
 set -o pipefail
+set -x
 
 # Start keywhiz (server)
 java -jar /opt/keysync/testing/keywhiz-server.jar server /opt/keysync/testing/keywhiz-config.yaml &
 
 # Start keysync (client)
 keysync --config /opt/keysync/testing/keysync-config.yaml &
 
-# Some time to finish sync
-# TODO(cs): use curl and hit status endpoint when openjdk/alpine image updates curl package
-sleep 20
+# Call the /sync endpoint which blocks until synced
+sleep 5
+curl --retry 20 -X POST http://localhost:31738/sync
 
 # Diff content & permissions
-cd /secrets/client1
-
-for file in *; do
-  perms_actual="$(stat -c '%U:%G:%a' "$file")"
-  perms_expected="$(cat /opt/keysync/testing/expected/ownership/"$file")"
-  content_actual="$(cat "$file")"
-  content_expected="$(cat /opt/keysync/testing/expected/content/"$file")"
-
-  if [ "$perms_actual" != "$perms_expected" ]; then
-    echo "ERROR: Incorrect ownership on file $file (expecting $perms_expected, got $perms_actual)"
-    exit 1
-  fi
-  if [ "$content_actual" != "$content_expected" ]; then
-    echo "ERROR: Incorrect content in file $file (expecting $content_expected, got $content_actual)"
-    exit 1
-  fi
-done
+function verify {
+  pushd $1
+  for file in *; do
+    perms_actual="$(stat -c '%U:%G:%a' "$file")"
+    perms_expected="$(cat /opt/keysync/testing/expected/ownership/"$file")"
+    content_actual="$(cat "$file")"
+    content_expected="$(cat /opt/keysync/testing/expected/content/"$file")"
+
+    if [ "$perms_actual" != "$perms_expected" ]; then
+      echo "ERROR: Incorrect ownership on file $file (expecting $perms_expected, got $perms_actual)"
+      exit 1
+    fi
+    if [ "$content_actual" != "$content_expected" ]; then
+      echo "ERROR: Incorrect content in file $file (expecting $content_expected, got $content_actual)"
+      exit 1
+    fi
+  done
+  echo "Verified $1"
+  popd
+}
+
+verify /secrets/client1
+
+# Make a second client
+sed 's/client1/client2/g' /opt/keysync/testing/clients/client.yaml > /opt/keysync/testing/clients/client2.yaml
+
+curl --fail -X POST http://localhost:31738/sync/client2
+
+verify /secrets/client2
+
+rm /opt/keysync/testing/clients/client2.yaml
+
+curl --fail -X POST http://localhost:31738/sync/client2
+
+if [ -d /secrets/client2 ]; then
+  echo "ERROR: Client 2 was not removed"
+  exit 1
+fi
+
+# Subsequent try should 404
+curl -X POST http://localhost:31738/sync/client2
+
+# Make sure client 1 still works
+curl --fail -X POST http://localhost:31738/sync/client1
+verify /secrets/client1
+
+echo "Test passed"