diff --git a/client/task_runner.go b/client/task_runner.go
index 56df71f619ae..c8c8196ef7c6 100644
--- a/client/task_runner.go
+++ b/client/task_runner.go
@@ -553,6 +553,10 @@ func (f *tokenFuture) Get() string {
 // allows setting the initial Vault token. This is useful when the Vault token
 // is recovered off disk.
 func (r *TaskRunner) vaultManager(token string) {
+	// Always stop renewing the token. If token is empty or untracked, it is a
+	// no-op so this is always safe.
+	defer r.vaultClient.StopRenewToken(r.vaultFuture.Get())
+
 	// updatedToken lets us store state between loops. If true, a new token
 	// has been retrieved and we need to apply the Vault change mode
 	var updatedToken bool
diff --git a/client/vaultclient/vaultclient.go b/client/vaultclient/vaultclient.go
index a5d0f512c72d..cb17ced33d28 100644
--- a/client/vaultclient/vaultclient.go
+++ b/client/vaultclient/vaultclient.go
@@ -457,9 +457,8 @@ func (c *vaultClient) renew(req *vaultClientRenewalRequest) error {
 			// item is tracked by the renewal loop, stop renewing
 			// it by removing the corresponding heap entry.
 			if err := c.heap.Remove(req.id); err != nil {
-				return fmt.Errorf("failed to remove heap entry. err: %v", err)
+				return fmt.Errorf("failed to remove heap entry: %v", err)
 			}
-			delete(c.heap.heapMap, req.id)
 
 			// Report the fatal error to the client
 			req.errCh <- renewalErr
@@ -578,15 +577,10 @@ func (c *vaultClient) stopRenew(id string) error {
 		return nil
 	}
 
-	// Remove the identifier from the heap
 	if err := c.heap.Remove(id); err != nil {
 		return fmt.Errorf("failed to remove heap entry: %v", err)
 	}
 
-	// Delete the identifier from the map only after the it is removed from
-	// the heap. Heap's remove method relies on the heap map.
-	delete(c.heap.heapMap, id)
-
 	// Signal an update to the renewal loop.
 	if c.running {
 		select {