Improve distributor rollouts (#604)

* sleep an hour

Signed-off-by: Joe Elliott <number101010@gmail.com>

* are we stopping shims

Signed-off-by: Joe Elliott <number101010@gmail.com>

* sleep for 30 seconds

Signed-off-by: Joe Elliott <number101010@gmail.com>

* 30 second sleeps everywhere

Signed-off-by: Joe Elliott <number101010@gmail.com>

* only sleep for 30 seconds once

Signed-off-by: Joe Elliott <number101010@gmail.com>

* added termination grace period

Signed-off-by: Joe Elliott <number101010@gmail.com>

* Slow roll it

Signed-off-by: Joe Elliott <number101010@gmail.com>

* undo change

Signed-off-by: Joe Elliott <number101010@gmail.com>

modules/distributor/receiver/shim.go

@@ -135,6 +135,12 @@ func (r *receiversShim) starting(ctx context.Context) error {
 
 // Called after distributor is asked to stop via StopAsync.
 func (r *receiversShim) stopping(_ error) error {
+	// when shutdown is called on the receiver it immediately shuts down its connection
+	// which drops requests on the floor. at this point in the shutdown process
+	// the readiness handler is already down so we are not receiving any more requests.
+	// sleep for 30 seconds to here to all pending requests to finish.
+	time.Sleep(30 * time.Second)
+
 	ctx, cancelFn := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancelFn()
 

operations/jsonnet/microservices/distributor.libsonnet

@@ -36,6 +36,9 @@
                      app: target_name,
                      [$._config.gossip_member_label]: 'true',
                    }) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) +
+    deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(60) +
     deployment.mixin.spec.template.metadata.withAnnotations({
       config_hash: std.md5(std.toString($.tempo_configmap.data['tempo.yaml'])),
     }) +