Wait on all ingesters to complete processing before returning success

Errors still immediately return an error

pkg/distributor/distributor.go

@@ -386,23 +386,27 @@ func (d *Distributor) sendSamples(ctx context.Context, ingester ring.InstanceDes
 	//
 	// The use of atomic increments here guarantees only a single sendSamples
 	// goroutine will write to either channel.
+	done := false
 	for i := range streamTrackers {
 		if err != nil {
 			if streamTrackers[i].failed.Inc() <= int32(streamTrackers[i].maxFailures) {
 				continue
 			}
-			if pushTracker.samplesFailed.Inc() == 1 {
+			if !done && pushTracker.samplesFailed.Inc() == 1 {
 				pushTracker.err <- err
 			}
 		} else {
 			if streamTrackers[i].succeeded.Inc() != int32(streamTrackers[i].minSuccess) {
 				continue
 			}
 			if pushTracker.samplesPending.Dec() == 0 {
-				pushTracker.done <- struct{}{}
+				done = true
 			}
 		}
 	}
+	if done {
+		pushTracker.done <- struct{}{}
+	}
 }
 
 // TODO taken from Cortex, see if we can refactor out an usable interface.