Loki: Fix query-frontend ready handler

pkg/loki/loki.go

@@ -302,6 +302,15 @@ func (t *Loki) readyHandler(sm *services.Manager) http.HandlerFunc {
 			}
 		}
 
+		// Query Frontend has a special check that makes sure that a querier is attached before it signals
+		// itself as ready
+		if t.frontend != nil {
+			if err := t.frontend.CheckReady(r.Context()); err != nil {
+				http.Error(w, "Query Frontend not ready: "+err.Error(), http.StatusServiceUnavailable)
+				return
+			}
+		}
+
 		http.Error(w, "ready", http.StatusOK)
 	}
 }

pkg/loki/modules.go

@@ -346,8 +346,10 @@ func (t *Loki) initQueryFrontend() (_ services.Service, err error) {
 	t.server.HTTP.Handle("/api/prom/label", frontendHandler)
 	t.server.HTTP.Handle("/api/prom/label/{name}/values", frontendHandler)
 	t.server.HTTP.Handle("/api/prom/series", frontendHandler)
-	// fallback route
-	t.server.HTTP.PathPrefix("/").Handler(defaultHandler)
+
+	// defer tail endpoints to the default handler
+	t.server.HTTP.Handle("/loki/api/v1/tail", defaultHandler)
+	t.server.HTTP.Handle("/api/prom/tail", defaultHandler)
 
 	return services.NewIdleService(nil, func(_ error) error {
 		t.frontend.Close()

production/ksonnet/loki/query-frontend.libsonnet

@@ -11,15 +11,19 @@
     container.new('query-frontend', $._images.query_frontend) +
     container.withPorts($.util.defaultPorts) +
     container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
+    container.mixin.readinessProbe.httpGet.withPath('/ready') +
+    container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) +
+    container.mixin.readinessProbe.withInitialDelaySeconds(15) +
+    container.mixin.readinessProbe.withTimeoutSeconds(1) +
     $.jaeger_mixin +
     if $._config.queryFrontend.sharded_queries_enabled then
-    $.util.resourcesRequests('2', '2Gi') +
-    $.util.resourcesLimits(null, '6Gi') +
-    container.withEnvMap({
-      JAEGER_REPORTER_MAX_QUEUE_SIZE: '5000',
-    })
+      $.util.resourcesRequests('2', '2Gi') +
+      $.util.resourcesLimits(null, '6Gi') +
+      container.withEnvMap({
+        JAEGER_REPORTER_MAX_QUEUE_SIZE: '5000',
+      })
     else $.util.resourcesRequests('2', '600Mi') +
-    $.util.resourcesLimits(null, '1200Mi'),
+         $.util.resourcesLimits(null, '1200Mi'),
 
   local deployment = $.apps.v1.deployment,
 
@@ -38,6 +42,10 @@
     // each query-frontend pod IP and NOT the service IP. To make it, we do NOT
     // use the service cluster IP so that when the service DNS is resolved it
     // returns the set of query-frontend IPs.
-    service.mixin.spec.withClusterIp('None'),
+    service.mixin.spec.withClusterIp('None') +
+    // Query frontend will not become ready until at least one querier connects
+    // which creates a chicken and egg scenario if we don't publish the
+    // query-frontend address before it's ready.
+    service.mixin.spec.withPublishNotReadyAddresses(true),
 
 }