[WIP] possible solution to propagate informative spawn failure messages from spawner to bhub ui

binderhub/builder.py

@@ -12,7 +12,7 @@
 import docker
 from tornado.concurrent import chain_future, Future
 from tornado import gen
-from tornado.web import Finish, authenticated
+from tornado.web import Finish, authenticated, HTTPError
 from tornado.queues import Queue
 from tornado.iostream import StreamClosedError
 from tornado.ioloop import IOLoop
@@ -518,8 +518,8 @@ async def launch(self, kube):
                         status=status, **self.repo_metric_labels,
                     ).inc()
 
-                if i + 1 == launcher.retries:
-                    # last attempt failed, let it raise
+                if i + 1 == launcher.retries or (isinstance(e, HTTPError) and e.status_code == 409):
+                    # last attempt failed or 409 client error, let it raise
                     raise
 
                 # not the last attempt, try again

binderhub/launcher.py

@@ -64,10 +64,6 @@ async def api_request(self, url, *args, **kwargs):
             try:
                 return await AsyncHTTPClient().fetch(req)
             except HTTPError as e:
-                # swallow 409 errors on retry only (not first attempt)
-                if i > 1 and e.code == 409 and e.response:
-                    self.log.warning("Treating 409 conflict on retry as success")
-                    return e.response
                 # retry requests that fail with error codes greater than 500
                 # because they are likely intermittent issues in the cluster
                 # e.g. 502,504 due to ingress issues or Hub relocating,
@@ -189,12 +185,17 @@ async def launch(self, image, username, server_name='', repo_url=''):
         except HTTPError as e:
             if e.response:
                 body = e.response.body
+                message = json.loads(body.decode('utf-8')).get('message', '')
             else:
                 body = ''
+                message = ''
 
-            app_log.error("Error starting server{} for user {}: {}\n{}".
-                          format(_server_name, username, e, body))
-            raise web.HTTPError(500, "Failed to launch image %s" % image)
+            if e.code == 409:
+                raise web.HTTPError(409, message)
+            else:
+                app_log.error("Error starting server{} for user {}: {}\n{}".
+                              format(_server_name, username, e, body))
+                raise web.HTTPError(500, "Failed to launch image %s" % image)
 
         data['url'] = self.hub_url + 'user/%s/%s' % (username, server_name)
         return data

helm-chart/binderhub/values.yaml

@@ -62,7 +62,10 @@ jupyterhub:
   rbac:
     enabled: true
   hub:
+    consecutiveFailureLimit: 0
     extraConfig:
+      hub: |
+        c.JupyterHub.tornado_settings['slow_spawn_timeout']= 10
       binder: |
         import os
         import sys