From c809acae1a7a0ff0e93d2c87dd15356736a6f257 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 30 Jul 2022 16:12:21 +0100 Subject: [PATCH 1/3] Check for errors when submitting builds --- binderhub/builder.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index fdbc391c9..26522cb5b 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -447,11 +447,25 @@ async def get(self, provider_prefix, _unescaped_spec): ) with BUILDS_INPROGRESS.track_inprogress(): + done = False + failed = False + + def _check_result(future): + nonlocal done + nonlocal failed + try: + r = future.result() + app_log.debug("task completed: %s", r) + except Exception: + app_log.error("task failed: %s", exc_info=True) + done = True + failed = True + build_starttime = time.perf_counter() pool = self.settings["build_pool"] # Start building submit_future = pool.submit(build.submit) - # TODO: hook up actual error handling when this fails + submit_future.add_done_callback(_check_result) IOLoop.current().add_callback(lambda: submit_future) log_future = None @@ -464,8 +478,6 @@ async def get(self, provider_prefix, _unescaped_spec): } ) - done = False - failed = False while not done: progress = await q.get() @@ -486,6 +498,7 @@ async def get(self, provider_prefix, _unescaped_spec): # start capturing build logs once the pod is running if log_future is None: log_future = pool.submit(build.stream_logs) + log_future.add_done_callback(_check_result) continue elif progress.payload == ProgressEvent.BuildStatus.COMPLETED: # Do nothing, is ok! From a5d29b66dad3ae7edd10dfa8743bf64c1e9cf971 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 1 Aug 2022 09:51:01 +0100 Subject: [PATCH 2/3] Add back todo about error propagation --- binderhub/builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/binderhub/builder.py b/binderhub/builder.py index 26522cb5b..53907a3ed 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -460,6 +460,7 @@ def _check_result(future): app_log.error("task failed: %s", exc_info=True) done = True failed = True + # TODO: Propagate error to front-end build_starttime = time.perf_counter() pool = self.settings["build_pool"] From 72e185235a7f1a05edcc2904711879377fbd575c Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 1 Aug 2022 19:27:46 +0100 Subject: [PATCH 3/3] Remove leftover format spec in error log --- binderhub/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index 53907a3ed..09dc3cb61 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -457,7 +457,7 @@ def _check_result(future): r = future.result() app_log.debug("task completed: %s", r) except Exception: - app_log.error("task failed: %s", exc_info=True) + app_log.error("task failed", exc_info=True) done = True failed = True # TODO: Propagate error to front-end