From 315e7b90ce05e833008874787f71d4fe7507e044 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 6 Jan 2025 12:53:02 +0100 Subject: [PATCH 1/8] Add support to image_prefix_push --- binderhub/builder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index f420febe7..118e108fa 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -385,7 +385,10 @@ async def get(self, provider_prefix, _unescaped_spec): # generate a complete build name (for GitHub: `build-{user}-{repo}-{ref}`) - image_prefix = self.settings["image_prefix"] + if "image_prefix_push" in self.settings: + image_prefix = self.settings["image_prefix_push"] + else: + image_prefix = self.settings["image_prefix"] # Enforces max 255 characters before image safe_build_slug = _safe_build_slug( From 35c656c61e1be17dac538b909ecc47230ea80c9e Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 6 Jan 2025 13:02:15 +0100 Subject: [PATCH 2/8] Add image_prefix_push and image_prefix_pull to app.py --- binderhub/app.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/binderhub/app.py b/binderhub/app.py index 560f54506..4dd08f34e 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -431,7 +431,37 @@ def _pod_quota_deprecated(self, change): image_prefix = Unicode( "", help=""" - Prefix for all built docker images. + Fallback prefix for all built docker images. + + If you are pushing to gcr.io, this would start with: + gcr.io// + + Set according to whatever registry you are pushing to. + + Defaults to "", which is probably not what you want :) + """, + config=True, + ) + + image_prefix_push = Unicode( + "", + help=""" + Prefix for built docker images push to container registry. + + If you are pushing to gcr.io, this would start with: + gcr.io// + + Set according to whatever registry you are pushing to. + + Defaults to "", which is probably not what you want :) + """, + config=True, + ) + + image_prefix_pull = Unicode( + "", + help=""" + Prefix for built docker images pull from container registry. If you are pushing to gcr.io, this would start with: gcr.io// From 419329ded9b891d197c4412d722fcf707dee6bce Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 13 Jan 2025 12:01:06 +0100 Subject: [PATCH 3/8] Add default value for image_prefix_push and image_prefix_pull. --- binderhub/app.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 4dd08f34e..42d2a2827 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -444,7 +444,6 @@ def _pod_quota_deprecated(self, change): ) image_prefix_push = Unicode( - "", help=""" Prefix for built docker images push to container registry. @@ -458,8 +457,11 @@ def _pod_quota_deprecated(self, change): config=True, ) + @default("image_prefix_push") + def _image_prefix_push_default(self): + return self.image_prefix + image_prefix_pull = Unicode( - "", help=""" Prefix for built docker images pull from container registry. @@ -473,6 +475,11 @@ def _pod_quota_deprecated(self, change): config=True, ) + @default("image_prefix_pull") + def _image_prefix_pull_default(self): + return self.image_prefix + + build_memory_request = ByteSpecification( 0, help=""" From 05d1068dfe5028fd504cbacc7bf36e1a49629433 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Mon, 13 Jan 2025 12:05:56 +0100 Subject: [PATCH 4/8] Pass values to tornado - image_prefix_pull - image_prefix_push --- binderhub/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/binderhub/app.py b/binderhub/app.py index 42d2a2827..32caf8261 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -972,7 +972,8 @@ def initialize(self, *args, **kwargs): self.tornado_settings.update( { "log_function": log_request, - "image_prefix": self.image_prefix, + "image_prefix_pull": self.image_prefix_pull, + "image_prefix_push": self.image_prefix_push, "debug": self.debug, "default_opengraph_title": self.default_opengraph_title, "launcher": self.launcher, From 1747e92ffdac8913fc35b059ec2a743bcd3387cd Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 14 Jan 2025 14:14:37 +0100 Subject: [PATCH 5/8] Use image_prefix_push and image_prefix_pull --- binderhub/builder.py | 5 +---- binderhub/health.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index 118e108fa..494b87444 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -385,10 +385,7 @@ async def get(self, provider_prefix, _unescaped_spec): # generate a complete build name (for GitHub: `build-{user}-{repo}-{ref}`) - if "image_prefix_push" in self.settings: - image_prefix = self.settings["image_prefix_push"] - else: - image_prefix = self.settings["image_prefix"] + image_prefix = self.settings["image_prefix_push"] # Enforces max 255 characters before image safe_build_slug = _safe_build_slug( diff --git a/binderhub/health.py b/binderhub/health.py index 1b65bf3e0..a7d182f84 100644 --- a/binderhub/health.py +++ b/binderhub/health.py @@ -153,7 +153,7 @@ async def check_docker_registry(self): registry = self.settings["registry"] # we are only interested in getting a response from the registry, we # don't care if the image actually exists or not - image_fullname = self.settings["image_prefix"] + "some-image-name:12345" + image_fullname = self.settings["image_prefix_pull"] + "some-image-name:12345" name, tag = _get_image_basename_and_tag(image_fullname) await registry.get_image_manifest(name, tag) return True From cdbc61f2903128485ddf1aaa16ee0089b8524db5 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Tue, 14 Jan 2025 14:37:44 +0100 Subject: [PATCH 6/8] Rename image_name to image_name_pull and image_name_push --- binderhub/builder.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index 494b87444..3914a0f18 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -385,38 +385,48 @@ async def get(self, provider_prefix, _unescaped_spec): # generate a complete build name (for GitHub: `build-{user}-{repo}-{ref}`) - image_prefix = self.settings["image_prefix_push"] + image_prefix_push = self.settings["image_prefix_push"] + image_prefix_pull = self.settings["image_prefix_pull"] # Enforces max 255 characters before image safe_build_slug = _safe_build_slug( - provider.get_build_slug(), limit=255 - len(image_prefix) + provider.get_build_slug(), + limit=255 - max(len(image_prefix_push), len(image_prefix_pull)) ) build_name = _generate_build_name( provider.get_build_slug(), ref, prefix="build-" ) - image_name = self.image_name = ( + image_name_push = self.image_name_push = ( "{prefix}{build_slug}:{ref}".format( - prefix=image_prefix, build_slug=safe_build_slug, ref=ref + prefix=image_prefix_push, build_slug=safe_build_slug, ref=ref + ) + .replace("_", "-") + .lower() + ) + image_name_pull = self.image_name_pull = ( + "{prefix}{build_slug}:{ref}".format( + prefix=image_prefix_pull, build_slug=safe_build_slug, ref=ref ) .replace("_", "-") .lower() ) - image_without_tag, image_tag = _get_image_basename_and_tag(image_name) + image_push_without_tag, image_push_tag = _get_image_basename_and_tag(image_name_push) + image_pull_without_tag, image_pull_tag = _get_image_basename_and_tag(image_name_pull) if self.settings["use_registry"]: for _ in range(3): try: image_manifest = await self.registry.get_image_manifest( - image_without_tag, image_tag + image_pull_without_tag, image_pull_tag ) image_found = bool(image_manifest) break except HTTPClientError: app_log.exception( "Failed to get image manifest for %s", - image_name, + image_name_pull, ) image_found = False else: @@ -424,7 +434,7 @@ async def get(self, provider_prefix, _unescaped_spec): # Assume we're running in single-node mode or all binder pods are assigned to the same node! docker_client = docker.from_env(version="auto") try: - docker_client.images.get(image_name) + docker_client.images.get(image_name_pull) except docker.errors.ImageNotFound: # image doesn't exist, so do a build! image_found = False @@ -437,7 +447,7 @@ async def get(self, provider_prefix, _unescaped_spec): await self.emit( { "phase": "ready", - "imageName": image_name, + "imageName": image_name_push, "message": "Done! Found built image\n", } ) @@ -445,7 +455,7 @@ async def get(self, provider_prefix, _unescaped_spec): await self.emit( { "phase": "built", - "imageName": image_name, + "imageName": image_name_push, "message": "Found built image, launching...\n", } ) @@ -490,12 +500,12 @@ async def get(self, provider_prefix, _unescaped_spec): name=build_name, repo_url=repo_url, ref=ref, - image_name=image_name, + image_name=image_name_push, git_credentials=provider.git_credentials, ) if self.settings["use_registry"]: push_token = await self.registry.get_credentials( - image_without_tag, image_tag + image_push_without_tag, image_push_tag ) if push_token: build.registry_credentials = push_token @@ -555,7 +565,7 @@ def _check_result(future): event = { "phase": phase, "message": message, - "imageName": image_name, + "imageName": image_name_push, } BUILD_TIME.labels(status="success").observe( time.perf_counter() - build_starttime @@ -647,7 +657,7 @@ async def check_quota(self, provider): launch_quota = self.settings["launch_quota"] try: return await launch_quota.check_repo_quota( - self.image_name, repo_config, self.repo_url + self.image_name_pull, repo_config, self.repo_url ) except LaunchQuotaExceeded as e: LAUNCH_COUNT.labels( @@ -715,7 +725,7 @@ async def handle_progress_event(event): "binder_persistent_request": self.binder_persistent_request, } server_info = await launcher.launch( - image=self.image_name, + image=self.image_name_pull, username=username, server_name=server_name, repo_url=self.repo_url, From 479ce72fe01b5a5a0557f4850b7429ddf08fbbe0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:20:17 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- binderhub/app.py | 1 - binderhub/builder.py | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 32caf8261..b5134a2d5 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -479,7 +479,6 @@ def _image_prefix_push_default(self): def _image_prefix_pull_default(self): return self.image_prefix - build_memory_request = ByteSpecification( 0, help=""" diff --git a/binderhub/builder.py b/binderhub/builder.py index 3914a0f18..1d0903a93 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -391,7 +391,7 @@ async def get(self, provider_prefix, _unescaped_spec): # Enforces max 255 characters before image safe_build_slug = _safe_build_slug( provider.get_build_slug(), - limit=255 - max(len(image_prefix_push), len(image_prefix_pull)) + limit=255 - max(len(image_prefix_push), len(image_prefix_pull)), ) build_name = _generate_build_name( @@ -413,8 +413,12 @@ async def get(self, provider_prefix, _unescaped_spec): .lower() ) - image_push_without_tag, image_push_tag = _get_image_basename_and_tag(image_name_push) - image_pull_without_tag, image_pull_tag = _get_image_basename_and_tag(image_name_pull) + image_push_without_tag, image_push_tag = _get_image_basename_and_tag( + image_name_push + ) + image_pull_without_tag, image_pull_tag = _get_image_basename_and_tag( + image_name_pull + ) if self.settings["use_registry"]: for _ in range(3): try: From c322eabfdd78574e6b2ea03863f04ad4593060d7 Mon Sep 17 00:00:00 2001 From: Raniere Silva Date: Wed, 15 Jan 2025 09:59:37 +0100 Subject: [PATCH 8/8] Improve grammar in documentation as suggested by @manics. --- binderhub/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index b5134a2d5..f15962520 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -445,7 +445,7 @@ def _pod_quota_deprecated(self, change): image_prefix_push = Unicode( help=""" - Prefix for built docker images push to container registry. + Prefix for built docker images being pushed to the to container registry. If you are pushing to gcr.io, this would start with: gcr.io// @@ -463,7 +463,7 @@ def _image_prefix_push_default(self): image_prefix_pull = Unicode( help=""" - Prefix for built docker images pull from container registry. + Prefix for built docker images being pulled from container registry. If you are pushing to gcr.io, this would start with: gcr.io//