From b3f9e7ce8e106c7c32052d94e7680debad433d0b Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Thu, 7 Mar 2024 02:32:29 +0000 Subject: [PATCH 01/14] Add CKAN provider --- binderhub/app.py | 2 + binderhub/event-schemas/launch.json | 3 +- binderhub/main.py | 1 + binderhub/repoproviders.py | 67 +++++++++++++++++++++++++ binderhub/static/js/src/form.js | 3 +- binderhub/tests/test_repoproviders.py | 29 +++++++++++ docs/source/developer/repoproviders.rst | 2 + docs/source/reference/repoproviders.rst | 5 ++ 8 files changed, 110 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 4c9e8f4c2..ae2632c9a 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -56,6 +56,7 @@ from .ratelimit import RateLimiter from .registry import DockerRegistry from .repoproviders import ( + CKANProvider, DataverseProvider, FigshareProvider, GistRepoProvider, @@ -586,6 +587,7 @@ def _default_build_namespace(self): "figshare": FigshareProvider, "hydroshare": HydroshareProvider, "dataverse": DataverseProvider, + "ckan": CKANProvider, }, config=True, help=""" diff --git a/binderhub/event-schemas/launch.json b/binderhub/event-schemas/launch.json index 16e277cf4..446182926 100644 --- a/binderhub/event-schemas/launch.json +++ b/binderhub/event-schemas/launch.json @@ -14,7 +14,8 @@ "Zenodo", "Figshare", "Hydroshare", - "Dataverse" + "Dataverse", + "CKAN" ], "description": "Provider for the repository being launched" }, diff --git a/binderhub/main.py b/binderhub/main.py index 2a2027598..f89d23d79 100644 --- a/binderhub/main.py +++ b/binderhub/main.py @@ -22,6 +22,7 @@ "figshare": "Figshare", "hydroshare": "Hydroshare", "dataverse": "Dataverse", + "ckan": "CKAN", } diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index be9dd75f4..fc3ca9cb5 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -448,6 +448,73 @@ def get_build_slug(self): return f"hydroshare-{self.record_id}" +class CKANProvider(RepoProvider): + """Provide contents of a CKAN dataset + Users must provide a spec consisting of the CKAN dataset URL. + """ + + name = Unicode("CKAN") + + display_name = "CKAN dataset" + + url_regex = r"/dataset/[a-z0-9_\\-]*$" + + labels = { + "text": "CKAN dataset URL (https://demo.ckan.org/dataset/sample-dataset-1)", + "tag_text": "Git ref (branch, tag, or commit)", + "ref_prop_disabled": True, + "label_prop_disabled": True, + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.repo = urllib.parse.unquote(self.spec) + + async def get_resolved_ref(self): + parsed_repo = urlparse(self.repo) + self.dataset_id = parsed_repo.path.rsplit("/", maxsplit=1)[1] + + client = AsyncHTTPClient() + + api = parsed_repo._replace( + path=re.sub(self.url_regex, "/api/3/action/", parsed_repo.path) + ).geturl() + + package_show_url = f"{api}package_show?id={self.dataset_id}" + + try: + r = await client.fetch(package_show_url, user_agent="BinderHub") + except HTTPError: + return None + + def parse_date(json_body): + json_response = json.loads(json_body) + date = json_response["result"]["metadata_modified"] + parsed_date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") + epoch = parsed_date.replace(tzinfo=timezone(timedelta(0))).timestamp() + # truncate the timestamp + return str(int(epoch)) + + self.record_id = f"{self.dataset_id}.v{parse_date(r.body)}" + + return self.record_id + + async def get_resolved_spec(self): + if not hasattr(self, "record_id"): + await self.get_resolved_ref() + return self.repo + + def get_repo_url(self): + return self.repo + + async def get_resolved_ref_url(self): + resolved_spec = await self.get_resolved_spec() + return resolved_spec + + def get_build_slug(self): + return f"ckan-{self.dataset_id}" + + class GitRepoProvider(RepoProvider): """Bare bones git repo provider. diff --git a/binderhub/static/js/src/form.js b/binderhub/static/js/src/form.js index cc00d7b45..1bf70e6f1 100644 --- a/binderhub/static/js/src/form.js +++ b/binderhub/static/js/src/form.js @@ -31,7 +31,8 @@ export function getBuildFormValues() { providerPrefix === "zenodo" || providerPrefix === "figshare" || providerPrefix === "dataverse" || - providerPrefix === "hydroshare" + providerPrefix === "hydroshare" || + providerPrefix === "ckan" ) { ref = ""; } diff --git a/binderhub/tests/test_repoproviders.py b/binderhub/tests/test_repoproviders.py index df5f63e0e..591bc0848 100644 --- a/binderhub/tests/test_repoproviders.py +++ b/binderhub/tests/test_repoproviders.py @@ -6,6 +6,7 @@ from tornado.ioloop import IOLoop from binderhub.repoproviders import ( + CKANProvider, DataverseProvider, FigshareProvider, GistRepoProvider, @@ -209,6 +210,34 @@ async def test_dataverse( assert spec == resolved_spec +@pytest.mark.parametrize( + "spec,resolved_spec,resolved_ref,resolved_ref_url,build_slug", + [ + [ + "https://demo.ckan.org/dataset/sample-dataset-1", + "https://demo.ckan.org/dataset/sample-dataset-1", + "sample-dataset-1.v", + "https://demo.ckan.org/dataset/sample-dataset-1", + "ckan-sample-dataset-1", + ], + ], +) +async def test_ckan(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug): + provider = CKANProvider(spec=spec) + + ref = await provider.get_resolved_ref() + assert resolved_ref in ref + + slug = provider.get_build_slug() + assert slug == build_slug + repo_url = provider.get_repo_url() + assert repo_url == spec + ref_url = await provider.get_resolved_ref_url() + assert ref_url == resolved_ref_url + spec = await provider.get_resolved_spec() + assert spec == resolved_spec + + @pytest.mark.github_api @pytest.mark.parametrize( "repo,unresolved_ref,resolved_ref", diff --git a/docs/source/developer/repoproviders.rst b/docs/source/developer/repoproviders.rst index 083db3e27..ab648f1c4 100644 --- a/docs/source/developer/repoproviders.rst +++ b/docs/source/developer/repoproviders.rst @@ -36,6 +36,8 @@ Currently supported providers, their prefixes and specs are: +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ | Dataverse | ``dataverse`` | ```` | `Dataverse `_ is open source research data repository software installed all over the world. | +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ + | CKAN | ``ckan`` | ``/`` | `CKAN `_ is an open source data management system. | + +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ | Git | ``git`` | ``/`` | A generic repository provider for URLs that point directly to a git repository. | +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/reference/repoproviders.rst b/docs/source/reference/repoproviders.rst index d0f5ca37c..40b230d70 100644 --- a/docs/source/reference/repoproviders.rst +++ b/docs/source/reference/repoproviders.rst @@ -65,6 +65,11 @@ Module: :mod:`binderhub.repoproviders` .. autoconfigurable:: DataverseProvider :members: +:class:`CKANProvider` +--------------------------- + +.. autoconfigurable:: CKANProvider + :members: :class:`GitRepoProvider` --------------------------- From 83768a1cdbef03a7412c0c82be424e270a012ab8 Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Mon, 25 Mar 2024 12:17:45 +0000 Subject: [PATCH 02/14] Inline the parse_date function --- binderhub/repoproviders.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index fc3ca9cb5..e2d950014 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -487,15 +487,14 @@ async def get_resolved_ref(self): except HTTPError: return None - def parse_date(json_body): - json_response = json.loads(json_body) - date = json_response["result"]["metadata_modified"] - parsed_date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") - epoch = parsed_date.replace(tzinfo=timezone(timedelta(0))).timestamp() - # truncate the timestamp - return str(int(epoch)) - - self.record_id = f"{self.dataset_id}.v{parse_date(r.body)}" + json_response = json.loads(r.body) + date = json_response["result"]["metadata_modified"] + parsed_date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") + epoch = parsed_date.replace(tzinfo=timezone(timedelta(0))).timestamp() + # truncate the timestamp + dataset_version = str(int(epoch)) + + self.record_id = f"{self.dataset_id}.v{dataset_version}" return self.record_id From 1ba7d017584cb2412ca6c5637fbf85b09986c5cd Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Mon, 25 Mar 2024 12:39:56 +0000 Subject: [PATCH 03/14] Make the splitting logic more straightforward --- binderhub/repoproviders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index e2d950014..5843e794d 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -457,8 +457,6 @@ class CKANProvider(RepoProvider): display_name = "CKAN dataset" - url_regex = r"/dataset/[a-z0-9_\\-]*$" - labels = { "text": "CKAN dataset URL (https://demo.ckan.org/dataset/sample-dataset-1)", "tag_text": "Git ref (branch, tag, or commit)", @@ -476,8 +474,10 @@ async def get_resolved_ref(self): client = AsyncHTTPClient() + url_parts = parsed_repo.path.split("/") + api_url_path = "/api/3/action/" api = parsed_repo._replace( - path=re.sub(self.url_regex, "/api/3/action/", parsed_repo.path) + path="/".join(url_parts[:-2]) + api_url_path ).geturl() package_show_url = f"{api}package_show?id={self.dataset_id}" From d0838308d48e8bb4e7cb7bf32b291d1e59f043fd Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Wed, 15 May 2024 03:58:33 +0000 Subject: [PATCH 04/14] Handle the activities --- binderhub/repoproviders.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index 5843e794d..955455671 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -15,7 +15,7 @@ import time import urllib.parse from datetime import datetime, timedelta, timezone -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse import escapism from prometheus_client import Gauge @@ -470,20 +470,35 @@ def __init__(self, *args, **kwargs): async def get_resolved_ref(self): parsed_repo = urlparse(self.repo) - self.dataset_id = parsed_repo.path.rsplit("/", maxsplit=1)[1] - client = AsyncHTTPClient() + url_parts_1 = parsed_repo.path.split("/history/") + url_parts_2 = url_parts_1[0].split("/") + if url_parts_2[-2] == "dataset": + self.dataset_id = url_parts_2[-1] + else: + return None - url_parts = parsed_repo.path.split("/") api_url_path = "/api/3/action/" api = parsed_repo._replace( - path="/".join(url_parts[:-2]) + api_url_path + path="/".join(url_parts_2[:-2]) + api_url_path, query="" ).geturl() - package_show_url = f"{api}package_show?id={self.dataset_id}" + # handle the activites + activity_id = None + if parse_qs(parsed_repo.query).get("activity_id") is not None: + activity_id = parse_qs(parsed_repo.query).get("activity_id")[0] + if len(url_parts_1) == 2: + activity_id = url_parts_1[-1] + if activity_id: + fetch_url = ( + f"{api}activity_data_show?" f"id={activity_id}&object_type=package" + ) + else: + fetch_url = f"{api}package_show?id={self.dataset_id}" + client = AsyncHTTPClient() try: - r = await client.fetch(package_show_url, user_agent="BinderHub") + r = await client.fetch(fetch_url, user_agent="BinderHub") except HTTPError: return None From 32a534100773b0a8c3fd45d2e39d75fb9e0f72c5 Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Thu, 27 Jun 2024 03:54:59 +0000 Subject: [PATCH 05/14] Use urlencode to construct query strings --- binderhub/repoproviders.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index 955455671..2fad9f15b 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -15,7 +15,7 @@ import time import urllib.parse from datetime import datetime, timedelta, timezone -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, urlencode, urlparse import escapism from prometheus_client import Gauge @@ -490,11 +490,11 @@ async def get_resolved_ref(self): if len(url_parts_1) == 2: activity_id = url_parts_1[-1] if activity_id: - fetch_url = ( - f"{api}activity_data_show?" f"id={activity_id}&object_type=package" + fetch_url = f"{api}activity_data_show?" + urlencode( + {"id": activity_id, "object_type": "package"} ) else: - fetch_url = f"{api}package_show?id={self.dataset_id}" + fetch_url = f"{api}package_show?" + urlencode({"id": self.dataset_id}) client = AsyncHTTPClient() try: From 4ad04eaa5a6ef333b01e4b0f5aa448f3367f0250 Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Thu, 27 Jun 2024 07:04:48 +0000 Subject: [PATCH 06/14] Cleanup URL parsing mechanisms --- binderhub/repoproviders.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/binderhub/repoproviders.py b/binderhub/repoproviders.py index 2fad9f15b..dbab18d5a 100644 --- a/binderhub/repoproviders.py +++ b/binderhub/repoproviders.py @@ -471,24 +471,29 @@ def __init__(self, *args, **kwargs): async def get_resolved_ref(self): parsed_repo = urlparse(self.repo) - url_parts_1 = parsed_repo.path.split("/history/") - url_parts_2 = url_parts_1[0].split("/") - if url_parts_2[-2] == "dataset": - self.dataset_id = url_parts_2[-1] - else: + if "/dataset/" not in parsed_repo.path: + # Not actually a dataset return None - api_url_path = "/api/3/action/" + # CKAN may be under a URL prefix, and we should accomodate that + url_prefix, dataset_url = parsed_repo.path.split("/dataset/") + + dataset_url_parts = dataset_url.split("/") + self.dataset_id = dataset_url_parts[0] + api = parsed_repo._replace( - path="/".join(url_parts_2[:-2]) + api_url_path, query="" + path=f"{url_prefix}/api/3/action/", query="" ).geturl() - # handle the activites + # Activity ID may be present either as a query parameter, activity_id + # or as part of the URL, under `/history/`. If `/history/` + # is present, that takes precedence over `activity_id` activity_id = None - if parse_qs(parsed_repo.query).get("activity_id") is not None: + if "history" in dataset_url_parts: + activity_id = dataset_url_parts[dataset_url_parts.index("history") + 1] + elif parse_qs(parsed_repo.query).get("activity_id") is not None: activity_id = parse_qs(parsed_repo.query).get("activity_id")[0] - if len(url_parts_1) == 2: - activity_id = url_parts_1[-1] + if activity_id: fetch_url = f"{api}activity_data_show?" + urlencode( {"id": activity_id, "object_type": "package"} From 2945d83ca686b7a68c28170c9eb4fcfd432702d2 Mon Sep 17 00:00:00 2001 From: Sol Lee Date: Thu, 27 Jun 2024 15:53:28 +0000 Subject: [PATCH 07/14] Add tests for dataset activities and non-CKAN datasets --- binderhub/tests/test_repoproviders.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/binderhub/tests/test_repoproviders.py b/binderhub/tests/test_repoproviders.py index 591bc0848..ea3b6e4c0 100644 --- a/binderhub/tests/test_repoproviders.py +++ b/binderhub/tests/test_repoproviders.py @@ -220,12 +220,31 @@ async def test_dataverse( "https://demo.ckan.org/dataset/sample-dataset-1", "ckan-sample-dataset-1", ], + [ + "https://demo.datashades.com/dataset/chart-test?activity_id=061888e9-e3c2-4769-b097-9c195a841e2f", + "https://demo.datashades.com/dataset/chart-test?activity_id=061888e9-e3c2-4769-b097-9c195a841e2f", + "chart-test.v1717501747", + "https://demo.datashades.com/dataset/chart-test?activity_id=061888e9-e3c2-4769-b097-9c195a841e2f", + "ckan-chart-test", + ], + [ + "https://demo.datashades.com/dataset/chart-test/history/061888e9-e3c2-4769-b097-9c195a841e2f", + "https://demo.datashades.com/dataset/chart-test/history/061888e9-e3c2-4769-b097-9c195a841e2f", + "chart-test.v1717501747", + "https://demo.datashades.com/dataset/chart-test/history/061888e9-e3c2-4769-b097-9c195a841e2f", + "ckan-chart-test", + ], + ["https://demo.ckan.org/group/roger", None, None, None, None], + ["https://demo.ckan.org/dataset/nosuchdataset", None, None, None, None], ], ) async def test_ckan(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug): provider = CKANProvider(spec=spec) ref = await provider.get_resolved_ref() + if not resolved_ref: + # We are done here if we don't expect to resolve + return assert resolved_ref in ref slug = provider.get_build_slug() From 0e9834d189950ef6a7be720f381ae20a0b10601c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 28 Jun 2024 17:29:52 -0700 Subject: [PATCH 08/14] Bump version of repo2docker being used Should pass once https://github.com/jupyterhub/repo2docker/pull/1356 is merged and released --- binderhub/app.py | 2 +- binderhub/build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 5eb97aa9c..00add9199 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -557,7 +557,7 @@ def _default_build_namespace(self): return os.environ.get("BUILD_NAMESPACE", "default") build_image = Unicode( - "quay.io/jupyterhub/repo2docker:2023.06.0", + "quay.io/jupyterhub/repo2docker:2024.06.0", help=""" DEPRECATED: Use c.KubernetesBuildExecutor.build_image diff --git a/binderhub/build.py b/binderhub/build.py index 5e46f7b58..fbcc14fa5 100644 --- a/binderhub/build.py +++ b/binderhub/build.py @@ -288,7 +288,7 @@ def _default_namespace(self): return os.getenv("BUILD_NAMESPACE", "default") build_image = Unicode( - "quay.io/jupyterhub/repo2docker:2023.06.0", + "quay.io/jupyterhub/repo2docker:2024.06.0", help="Docker image containing repo2docker that is used to spawn the build pods.", config=True, ) From 1b4cfae7c8629e37a4ed4cf5e6b48e3a670dea2b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 1 Jul 2024 14:59:33 -0700 Subject: [PATCH 09/14] Bump repo2docker version to match release name --- binderhub/app.py | 2 +- binderhub/build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 00add9199..384752751 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -557,7 +557,7 @@ def _default_build_namespace(self): return os.environ.get("BUILD_NAMESPACE", "default") build_image = Unicode( - "quay.io/jupyterhub/repo2docker:2024.06.0", + "quay.io/jupyterhub/repo2docker:2024.07.0", help=""" DEPRECATED: Use c.KubernetesBuildExecutor.build_image diff --git a/binderhub/build.py b/binderhub/build.py index fbcc14fa5..a5be78d3d 100644 --- a/binderhub/build.py +++ b/binderhub/build.py @@ -288,7 +288,7 @@ def _default_namespace(self): return os.getenv("BUILD_NAMESPACE", "default") build_image = Unicode( - "quay.io/jupyterhub/repo2docker:2024.06.0", + "quay.io/jupyterhub/repo2docker:2024.07.0", help="Docker image containing repo2docker that is used to spawn the build pods.", config=True, ) From c33850405073ab48aec2c639783d9943db092868 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 00:11:53 +0000 Subject: [PATCH 10/14] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.15.2 → v3.16.0](https://github.com/asottile/pyupgrade/compare/v3.15.2...v3.16.0) - [github.com/PyCQA/flake8: 7.0.0 → 7.1.0](https://github.com/PyCQA/flake8/compare/7.0.0...7.1.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ddc56f083..c899556ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.16.0 hooks: - id: pyupgrade args: @@ -71,7 +71,7 @@ repos: # Lint: Python code - repo: https://github.com/PyCQA/flake8 - rev: "7.0.0" + rev: "7.1.0" hooks: - id: flake8 From 97701b051dc1198ee4bfda6dd51fca393e48dc09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 01:49:27 +0000 Subject: [PATCH 11/14] chore(deps): bump certifi in /helm-chart/images/binderhub Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] --- helm-chart/images/binderhub/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/images/binderhub/requirements.txt b/helm-chart/images/binderhub/requirements.txt index 450e70409..e951ac918 100644 --- a/helm-chart/images/binderhub/requirements.txt +++ b/helm-chart/images/binderhub/requirements.txt @@ -14,7 +14,7 @@ attrs==23.2.0 # referencing cachetools==5.3.3 # via google-auth -certifi==2024.2.2 +certifi==2024.7.4 # via # kubernetes # requests From 9bfd6ea1eb4845f907f63086d1631bff0a909fdb Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Jul 2024 13:14:03 +0200 Subject: [PATCH 12/14] Update message about repo not allowed to launch --- binderhub/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binderhub/builder.py b/binderhub/builder.py index c97704726..f420febe7 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -300,7 +300,7 @@ async def get(self, provider_prefix, _unescaped_spec): await self.emit( { "phase": "failed", - "message": f"Sorry, {spec} has been temporarily disabled from launching. Please contact admins for more info!", + "message": f"Sorry, {spec} is not allowed to launch. Please contact admins for more info!", } ) return From e5bac94befd1b1fb28b52bc017b6332082dbf879 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 13:51:28 +0200 Subject: [PATCH 13/14] Bump pip-tools to v7 used by ci/refreeze script updating requirements.txt files The changes in flags are to respect deprecation notices in https://github.com/jazzband/pip-tools?tab=readme-ov-file#deprecations. --- ci/refreeze | 2 +- helm-chart/images/binderhub/requirements.txt | 83 ++++++++++---------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/ci/refreeze b/ci/refreeze index 561a789fe..903646f52 100755 --- a/ci/refreeze +++ b/ci/refreeze @@ -11,4 +11,4 @@ docker run --rm \ --workdir=/io \ --user=root \ python:3.11-bullseye \ - sh -c 'pip install pip-tools==6.* && pip-compile --upgrade helm-chart/images/binderhub/requirements.in' + sh -c 'pip install pip-tools==7.* && pip-compile --allow-unsafe --strip-extras --upgrade helm-chart/images/binderhub/requirements.in' diff --git a/helm-chart/images/binderhub/requirements.txt b/helm-chart/images/binderhub/requirements.txt index e951ac918..8e51e7279 100644 --- a/helm-chart/images/binderhub/requirements.txt +++ b/helm-chart/images/binderhub/requirements.txt @@ -4,7 +4,7 @@ # # Use the "Run workflow" button at https://github.com/jupyterhub/binderhub/actions/workflows/watch-dependencies.yaml # -alembic==1.13.1 +alembic==1.13.2 # via jupyterhub async-generator==1.10 # via jupyterhub @@ -24,33 +24,33 @@ cffi==1.16.0 # via cryptography charset-normalizer==3.3.2 # via requests -cryptography==42.0.5 +cryptography==42.0.8 # via pyopenssl -docker==7.0.0 - # via -r ../../../requirements.txt +docker==7.1.0 + # via -r helm-chart/images/binderhub/../../../requirements.txt escapism==1.0.1 - # via -r ../../../requirements.txt -google-api-core[grpc]==2.18.0 + # via -r helm-chart/images/binderhub/../../../requirements.txt +google-api-core==2.19.1 # via # google-cloud-appengine-logging # google-cloud-core # google-cloud-logging -google-auth==2.29.0 +google-auth==2.32.0 # via # google-api-core # google-cloud-appengine-logging # google-cloud-core # google-cloud-logging # kubernetes -google-cloud-appengine-logging==1.4.3 +google-cloud-appengine-logging==1.4.4 # via google-cloud-logging google-cloud-audit-log==0.2.5 # via google-cloud-logging google-cloud-core==2.4.1 # via google-cloud-logging google-cloud-logging==3.10.0 - # via -r requirements.in -googleapis-common-protos[grpc]==1.63.0 + # via -r helm-chart/images/binderhub/requirements.in +googleapis-common-protos==1.63.2 # via # google-api-core # google-cloud-audit-log @@ -58,39 +58,39 @@ googleapis-common-protos[grpc]==1.63.0 # grpcio-status greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.13.0 +grpc-google-iam-v1==0.13.1 # via google-cloud-logging -grpcio==1.62.1 +grpcio==1.65.0 # via # google-api-core # googleapis-common-protos # grpc-google-iam-v1 # grpcio-status -grpcio-status==1.62.1 +grpcio-status==1.62.2 # via google-api-core idna==3.7 # via requests jinja2==3.1.4 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyterhub -jsonschema==4.21.1 +jsonschema==4.23.0 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyter-telemetry jsonschema-specifications==2023.12.1 # via jsonschema jupyter-telemetry==0.1.0 # via jupyterhub -jupyterhub==4.1.3 +jupyterhub==4.1.5 # via - # -r ../../../requirements.txt - # -r requirements.in + # -r helm-chart/images/binderhub/../../../requirements.txt + # -r helm-chart/images/binderhub/requirements.in kubernetes==9.0.1 # via - # -r ../../../requirements.txt - # -r requirements.in -mako==1.3.2 + # -r helm-chart/images/binderhub/../../../requirements.txt + # -r helm-chart/images/binderhub/requirements.in +mako==1.3.5 # via alembic markupsafe==2.1.5 # via @@ -100,17 +100,15 @@ oauthlib==3.2.2 # via # jupyterhub # requests-oauthlib -packaging==24.0 - # via - # docker - # jupyterhub +packaging==24.1 + # via jupyterhub pamela==1.1.0 # via jupyterhub prometheus-client==0.20.0 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyterhub -proto-plus==1.23.0 +proto-plus==1.24.0 # via # google-api-core # google-cloud-appengine-logging @@ -131,12 +129,12 @@ pyasn1==0.6.0 # rsa pyasn1-modules==0.4.0 # via google-auth -pycparser==2.21 +pycparser==2.22 # via cffi pycurl==7.45.3 - # via -r requirements.in + # via -r helm-chart/images/binderhub/requirements.in pyjwt==2.8.0 - # via -r ../../../requirements.txt + # via -r helm-chart/images/binderhub/../../../requirements.txt pyopenssl==24.1.0 # via certipy python-dateutil==2.9.0.post0 @@ -145,15 +143,15 @@ python-dateutil==2.9.0.post0 # kubernetes python-json-logger==2.0.7 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyter-telemetry pyyaml==6.0.1 # via kubernetes -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # docker # google-api-core @@ -162,7 +160,7 @@ requests==2.31.0 # requests-oauthlib requests-oauthlib==2.0.0 # via kubernetes -rpds-py==0.18.0 +rpds-py==0.19.0 # via # jsonschema # referencing @@ -176,20 +174,20 @@ six==1.16.0 # via # kubernetes # python-dateutil -sqlalchemy==2.0.29 +sqlalchemy==2.0.31 # via # alembic # jupyterhub tornado==6.4.1 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyterhub -traitlets==5.14.2 +traitlets==5.14.3 # via - # -r ../../../requirements.txt + # -r helm-chart/images/binderhub/../../../requirements.txt # jupyter-telemetry # jupyterhub -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # alembic # sqlalchemy @@ -198,8 +196,9 @@ urllib3==2.2.2 # docker # kubernetes # requests -websocket-client==1.7.0 +websocket-client==1.8.0 # via kubernetes # The following packages are considered to be unsafe in a requirements file: -# setuptools +setuptools==70.3.0 + # via kubernetes From 16a5e165ef6d6629390d439434b65c3b9cd0c2db Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 14:04:09 +0200 Subject: [PATCH 14/14] Make pre-commit-hook not work on generated requirements.txt --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c899556ae..1e30f7072 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,8 @@ repos: - id: check-case-conflict - id: check-executables-have-shebangs - id: requirements-txt-fixer + # exclude ci/refreeze generated requirements.txt + exclude: ^.*images\/.*\/requirements\.txt$ # Lint: Python code - repo: https://github.com/PyCQA/flake8