Merge pull request #1637 from manics/registry-dynamic-token

Support dynamic registry tokens, add wrapper for working with registry microservice

binderhub/registry.py

@@ -346,3 +346,102 @@ class FakeRegistry(DockerRegistry):
 
     async def get_image_manifest(self, image, tag):
         return None
+
+
+class ExternalRegistryHelper(DockerRegistry):
+    """
+    A registry that uses a micro-service to check and create image
+    repositories.
+
+    Also handles creation of tokens for pushing to a registry if required.
+    """
+
+    service_url = Unicode(
+        "http://binderhub-container-registry-helper:8080",
+        allow_none=False,
+        help="The URL of the registry helper micro-service.",
+        config=True,
+    )
+
+    auth_token = Unicode(
+        os.getenv("BINDERHUB_CONTAINER_REGISTRY_HELPER_AUTH_TOKEN"),
+        help="The auth token to use when accessing the registry helper micro-service.",
+        config=True,
+    )
+
+    async def _request(self, endpoint, **kwargs):
+        client = httpclient.AsyncHTTPClient()
+        repo_url = f"{self.service_url}{endpoint}"
+        headers = {"Authorization": f"Bearer {self.auth_token}"}
+        repo = await client.fetch(repo_url, headers=headers, **kwargs)
+        return json.loads(repo.body.decode("utf-8"))
+
+    async def _get_image(self, image, tag):
+        repo_url = f"/image/{image}:{tag}"
+        self.log.debug(f"Checking whether image exists: {repo_url}")
+        try:
+            image_json = await self._request(repo_url)
+            return image_json
+        except httpclient.HTTPError as e:
+            if e.code == 404:
+                return None
+            raise
+
+    async def get_image_manifest(self, image, tag):
+        """
+        Checks whether the image exists in the registry.
+
+        If the container repository doesn't exist create the repository.
+
+        The container repository name may not be the same as the BinderHub image name.
+
+        E.g. Oracle Container Registry (OCIR) has the form:
+        OCIR_NAMESPACE/OCIR_REPOSITORY_NAME:TAG
+
+        These extra components are handled automatically by the registry helper
+        so BinderHub repository names such as OCIR_NAMESPACE/OCIR_REPOSITORY_NAME
+        can be used directly, it is not necessary to remove the extra components.
+
+        Returns the image manifest if the image exists, otherwise None
+        """
+
+        repo_url = f"/repo/{image}"
+        self.log.debug(f"Checking whether repository exists: {repo_url}")
+        try:
+            repo_json = await self._request(repo_url)
+        except httpclient.HTTPError as e:
+            if e.code == 404:
+                repo_json = None
+            else:
+                raise
+
+        if repo_json:
+            return await self._get_image(image, tag)
+        else:
+            self.log.debug(f"Creating repository: {repo_url}")
+            await self._request(repo_url, method="POST", body="")
+            return None
+
+    async def get_credentials(self, image, tag):
+        """
+        Get the registry credentials for the given image and tag if supported
+        by the remote helper, otherwise returns None
+
+        Returns a dictionary of login fields.
+        """
+        token_url = f"/token/{image}:{tag}"
+        self.log.debug(f"Getting registry token: {token_url}")
+        token_json = None
+        try:
+            token_json = await self._request(token_url, method="POST", body="")
+        except httpclient.HTTPError as e:
+            if e.code == 404:
+                return None
+            raise
+        self.log.debug(f"Token: {*token_json.keys(),}")
+        token = {
+            k: v
+            for (k, v) in token_json.items()
+            if k in ["username", "password", "registry"]
+        }
+        return token

binderhub/tests/test_registry.py

@@ -9,7 +9,7 @@
 from tornado import httpclient
 from tornado.web import Application, HTTPError, RequestHandler
 
-from binderhub.registry import DockerRegistry
+from binderhub.registry import DockerRegistry, ExternalRegistryHelper
 
 
 def test_registry_defaults(tmpdir):
@@ -242,3 +242,149 @@ async def test_get_image_manifest(tmpdir, token_url_known):
     assert registry.password == password
     manifest = await registry.get_image_manifest("myimage", "abc123")
     assert manifest == {"image": "myimage", "tag": "abc123"}
+
+
+class FakeExternalRegistryHandler(RequestHandler):
+    def initialize(self, store):
+        self.store = store
+
+
+class FakeRegistryRepoHandler(FakeExternalRegistryHandler):
+    def get(self, repo):
+        print(f"GET {repo} request received\n")
+        self.store.append(self.request)
+        if self.request.headers.get("Authorization") != "Bearer registry-token":
+            self.set_status(403)
+        if repo == "owner/my-repo":
+            self.write(json.dumps({"RepositoryName": "owner/my-repo"}))
+        else:
+            self.set_status(404)
+
+    def post(self, repo):
+        print(f"POST {repo} request received\n")
+        self.store.append(self.request)
+        if self.request.headers.get("Authorization") != "Bearer registry-token":
+            self.set_status(403)
+        if repo == "owner/new-repo":
+            self.write(json.dumps({"RepositoryName": "owner/my-repo"}))
+        else:
+            self.set_status(
+                499, f"Unexpected test request {self.request.method} {self.request.uri}"
+            )
+
+
+class FakeRegistryImageHandler(FakeExternalRegistryHandler):
+    def get(self, image):
+        print(f"GET {image} request received\n")
+        self.store.append(self.request)
+        if self.request.headers.get("Authorization") != "Bearer registry-token":
+            self.set_status(403)
+        if image in ("owner/my-repo", "owner/my-repo:latest", "owner/my-repo:tag"):
+            self.write(json.dumps({"ImageTags": ["latest", "tag"]}))
+        else:
+            self.set_status(404)
+
+
+class FakeRegistryTokenHandler(FakeExternalRegistryHandler):
+    def post(self, repo):
+        print(f"POST {repo} request received\n")
+        self.store.append(self.request)
+        if self.request.headers.get("Authorization") != "Bearer registry-token":
+            self.set_status(403)
+        if repo == "owner/my-repo:tag":
+            self.write(
+                json.dumps(
+                    {
+                        "username": "user",
+                        "password": "token",
+                        "registry": "registry.example.org",
+                    }
+                )
+            )
+        else:
+            self.set_status(
+                499, f"Unexpected test request {self.request.method} {self.request.uri}"
+            )
+
+
+@pytest.fixture
+async def fake_external_registry():
+    request_store = []
+    app = Application(
+        [
+            (r"/repo/(.+)", FakeRegistryRepoHandler, {"store": request_store}),
+            (r"/image/(.+)", FakeRegistryImageHandler, {"store": request_store}),
+            (r"/token/(.+)", FakeRegistryTokenHandler, {"store": request_store}),
+        ]
+    )
+    ip = "127.0.0.1"
+    port = None
+    for _ in range(100):
+        port = randint(10000, 65535)
+        try:
+            server = app.listen(port, ip)
+            break
+        except OSError:
+            port = None
+    if port is None:
+        raise Exception("Failed to find a free port")
+
+    yield f"http://{ip}:{port}", request_store
+
+    server.stop()
+
+
+async def test_external_registry_helper_exists(fake_external_registry):
+    service, request_store = fake_external_registry
+
+    registry = ExternalRegistryHelper(
+        service_url=service,
+        auth_token="registry-token",
+    )
+
+    r = await registry.get_image_manifest("owner/my-repo", "tag")
+    assert r == {"ImageTags": ["latest", "tag"]}
+
+    assert len(request_store) == 2
+    assert request_store[0].method == "GET"
+    assert request_store[0].uri == "/repo/owner/my-repo"
+    assert request_store[1].method == "GET"
+    assert request_store[1].uri == "/image/owner/my-repo:tag"
+
+
+async def test_external_registry_helper_not_exists(fake_external_registry):
+    service, request_store = fake_external_registry
+
+    registry = ExternalRegistryHelper(
+        service_url=service,
+        auth_token="registry-token",
+    )
+
+    r = await registry.get_image_manifest("owner/new-repo", "tag")
+    assert r is None
+
+    assert len(request_store) == 2
+    assert request_store[0].method == "GET"
+    assert request_store[0].uri == "/repo/owner/new-repo"
+    assert request_store[1].method == "POST"
+    assert request_store[1].uri == "/repo/owner/new-repo"
+
+
+async def test_external_registry_helper_token(fake_external_registry):
+    service, request_store = fake_external_registry
+
+    registry = ExternalRegistryHelper(
+        service_url=service,
+        auth_token="registry-token",
+    )
+
+    r = await registry.get_credentials("owner/my-repo", "tag")
+    assert r == {
+        "username": "user",
+        "password": "token",
+        "registry": "registry.example.org",
+    }
+
+    assert len(request_store) == 1
+    assert request_store[0].method == "POST"
+    assert request_store[0].uri == "/token/owner/my-repo:tag"