diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da406f61a..a3b830add 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,7 @@ jobs: # jobs or have "template" jobs, so we use `if` conditions on steps tests: runs-on: ubuntu-20.04 + timeout-minutes: 10 strategy: # keep running so we can see if tests with other k3s/k8s/helm versions pass fail-fast: false @@ -278,6 +279,7 @@ jobs: test-local: runs-on: ubuntu-20.04 + timeout-minutes: 5 steps: - uses: actions/checkout@v3 diff --git a/binderhub/app.py b/binderhub/app.py index 47198e60c..ae57543a2 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -41,7 +41,7 @@ from traitlets.config import Application from .base import AboutHandler, Custom404, VersionHandler -from .build import Build +from .build import Build, BuildExecutor from .builder import BuildHandler from .config import ConfigHandler from .events import EventLog @@ -270,10 +270,11 @@ def _valid_badge_base_url(self, proposal): build_class = Type( Build, + klass=BuildExecutor, help=""" The class used to build repo2docker images. - Must inherit from binderhub.build.Build + Must inherit from binderhub.build.BuildExecutor """, config=True, ) @@ -818,6 +819,7 @@ def initialize(self, *args, **kwargs): "build_class": self.build_class, "registry": registry, "traitlets_config": self.config, + "traitlets_parent": self, "google_analytics_code": self.google_analytics_code, "google_analytics_domain": self.google_analytics_domain, "about_message": self.about_message, diff --git a/binderhub/build.py b/binderhub/build.py index 6baa09d25..c7dad35ca 100644 --- a/binderhub/build.py +++ b/binderhub/build.py @@ -4,6 +4,7 @@ import datetime import json +import os import threading import warnings from collections import defaultdict @@ -11,9 +12,12 @@ from typing import Union from urllib.parse import urlparse +import kubernetes.config from kubernetes import client, watch from tornado.ioloop import IOLoop from tornado.log import app_log +from traitlets import Any, Bool, Dict, Integer, Unicode, default +from traitlets.config import LoggingConfigurable from .utils import KUBE_REQUEST_TIMEOUT, rendezvous_rank @@ -49,133 +53,65 @@ def __init__(self, kind: Kind, payload: Union[str, BuildStatus]): self.payload = payload -class Build: - """Represents a build of a git repository into a docker image. - - This ultimately maps to a single pod on a kubernetes cluster. Many - different build objects can point to this single pod and perform - operations on the pod. The code in this class needs to be careful and take - this into account. - - For example, operations a Build object tries might not succeed because - another Build object pointing to the same pod might have done something - else. This should be handled gracefully, and the build object should - reflect the state of the pod as quickly as possible. - - ``name`` - The ``name`` should be unique and immutable since it is used to - sync to the pod. The ``name`` should be unique for a - ``(repo_url, ref)`` tuple, and the same tuple should correspond - to the same ``name``. This allows use of the locking provided by k8s - API instead of having to invent our own locking code. - +class BuildExecutor(LoggingConfigurable): + """Base class for a build of a version controlled repository to a self-contained + environment """ - def __init__( - self, - q, - api, - name, - *, - namespace, - repo_url, - ref, - build_image, - docker_host, - image_name, - git_credentials=None, - push_secret=None, - memory_limit=0, - memory_request=0, - node_selector=None, - appendix="", - log_tail_lines=100, - sticky_builds=False, - ): - """ - Parameters - ---------- - - q : tornado.queues.Queue - Queue that receives progress events after the build has been submitted - api : kubernetes.client.CoreV1Api() - Api object to make kubernetes requests via - name : str - A unique name for the thing (repo, ref) being built. Used to coalesce - builds, make sure they are not being unnecessarily repeated - namespace : str - Kubernetes namespace to spawn build pods into - repo_url : str - URL of repository to build. - Passed through to repo2docker. - ref : str - Ref of repository to build - Passed through to repo2docker. - build_image : str - Docker image containing repo2docker that is used to spawn the build - pods. - docker_host : str - The docker socket to use for building the image. - Must be a unix domain socket on a filesystem path accessible on the - node in which the build pod is running. - image_name : str - Full name of the image to build. Includes the tag. - Passed through to repo2docker. - git_credentials : str - Git credentials to use to clone private repositories. Passed - through to repo2docker via the GIT_CREDENTIAL_ENV environment - variable. Can be anything that will be accepted by git as - a valid output from a git-credential helper. See - https://git-scm.com/docs/gitcredentials for more information. - push_secret : str - Kubernetes secret containing credentials to push docker image to registry. - memory_limit - Memory limit for the docker build process. Can be an integer in - bytes, or a byte specification (like 6M). - Passed through to repo2docker. - memory_request - Memory request of the build pod. The actual building happens in the - docker daemon, but setting request in the build pod makes sure that - memory is reserved for the docker build in the node by the kubernetes - scheduler. - node_selector : dict - Node selector for the kubernetes build pod. - appendix : str - Appendix to be added at the end of the Dockerfile used by repo2docker. - Passed through to repo2docker. - log_tail_lines : int - Number of log lines to fetch from a currently running build. - If a build with the same name is already running when submitted, - only the last `log_tail_lines` number of lines will be fetched and - displayed to the end user. If not, all log lines will be streamed. - sticky_builds : bool - If true, builds for the same repo (but different refs) will try to - schedule on the same node, to reuse cache layers in the docker daemon - being used. - """ - self.q = q - self.api = api - self.repo_url = repo_url - self.ref = ref - self.name = name - self.namespace = namespace - self.image_name = image_name - self.push_secret = push_secret - self.build_image = build_image + q = Any( + help="Queue that receives progress events after the build has been submitted", + ) + + name = Unicode( + help=( + "A unique name for the thing (repo, ref) being built." + "Used to coalesce builds, make sure they are not being unnecessarily repeated." + ), + ) + + repo_url = Unicode(help="URL of repository to build.") + + ref = Unicode(help="Ref of repository to build.") + + image_name = Unicode(help="Full name of the image to build. Includes the tag.") + + git_credentials = Unicode( + "", + allow_none=True, + help=( + "Git credentials to use when cloning the repository, passed via the GIT_CREDENTIAL_ENV environment variable." + "Can be anything that will be accepted by git as a valid output from a git-credential helper. " + "See https://git-scm.com/docs/gitcredentials for more information." + ), + config=True, + ) + + push_secret = Unicode( + "", + allow_none=True, + help="Implementation dependent secret for pushing image to a registry.", + config=True, + ) + + memory_limit = Integer( + 0, help="Memory limit for the build process in bytes", config=True + ) + + appendix = Unicode( + "", + help="Appendix to be added at the end of the Dockerfile used by repo2docker.", + config=True, + ) + + def __init__(self, **kwargs): + super().__init__(**kwargs) self.main_loop = IOLoop.current() - self.memory_limit = memory_limit - self.memory_request = memory_request - self.docker_host = docker_host - self.node_selector = node_selector - self.appendix = appendix - self.log_tail_lines = log_tail_lines - self.stop_event = threading.Event() - self.git_credentials = git_credentials - - self.sticky_builds = sticky_builds + stop_event = Any() - self._component_label = "binderhub-build" + @default("stop_event") + def _default_stop_event(self): + return threading.Event() def get_r2d_cmd_options(self): """Get options/flags for repo2docker""" @@ -213,72 +149,138 @@ def get_cmd(self): return cmd - @classmethod - def cleanup_builds(cls, kube, namespace, max_age): - """Delete stopped build pods and build pods that have aged out""" - builds = kube.list_namespaced_pod( - namespace=namespace, - label_selector="component=binderhub-build", - ).items - phases = defaultdict(int) - app_log.debug("%i build pods", len(builds)) - now = datetime.datetime.now(tz=datetime.timezone.utc) - start_cutoff = now - datetime.timedelta(seconds=max_age) - deleted = 0 - for build in builds: - phase = build.status.phase - phases[phase] += 1 - annotations = build.metadata.annotations or {} - repo = annotations.get("binder-repo", "unknown") - delete = False - if build.status.phase in {"Failed", "Succeeded", "Evicted"}: - # log Deleting Failed build build-image-... - # print(build.metadata) - app_log.info( - "Deleting %s build %s (repo=%s)", - build.status.phase, - build.metadata.name, - repo, - ) - delete = True - else: - # check age - started = build.status.start_time - if max_age and started and started < start_cutoff: - app_log.info( - "Deleting long-running build %s (repo=%s)", - build.metadata.name, - repo, - ) - delete = True - - if delete: - deleted += 1 - try: - kube.delete_namespaced_pod( - name=build.metadata.name, - namespace=namespace, - body=client.V1DeleteOptions(grace_period_seconds=0), - ) - except client.rest.ApiException as e: - if e.status == 404: - # Is ok, someone else has already deleted it - pass - else: - raise - - if deleted: - app_log.info("Deleted %i/%i build pods", deleted, len(builds)) - app_log.debug( - "Build phase summary: %s", json.dumps(phases, sort_keys=True, indent=1) - ) - def progress(self, kind: ProgressEvent.Kind, payload: str): """ Put current progress info into the queue on the main thread """ self.main_loop.add_callback(self.q.put, ProgressEvent(kind, payload)) + def submit(self): + """ + Run a build to create the image for the repository. + + Progress of the build can be monitored by listening for items in + the Queue passed to the constructor as `q`. + """ + raise NotImplementedError() + + def stream_logs(self): + """ + Stream build logs to the queue in self.q + """ + pass + + def cleanup(self): + """ + Stream build logs to the queue in self.q + """ + pass + + def stop(self): + """ + Stop watching progress of build + + Frees up build watchers that are no longer hooked up to any current requests. + This is not related to stopping the build. + """ + self.stop_event.set() + + +class KubernetesBuildExecutor(BuildExecutor): + """Represents a build of a git repository into a docker image. + + This ultimately maps to a single pod on a kubernetes cluster. Many + different build objects can point to this single pod and perform + operations on the pod. The code in this class needs to be careful and take + this into account. + + For example, operations a Build object tries might not succeed because + another Build object pointing to the same pod might have done something + else. This should be handled gracefully, and the build object should + reflect the state of the pod as quickly as possible. + + ``name`` + The ``name`` should be unique and immutable since it is used to + sync to the pod. The ``name`` should be unique for a + ``(repo_url, ref)`` tuple, and the same tuple should correspond + to the same ``name``. This allows use of the locking provided by k8s + API instead of having to invent our own locking code. + + """ + + api = Any( + help="Kubernetes API object to make requests (kubernetes.client.CoreV1Api())", + ) + + @default("api") + def _default_api(self): + try: + kubernetes.config.load_incluster_config() + except kubernetes.config.ConfigException: + kubernetes.config.load_kube_config() + return client.CoreV1Api() + + namespace = Unicode( + help="Kubernetes namespace to spawn build pods into", config=True + ) + + @default("namespace") + def _default_namespace(self): + return os.getenv("BUILD_NAMESPACE", "default") + + build_image = Unicode( + "quay.io/jupyterhub/repo2docker:2022.02.0", + help="Docker image containing repo2docker that is used to spawn the build pods.", + config=True, + ) + + docker_host = Unicode( + "/var/run/docker.sock", + help=( + "The docker socket to use for building the image. " + "Must be a unix domain socket on a filesystem path accessible on the node " + "in which the build pod is running." + ), + config=True, + ) + + memory_request = Integer( + 0, + help=( + "Memory request of the build pod. " + "The actual building happens in the docker daemon, " + "but setting request in the build pod makes sure that memory is reserved for the docker build " + "in the node by the kubernetes scheduler." + ), + config=True, + ) + + node_selector = Dict( + {}, help="Node selector for the kubernetes build pod.", config=True + ) + + log_tail_lines = Integer( + 100, + help=( + "Number of log lines to fetch from a currently running build. " + "If a build with the same name is already running when submitted, " + "only the last `log_tail_lines` number of lines will be fetched and displayed to the end user. " + "If not, all log lines will be streamed." + ), + config=True, + ) + + sticky_builds = Bool( + False, + help=( + "If true, builds for the same repo (but different refs) will try to schedule on the same node, " + "to reuse cache layers in the docker daemon being used." + ), + config=True, + ) + + _component_label = Unicode("binderhub-build") + def get_affinity(self): """Determine the affinity term for the build pod. @@ -569,16 +571,94 @@ def cleanup(self): else: raise - def stop(self): - """ - Stop wathcing for progress of build. - """ - self.stop_event.set() + +class KubernetesCleaner(LoggingConfigurable): + """Regular cleanup utility for kubernetes builds + + Instantiate this class, and call cleanup() periodically. + """ + + kube = Any(help="kubernetes API client") + + @default("kube") + def _default_kube(self): + try: + kubernetes.config.load_incluster_config() + except kubernetes.config.ConfigException: + kubernetes.config.load_kube_config() + return client.CoreV1Api() + + namespace = Unicode(help="Kubernetes namespace", config=True) + + @default("namespace") + def _default_namespace(self): + return os.getenv("BUILD_NAMESPACE", "default") + + max_age = Integer(help="Maximum age of build pods to keep", config=True) + + def cleanup(self): + """Delete stopped build pods and build pods that have aged out""" + builds = self.kube.list_namespaced_pod( + namespace=self.namespace, + label_selector="component=binderhub-build", + ).items + phases = defaultdict(int) + app_log.debug("%i build pods", len(builds)) + now = datetime.datetime.now(tz=datetime.timezone.utc) + start_cutoff = now - datetime.timedelta(seconds=self.max_age) + deleted = 0 + for build in builds: + phase = build.status.phase + phases[phase] += 1 + annotations = build.metadata.annotations or {} + repo = annotations.get("binder-repo", "unknown") + delete = False + if build.status.phase in {"Failed", "Succeeded", "Evicted"}: + # log Deleting Failed build build-image-... + # print(build.metadata) + app_log.info( + "Deleting %s build %s (repo=%s)", + build.status.phase, + build.metadata.name, + repo, + ) + delete = True + else: + # check age + started = build.status.start_time + if self.max_age and started and started < start_cutoff: + app_log.info( + "Deleting long-running build %s (repo=%s)", + build.metadata.name, + repo, + ) + delete = True + + if delete: + deleted += 1 + try: + self.kube.delete_namespaced_pod( + name=build.metadata.name, + namespace=self.namespace, + body=client.V1DeleteOptions(grace_period_seconds=0), + ) + except client.rest.ApiException as e: + if e.status == 404: + # Is ok, someone else has already deleted it + pass + else: + raise + + if deleted: + app_log.info("Deleted %i/%i build pods", deleted, len(builds)) + app_log.debug( + "Build phase summary: %s", json.dumps(phases, sort_keys=True, indent=1) + ) -class FakeBuild(Build): +class FakeBuild(BuildExecutor): """ - Fake Building process to be able to work on the UI without a running Minikube. + Fake Building process to be able to work on the UI without a builder. """ def submit(self): @@ -630,3 +710,89 @@ def stream_logs(self): } ), ) + + +class Build(KubernetesBuildExecutor): + """DEPRECATED: Use KubernetesBuildExecutor and configure with Traitlets + + Represents a build of a git repository into a docker image. + + This ultimately maps to a single pod on a kubernetes cluster. Many + different build objects can point to this single pod and perform + operations on the pod. The code in this class needs to be careful and take + this into account. + + For example, operations a Build object tries might not succeed because + another Build object pointing to the same pod might have done something + else. This should be handled gracefully, and the build object should + reflect the state of the pod as quickly as possible. + + ``name`` + The ``name`` should be unique and immutable since it is used to + sync to the pod. The ``name`` should be unique for a + ``(repo_url, ref)`` tuple, and the same tuple should correspond + to the same ``name``. This allows use of the locking provided by k8s + API instead of having to invent our own locking code. + + """ + + """ + For backwards compatibility: BinderHub previously assumed a single cleaner for all builds + """ + _cleaners = {} + + def __init__( + self, + q, + api, + name, + *, + namespace, + repo_url, + ref, + build_image, + docker_host, + image_name, + git_credentials=None, + push_secret=None, + memory_limit=0, + memory_request=0, + node_selector=None, + appendix="", + log_tail_lines=100, + sticky_builds=False, + ): + warnings.warn( + "Class Build is deprecated, use KubernetesBuildExecutor and configure with Traitlets", + DeprecationWarning, + ) + + super().__init__() + + self.q = q + self.api = api + self.repo_url = repo_url + self.ref = ref + self.name = name + self.namespace = namespace + self.image_name = image_name + self.push_secret = push_secret + self.build_image = build_image + self.memory_limit = memory_limit + self.memory_request = memory_request + self.docker_host = docker_host + self.node_selector = node_selector + self.appendix = appendix + self.log_tail_lines = log_tail_lines + self.git_credentials = git_credentials + self.sticky_builds = sticky_builds + + @classmethod + def cleanup_builds(cls, kube, namespace, max_age): + """Delete stopped build pods and build pods that have aged out""" + key = (kube, namespace, max_age) + if key not in Build._cleaners: + Build._cleaners[key] = KubernetesCleaner( + kube=kube, namespace=namespace, max_age=max_age + ) + Build._cleaners[key].cleanup() diff --git a/binderhub/build_local.py b/binderhub/build_local.py index b356e4138..f840acb11 100644 --- a/binderhub/build_local.py +++ b/binderhub/build_local.py @@ -8,12 +8,11 @@ # These methods are synchronous so don't use tornado.queue import queue import subprocess -from threading import Event, Thread +from threading import Thread -from tornado.ioloop import IOLoop from tornado.log import app_log -from .build import Build, ProgressEvent +from .build import BuildExecutor, ProgressEvent DEFAULT_READ_TIMEOUT = 1 @@ -104,7 +103,7 @@ def read_to_queue(proc, capture, q): raise subprocess.CalledProcessError(proc.returncode, cmd) -class LocalRepo2dockerBuild(Build): +class LocalRepo2dockerBuild(BuildExecutor): """Represents a build of a git repository into a docker image. This runs a build using the repo2docker command line tool. @@ -112,104 +111,6 @@ class LocalRepo2dockerBuild(Build): WARNING: This is still under development. Breaking changes may be made at any time. """ - def __init__( - self, - q, - api, - name, - *, - namespace, - repo_url, - ref, - build_image, - docker_host, - image_name, - git_credentials=None, - push_secret=None, - memory_limit=0, - memory_request=0, - node_selector=None, - appendix="", - log_tail_lines=100, - sticky_builds=False, - ): - """ - Parameters - ---------- - - q : tornado.queues.Queue - Queue that receives progress events after the build has been submitted - api : ignored - name : str - A unique name for the thing (repo, ref) being built. Used to coalesce - builds, make sure they are not being unnecessarily repeated - namespace : ignored - repo_url : str - URL of repository to build. - Passed through to repo2docker. - ref : str - Ref of repository to build - Passed through to repo2docker. - build_image : ignored - docker_host : ignored - image_name : str - Full name of the image to build. Includes the tag. - Passed through to repo2docker. - git_credentials : str - Git credentials to use to clone private repositories. Passed - through to repo2docker via the GIT_CREDENTIAL_ENV environment - variable. Can be anything that will be accepted by git as - a valid output from a git-credential helper. See - https://git-scm.com/docs/gitcredentials for more information. - push_secret : ignored - memory_limit - Memory limit for the docker build process. Can be an integer in - bytes, or a byte specification (like 6M). - Passed through to repo2docker. - memory_request - Memory request of the build pod. The actual building happens in the - docker daemon, but setting request in the build pod makes sure that - memory is reserved for the docker build in the node by the kubernetes - scheduler. - node_selector : ignored - appendix : str - Appendix to be added at the end of the Dockerfile used by repo2docker. - Passed through to repo2docker. - log_tail_lines : int - Number of log lines to fetch from a currently running build. - If a build with the same name is already running when submitted, - only the last `log_tail_lines` number of lines will be fetched and - displayed to the end user. If not, all log lines will be streamed. - sticky_builds : ignored - """ - self.q = q - self.repo_url = repo_url - self.ref = ref - self.name = name - self.image_name = image_name - self.push_secret = push_secret - self.main_loop = IOLoop.current() - self.memory_limit = memory_limit - self.memory_request = memory_request - self.appendix = appendix - self.log_tail_lines = log_tail_lines - - self.stop_event = Event() - self.git_credentials = git_credentials - - @classmethod - def cleanup_builds(cls, kube, namespace, max_age): - app_log.debug("Not implemented") - - def progress(self, kind: ProgressEvent.Kind, payload: str): - """ - Put current progress info into the queue on the main thread - """ - self.main_loop.add_callback(self.q.put, ProgressEvent(kind, payload)) - - def get_affinity(self): - raise NotImplementedError() - def submit(self): """ Run a build to create the image for the repository. @@ -266,12 +167,3 @@ def _handle_log(self, line): } ) self.progress(ProgressEvent.Kind.LOG_MESSAGE, line) - - def stream_logs(self): - pass - - def cleanup(self): - pass - - def stop(self): - self.stop_event.set() diff --git a/binderhub/builder.py b/binderhub/builder.py index 09dc3cb61..6b49bd593 100644 --- a/binderhub/builder.py +++ b/binderhub/builder.py @@ -22,7 +22,7 @@ from tornado.web import Finish, authenticated from .base import BaseHandler -from .build import ProgressEvent +from .build import Build, ProgressEvent from .utils import KUBE_REQUEST_TIMEOUT # Separate buckets for builds and launches. @@ -425,26 +425,50 @@ async def get(self, provider_prefix, _unescaped_spec): ref_url=self.ref_url, ) - self.build = build = BuildClass( - q=q, - # api object can be None if we are using FakeBuild - api=self.settings.get("kubernetes_client"), - name=build_name, - namespace=self.settings["build_namespace"], - repo_url=repo_url, - ref=ref, - image_name=image_name, - push_secret=push_secret, - build_image=self.settings["build_image"], - memory_limit=self.settings["build_memory_limit"], - memory_request=self.settings["build_memory_request"], - docker_host=self.settings["build_docker_host"], - node_selector=self.settings["build_node_selector"], - appendix=appendix, - log_tail_lines=self.settings["log_tail_lines"], - git_credentials=provider.git_credentials, - sticky_builds=self.settings["sticky_builds"], - ) + if issubclass(BuildClass, Build): + # Deprecated, see docstring of the Build class for more details + build = BuildClass( + q=q, + # api object can be None if we are using FakeBuild + api=self.settings.get("kubernetes_client"), + name=build_name, + namespace=self.settings["build_namespace"], + repo_url=repo_url, + ref=ref, + image_name=image_name, + push_secret=push_secret, + build_image=self.settings["build_image"], + memory_limit=self.settings["build_memory_limit"], + memory_request=self.settings["build_memory_request"], + docker_host=self.settings["build_docker_host"], + node_selector=self.settings["build_node_selector"], + appendix=appendix, + log_tail_lines=self.settings["log_tail_lines"], + git_credentials=provider.git_credentials, + sticky_builds=self.settings["sticky_builds"], + ) + else: + build = BuildClass( + # Commented properties should be set in traitlets config + parent=self.settings["traitlets_parent"], + q=q, + name=build_name, + # namespace=self.settings["build_namespace"], + repo_url=repo_url, + ref=ref, + image_name=image_name, + # push_secret=push_secret, + # build_image=self.settings["build_image"], + # memory_limit=self.settings["build_memory_limit"], + # memory_request=self.settings["build_memory_request"], + # docker_host=self.settings["build_docker_host"], + # node_selector=self.settings["build_node_selector"], + # appendix=appendix, + # log_tail_lines=self.settings["log_tail_lines"], + git_credentials=provider.git_credentials, + # sticky_builds=self.settings["sticky_builds"], + ) + self.build = build with BUILDS_INPROGRESS.track_inprogress(): done = False diff --git a/binderhub/tests/conftest.py b/binderhub/tests/conftest.py index 6bdf13186..4cde689f1 100644 --- a/binderhub/tests/conftest.py +++ b/binderhub/tests/conftest.py @@ -128,14 +128,12 @@ def mock_asynchttpclient(request): @pytest.fixture -def io_loop(event_loop, request): +async def io_loop(event_loop, request): """Same as pytest-tornado.io_loop, but runs with pytest-asyncio""" io_loop = AsyncIOMainLoop() - io_loop.make_current() assert io_loop.asyncio_loop is event_loop def _close(): - io_loop.clear_current() io_loop.close(all_fds=True) request.addfinalizer(_close) diff --git a/binderhub/tests/test_build.py b/binderhub/tests/test_build.py index 2f9477ef3..83f9a0990 100644 --- a/binderhub/tests/test_build.py +++ b/binderhub/tests/test_build.py @@ -122,15 +122,15 @@ def test_default_affinity(): api=mock_k8s_api, name="test_build", namespace="build_namespace", - repo_url=mock.MagicMock(), - ref=mock.MagicMock(), - build_image=mock.MagicMock(), - image_name=mock.MagicMock(), - push_secret=mock.MagicMock(), - memory_limit=mock.MagicMock(), + repo_url="repo", + ref="ref", + build_image="image", + image_name="name", + push_secret="", + memory_limit=0, git_credentials=None, docker_host="http://mydockerregistry.local", - node_selector=mock.MagicMock(), + node_selector={}, ) affinity = build.get_affinity() @@ -150,15 +150,15 @@ def test_sticky_builds_affinity(): api=mock_k8s_api, name="test_build", namespace="build_namespace", - repo_url=mock.MagicMock(), - ref=mock.MagicMock(), - build_image=mock.MagicMock(), - image_name=mock.MagicMock(), - push_secret=mock.MagicMock(), - memory_limit=mock.MagicMock(), + repo_url="repo", + ref="ref", + build_image="image", + image_name="name", + push_secret="", + memory_limit=0, git_credentials=None, docker_host="http://mydockerregistry.local", - node_selector=mock.MagicMock(), + node_selector={}, sticky_builds=True, ) @@ -176,10 +176,10 @@ def test_sticky_builds_affinity(): def test_git_credentials_passed_to_podspec_upon_submit(): - git_credentials = { + git_credentials = """{ "client_id": "my_username", "access_token": "my_access_token", - } + }""" mock_k8s_api = _list_dind_pods_mock() @@ -188,15 +188,15 @@ def test_git_credentials_passed_to_podspec_upon_submit(): api=mock_k8s_api, name="test_build", namespace="build_namespace", - repo_url=mock.MagicMock(), - ref=mock.MagicMock(), + repo_url="repo", + ref="ref", + build_image="image", + image_name="name", + push_secret="", + memory_limit=0, git_credentials=git_credentials, - build_image=mock.MagicMock(), - image_name=mock.MagicMock(), - push_secret=mock.MagicMock(), - memory_limit=mock.MagicMock(), docker_host="http://mydockerregistry.local", - node_selector=mock.MagicMock(), + node_selector={}, ) with mock.patch.object(build.stop_event, "is_set", return_value=True): @@ -222,14 +222,10 @@ async def test_local_repo2docker_build(): name = str(uuid4()) build = LocalRepo2dockerBuild( - q, - None, - name, - namespace=None, + q=q, + name=name, repo_url=repo_url, ref=ref, - build_image=None, - docker_host=None, image_name=name, ) build.submit() @@ -250,7 +246,8 @@ async def test_local_repo2docker_build(): @pytest.mark.asyncio(timeout=20) -async def test_local_repo2docker_build_stop(event_loop): +async def test_local_repo2docker_build_stop(io_loop): + io_loop = await io_loop q = Queue() # We need a slow build here so that we can interrupt it, so pick a large repo that # will take several seconds to clone @@ -259,17 +256,13 @@ async def test_local_repo2docker_build_stop(event_loop): name = str(uuid4()) build = LocalRepo2dockerBuild( - q, - None, - name, - namespace=None, + q=q, + name=name, repo_url=repo_url, ref=ref, - build_image=None, - docker_host=None, image_name=name, ) - event_loop.run_in_executor(None, build.submit) + io_loop.run_in_executor(None, build.submit) # Get first few log messages to check it successfully stared event = await q.get()