diff --git a/.github/integration-test.py b/.github/integration-test.py index 9ea09a8ae..67b8a9d3a 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -4,15 +4,16 @@ import os -def build_systemd_image(image_name, source_path): +def build_systemd_image(image_name, source_path, build_args=None): """ Build docker image with systemd at source_path. Built image is tagged with image_name """ - subprocess.check_call([ - 'docker', 'build', '-t', image_name, source_path - ]) + cmd = ['docker', 'build', '-t', image_name, source_path] + if build_args: + cmd.extend([f"--build-arg={ba}" for ba in build_args]) + subprocess.check_call(cmd) def run_systemd_image(image_name, container_name, bootstrap_pip_spec): @@ -94,6 +95,10 @@ def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, ins copy_to_container(test_name, os.path.join(source_path, 'bootstrap/.'), '/srv/src') copy_to_container(test_name, os.path.join(source_path, 'integration-tests/'), '/srv/src') + # These logs can be very relevant to debug a container startup failure + print(f"--- Start of logs from the container: {test_name}") + print(subprocess.check_output(['docker', 'logs', test_name]).decode()) + print(f"--- End of logs from the container: {test_name}") # Install TLJH from the default branch first to test upgrades if upgrade: @@ -115,7 +120,10 @@ def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, ins ) run_container_command( test_name, - '/opt/tljh/hub/bin/python3 -m pytest -v {}'.format( + # We abort pytest after two failures as a compromise between wanting to + # avoid a flood of logs while still understanding if multiple tests + # would fail. + '/opt/tljh/hub/bin/python3 -m pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no {}'.format( ' '.join([os.path.join('/srv/src/integration-tests/', f) for f in test_files]) ) ) @@ -138,13 +146,21 @@ def main(): argparser = argparse.ArgumentParser() subparsers = argparser.add_subparsers(dest='action') - subparsers.add_parser('build-image') + build_image_parser = subparsers.add_parser('build-image') + build_image_parser.add_argument( + "--build-arg", + action="append", + dest="build_args", + ) + subparsers.add_parser('stop-container').add_argument( 'container_name' ) + subparsers.add_parser('start-container').add_argument( 'container_name' ) + run_parser = subparsers.add_parser('run') run_parser.add_argument('container_name') run_parser.add_argument('command') @@ -181,7 +197,7 @@ def main(): elif args.action == 'stop-container': stop_container(args.container_name) elif args.action == 'build-image': - build_systemd_image(image_name, 'integration-tests') + build_systemd_image(image_name, 'integration-tests', args.build_args) if __name__ == '__main__': diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 75e2f6393..cd910a455 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -1,41 +1,182 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +# +name: Integration tests + on: pull_request: + paths-ignore: + - "docs/**" + - "**.md" + - "**.rst" + - ".github/workflows/*" + - "!.github/workflows/integration-test.yaml" push: + paths-ignore: + - "docs/**" + - "**.md" + - "**.rst" + - ".github/workflows/*" + - "!.github/workflows/integration-test.yaml" + branches-ignore: + - "dependabot/**" + - "pre-commit-ci-update-config" workflow_dispatch: jobs: - integration-test: - runs-on: ubuntu-18.04 + + # This job is used as a workaround to a limitation when using a matrix of + # variations that a job should be executed against. The limitation is that a + # matrix once defined can't include any conditions. + # + # What this job does before our real test job with a matrix of variations run, + # is to decide on that matrix of variations a conditional logic of our choice. + # + # For more details, see this excellent stack overflow answer: + # https://stackoverflow.com/a/65434401/2220152 + # + decide-on-test-jobs-to-run: + name: Decide on test jobs to run + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + # Currently, this logic filters out a matrix entry equaling a specific git + # reference identified by "dont_run_on_ref". + - name: Decide on test jobs to run + id: set-matrix + run: | + matrix_post_filter=$( + echo "$matrix_include_pre_filter" \ + | yq e --output-format=json '.' - \ + | jq '{"include": map( . | select(.dont_run_on_ref != "${{ github.ref }}" ))}' + ) + echo ::set-output name=matrix::$(echo "$matrix_post_filter") + + echo "The subsequent job's matrix are:" + echo $matrix_post_filter | jq '.' + env: + matrix_include_pre_filter: | + - name: "Int. tests: Ubuntu 18.04, Py 3.6" + ubuntu_version: "18.04" + python_version: "3.6" + extra_flags: "" + - name: "Int. tests: Ubuntu 20.04, Py 3.9" + ubuntu_version: "20.04" + python_version: "3.9" + extra_flags: "" + - name: "Int. tests: Ubuntu 21.10, Py 3.9" + runs_on: "20.04" + ubuntu_version: "21.10" + python_version: "3.9" + extra_flags: "" + - name: "Int. tests: Ubuntu 20.04, Py 3.9, --upgrade" + ubuntu_version: "20.04" + python_version: "3.9" + extra_flags: --upgrade + dont_run_on_ref: refs/heads/master + + integration-tests: + needs: decide-on-test-jobs-to-run + + # runs-on can only be configured to the LTS releases of ubuntu (18.04, + # 20.04, ...), so if we want to test against the latest non-LTS release, we + # must compromise when configuring runs-on and configure runs-on to be the + # latest LTS release instead. + # + # This can have consequences because actions like actions/setup-python will + # mount cached installations associated with the chosen runs-on environment. + # + runs-on: ${{ format('ubuntu-{0}', matrix.runs_on || matrix.ubuntu_version) }} + + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.decide-on-test-jobs-to-run.outputs.matrix) }} + steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.6 - - name: Set BOOTSTRAP_PIP_SPEC value + python-version: ${{ matrix.python_version }} + + - name: Install pytest + run: python3 -m pip install pytest + + # We abort pytest after two failures as a compromise between wanting to + # avoid a flood of logs while still understanding if multiple tests would + # fail. + - name: Run bootstrap tests (Runs in/Builds ubuntu:${{ matrix.ubuntu_version }} derived image) run: | - echo "BOOTSTRAP_PIP_SPEC=git+https://github.com/$GITHUB_REPOSITORY.git@$GITHUB_REF" >> $GITHUB_ENV - - name: Build systemd image + pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no \ + integration-tests/test_bootstrap.py + timeout-minutes: 15 + env: + # integration-tests/test_bootstrap.py will build and start containers + # based on this environment variable. This is similar to how + # .github/integration-test.py build-image can take a --build-arg + # setting the ubuntu_version. + UBUNTU_VERSION: ${{ matrix.ubuntu_version }} + + # We build a docker image from wherein we will work + - name: Build systemd image (Builds ubuntu:${{ matrix.ubuntu_version }} derived image) run: | - .github/integration-test.py build-image - - name: Run bootstrap checks + .github/integration-test.py build-image \ + --build-arg "ubuntu_version=${{ matrix.ubuntu_version }}" + + # FIXME: Make the logic below easier to follow. + # - In short, setting BOOTSTRAP_PIP_SPEC here, specifies from what + # location the tljh python package should be installed from. In this + # GitHub Workflow's test job, we provide a remote reference to itself as + # found on GitHub - this could be the HEAD of a PR branch or the default + # branch on merge. + # + # Overview of how this logic influences the end result. + # - integration-test.yaml: + # Runs integration-test.py by passing --bootstrap-pip-spec flag with a + # reference to the pull request on GitHub. + # - integration-test.py: + # Starts a pre-build systemd container, setting the + # TLJH_BOOTSTRAP_PIP_SPEC based on its passed --bootstrap-pip-spec value. + # - systemd container: + # Runs bootstrap.py + # - bootstrap.py + # Makes use of TLJH_BOOTSTRAP_PIP_SPEC environment variable to install + # the tljh package from a given location, which could be a local git + # clone of this repo where setup.py resides, or a reference to some + # GitHub branch for example. + - name: Set BOOTSTRAP_PIP_SPEC value run: | - python3 -m pip install pytest - pytest integration-tests/test_bootstrap.py -s - - name: Run basic tests + echo "BOOTSTRAP_PIP_SPEC=git+https://github.com/$GITHUB_REPOSITORY.git@$GITHUB_REF" >> $GITHUB_ENV + echo $BOOTSTRAP_PIP_SPEC + + - name: Run basic tests (Runs in ubuntu:${{ matrix.ubuntu_version }} derived image) run: | - .github/integration-test.py run-test \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - basic-tests test_hub.py test_proxy.py \ - test_install.py test_extensions.py - - name: Run admin tests + .github/integration-test.py run-test basic-tests \ + --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ + ${{ matrix.extra_flags }} \ + test_hub.py \ + test_proxy.py \ + test_install.py \ + test_extensions.py + timeout-minutes: 15 + + - name: Run admin tests (Runs in ubuntu:${{ matrix.ubuntu_version }} derived image) run: | - .github/integration-test.py run-test \ - --installer-args "--admin admin:admin" \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - basic-tests test_admin_installer.py \ - - name: Run plugin tests + .github/integration-test.py run-test admin-tests \ + --installer-args "--admin admin:admin" \ + --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ + ${{ matrix.extra_flags }} \ + test_admin_installer.py + timeout-minutes: 15 + + - name: Run plugin tests (Runs in ubuntu:${{ matrix.ubuntu_version }} derived image) run: | - .github/integration-test.py run-test \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ - plugins test_simplest_plugin.py \ + .github/integration-test.py run-test plugin-tests \ + --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ + --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ + ${{ matrix.extra_flags }} \ + test_simplest_plugin.py + timeout-minutes: 15 diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index ce047fda5..5de90ce1f 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,36 +1,108 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +# +name: Unit tests + on: pull_request: + paths-ignore: + - "docs/**" + - "**.md" + - "**.rst" + - ".github/workflows/*" + - "!.github/workflows/unit-test.yaml" push: + paths-ignore: + - "docs/**" + - "**.md" + - "**.rst" + - ".github/workflows/*" + - "!.github/workflows/unit-test.yaml" + branches-ignore: + - "dependabot/**" + - "pre-commit-ci-update-config" workflow_dispatch: jobs: - unit-test: - runs-on: ubuntu-18.04 - container: ubuntu:18.04 + unit-tests: + name: ${{ matrix.name }} + + # runs-on can only be configured to the LTS releases of ubuntu (18.04, + # 20.04, ...), so if we want to test against the latest non-LTS release, we + # must compromise when configuring runs-on and configure runs-on to be the + # latest LTS release instead. + # + # This can have consequences because actions like actions/setup-python will + # mount cached installations associated with the chosen runs-on environment. + # + runs-on: ${{ format('ubuntu-{0}', matrix.runs_on || matrix.ubuntu_version) }} + container: ubuntu:${{ matrix.ubuntu_version }} + + strategy: + fail-fast: false + matrix: + include: + - name: "Unit tests: Ubuntu 18.04, Py 3.6" + ubuntu_version: "18.04" + python_version: "3.6" + - name: "Unit tests: Ubuntu 20.04, Py 3.9" + ubuntu_version: "20.04" + python_version: "3.9" + # Test against Ubuntu 21.10 fails as of 2021-10-18 fail with the error + # described in: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/714#issuecomment-945154101 + # + # - name: "Unit tests: Ubuntu 21.10, Py 3.9" + # runs_on: "20.04" + # ubuntu_version: "21.10" + # python_version: "3.9" + steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python_version }} + - name: Install venv, git and setup venv run: | - apt-get update --yes && apt-get install --yes python3-venv git + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install --yes \ + python3-venv \ + git + python3 -m venv /srv/venv echo '/srv/venv/bin' >> $GITHUB_PATH - - name: Cache pip deps + + # WARNING: This action loads a cache of pip dependencies based on the + # declared key, and it will save a cache for that key on job + # completion. Make sure to update the key to bust the cache + # properly if you make a change that should influence it. + - name: Load cached Python dependencies uses: actions/cache@v2 with: path: /srv/venv/ - key: ${{ runner.os }}-pip-dependencies-${{ hashFiles('*setup.py', '*dev-requirements.txt') }} + key: >- + pip- + ${{ matrix.runs_on }}- + ${{ matrix.ubuntu_version }}- + ${{ matrix.python_version }}- + ${{ hashFiles('setup.py', 'dev-requirements.txt', '.github/workflows/unit-test.yaml') }} + - name: Install Python dependencies + # Keep pip version pinning in sync with the one in bootstrap.py! + # See changelog at https://pip.pypa.io/en/latest/news/#changelog run: | - python3 -m pip install -U pip==20.0.* + python3 -m pip install -U "pip==21.3.*" python3 -m pip install -r dev-requirements.txt python3 -m pip install -e . pip freeze + + # We abort pytest after two failures as a compromise between wanting to + # avoid a flood of logs while still understanding if multiple tests would + # fail. - name: Run unit tests - run: | - pytest --cov=tljh tests/ + run: pytest --verbose --maxfail=2 --color=yes --durations=10 --cov=tljh tests/ + timeout-minutes: 15 + - name: Upload code coverage stats - run: | - codecov + run: codecov diff --git a/.github/workflows/upgrade-test.yaml b/.github/workflows/upgrade-test.yaml deleted file mode 100644 index 543a909db..000000000 --- a/.github/workflows/upgrade-test.yaml +++ /dev/null @@ -1,42 +0,0 @@ -on: - pull_request: - push: - workflow_dispatch: - -jobs: - upgrade-test: - if: ${{ github.ref != 'refs/heads/master' }} - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - name: Set BOOTSTRAP_PIP_SPEC value - run: | - echo "BOOTSTRAP_PIP_SPEC=git+https://github.com/$GITHUB_REPOSITORY.git@$GITHUB_REF" >> $GITHUB_ENV - echo $BOOTSTRAP_PIP_SPEC - - name: Build systemd image - run: | - .github/integration-test.py build-image - - name: Run basic tests - run: | - .github/integration-test.py run-test \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - basic-tests test_hub.py test_proxy.py \ - test_install.py test_extensions.py \ - --upgrade - - name: Run admin tests - run: | - .github/integration-test.py run-test \ - --installer-args "--admin admin:admin" \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - basic-tests test_admin_installer.py \ - --upgrade - - name: Run plugin tests - run: | - .github/integration-test.py run-test \ - --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ - --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ - plugins test_simplest_plugin.py \ - --upgrade diff --git a/README.md b/README.md new file mode 100644 index 000000000..361e8be40 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# The Littlest JupyterHub + +[![Documentation build status](https://img.shields.io/readthedocs/the-littlest-jupyterhub?logo=read-the-docs)](https://tljh.jupyter.org/en/latest/?badge=latest) +[![GitHub Workflow Status - Test](https://img.shields.io/github/workflow/status/jupyterhub/the-littlest-jupyterhub/Unit%20tests?logo=github&label=tests)](https://github.com/jupyterhub/the-littlest-jupyterhub/actions) +[![Test coverage of code](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/master/graph/badge.svg)](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub) +[![GitHub](https://img.shields.io/badge/issue_tracking-github-blue?logo=github)](https://github.com/jupyterhub/the-littlest-jupyterhub/issues) +[![Discourse](https://img.shields.io/badge/help_forum-discourse-blue?logo=discourse)](https://discourse.jupyter.org/c/jupyterhub/tljh) +[![Gitter](https://img.shields.io/badge/social_chat-gitter-blue?logo=gitter)](https://gitter.im/jupyterhub/jupyterhub) +[![Contribute](https://img.shields.io/badge/I_want_to_contribute!-grey?logo=jupyter)](https://tljh.jupyter.org/en/latest/contributing/index.html) + +**The Littlest JupyterHub** (TLJH) distribution helps you provide Jupyter Notebooks +to 1-100 users on a single server. + +The primary audience are people who do not consider themselves 'system administrators' +but would like to provide hosted Jupyter Notebooks for their students or users. +All users are provided with the same environment, and administrators +can easily install libraries into this environment without any specialized knowledge. + +See the [latest documentation](https://the-littlest-jupyterhub.readthedocs.io) +for more information on using The Littlest JupyterHub. + +For support questions please search or post to [our forum](https://discourse.jupyter.org/c/jupyterhub/). + +See the [contributing guide](https://the-littlest-jupyterhub.readthedocs.io/en/latest/contributing/index.html) +for information on the different ways of contributing to The Littlest JupyterHub. + +See [this blog post](http://words.yuvi.in/post/the-littlest-jupyterhub/) for +more information. + +## Development Status + +This project is currently in **beta** state. Folks have been using installations +of TLJH for more than a year now to great success. While we try hard not to, we +might still make breaking changes that have no clear upgrade pathway. + +## Installation + +The Littlest JupyterHub (TLJH) can run on any server that is running at least +**Ubuntu 18.04**. Earlier versions of Ubuntu are not supported. +We have several tutorials to get you started. + +- Tutorials to create a new server from scratch on a cloud provider & run TLJH + on it. These are **recommended** if you do not have much experience setting up + servers. + + - [Digital Ocean](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/digitalocean.html) + - [OVH](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/ovh.html) + - [Google Cloud](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/google.html) + - [Jetstream](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/jetstream.html) + - [Amazon Web Services](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/amazon.html) + - [Microsoft Azure](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/azure.html) + - ... your favorite provider here, if you can contribute! + +- [Tutorial to install TLJH on an already running server you have root access to](https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/custom-server.html). + You should use this if your cloud provider does not already have a direct tutorial, + or if you have experience setting up servers. + +## Documentation + +Our latest documentation is at: https://the-littlest-jupyterhub.readthedocs.io + +We place a high importance on consistency, readability and completeness of +documentation. If a feature is not documented, it does not exist. If a behavior +is not documented, it is a bug! We try to treat our documentation like we treat +our code: we aim to improve it as often as possible. + +If something is confusing to you in the documentation, it is a bug. We would be +happy if you could [file an issue](https://github.com/jupyterhub/the-littlest-jupyterhub/issues) about it - or +even better, [contribute a documentation fix](http://the-littlest-jupyterhub.readthedocs.io/en/latest/contributing/docs.html)! diff --git a/README.rst b/README.rst deleted file mode 100644 index 5f8866fea..000000000 --- a/README.rst +++ /dev/null @@ -1,81 +0,0 @@ -======================= -The Littlest JupyterHub -======================= - -.. image:: https://circleci.com/gh/jupyterhub/the-littlest-jupyterhub.svg?style=shield - :target: https://circleci.com/gh/jupyterhub/the-littlest-jupyterhub -.. image:: https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/master/graph/badge.svg - :target: https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub -.. image:: https://readthedocs.org/projects/the-littlest-jupyterhub/badge/?version=latest - :target: https://the-littlest-jupyterhub.readthedocs.io -.. image:: https://badges.gitter.im/jupyterhub/jupyterhub.svg - :target: https://gitter.im/jupyterhub/jupyterhub -.. image:: https://img.shields.io/badge/I_want_to_contribute!-grey?logo=jupyter - :target: https://the-littlest-jupyterhub.readthedocs.io/en/latest/contributing/index.html - -**The Littlest JupyterHub** (TLJH) distribution helps you provide Jupyter Notebooks -to 1-100 users on a single server. - -The primary audience are people who do not consider themselves 'system administrators' -but would like to provide hosted Jupyter Notebooks for their students or users. -All users are provided with the same environment, and administrators -can easily install libraries into this environment without any specialized knowledge. - -See the `latest documentation `_ -for more information on using The Littlest JupyterHub. - -For support questions please search or post to `our forum `_. - -See the `contributing guide `_ -for information on the different ways of contributing to The Littlest JupyterHub. - -See `this blog post `_ for -more information. - - -Development Status -================== - -This project is currently in **beta** state. Folks have been using installations -of TLJH for more than a year now to great success. While we try hard not to, we -might still make breaking changes that have no clear upgrade pathway. - -Installation -============ - -The Littlest JupyterHub (TLJH) can run on any server that is running at least -**Ubuntu 18.04**. Earlier versions of Ubuntu are not supported. -We have several tutorials to get you started. - -- Tutorials to create a new server from scratch on a cloud provider & run TLJH - on it. These are **recommended** if you do not have much experience setting up - servers. - - - `Digital Ocean `_ - - `OVH `_ - - `Google Cloud `_ - - `Jetstream `_ - - `Amazon Web Services `_ - - `Microsoft Azure `_ - - ... your favorite provider here, if you can contribute! - -- `Tutorial to install TLJH on an already running server you have root access to - `_. - You should use this if your cloud provider does not already have a direct tutorial, - or if you have experience setting up servers. - -Documentation -============= - -Our latest documentation is at: https://the-littlest-jupyterhub.readthedocs.io - -We place a high importance on consistency, readability and completeness of -documentation. If a feature is not documented, it does not exist. If a behavior -is not documented, it is a bug! We try to treat our documentation like we treat -our code: we aim to improve it as often as possible. - -If something is confusing to you in the documentation, it is a bug. We would be -happy if you could `file an issue -`_ about it - or -even better, `contribute a documentation fix -`_! diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 4eb718cd3..23739df5b 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -311,22 +311,23 @@ def serve_forever(server): logger.info('Existing TLJH installation not detected, installing...') logger.info('Setting up hub environment...') logger.info('Installing Python, venv, pip, and git via apt-get...') - # Install software-properties-common, so we can get add-apt-repository - # That helps us make sure the universe repository is enabled, since - # that's where the python3-pip package lives. In some very minimal base - # VM images, it looks like the universe repository is disabled by default, - # causing bootstrapping to fail. - run_subprocess(['apt-get', 'update', '--yes']) - run_subprocess(['apt-get', 'install', '--yes', 'software-properties-common']) - run_subprocess(['add-apt-repository', 'universe']) - - run_subprocess(['apt-get', 'update', '--yes']) - run_subprocess(['apt-get', 'install', '--yes', - 'python3', - 'python3-venv', - 'python3-pip', - 'git' - ]) + + # In some very minimal base VM images, it looks like the "universe" apt + # package repository is disabled by default, causing bootstrapping to + # fail. We install the software-properties-common package so we can get + # the add-apt-repository command to make sure the universe repository is + # enabled, since that's where the python3-pip package lives. + # + # In Ubuntu 21.10 DEBIAN_FRONTEND has found to be needed to avoid + # getting stuck on an input prompt during apt-get install. + # + apt_get_adjusted_env = os.environ.copy() + apt_get_adjusted_env["DEBIAN_FRONTEND"] = "noninteractive" + run_subprocess(['apt-get', 'update']) + run_subprocess(['apt-get', 'install', '--yes', 'software-properties-common'], env=apt_get_adjusted_env) + run_subprocess(['add-apt-repository', 'universe', '--yes']) + run_subprocess(['apt-get', 'update']) + run_subprocess(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'python3-pip', 'git'], env=apt_get_adjusted_env) logger.info('Setting up virtual environment at {}'.format(hub_prefix)) os.makedirs(hub_prefix, exist_ok=True) @@ -334,8 +335,10 @@ def serve_forever(server): # Upgrade pip + # Keep pip version pinning in sync with the one in unit-test.yml! + # See changelog at https://pip.pypa.io/en/latest/news/#changelog logger.info('Upgrading pip...') - run_subprocess([pip_bin, 'install', '--upgrade', 'pip==20.0.*']) + run_subprocess([pip_bin, 'install', '--upgrade', 'pip==21.3.*']) # Install/upgrade TLJH installer diff --git a/docs/contributing/tests.rst b/docs/contributing/tests.rst index bbd8f066b..f093ef412 100644 --- a/docs/contributing/tests.rst +++ b/docs/contributing/tests.rst @@ -20,7 +20,7 @@ that the various components fit together and work as they should. So we write a lot of integration tests, and put in more effort towards them than unit tests. -All integration tests are run on `CircleCI `_ +All integration tests are run in `GitHub Actions `_ for each PR and merge, making sure we don't have broken tests for too long. @@ -39,7 +39,7 @@ You can then run the tests with: .. code-block:: bash - .circleci/integration-test.py run-test + .github/integration-test.py run-test - ```` is an identifier for the tests - you can choose anything you want - ``>`` is list of test files (under ``integration-tests``) that should be run in one go. @@ -48,10 +48,11 @@ For example, to run all the basic tests, you would write: .. code-block:: bash - .circleci/integration-test.py run-test basic-tests \ - test_hub.py \ - test_install.py \ - test_extensions.py + .github/integration-test.py run-test basic-tests \ + test_hub.py \ + test_proxy.py \ + test_install.py \ + test_extensions.py This will run the tests in the three files against the same installation of TLJH and report errors. @@ -61,5 +62,5 @@ parameter: .. code-block:: bash - .circleci/integration-test.py run-test \ + .github/integration-test.py run-test \ --bootstrap-pip-spec="git+https://github.com/your-username/the-littlest-jupyterhub.git@branch-name" diff --git a/integration-tests/Dockerfile b/integration-tests/Dockerfile index 897218901..64d5d0b68 100644 --- a/integration-tests/Dockerfile +++ b/integration-tests/Dockerfile @@ -1,9 +1,17 @@ # Systemd inside a Docker container, for CI only -FROM ubuntu:18.04 - -RUN apt-get update --yes - -RUN apt-get install --yes systemd curl git sudo +ARG ubuntu_version=20.04 +FROM ubuntu:${ubuntu_version} + +# DEBIAN_FRONTEND is set to avoid being asked for input and hang during build: +# https://anonoz.github.io/tech/2020/04/24/docker-build-stuck-tzdata.html +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install --yes \ + systemd \ + curl \ + git \ + sudo \ + && rm -rf /var/lib/apt/lists/* # Kill all the things we don't need RUN find /etc/systemd/system \ @@ -25,4 +33,4 @@ STOPSIGNAL SIGRTMIN+3 #ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src #ENV PATH=/opt/tljh/hub/bin:${PATH} -CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"] +CMD ["/bin/bash", "-c", "exec /lib/systemd/systemd --log-target=journal 3>&1"] diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 4d3aea592..de95a46bb 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -4,7 +4,6 @@ import concurrent.futures import os import subprocess -from textwrap import dedent import time @@ -39,8 +38,43 @@ def get_bootstrap_script_location(container_name, show_progress_page): subprocess.check_call(["docker", "cp", source_path, f"{container_name}:/srv/src"]) return bootstrap_script - -def run_bootstrap(container_name, image, show_progress_page=False): +# FIXME: Refactor this function to easier to understand using the following +# parameters +# +# - param: container_apt_packages +# - param: bootstrap_tljh_source +# - local: copies local tljh repo to container and configures bootstrap to +# install tljh from copied repo +# - github: configures bootstrap to install tljh from the official github repo +# - : configures bootstrap to install tljh from any given remote location +# - param: bootstrap_flags +# +# FIXME: Consider stripping logic in this file to only testing if the bootstrap +# script successfully detects the too old Ubuntu version and the lack of +# systemd. The remaining test named test_progress_page could rely on +# running against the systemd container that cab be built by +# integration-test.py. +# +def run_bootstrap_after_preparing_container(container_name, image, show_progress_page=False): + """ + 1. Stops old container + 2. Starts --detached container + 3. Installs apt packages in container + 4. Two situations + + A) limited test (--show-progress-page=false) + - Copies ./bootstrap/ folder content to container /srv/src + - Runs copied bootstrap/bootstrap.py without flags + + B) full test (--show-progress-page=true) + - Copies ./ folder content to the container /srv/src + - Runs copied bootstrap/bootstrap.py with environment variables + - TLJH_BOOTSTRAP_DEV=yes + This makes --editable be used when installing the tljh package + - TLJH_BOOTSTRAP_PIP_SPEC=/srv/src + This makes us install tljh from the given location instead of from + github.com/jupyterhub/the-littlest-jupyterhub + """ # stop container if it is already running subprocess.run(["docker", "rm", "-f", container_name]) @@ -49,9 +83,9 @@ def run_bootstrap(container_name, image, show_progress_page=False): [ "docker", "run", + "--env=DEBIAN_FRONTEND=noninteractive", "--detach", - "--name", - container_name, + f"--name={container_name}", image, "/bin/bash", "-c", @@ -84,24 +118,17 @@ def test_ubuntu_too_old(): """ Error with a useful message when running in older Ubuntu """ - output = run_bootstrap("old-distro-test", "ubuntu:16.04") + output = run_bootstrap_after_preparing_container("old-distro-test", "ubuntu:16.04") assert output.stdout == "The Littlest JupyterHub requires Ubuntu 18.04 or higher\n" assert output.returncode == 1 def test_inside_no_systemd_docker(): - output = run_bootstrap("plain-docker-test", "ubuntu:18.04") - assert ( - output.stdout.strip() - == dedent( - """ - Systemd is required to run TLJH - Running inside a docker container without systemd isn't supported - We recommend against running a production TLJH instance inside a docker container - For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html - """ - ).strip() + output = run_bootstrap_after_preparing_container( + "plain-docker-test", + f"ubuntu:{os.getenv('UBUNTU_VERSION', '20.04')}", ) + assert "Systemd is required to run TLJH" in output.stdout assert output.returncode == 1 @@ -133,7 +160,10 @@ def verify_progress_page(expected_status_code, timeout): def test_progress_page(): with concurrent.futures.ThreadPoolExecutor() as executor: installer = executor.submit( - run_bootstrap, "progress-page", "ubuntu:18.04", True + run_bootstrap_after_preparing_container, + "progress-page", + f"ubuntu:{os.getenv('UBUNTU_VERSION', '20.04')}", + True ) # Check if progress page started diff --git a/setup.py b/setup.py index f73290c0f..00d7402e6 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'backoff', 'requests', 'bcrypt', - 'jupyterhub-traefik-proxy==0.2.*', + 'jupyterhub-traefik-proxy==0.3.*', ], entry_points={ 'console_scripts': [