diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..87f9b977 --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +[flake8] +# Ignore style and complexity +# E: style errors +# W: style warnings +# C: complexity +# F401: module imported but unused +# F403: import * +# F811: redefinition of unused `name` from line `N` +# F841: local variable assigned but never used +# E402: module level import not at top of file +# I100: Import statements are in the wrong order +# I101: Imported names are in the wrong order. Should be +ignore = E, W, C, F401, F403, F811, F841, E402, I100, I101, D400 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 60726f6f..6fe59fde 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,23 +9,22 @@ on: jobs: deploy: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: '__token__' - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6e85398..919430e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,55 +1,77 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions +# +name: Test -name: Python package - -on: [push, pull_request] +on: + pull_request: + push: + workflow_dispatch: jobs: - build: + pre-commit: + name: Run pre-commit + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 + pytest: + name: "Run pytest" runs-on: ubuntu-20.04 continue-on-error: ${{ matrix.allow_failure }} strategy: + # Keep running even if one variation of the job fail + fail-fast: false matrix: python-version: - - '3.5' - - '3.9' + - "3.5" + - "3.9" JHUB_VER: - - '0.9.6' - - '1.0.0' - - '1.1.0' - - '1.2.0' + - "0.9.6" + - "1.0.0" + - "1.1.0" + - "1.2.0" + - "1.3.0" allow_failure: [false] + + exclude: + # JupyterHub 1.3.0 requires python 3.6+ + - JHUB_VER: "1.3.0" + python-version: "3.5" include: - - JHUB_VER: 'master' - python-version: '3.9' + - JHUB_VER: "master" + python-version: "3.9" allow_failure: true steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install pytest - pip install -r requirements.txt - - name: Install nodejs dependencies - run: | - sudo npm install -g configurable-http-proxy - # We need to check compatibility with different versions of the JH - # API, including latest development. For that, we also need to - # pull in the dependencies of that old JH version (but we don't - # need conda/npm for our tests). - - name: install JupyterHub - run: | - git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub - pip install --pre -r jupyterhub/dev-requirements.txt - pip install --upgrade pytest - pip install --pre -e jupyterhub - - - name: pytest - run: | - pytest --lf --cov batchspawner batchspawner/tests -v + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + pip install -r requirements.txt + - name: Install nodejs dependencies + run: | + sudo npm install -g configurable-http-proxy + + # We need to check compatibility with different versions of the JH + # API, including latest development. For that, we also need to + # pull in the dependencies of that old JH version (but we don't + # need conda/npm for our tests). + - name: install JupyterHub + run: | + git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub + pip install --pre -r jupyterhub/dev-requirements.txt + pip install --upgrade pytest + pip install --pre -e jupyterhub + + - name: pytest + run: | + pytest --verbose --color=yes --last-failed --cov batchspawner batchspawner/tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..92d453e6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +# pre-commit is a tool to perform a predefined set of tasks manually and/or +# automatically before git commits are made. +# +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all-files +# - Register git hooks: pre-commit install --install-hooks +# +repos: + # Autoformat: Python code + - repo: https://github.com/ambv/black + rev: 20.8b1 + hooks: + - id: black + args: [--target-version=py36] + + # Autoformat: markdown, yaml + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.2.1 + hooks: + - id: prettier + + # Lint: Python code + - repo: https://gitlab.com/pycqa/flake8 + rev: "3.8.4" + hooks: + - id: flake8 + + # Misc... + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available + hooks: + # Autoformat: Makes sure files end in a newline and only a newline. + - id: end-of-file-fixer + + # Autoformat: Sorts entries in requirements.txt. + - id: requirements-txt-fixer + + # Prevent giant (500kB) files from being committed. + - id: check-added-large-files + + # Lint: Check for files with names that would conflict on a + # case-insensitive filesystem like MacOS HFS+ or Windows FAT. + - id: check-case-conflict + + # Lint: Checks that non-binary executables have a proper shebang. + - id: check-executables-have-shebangs diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1f88a5..ab49bb4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,85 +14,81 @@ Fixed Added (user) -* PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`. -* PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH` +- PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`. +- PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH` Added (developer) -* PR #187: support for unknown job state +- PR #187: support for unknown job state Changed -* PR #177: Fail on first error in batch script by setting `set -e` to script templates. -* PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally. -* PR #189: remove bashism from default script template -* PR #195: fix exception handling in run_command -* PR #198: change from Travis to gh-actions for testing -* PR #196: documentation -* PR #199: update setup.py +- PR #177: Fail on first error in batch script by setting `set -e` to script templates. +- PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally. +- PR #189: remove bashism from default script template +- PR #195: fix exception handling in run_command +- PR #198: change from Travis to gh-actions for testing +- PR #196: documentation +- PR #199: update setup.py ## v1.0 (requires minimum JupyterHub 0.9 and Python 3.5) Added (user) -* Add support for JupyterHub named servers. #167 -* Add Jinja2 templating as an option for all scripts and commands. If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template. -* Add new option exec_prefix, which defaults to `sudo -E -u {username}`. This replaces explicit `sudo` in every batch command - changes in local commands may be needed. -* New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement). #99 -* Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script. #96 -* SlurmSpawner: add the `req_reservation` option. #91 -* Add basic support for JupyterHub progress updates, but this is not used much yet. #86 +- Add support for JupyterHub named servers. #167 +- Add Jinja2 templating as an option for all scripts and commands. If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template. +- Add new option exec_prefix, which defaults to `sudo -E -u {username}`. This replaces explicit `sudo` in every batch command - changes in local commands may be needed. +- New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement). #99 +- Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script. #96 +- SlurmSpawner: add the `req_reservation` option. #91 +- Add basic support for JupyterHub progress updates, but this is not used much yet. #86 Added (developer) -* Add many more tests. -* Add a new page `SPAWNERS.md` which information on specific spawners. Begin trying to collect a list of spawner-specific contacts. #97 -* Rename `current_ip` and `current_port` commands to `ip` and `port`. No user impact. #139 -* Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates. #90 +- Add many more tests. +- Add a new page `SPAWNERS.md` which information on specific spawners. Begin trying to collect a list of spawner-specific contacts. #97 +- Rename `current_ip` and `current_port` commands to `ip` and `port`. No user impact. #139 +- Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates. #90 Changed -* PR #58 and #141 changes logic of port selection, so that it is selected *after* the singleuser server starts. This means that the port number has to be conveyed back to JupyterHub. This requires the following changes: - - `jupyterhub_config.py` *must* explicitely import `batchspawner` - - Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub. This is now an integral part of the spawn process. +- PR #58 and #141 changes logic of port selection, so that it is selected _after_ the singleuser server starts. This means that the port number has to be conveyed back to JupyterHub. This requires the following changes: + - `jupyterhub_config.py` _must_ explicitely import `batchspawner` + - Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub. This is now an integral part of the spawn process. - If you have installed with `pip install -e`, you will have to re-install so that the new script `batchspawner-singleuser` is added to `$PATH`. -* Update minimum requirements to JupyterHub 0.9 and Python 3.5. #143 -* Update Slurm batch script. Now, the single-user notebook is run in a job step, with a wrapper of `srun`. This may need to be removed using `req_srun=''` if you don't want environment variables limited. -* Pass the environment dictionary to the queue and cancel commands as well. This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111 If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82. - +- Update minimum requirements to JupyterHub 0.9 and Python 3.5. #143 +- Update Slurm batch script. Now, the single-user notebook is run in a job step, with a wrapper of `srun`. This may need to be removed using `req_srun=''` if you don't want environment variables limited. +- Pass the environment dictionary to the queue and cancel commands as well. This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111 If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82. Fixed -* Improve debugging on failed submission by raising errors including error messages from the commands. #106 -* Many other non-user or developer visible changes. #107 #106 #100 -* In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests +- Improve debugging on failed submission by raising errors including error messages from the commands. #106 +- Many other non-user or developer visible changes. #107 #106 #100 +- In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests Removed - ## v0.8.1 (bugfix release) -* Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start. Issue #76 +- Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start. Issue #76 ## v0.8.0 (compatible with JupyterHub 0.5.0 through 0.8.1/0.9dev) -* SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility. If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect. If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`. -* add base options `req_ngpus` `req_partition` `req_account` and `req_options` -* Fix up logging -* Merge `user_options` with the template substitution vars instead of having it as a separate key -* Update ip/port handling for JupyterHub 0.8 -* Add `LICENSE` (BSD3) and `CONTRIBUTING.md` -* Add `LsfSpawner` for IBM LFS -* Add `MultiSlurmSpawner` -* Add `MoabSpawner` -* Add `condorSpawner` -* Add `GridEngineSpawner` -* SlurmSpawner: add `req_qos` option -* WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package -* Enable CI testing via Travis-CI - +- SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility. If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect. If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`. +- add base options `req_ngpus` `req_partition` `req_account` and `req_options` +- Fix up logging +- Merge `user_options` with the template substitution vars instead of having it as a separate key +- Update ip/port handling for JupyterHub 0.8 +- Add `LICENSE` (BSD3) and `CONTRIBUTING.md` +- Add `LsfSpawner` for IBM LFS +- Add `MultiSlurmSpawner` +- Add `MoabSpawner` +- Add `condorSpawner` +- Add `GridEngineSpawner` +- SlurmSpawner: add `req_qos` option +- WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package +- Enable CI testing via Travis-CI ## v0.3 (tag: jhub-0.3, compatible with JupyterHub 0.3.0) -* initial release containing `TorqueSpawner` and `SlurmSpawner` - +- initial release containing `TorqueSpawner` and `SlurmSpawner` diff --git a/README.md b/README.md index a83a3b98..4d6ca843 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,24 @@ # batchspawner for Jupyterhub -[![Build Status](https://img.shields.io/travis/com/jupyterhub/batchspawner?logo=travis)](https://travis-ci.com/jupyterhub/batchspawner) +[![GitHub Workflow Status - Test](https://img.shields.io/github/workflow/status/jupyterhub/batchspawner/Test?logo=github&label=tests)](https://github.com/jupyterhub/batchspawner/actions) +[![Latest PyPI version](https://img.shields.io/pypi/v/batchspawner?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/batchspawner) +[![GitHub](https://img.shields.io/badge/issue_tracking-github-blue?logo=github)](https://github.com/jupyterhub/batchspawner/issues) +[![Discourse](https://img.shields.io/badge/help_forum-discourse-blue?logo=discourse)](https://discourse.jupyter.org/c/jupyterhub) +[![Gitter](https://img.shields.io/badge/social_chat-gitter-blue?logo=gitter)](https://gitter.im/jupyterhub/jupyterhub) +[![Contribute](https://img.shields.io/badge/I_want_to_contribute!-grey?logo=jupyter)](https://github.com/jupyterhub/batchspawner/blob/master/CONTRIBUTING.md) This is a custom spawner for [Jupyterhub](https://jupyterhub.readthedocs.io/) that is designed for installations on clusters using batch scheduling software. -This began as a generalization of [mkgilbert's batchspawner](https://github.com/mkgilbert/slurmspawner) which in turn was inspired by [Andrea Zonca's blog post](http://zonca.github.io/2015/04/jupyterhub-hpc.html 'Run jupyterhub on a Supercomputer') where he explains his implementation for a spawner that uses SSH and Torque. His github repo is found [here](http://www.github.com/zonca/remotespawner 'RemoteSpawner'). +This began as a generalization of [mkgilbert's batchspawner](https://github.com/mkgilbert/slurmspawner) which in turn was inspired by [Andrea Zonca's blog post](http://zonca.github.io/2015/04/jupyterhub-hpc.html "Run jupyterhub on a Supercomputer") where he explains his implementation for a spawner that uses SSH and Torque. His github repo is found [here](http://www.github.com/zonca/remotespawner "RemoteSpawner"). -This package formerly included WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners. These have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package. +This package formerly included WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners. These have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package. ## Installation + 1. from root directory of this repo (where setup.py is), run `pip install -e .` If you don't actually need an editable version, you can simply run - `pip install batchspawner` + `pip install batchspawner` 2. add lines in jupyterhub_config.py for the spawner you intend to use, e.g. @@ -21,6 +27,7 @@ This package formerly included WrapSpawner and ProfilesSpawner, which provide me c.JupyterHub.spawner_class = 'batchspawner.TorqueSpawner' import batchspawner # Even though not used, needed to register batchspawner interface ``` + 3. Depending on the spawner, additional configuration will likely be needed. ## Batch Spawners @@ -32,18 +39,20 @@ For information on the specific spawners, see [SPAWNERS.md](SPAWNERS.md). This file contains an abstraction layer for batch job queueing systems (`BatchSpawnerBase`), and implements Jupyterhub spawners for Torque, Moab, SLURM, SGE, HTCondor, LSF, and eventually others. Common attributes of batch submission / resource manager environments will include notions of: - * queue names, resource manager addresses - * resource limits including runtime, number of processes, memory - * singleuser child process running on (usually remote) host not known until runtime - * job submission and monitoring via resource manager utilities - * remote execution via submission of templated scripts - * job names instead of PIDs + +- queue names, resource manager addresses +- resource limits including runtime, number of processes, memory +- singleuser child process running on (usually remote) host not known until runtime +- job submission and monitoring via resource manager utilities +- remote execution via submission of templated scripts +- job names instead of PIDs `BatchSpawnerBase` provides several general mechanisms: - * configurable traits `req_foo` that are exposed as `{foo}` in job template scripts. Templates (submit scripts in particular) may also use the full power of [jinja2](http://jinja.pocoo.org/). Templates are automatically detected if a `{{` or `{%` is present, otherwise str.format() used. - * configurable command templates for submitting/querying/cancelling jobs - * a generic concept of job-ID and ID-based job state tracking - * overrideable hooks for subclasses to plug in logic at numerous points + +- configurable traits `req_foo` that are exposed as `{foo}` in job template scripts. Templates (submit scripts in particular) may also use the full power of [jinja2](http://jinja.pocoo.org/). Templates are automatically detected if a `{{` or `{%` is present, otherwise str.format() used. +- configurable command templates for submitting/querying/cancelling jobs +- a generic concept of job-ID and ID-based job state tracking +- overrideable hooks for subclasses to plug in logic at numerous points ### Example @@ -51,54 +60,53 @@ Every effort has been made to accommodate highly diverse systems through configu only. This example consists of the (lightly edited) configuration used by the author to run Jupyter notebooks on an academic supercomputer cluster. - ```python - # Select the Torque backend and increase the timeout since batch jobs may take time to start - import batchspawner - c.JupyterHub.spawner_class = 'batchspawner.TorqueSpawner' - c.Spawner.http_timeout = 120 - - #------------------------------------------------------------------------------ - # BatchSpawnerBase configuration - # These are simply setting parameters used in the job script template below - #------------------------------------------------------------------------------ - c.BatchSpawnerBase.req_nprocs = '2' - c.BatchSpawnerBase.req_queue = 'mesabi' - c.BatchSpawnerBase.req_host = 'mesabi.xyz.edu' - c.BatchSpawnerBase.req_runtime = '12:00:00' - c.BatchSpawnerBase.req_memory = '4gb' - #------------------------------------------------------------------------------ - # TorqueSpawner configuration - # The script below is nearly identical to the default template, but we needed - # to add a line for our local environment. For most sites the default templates - # should be a good starting point. - #------------------------------------------------------------------------------ - c.TorqueSpawner.batch_script = '''#!/bin/sh - #PBS -q {queue}@{host} - #PBS -l walltime={runtime} - #PBS -l nodes=1:ppn={nprocs} - #PBS -l mem={memory} - #PBS -N jupyterhub-singleuser - #PBS -v {keepvars} - module load python3 - {cmd} - ''' - # For our site we need to munge the execution hostname returned by qstat - c.TorqueSpawner.state_exechost_exp = r'int-\1.mesabi.xyz.edu' - ``` +```python +# Select the Torque backend and increase the timeout since batch jobs may take time to start +import batchspawner +c.JupyterHub.spawner_class = 'batchspawner.TorqueSpawner' +c.Spawner.http_timeout = 120 + +#------------------------------------------------------------------------------ +# BatchSpawnerBase configuration +# These are simply setting parameters used in the job script template below +#------------------------------------------------------------------------------ +c.BatchSpawnerBase.req_nprocs = '2' +c.BatchSpawnerBase.req_queue = 'mesabi' +c.BatchSpawnerBase.req_host = 'mesabi.xyz.edu' +c.BatchSpawnerBase.req_runtime = '12:00:00' +c.BatchSpawnerBase.req_memory = '4gb' +#------------------------------------------------------------------------------ +# TorqueSpawner configuration +# The script below is nearly identical to the default template, but we needed +# to add a line for our local environment. For most sites the default templates +# should be a good starting point. +#------------------------------------------------------------------------------ +c.TorqueSpawner.batch_script = '''#!/bin/sh +#PBS -q {queue}@{host} +#PBS -l walltime={runtime} +#PBS -l nodes=1:ppn={nprocs} +#PBS -l mem={memory} +#PBS -N jupyterhub-singleuser +#PBS -v {keepvars} +module load python3 +{cmd} +''' +# For our site we need to munge the execution hostname returned by qstat +c.TorqueSpawner.state_exechost_exp = r'int-\1.mesabi.xyz.edu' +``` ### Security Unless otherwise stated for a specific spawner, assume that spawners -*do* evaluate shell environment for users and thus the [security +_do_ evaluate shell environment for users and thus the [security requirements of JupyterHub security for untrusted users](https://jupyterhub.readthedocs.io/en/stable/reference/websecurity.html) -are not fulfilled because some (most?) spawners *do* start a user +are not fulfilled because some (most?) spawners _do_ start a user shell which will execute arbitrary user environment configuration -(``.profile``, ``.bashrc`` and the like) unless users do not have -access to their own cluster user account. This is something which we +(`.profile`, `.bashrc` and the like) unless users do not have +access to their own cluster user account. This is something which we are working on. - ## Provide different configurations of BatchSpawner ### Overview @@ -117,85 +125,82 @@ The following is based on the author's configuration (at the same site as the ex showing how to give users access to multiple job configurations on the batch scheduled clusters, as well as an option to run a local notebook directly on the jupyterhub server. - ```python - # Same initial setup as the previous example - import batchspawner - c.JupyterHub.spawner_class = 'wrapspawner.ProfilesSpawner' - c.Spawner.http_timeout = 120 - #------------------------------------------------------------------------------ - # BatchSpawnerBase configuration - # Providing default values that we may omit in the profiles - #------------------------------------------------------------------------------ - c.BatchSpawnerBase.req_host = 'mesabi.xyz.edu' - c.BatchSpawnerBase.req_runtime = '12:00:00' - c.TorqueSpawner.state_exechost_exp = r'in-\1.mesabi.xyz.edu' - #------------------------------------------------------------------------------ - # ProfilesSpawner configuration - #------------------------------------------------------------------------------ - # List of profiles to offer for selection. Signature is: - # List(Tuple( Unicode, Unicode, Type(Spawner), Dict )) - # corresponding to profile display name, unique key, Spawner class, - # dictionary of spawner config options. - # - # The first three values will be exposed in the input_template as {display}, - # {key}, and {type} - # - c.ProfilesSpawner.profiles = [ - ( "Local server", 'local', 'jupyterhub.spawner.LocalProcessSpawner', {'ip':'0.0.0.0'} ), - ('Mesabi - 2 cores, 4 GB, 8 hours', 'mesabi2c4g12h', 'batchspawner.TorqueSpawner', - dict(req_nprocs='2', req_queue='mesabi', req_runtime='8:00:00', req_memory='4gb')), - ('Mesabi - 12 cores, 128 GB, 4 hours', 'mesabi128gb', 'batchspawner.TorqueSpawner', - dict(req_nprocs='12', req_queue='ram256g', req_runtime='4:00:00', req_memory='125gb')), - ('Mesabi - 2 cores, 4 GB, 24 hours', 'mesabi2c4gb24h', 'batchspawner.TorqueSpawner', - dict(req_nprocs='2', req_queue='mesabi', req_runtime='24:00:00', req_memory='4gb')), - ('Interactive Cluster - 2 cores, 4 GB, 8 hours', 'lab', 'batchspawner.TorqueSpawner', - dict(req_nprocs='2', req_host='labhost.xyz.edu', req_queue='lab', - req_runtime='8:00:00', req_memory='4gb', state_exechost_exp='')), - ] - c.ProfilesSpawner.ip = '0.0.0.0' - ``` - +```python +# Same initial setup as the previous example +import batchspawner +c.JupyterHub.spawner_class = 'wrapspawner.ProfilesSpawner' +c.Spawner.http_timeout = 120 +#------------------------------------------------------------------------------ +# BatchSpawnerBase configuration +# Providing default values that we may omit in the profiles +#------------------------------------------------------------------------------ +c.BatchSpawnerBase.req_host = 'mesabi.xyz.edu' +c.BatchSpawnerBase.req_runtime = '12:00:00' +c.TorqueSpawner.state_exechost_exp = r'in-\1.mesabi.xyz.edu' +#------------------------------------------------------------------------------ +# ProfilesSpawner configuration +#------------------------------------------------------------------------------ +# List of profiles to offer for selection. Signature is: +# List(Tuple( Unicode, Unicode, Type(Spawner), Dict )) +# corresponding to profile display name, unique key, Spawner class, +# dictionary of spawner config options. +# +# The first three values will be exposed in the input_template as {display}, +# {key}, and {type} +# +c.ProfilesSpawner.profiles = [ + ( "Local server", 'local', 'jupyterhub.spawner.LocalProcessSpawner', {'ip':'0.0.0.0'} ), + ('Mesabi - 2 cores, 4 GB, 8 hours', 'mesabi2c4g12h', 'batchspawner.TorqueSpawner', + dict(req_nprocs='2', req_queue='mesabi', req_runtime='8:00:00', req_memory='4gb')), + ('Mesabi - 12 cores, 128 GB, 4 hours', 'mesabi128gb', 'batchspawner.TorqueSpawner', + dict(req_nprocs='12', req_queue='ram256g', req_runtime='4:00:00', req_memory='125gb')), + ('Mesabi - 2 cores, 4 GB, 24 hours', 'mesabi2c4gb24h', 'batchspawner.TorqueSpawner', + dict(req_nprocs='2', req_queue='mesabi', req_runtime='24:00:00', req_memory='4gb')), + ('Interactive Cluster - 2 cores, 4 GB, 8 hours', 'lab', 'batchspawner.TorqueSpawner', + dict(req_nprocs='2', req_host='labhost.xyz.edu', req_queue='lab', + req_runtime='8:00:00', req_memory='4gb', state_exechost_exp='')), + ] +c.ProfilesSpawner.ip = '0.0.0.0' +``` ## Debugging batchspawner Sometimes it can be hard to debug batchspawner, but it's not really -once you know how the pieces interact. Check the following places for +once you know how the pieces interact. Check the following places for error messages: -* Check the JupyterHub logs for errors. +- Check the JupyterHub logs for errors. -* Check the JupyterHub logs for the batch script that got submitted - and the command used to submit it. Are these correct? (Note that +- Check the JupyterHub logs for the batch script that got submitted + and the command used to submit it. Are these correct? (Note that there are submission environment variables too, which aren't displayed.) -* At this point, it's a matter of checking the batch system. Is the - job ever scheduled? Does it run? Does it succeed? Check the batch - system status and output of the job. The most comon failure +- At this point, it's a matter of checking the batch system. Is the + job ever scheduled? Does it run? Does it succeed? Check the batch + system status and output of the job. The most comon failure patterns are a) job never starting due to bad scheduler options, b) job waiting in the queue beyond the `start_timeout`, causing JupyterHub to kill the job. -* At this point the job starts. Does it fail immediately, or before - Jupyter starts? Check the scheduler output files (stdout/stderr of - the job), wherever it is stored. To debug the job script, you can - add debugging into the batch script, such as an `env` or `set - -x`. - -* At this point Jupyter itself starts - check its error messages. Is - it starting with the right options? Can it communicate with the - hub? At this point there usually isn't anything - batchspawner-specific, with the one exception below. The error log - would be in the batch script output (same file as above). There may +- At this point the job starts. Does it fail immediately, or before + Jupyter starts? Check the scheduler output files (stdout/stderr of + the job), wherever it is stored. To debug the job script, you can + add debugging into the batch script, such as an `env` or `set -x`. + +- At this point Jupyter itself starts - check its error messages. Is + it starting with the right options? Can it communicate with the + hub? At this point there usually isn't anything + batchspawner-specific, with the one exception below. The error log + would be in the batch script output (same file as above). There may also be clues in the JupyterHub logfile. Common problems: -* Did you `import batchspawner` in the `jupyterhub_config.py` file? +- Did you `import batchspawner` in the `jupyterhub_config.py` file? This is needed in order to activate the batchspawer API in JupyterHub. - ## Changelog See [CHANGELOG.md](CHANGELOG.md). diff --git a/SPAWNERS.md b/SPAWNERS.md index cce52e6e..15ecf78b 100644 --- a/SPAWNERS.md +++ b/SPAWNERS.md @@ -1,12 +1,12 @@ # Notes on specific spawners **Spawner maintainers**: Included below are "spawner maintainers", -when available. There aren't official obligations, but the general +when available. There aren't official obligations, but the general idea is that you should watch the repository and feel especially empowered to comment on issues when you think it might be relevant to you (obviously everyone should be, but this is our attempt at even -more outreach). You should let us know when we break something and -provide a diversity of opinions in general. Submitting PRs and +more outreach). You should let us know when we break something and +provide a diversity of opinions in general. Submitting PRs and testing is nice but not required. To be listed as a maintainer, just submit an issue or PR adding you, @@ -16,57 +16,50 @@ and please watch the repository on Github. Maintainers: - ## `MoabSpawner` Subclass of TorqueSpawner Maintainers: - ## `SlurmSpawner` Maintainers: @rkdarst This spawner enforces the environment if `srun` is used to wrap the -spawner command, which is the default. If you *do* want user -environment to be used, set `req_srun=''`. However, this is not +spawner command, which is the default. If you _do_ want user +environment to be used, set `req_srun=''`. However, this is not perfect: there is still a bash shell begun as the user which could run arbitrary startup, define shell aliases for `srun`, etc. Use of `srun` is required to gracefully terminate. - ## `GridengineSpawner` Maintainers: - ## `CondorSpawner` Maintainers: - ## `LsfSpawner` Maintainers: - # Checklist for making spawners Please document each of these things under the spawner list above, - -even if it is "OK", we need to track status of all spawners. If it is +even if it is "OK", we need to track status of all spawners. If it is a bug, users really need to know. -- Does your spawner read shell environment before starting? (See +- Does your spawner read shell environment before starting? (See [Jupyterhub Security](https://jupyterhub.readthedocs.io/en/stable/reference/websecurity.html). - Does your spawner send SIGTERM to the jupyterhub-singleuser process - before SIGKILL? It should, so that the process can terminate - gracefully. Add `echo "terminated gracefully"` to the end of the + before SIGKILL? It should, so that the process can terminate + gracefully. Add `echo "terminated gracefully"` to the end of the batch script - if you see this in your singleuser server output, you - know that you DO receive SIGTERM and terminate gracefully. If your + know that you DO receive SIGTERM and terminate gracefully. If your batch system can not automatically send SIGTERM before SIGKILL, PR #75 might help here, ask for it to be finished. - diff --git a/batchspawner/__init__.py b/batchspawner/__init__.py index e6b50a69..976081ee 100644 --- a/batchspawner/__init__.py +++ b/batchspawner/__init__.py @@ -1,2 +1,2 @@ from .batchspawner import * -from . import api \ No newline at end of file +from . import api diff --git a/batchspawner/api.py b/batchspawner/api.py index 514dd0a5..d5459d12 100644 --- a/batchspawner/api.py +++ b/batchspawner/api.py @@ -1,12 +1,13 @@ import json from tornado import web -from jupyterhub.apihandlers import APIHandler, default_handlers +from jupyterhub.apihandlers import APIHandler, default_handlers + class BatchSpawnerAPIHandler(APIHandler): @web.authenticated def post(self): """POST set user spawner data""" - if hasattr(self, 'current_user'): + if hasattr(self, "current_user"): # Jupyterhub compatability, (september 2018, d79a99323ef1d) user = self.current_user else: @@ -25,4 +26,5 @@ def post(self): self.finish(json.dumps({"message": "BatchSpawner data configured"})) self.set_status(201) + default_handlers.append((r"/api/batchspawner", BatchSpawnerAPIHandler)) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 5a26cf37..5c7c971b 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -35,9 +35,7 @@ from jupyterhub.spawner import Spawner from jupyterhub.traitlets import Command -from traitlets import ( - Integer, Unicode, Float, Dict, default -) +from traitlets import Integer, Unicode, Float, Dict, default from jupyterhub.utils import random_port from jupyterhub.spawner import set_user_setuid @@ -53,15 +51,17 @@ def format_template(template, *args, **kwargs): """ if isinstance(template, Template): return template.render(*args, **kwargs) - elif '{{' in template or '{%' in template: + elif "{{" in template or "{%" in template: return Template(template).render(*args, **kwargs) return template.format(*args, **kwargs) + class JobStatus(Enum): - NOTFOUND = 0 - RUNNING = 1 - PENDING = 2 - UNKNOWN = 3 + NOTFOUND = 0 + RUNNING = 1 + PENDING = 2 + UNKNOWN = 3 + class BatchSpawnerBase(Spawner): """Base class for spawners using resource manager batch job submission mechanisms @@ -88,89 +88,110 @@ class BatchSpawnerBase(Spawner): start_timeout = Integer(300).tag(config=True) # override default server ip since batch jobs normally running remotely - ip = Unicode("0.0.0.0", help="Address for singleuser server to listen at").tag(config=True) + ip = Unicode( + "0.0.0.0", + help="Address for singleuser server to listen at", + ).tag(config=True) - exec_prefix = Unicode('sudo -E -u {username}', - help="Standard executon prefix (e.g. the default sudo -E -u {username})" - ).tag(config=True) + exec_prefix = Unicode( + "sudo -E -u {username}", + help="Standard executon prefix (e.g. the default sudo -E -u {username})", + ).tag(config=True) # all these req_foo traits will be available as substvars for templated strings - req_queue = Unicode('', - help="Queue name to submit job to resource manager" - ).tag(config=True) + req_queue = Unicode( + "", + help="Queue name to submit job to resource manager", + ).tag(config=True) - req_host = Unicode('', - help="Host name of batch server to submit job to resource manager" - ).tag(config=True) + req_host = Unicode( + "", + help="Host name of batch server to submit job to resource manager", + ).tag(config=True) - req_memory = Unicode('', - help="Memory to request from resource manager" - ).tag(config=True) + req_memory = Unicode( + "", + help="Memory to request from resource manager", + ).tag(config=True) - req_nprocs = Unicode('', - help="Number of processors to request from resource manager" - ).tag(config=True) + req_nprocs = Unicode( + "", + help="Number of processors to request from resource manager", + ).tag(config=True) - req_ngpus = Unicode('', - help="Number of GPUs to request from resource manager" - ).tag(config=True) + req_ngpus = Unicode( + "", + help="Number of GPUs to request from resource manager", + ).tag(config=True) - req_runtime = Unicode('', - help="Length of time for submitted job to run" - ).tag(config=True) + req_runtime = Unicode( + "", + help="Length of time for submitted job to run", + ).tag(config=True) - req_partition = Unicode('', - help="Partition name to submit job to resource manager" - ).tag(config=True) + req_partition = Unicode( + "", + help="Partition name to submit job to resource manager", + ).tag(config=True) - req_account = Unicode('', - help="Account name string to pass to the resource manager" - ).tag(config=True) + req_account = Unicode( + "", + help="Account name string to pass to the resource manager", + ).tag(config=True) - req_options = Unicode('', - help="Other options to include into job submission script" - ).tag(config=True) + req_options = Unicode( + "", + help="Other options to include into job submission script", + ).tag(config=True) - req_prologue = Unicode('', - help="Script to run before single user server starts." - ).tag(config=True) + req_prologue = Unicode( + "", + help="Script to run before single user server starts.", + ).tag(config=True) - req_epilogue = Unicode('', - help="Script to run after single user server ends." - ).tag(config=True) + req_epilogue = Unicode( + "", + help="Script to run after single user server ends.", + ).tag(config=True) req_username = Unicode() - @default('req_username') + + @default("req_username") def _req_username_default(self): return self.user.name # Useful IF getpwnam on submit host returns correct info for exec host req_homedir = Unicode() - @default('req_homedir') + + @default("req_homedir") def _req_homedir_default(self): return pwd.getpwnam(self.user.name).pw_dir req_keepvars = Unicode() - @default('req_keepvars') + + @default("req_keepvars") def _req_keepvars_default(self): - return ','.join(self.get_env().keys()) + return ",".join(self.get_env().keys()) req_keepvars_extra = Unicode( help="Extra environment variables which should be configured, " - "added to the defaults in keepvars, " - "comma separated list.") + "added to the defaults in keepvars, " + "comma separated list.", + ) - batch_script = Unicode('', + batch_script = Unicode( + "", help="Template for job submission script. Traits on this class named like req_xyz " - "will be substituted in the template for {xyz} using string.Formatter. " - "Must include {cmd} which will be replaced with the jupyterhub-singleuser command line." - ).tag(config=True) + "will be substituted in the template for {xyz} using string.Formatter. " + "Must include {cmd} which will be replaced with the jupyterhub-singleuser command line.", + ).tag(config=True) - batchspawner_singleuser_cmd = Unicode('batchspawner-singleuser', + batchspawner_singleuser_cmd = Unicode( + "batchspawner-singleuser", help="A wrapper which is capable of special batchspawner setup: currently sets the port on " - "the remote host. Not needed to be set under normal circumstances, unless path needs " - "specification." - ).tag(config=True) + "the remote host. Not needed to be set under normal circumstances, unless path needs " + "specification.", + ).tag(config=True) # Raw output of job submission command unless overridden job_id = Unicode() @@ -180,17 +201,18 @@ def _req_keepvars_default(self): # Prepare substitution variables for templates using req_xyz traits def get_req_subvars(self): - reqlist = [ t for t in self.trait_names() if t.startswith('req_') ] + reqlist = [t for t in self.trait_names() if t.startswith("req_")] subvars = {} for t in reqlist: subvars[t[4:]] = getattr(self, t) - if subvars.get('keepvars_extra'): - subvars['keepvars'] += ',' + subvars['keepvars_extra'] + if subvars.get("keepvars_extra"): + subvars["keepvars"] += "," + subvars["keepvars_extra"] return subvars - batch_submit_cmd = Unicode('', - help="Command to run to submit batch scripts. Formatted using req_xyz traits as {xyz}." - ).tag(config=True) + batch_submit_cmd = Unicode( + "", + help="Command to run to submit batch scripts. Formatted using req_xyz traits as {xyz}.", + ).tag(config=True) def parse_job_id(self, output): "Parse output of submit command to get job id." @@ -198,17 +220,20 @@ def parse_job_id(self, output): def cmd_formatted_for_batch(self): """The command which is substituted inside of the batch script""" - return ' '.join([self.batchspawner_singleuser_cmd] + self.cmd + self.get_args()) + return " ".join([self.batchspawner_singleuser_cmd] + self.cmd + self.get_args()) async def run_command(self, cmd, input=None, env=None): - proc = await asyncio.create_subprocess_shell(cmd, env=env, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) - inbytes=None + proc = await asyncio.create_subprocess_shell( + cmd, + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + inbytes = None if input: - inbytes=input.encode() + inbytes = input.encode() try: out, eout = await proc.communicate(input=inbytes) @@ -221,16 +246,23 @@ async def run_command(self, cmd, input=None, env=None): out = out.decode().strip() eout = eout.decode().strip() self.log.error("Subprocess returned exitcode %s" % proc.returncode) - self.log.error('Stdout:') + self.log.error("Stdout:") self.log.error(out) - self.log.error('Stderr:') + self.log.error("Stderr:") self.log.error(eout) - raise RuntimeError('{} exit status {}: {}'.format(cmd, proc.returncode, eout)) + raise RuntimeError( + "{} exit status {}: {}".format(cmd, proc.returncode, eout) + ) except asyncio.TimeoutError: - self.log.error('Encountered timeout trying to clean up command, process probably killed already: %s' % cmd) + self.log.error( + "Encountered timeout trying to clean up command, process probably killed already: %s" + % cmd + ) return "" except: - self.log.error('Encountered exception trying to clean up command: %s' % cmd) + self.log.error( + "Encountered exception trying to clean up command: %s" % cmd + ) raise else: eout = eout.decode().strip() @@ -251,49 +283,58 @@ async def _get_batch_script(self, **subvars): async def submit_batch_script(self): subvars = self.get_req_subvars() # `cmd` is submitted to the batch system - cmd = ' '.join((format_template(self.exec_prefix, **subvars), - format_template(self.batch_submit_cmd, **subvars))) + cmd = " ".join( + ( + format_template(self.exec_prefix, **subvars), + format_template(self.batch_submit_cmd, **subvars), + ) + ) # `subvars['cmd']` is what is run _inside_ the batch script, # put into the template. - subvars['cmd'] = self.cmd_formatted_for_batch() - if hasattr(self, 'user_options'): + subvars["cmd"] = self.cmd_formatted_for_batch() + if hasattr(self, "user_options"): subvars.update(self.user_options) script = await self._get_batch_script(**subvars) - self.log.info('Spawner submitting job using ' + cmd) - self.log.info('Spawner submitted script:\n' + script) + self.log.info("Spawner submitting job using " + cmd) + self.log.info("Spawner submitted script:\n" + script) out = await self.run_command(cmd, input=script, env=self.get_env()) try: - self.log.info('Job submitted. cmd: ' + cmd + ' output: ' + out) + self.log.info("Job submitted. cmd: " + cmd + " output: " + out) self.job_id = self.parse_job_id(out) except: - self.log.error('Job submission failed with exit code ' + out) - self.job_id = '' + self.log.error("Job submission failed with exit code " + out) + self.job_id = "" return self.job_id # Override if your batch system needs something more elaborate to query the job status - batch_query_cmd = Unicode('', + batch_query_cmd = Unicode( + "", help="Command to run to query job status. Formatted using req_xyz traits as {xyz} " - "and self.job_id as {job_id}." - ).tag(config=True) + "and self.job_id as {job_id}.", + ).tag(config=True) async def query_job_status(self): """Check job status, return JobStatus object.""" if self.job_id is None or len(self.job_id) == 0: - self.job_status = '' + self.job_status = "" return JobStatus.NOTFOUND subvars = self.get_req_subvars() - subvars['job_id'] = self.job_id - cmd = ' '.join((format_template(self.exec_prefix, **subvars), - format_template(self.batch_query_cmd, **subvars))) - self.log.debug('Spawner querying job: ' + cmd) + subvars["job_id"] = self.job_id + cmd = " ".join( + ( + format_template(self.exec_prefix, **subvars), + format_template(self.batch_query_cmd, **subvars), + ) + ) + self.log.debug("Spawner querying job: " + cmd) try: self.job_status = await self.run_command(cmd) except RuntimeError as e: # e.args[0] is stderr from the process self.job_status = e.args[0] except Exception as e: - self.log.error('Error querying job ' + self.job_id) - self.job_status = '' + self.log.error("Error querying job " + self.job_id) + self.job_status = "" if self.state_isrunning(): return JobStatus.RUNNING @@ -304,38 +345,43 @@ async def query_job_status(self): else: return JobStatus.NOTFOUND - batch_cancel_cmd = Unicode('', - help="Command to stop/cancel a previously submitted job. Formatted like batch_query_cmd." - ).tag(config=True) + batch_cancel_cmd = Unicode( + "", + help="Command to stop/cancel a previously submitted job. Formatted like batch_query_cmd.", + ).tag(config=True) async def cancel_batch_job(self): subvars = self.get_req_subvars() - subvars['job_id'] = self.job_id - cmd = ' '.join((format_template(self.exec_prefix, **subvars), - format_template(self.batch_cancel_cmd, **subvars))) - self.log.info('Cancelling job ' + self.job_id + ': ' + cmd) + subvars["job_id"] = self.job_id + cmd = " ".join( + ( + format_template(self.exec_prefix, **subvars), + format_template(self.batch_cancel_cmd, **subvars), + ) + ) + self.log.info("Cancelling job " + self.job_id + ": " + cmd) await self.run_command(cmd) def load_state(self, state): """load job_id from state""" super(BatchSpawnerBase, self).load_state(state) - self.job_id = state.get('job_id', '') - self.job_status = state.get('job_status', '') + self.job_id = state.get("job_id", "") + self.job_status = state.get("job_status", "") def get_state(self): """add job_id to state""" state = super(BatchSpawnerBase, self).get_state() if self.job_id: - state['job_id'] = self.job_id + state["job_id"] = self.job_id if self.job_status: - state['job_status'] = self.job_status + state["job_status"] = self.job_status return state def clear_state(self): """clear job_id state""" super(BatchSpawnerBase, self).clear_state() self.job_id = "" - self.job_status = '' + self.job_status = "" def make_preexec_fn(self, name): """make preexec fn to change uid (if running as root) before job submission""" @@ -366,16 +412,17 @@ async def poll(self): self.clear_state() return 1 - startup_poll_interval = Float(0.5, - help="Polling interval (seconds) to check job state during startup" - ).tag(config=True) + startup_poll_interval = Float( + 0.5, + help="Polling interval (seconds) to check job state during startup", + ).tag(config=True) async def start(self): """Start the process""" - self.ip = self.traits()['ip'].default_value - self.port = self.traits()['port'].default_value + self.ip = self.traits()["ip"].default_value + self.port = self.traits()["port"].default_value - if jupyterhub.version_info >= (0,8) and self.server: + if jupyterhub.version_info >= (0, 8) and self.server: self.server.port = self.port job = await self.submit_batch_script() @@ -385,22 +432,30 @@ async def start(self): # So this function should not return unless successful, and if unsuccessful # should either raise and Exception or loop forever. if len(self.job_id) == 0: - raise RuntimeError("Jupyter batch job submission failure (no jobid in output)") + raise RuntimeError( + "Jupyter batch job submission failure (no jobid in output)" + ) while True: status = await self.query_job_status() if status == JobStatus.RUNNING: break elif status == JobStatus.PENDING: - self.log.debug('Job ' + self.job_id + ' still pending') + self.log.debug("Job " + self.job_id + " still pending") elif status == JobStatus.UNKNOWN: - self.log.debug('Job ' + self.job_id + ' still unknown') + self.log.debug("Job " + self.job_id + " still unknown") else: - self.log.warning('Job ' + self.job_id + ' neither pending nor running.\n' + - self.job_status) + self.log.warning( + "Job " + + self.job_id + + " neither pending nor running.\n" + + self.job_status + ) self.clear_state() - raise RuntimeError('The Jupyter batch job has disappeared' - ' while pending in the queue or died immediately' - ' after starting.') + raise RuntimeError( + "The Jupyter batch job has disappeared" + " while pending in the queue or died immediately" + " after starting." + ) await gen.sleep(self.startup_poll_interval) self.ip = self.state_gethost() @@ -408,17 +463,19 @@ async def start(self): await gen.sleep(self.startup_poll_interval) # Test framework: For testing, mock_port is set because we # don't actually run the single-user server yet. - if hasattr(self, 'mock_port'): + if hasattr(self, "mock_port"): self.port = self.mock_port - if jupyterhub.version_info < (0,7): + if jupyterhub.version_info < (0, 7): # store on user for pre-jupyterhub-0.7: self.user.server.port = self.port self.user.server.ip = self.ip self.db.commit() - self.log.info("Notebook server job {0} started at {1}:{2}".format( - self.job_id, self.ip, self.port) + self.log.info( + "Notebook server job {0} started at {1}:{2}".format( + self.job_id, self.ip, self.port ) + ) return self.ip, self.port @@ -438,28 +495,37 @@ async def stop(self, now=False): return await gen.sleep(1.0) if self.job_id: - self.log.warning("Notebook server job {0} at {1}:{2} possibly failed to terminate".format( - self.job_id, self.ip, self.port) + self.log.warning( + "Notebook server job {0} at {1}:{2} possibly failed to terminate".format( + self.job_id, self.ip, self.port ) + ) @async_generator async def progress(self): while True: if self.state_ispending(): - await yield_({ - "message": "Pending in queue...", - }) + await yield_( + { + "message": "Pending in queue...", + } + ) elif self.state_isrunning(): - await yield_({ - "message": "Cluster job running... waiting to connect", - }) + await yield_( + { + "message": "Cluster job running... waiting to connect", + } + ) return else: - await yield_({ - "message": "Unknown status...", - }) + await yield_( + { + "message": "Unknown status...", + } + ) await gen.sleep(1) + class BatchSpawnerRegexStates(BatchSpawnerBase): """Subclass of BatchSpawnerBase that uses config-supplied regular expressions to interact with batch submission system state. Provides implementations of @@ -477,22 +543,33 @@ class BatchSpawnerRegexStates(BatchSpawnerBase): will be expanded using this string to obtain the notebook IP. See Python docs: re.match.expand """ - state_pending_re = Unicode('', - help="Regex that matches job_status if job is waiting to run").tag(config=True) - state_running_re = Unicode('', - help="Regex that matches job_status if job is running").tag(config=True) - state_exechost_re = Unicode('', + + state_pending_re = Unicode( + "", + help="Regex that matches job_status if job is waiting to run", + ).tag(config=True) + state_running_re = Unicode( + "", + help="Regex that matches job_status if job is running", + ).tag(config=True) + state_exechost_re = Unicode( + "", help="Regex with at least one capture group that extracts " - "the execution host from job_status output").tag(config=True) - state_exechost_exp = Unicode('', + "the execution host from job_status output", + ).tag(config=True) + state_exechost_exp = Unicode( + "", help="""If empty, notebook IP will be set to the contents of the first capture group. If this variable is set, the match object will be expanded using this string to obtain the notebook IP. - See Python docs: re.match.expand""").tag(config=True) - state_unknown_re = Unicode('', + See Python docs: re.match.expand""", + ).tag(config=True) + state_unknown_re = Unicode( + "", help="Regex that matches job_status if the resource manager is not answering." - "Blank indicates not used.").tag(config=True) + "Blank indicates not used.", + ).tag(config=True) def state_ispending(self): assert self.state_pending_re, "Misconfigured: define state_running_re" @@ -511,7 +588,9 @@ def state_gethost(self): assert self.state_exechost_re, "Misconfigured: define state_exechost_re" match = re.search(self.state_exechost_re, self.job_status) if not match: - self.log.error("Spawner unable to match host addr in job status: " + self.job_status) + self.log.error( + "Spawner unable to match host addr in job status: " + self.job_status + ) return if not self.state_exechost_exp: return match.groups()[0] @@ -520,7 +599,8 @@ def state_gethost(self): class TorqueSpawner(BatchSpawnerRegexStates): - batch_script = Unicode("""#!/bin/sh + batch_script = Unicode( + """#!/bin/sh #PBS -q {queue}@{host} #PBS -l walltime={runtime} #PBS -l nodes=1:ppn={nprocs} @@ -534,32 +614,34 @@ class TorqueSpawner(BatchSpawnerRegexStates): {prologue} {cmd} {epilogue} -""").tag(config=True) +""" + ).tag(config=True) # outputs job id string - batch_submit_cmd = Unicode('qsub').tag(config=True) + batch_submit_cmd = Unicode("qsub").tag(config=True) # outputs job data XML string - batch_query_cmd = Unicode('qstat -x {job_id}').tag(config=True) - batch_cancel_cmd = Unicode('qdel {job_id}').tag(config=True) + batch_query_cmd = Unicode("qstat -x {job_id}").tag(config=True) + batch_cancel_cmd = Unicode("qdel {job_id}").tag(config=True) # search XML string for job_state - [QH] = pending, R = running, [CE] = done - state_pending_re = Unicode(r'[QH]').tag(config=True) - state_running_re = Unicode(r'R').tag(config=True) - state_exechost_re = Unicode(r'((?:[\w_-]+\.?)+)/\d+').tag(config=True) + state_pending_re = Unicode(r"[QH]").tag(config=True) + state_running_re = Unicode(r"R").tag(config=True) + state_exechost_re = Unicode(r"((?:[\w_-]+\.?)+)/\d+").tag(config=True) class MoabSpawner(TorqueSpawner): # outputs job id string - batch_submit_cmd = Unicode('msub').tag(config=True) + batch_submit_cmd = Unicode("msub").tag(config=True) # outputs job data XML string - batch_query_cmd = Unicode('mdiag -j {job_id} --xml').tag(config=True) - batch_cancel_cmd = Unicode('mjobctl -c {job_id}').tag(config=True) + batch_query_cmd = Unicode("mdiag -j {job_id} --xml").tag(config=True) + batch_cancel_cmd = Unicode("mjobctl -c {job_id}").tag(config=True) state_pending_re = Unicode(r'State="Idle"').tag(config=True) state_running_re = Unicode(r'State="Running"').tag(config=True) state_exechost_re = Unicode(r'AllocNodeList="([^\r\n\t\f :"]*)').tag(config=True) class PBSSpawner(TorqueSpawner): - batch_script = Unicode("""#!/bin/sh + batch_script = Unicode( + """#!/bin/sh {% if queue or host %}#PBS -q {% if queue %}{{queue}}{% endif %}\ {% if host %}@{{host}}{% endif %}{% endif %} #PBS -l walltime={{runtime}} @@ -575,14 +657,15 @@ class PBSSpawner(TorqueSpawner): {{prologue}} {{cmd}} {{epilogue}} -""").tag(config=True) +""" + ).tag(config=True) # outputs job data XML string - batch_query_cmd = Unicode('qstat -fx {job_id}').tag(config=True) + batch_query_cmd = Unicode("qstat -fx {job_id}").tag(config=True) - state_pending_re = Unicode(r'job_state = [QH]').tag(config=True) - state_running_re = Unicode(r'job_state = R').tag(config=True) - state_exechost_re = Unicode(r'exec_host = ([\w_-]+)/').tag(config=True) + state_pending_re = Unicode(r"job_state = [QH]").tag(config=True) + state_running_re = Unicode(r"job_state = R").tag(config=True) + state_exechost_re = Unicode(r"exec_host = ([\w_-]+)/").tag(config=True) class UserEnvMixin: @@ -591,13 +674,13 @@ class UserEnvMixin: def user_env(self, env): """get user environment""" - env['USER'] = self.user.name + env["USER"] = self.user.name home = pwd.getpwnam(self.user.name).pw_dir shell = pwd.getpwnam(self.user.name).pw_shell if home: - env['HOME'] = home + env["HOME"] = home if shell: - env['SHELL'] = shell + env["SHELL"] = shell return env def get_env(self): @@ -613,8 +696,9 @@ def get_env(self): return env -class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates): - batch_script = Unicode("""#!/bin/bash +class SlurmSpawner(UserEnvMixin, BatchSpawnerRegexStates): + batch_script = Unicode( + """#!/bin/bash #SBATCH --output={{homedir}}/jupyterhub_slurmspawner_%j.log #SBATCH --job-name=spawner-jupyterhub #SBATCH --chdir={{homedir}} @@ -636,49 +720,57 @@ class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates): {% if srun %}{{srun}} {% endif %}{{cmd}} echo "jupyterhub-singleuser ended gracefully" {{epilogue}} -""").tag(config=True) +""" + ).tag(config=True) # all these req_foo traits will be available as substvars for templated strings - req_cluster = Unicode('', - help="Cluster name to submit job to resource manager" - ).tag(config=True) + req_cluster = Unicode( + "", + help="Cluster name to submit job to resource manager", + ).tag(config=True) - req_qos = Unicode('', - help="QoS name to submit job to resource manager" - ).tag(config=True) + req_qos = Unicode( + "", + help="QoS name to submit job to resource manager", + ).tag(config=True) - req_srun = Unicode('srun', + req_srun = Unicode( + "srun", help="Set req_srun='' to disable running in job step, and note that " - "this affects environment handling. This is effectively a " - "prefix for the singleuser command." - ).tag(config=True) + "this affects environment handling. This is effectively a " + "prefix for the singleuser command.", + ).tag(config=True) - req_reservation = Unicode('', \ - help="Reservation name to submit to resource manager" - ).tag(config=True) + req_reservation = Unicode( + "", + help="Reservation name to submit to resource manager", + ).tag(config=True) - req_gres = Unicode('', \ - help="Additional resources (e.g. GPUs) requested" + req_gres = Unicode( + "", + help="Additional resources (e.g. GPUs) requested", ).tag(config=True) # outputs line like "Submitted batch job 209" - batch_submit_cmd = Unicode('sbatch --parsable').tag(config=True) + batch_submit_cmd = Unicode("sbatch --parsable").tag(config=True) # outputs status and exec node like "RUNNING hostname" - batch_query_cmd = Unicode("squeue -h -j {job_id} -o '%T %B'").tag(config=True) # - batch_cancel_cmd = Unicode('scancel {job_id}').tag(config=True) + batch_query_cmd = Unicode("squeue -h -j {job_id} -o '%T %B'").tag(config=True) + batch_cancel_cmd = Unicode("scancel {job_id}").tag(config=True) # use long-form states: PENDING, CONFIGURING = pending # RUNNING, COMPLETING = running - state_pending_re = Unicode(r'^(?:PENDING|CONFIGURING)').tag(config=True) - state_running_re = Unicode(r'^(?:RUNNING|COMPLETING)').tag(config=True) - state_unknown_re = Unicode(r'^slurm_load_jobs error: (?:Socket timed out on send/recv|Unable to contact slurm controller)').tag(config=True) - state_exechost_re = Unicode(r'\s+((?:[\w_-]+\.?)+)$').tag(config=True) + state_pending_re = Unicode(r"^(?:PENDING|CONFIGURING)").tag(config=True) + state_running_re = Unicode(r"^(?:RUNNING|COMPLETING)").tag(config=True) + state_unknown_re = Unicode( + r"^slurm_load_jobs error: (?:Socket timed out on send/recv|Unable to contact slurm controller)" + ).tag(config=True) + state_exechost_re = Unicode(r"\s+((?:[\w_-]+\.?)+)$").tag(config=True) def parse_job_id(self, output): # make sure jobid is really a number try: # use only last line to circumvent slurm bug output = output.splitlines()[-1] - id = output.split(';')[0] + id = output.split(";")[0] int(id) except Exception as e: self.log.error("SlurmSpawner unable to parse job ID from text: " + output) @@ -687,12 +779,16 @@ def parse_job_id(self, output): class MultiSlurmSpawner(SlurmSpawner): - '''When slurm has been compiled with --enable-multiple-slurmd, the - administrator sets the name of the slurmd instance via the slurmd -N - option. This node name is usually different from the hostname and may - not be resolvable by JupyterHub. Here we enable the administrator to - map the node names onto the real hostnames via a traitlet.''' - daemon_resolver = Dict({}, help="Map node names to hostnames").tag(config=True) + """When slurm has been compiled with --enable-multiple-slurmd, the + administrator sets the name of the slurmd instance via the slurmd -N + option. This node name is usually different from the hostname and may + not be resolvable by JupyterHub. Here we enable the administrator to + map the node names onto the real hostnames via a traitlet.""" + + daemon_resolver = Dict( + {}, + help="Map node names to hostnames", + ).tag(config=True) def state_gethost(self): host = SlurmSpawner.state_gethost(self) @@ -700,7 +796,8 @@ def state_gethost(self): class GridengineSpawner(BatchSpawnerBase): - batch_script = Unicode("""#!/bin/bash + batch_script = Unicode( + """#!/bin/bash #$ -j yes #$ -N spawner-jupyterhub #$ -o {homedir}/.jupyterhub.sge.out @@ -713,41 +810,49 @@ class GridengineSpawner(BatchSpawnerBase): {prologue} {cmd} {epilogue} -""").tag(config=True) +""" + ).tag(config=True) # outputs job id string - batch_submit_cmd = Unicode('qsub').tag(config=True) + batch_submit_cmd = Unicode("qsub").tag(config=True) # outputs job data XML string - batch_query_cmd = Unicode('qstat -xml').tag(config=True) - batch_cancel_cmd = Unicode('qdel {job_id}').tag(config=True) + batch_query_cmd = Unicode("qstat -xml").tag(config=True) + batch_cancel_cmd = Unicode("qdel {job_id}").tag(config=True) def parse_job_id(self, output): - return output.split(' ')[2] + return output.split(" ")[2] def state_ispending(self): if self.job_status: job_info = ET.fromstring(self.job_status).find( - ".//job_list[JB_job_number='{0}']".format(self.job_id)) + ".//job_list[JB_job_number='{0}']".format(self.job_id) + ) if job_info is not None: - return job_info.attrib.get('state') == 'pending' + return job_info.attrib.get("state") == "pending" return False def state_isrunning(self): if self.job_status: job_info = ET.fromstring(self.job_status).find( - ".//job_list[JB_job_number='{0}']".format(self.job_id)) + ".//job_list[JB_job_number='{0}']".format(self.job_id) + ) if job_info is not None: - return job_info.attrib.get('state') == 'running' + return job_info.attrib.get("state") == "running" return False def state_gethost(self): if self.job_status: queue_name = ET.fromstring(self.job_status).find( - ".//job_list[JB_job_number='{0}']/queue_name".format(self.job_id)) + ".//job_list[JB_job_number='{0}']/queue_name".format(self.job_id) + ) if queue_name is not None and queue_name.text: - return queue_name.text.split('@')[1] + return queue_name.text.split("@")[1] - self.log.error("Spawner unable to match host addr in job {0} with status {1}".format(self.job_id, self.job_status)) + self.log.error( + "Spawner unable to match host addr in job {0} with status {1}".format( + self.job_id, self.job_status + ) + ) return def get_env(self): @@ -755,14 +860,23 @@ def get_env(self): # SGE relies on environment variables to launch local jobs. Ensure that these values are included # in the environment used to run the spawner. - for key in ['SGE_CELL','SGE_EXECD','SGE_ROOT','SGE_CLUSTER_NAME','SGE_QMASTER_PORT', 'SGE_EXECD_PORT','PATH']: + for key in [ + "SGE_CELL", + "SGE_EXECD", + "SGE_ROOT", + "SGE_CLUSTER_NAME", + "SGE_QMASTER_PORT", + "SGE_EXECD_PORT", + "PATH", + ]: if key in os.environ and key not in env: env[key] = os.environ[key] return env -class CondorSpawner(UserEnvMixin,BatchSpawnerRegexStates): - batch_script = Unicode(""" +class CondorSpawner(UserEnvMixin, BatchSpawnerRegexStates): + batch_script = Unicode( + """ Executable = /bin/sh RequestMemory = {memory} RequestCpus = {nprocs} @@ -774,20 +888,23 @@ class CondorSpawner(UserEnvMixin,BatchSpawnerRegexStates): GetEnv = True {options} Queue -""").tag(config=True) +""" + ).tag(config=True) # outputs job id string - batch_submit_cmd = Unicode('condor_submit').tag(config=True) + batch_submit_cmd = Unicode("condor_submit").tag(config=True) # outputs job data XML string - batch_query_cmd = Unicode('condor_q {job_id} -format "%s, " JobStatus -format "%s" RemoteHost -format "\n" True').tag(config=True) - batch_cancel_cmd = Unicode('condor_rm {job_id}').tag(config=True) + batch_query_cmd = Unicode( + 'condor_q {job_id} -format "%s, " JobStatus -format "%s" RemoteHost -format "\n" True' + ).tag(config=True) + batch_cancel_cmd = Unicode("condor_rm {job_id}").tag(config=True) # job status: 1 = pending, 2 = running - state_pending_re = Unicode(r'^1,').tag(config=True) - state_running_re = Unicode(r'^2,').tag(config=True) - state_exechost_re = Unicode(r'^\w*, .*@([^ ]*)').tag(config=True) + state_pending_re = Unicode(r"^1,").tag(config=True) + state_running_re = Unicode(r"^2,").tag(config=True) + state_exechost_re = Unicode(r"^\w*, .*@([^ ]*)").tag(config=True) def parse_job_id(self, output): - match = re.search(r'.*submitted to cluster ([0-9]+)', output) + match = re.search(r".*submitted to cluster ([0-9]+)", output) if match: return match.groups()[0] @@ -796,13 +913,19 @@ def parse_job_id(self, output): raise Exception(error_msg) def cmd_formatted_for_batch(self): - return super(CondorSpawner,self).cmd_formatted_for_batch().replace('"','""').replace("'","''") + return ( + super(CondorSpawner, self) + .cmd_formatted_for_batch() + .replace('"', '""') + .replace("'", "''") + ) class LsfSpawner(BatchSpawnerBase): - '''A Spawner that uses IBM's Platform Load Sharing Facility (LSF) to launch notebooks.''' + """A Spawner that uses IBM's Platform Load Sharing Facility (LSF) to launch notebooks.""" - batch_script = Unicode('''#!/bin/sh + batch_script = Unicode( + """#!/bin/sh #BSUB -R "select[type==any]" # Allow spawning on non-uniform hardware #BSUB -R "span[hosts=1]" # Only spawn job on one server #BSUB -q {queue} @@ -815,19 +938,27 @@ class LsfSpawner(BatchSpawnerBase): {prologue} {cmd} {epilogue} -''').tag(config=True) - +""" + ).tag(config=True) - batch_submit_cmd = Unicode('bsub').tag(config=True) - batch_query_cmd = Unicode('bjobs -a -noheader -o "STAT EXEC_HOST" {job_id}').tag(config=True) - batch_cancel_cmd = Unicode('bkill {job_id}').tag(config=True) + batch_submit_cmd = Unicode("bsub").tag(config=True) + batch_query_cmd = Unicode('bjobs -a -noheader -o "STAT EXEC_HOST" {job_id}').tag( + config=True + ) + batch_cancel_cmd = Unicode("bkill {job_id}").tag(config=True) def get_env(self): env = super().get_env() # LSF relies on environment variables to launch local jobs. Ensure that these values are included # in the environment used to run the spawner. - for key in ['LSF_ENVDIR','LSF_SERVERDIR','LSF_FULL_VERSION','LSF_LIBDIR','LSF_BINDIR']: + for key in [ + "LSF_ENVDIR", + "LSF_SERVERDIR", + "LSF_FULL_VERSION", + "LSF_LIBDIR", + "LSF_BINDIR", + ]: if key in os.environ and key not in env: env[key] = os.environ[key] return env @@ -835,23 +966,28 @@ def get_env(self): def parse_job_id(self, output): # Assumes output in the following form: # "Job <1815> is submitted to default queue ." - return output.split(' ')[1].strip('<>') + return output.split(" ")[1].strip("<>") def state_ispending(self): # Parse results of batch_query_cmd # Output determined by results of self.batch_query_cmd if self.job_status: - return self.job_status.split(' ')[0].upper() in {'PEND', 'PUSP'} + return self.job_status.split(" ")[0].upper() in {"PEND", "PUSP"} def state_isrunning(self): if self.job_status: - return self.job_status.split(' ')[0].upper() == 'RUN' + return self.job_status.split(" ")[0].upper() == "RUN" def state_gethost(self): if self.job_status: - return self.job_status.split(' ')[1].strip() + return self.job_status.split(" ")[1].strip() - self.log.error("Spawner unable to match host addr in job {0} with status {1}".format(self.job_id, self.job_status)) + self.log.error( + "Spawner unable to match host addr in job {0} with status {1}".format( + self.job_id, self.job_status + ) + ) return + # vim: set ai expandtab softtabstop=4: diff --git a/batchspawner/singleuser.py b/batchspawner/singleuser.py index e3ffab22..d2bd338e 100644 --- a/batchspawner/singleuser.py +++ b/batchspawner/singleuser.py @@ -7,19 +7,23 @@ from jupyterhub.utils import random_port, url_path_join from jupyterhub.services.auth import HubAuth + def main(argv=None): port = random_port() hub_auth = HubAuth() - hub_auth.client_ca = os.environ.get('JUPYTERHUB_SSL_CLIENT_CA', '') - hub_auth.certfile = os.environ.get('JUPYTERHUB_SSL_CERTFILE', '') - hub_auth.keyfile = os.environ.get('JUPYTERHUB_SSL_KEYFILE', '') - hub_auth._api_request(method='POST', - url=url_path_join(hub_auth.api_url, 'batchspawner'), - json={'port' : port}) + hub_auth.client_ca = os.environ.get("JUPYTERHUB_SSL_CLIENT_CA", "") + hub_auth.certfile = os.environ.get("JUPYTERHUB_SSL_CERTFILE", "") + hub_auth.keyfile = os.environ.get("JUPYTERHUB_SSL_KEYFILE", "") + hub_auth._api_request( + method="POST", + url=url_path_join(hub_auth.api_url, "batchspawner"), + json={"port": port}, + ) cmd_path = which(sys.argv[1]) - sys.argv = sys.argv[1:] + ['--port={}'.format(port)] + sys.argv = sys.argv[1:] + ["--port={}".format(port)] run_path(cmd_path, run_name="__main__") + if __name__ == "__main__": main() diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index 8b43df36..6ef75680 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -16,22 +16,24 @@ pass testhost = "userhost123" -testjob = "12345" +testjob = "12345" testport = 54321 + class BatchDummy(BatchSpawnerRegexStates): - exec_prefix = '' - batch_submit_cmd = Unicode('cat > /dev/null; echo '+testjob) - batch_query_cmd = Unicode('echo RUN '+testhost) - batch_cancel_cmd = Unicode('echo STOP') - batch_script = Unicode('{cmd}') - state_pending_re = Unicode('PEND') - state_running_re = Unicode('RUN') - state_exechost_re = Unicode('RUN (.*)$') - state_unknown_re = Unicode('UNKNOWN') + exec_prefix = "" + batch_submit_cmd = Unicode("cat > /dev/null; echo " + testjob) + batch_query_cmd = Unicode("echo RUN " + testhost) + batch_cancel_cmd = Unicode("echo STOP") + batch_script = Unicode("{cmd}") + state_pending_re = Unicode("PEND") + state_running_re = Unicode("RUN") + state_exechost_re = Unicode("RUN (.*)$") + state_unknown_re = Unicode("UNKNOWN") cmd_expectlist = None out_expectlist = None + def run_command(self, *args, **kwargs): """Overwriten run command to test templating and outputs""" cmd = args[0] @@ -39,62 +41,68 @@ def run_command(self, *args, **kwargs): if self.cmd_expectlist: run_re = self.cmd_expectlist.pop(0) if run_re: - print('run:', run_re) - assert run_re.search(cmd) is not None, \ - "Failed test: re={0} cmd={1}".format(run_re, cmd) + print("run:", run_re) + assert ( + run_re.search(cmd) is not None + ), "Failed test: re={0} cmd={1}".format(run_re, cmd) # Run command normally out = super().run_command(*args, **kwargs) # Test that the command matches the expectations if self.out_expectlist: out_re = self.out_expectlist.pop(0) if out_re: - print('out:', out_re) - assert out_re.search(cmd) is not None, \ - "Failed output: re={0} cmd={1} out={2}".format(out_re, cmd, out) + print("out:", out_re) + assert ( + out_re.search(cmd) is not None + ), "Failed output: re={0} cmd={1} out={2}".format(out_re, cmd, out) return out + def new_spawner(db, spawner_class=BatchDummy, **kwargs): - kwargs.setdefault('cmd', ['singleuser_command']) + kwargs.setdefault("cmd", ["singleuser_command"]) user = db.query(orm.User).first() - if version_info < (0,8): + if version_info < (0, 8): hub = db.query(orm.Hub).first() else: hub = Hub() user = User(user, {}) server = Server() # Set it after constructions because it isn't a traitlet. - kwargs.setdefault('hub', hub) - kwargs.setdefault('user', user) - kwargs.setdefault('poll_interval', 1) - if version_info < (0,8): + kwargs.setdefault("hub", hub) + kwargs.setdefault("user", user) + kwargs.setdefault("poll_interval", 1) + if version_info < (0, 8): spawner = spawner_class(db=db, **kwargs) spawner.mock_port = testport else: print("JupyterHub >=0.8 detected, using new spawner creation") # These are not traitlets so we have to set them here - spawner = user._new_spawner('', spawner_class=spawner_class, **kwargs) + spawner = user._new_spawner("", spawner_class=spawner_class, **kwargs) spawner.server = server spawner.mock_port = testport return spawner + @pytest.mark.slow def test_stress_submit(db, io_loop): for i in range(200): time.sleep(0.01) test_spawner_start_stop_poll(db, io_loop) + def check_ip(spawner, value): - if version_info < (0,7): + if version_info < (0, 7): assert spawner.user.server.ip == value else: assert spawner.ip == value + def test_spawner_start_stop_poll(db, io_loop): spawner = new_spawner(db=db) status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 - assert spawner.job_id == '' + assert spawner.job_id == "" assert spawner.get_state() == {} io_loop.run_sync(spawner.start, timeout=5) @@ -103,12 +111,13 @@ def test_spawner_start_stop_poll(db, io_loop): status = io_loop.run_sync(spawner.poll, timeout=5) assert status is None - spawner.batch_query_cmd = 'echo NOPE' + spawner.batch_query_cmd = "echo NOPE" io_loop.run_sync(spawner.stop, timeout=5) status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 assert spawner.get_state() == {} + def test_spawner_state_reload(db, io_loop): spawner = new_spawner(db=db) assert spawner.get_state() == {} @@ -118,7 +127,7 @@ def test_spawner_state_reload(db, io_loop): assert spawner.job_id == testjob state = spawner.get_state() - assert state == dict(job_id=testjob, job_status='RUN '+testhost) + assert state == dict(job_id=testjob, job_status="RUN " + testhost) spawner = new_spawner(db=db) spawner.clear_state() assert spawner.get_state() == {} @@ -127,26 +136,29 @@ def test_spawner_state_reload(db, io_loop): # and is not part of the spawner's persistent state assert spawner.job_id == testjob + def test_submit_failure(db, io_loop): spawner = new_spawner(db=db) assert spawner.get_state() == {} - spawner.batch_submit_cmd = 'cat > /dev/null; true' + spawner.batch_submit_cmd = "cat > /dev/null; true" with pytest.raises(RuntimeError) as e_info: io_loop.run_sync(spawner.start, timeout=30) - assert spawner.job_id == '' - assert spawner.job_status == '' + assert spawner.job_id == "" + assert spawner.job_status == "" + def test_submit_pending_fails(db, io_loop): """Submission works, but the batch query command immediately fails""" spawner = new_spawner(db=db) assert spawner.get_state() == {} - spawner.batch_query_cmd = 'echo xyz' + spawner.batch_query_cmd = "echo xyz" with pytest.raises(RuntimeError) as e_info: io_loop.run_sync(spawner.start, timeout=30) status = io_loop.run_sync(spawner.query_job_status, timeout=30) assert status == JobStatus.NOTFOUND - assert spawner.job_id == '' - assert spawner.job_status == '' + assert spawner.job_id == "" + assert spawner.job_status == "" + def test_poll_fails(db, io_loop): """Submission works, but a later .poll() fails""" @@ -154,12 +166,13 @@ def test_poll_fails(db, io_loop): assert spawner.get_state() == {} # The start is successful: io_loop.run_sync(spawner.start, timeout=30) - spawner.batch_query_cmd = 'echo xyz' + spawner.batch_query_cmd = "echo xyz" # Now, the poll fails: io_loop.run_sync(spawner.poll, timeout=30) # .poll() will run self.clear_state() if it's not found: - assert spawner.job_id == '' - assert spawner.job_status == '' + assert spawner.job_id == "" + assert spawner.job_status == "" + def test_unknown_status(db, io_loop): """Polling returns an unknown status""" @@ -167,13 +180,13 @@ def test_unknown_status(db, io_loop): assert spawner.get_state() == {} # The start is successful: io_loop.run_sync(spawner.start, timeout=30) - spawner.batch_query_cmd = 'echo UNKNOWN' + spawner.batch_query_cmd = "echo UNKNOWN" # This poll should not fail: io_loop.run_sync(spawner.poll, timeout=30) status = io_loop.run_sync(spawner.query_job_status, timeout=30) assert status == JobStatus.UNKNOWN - assert spawner.job_id == '12345' - assert spawner.job_status != '' + assert spawner.job_id == "12345" + assert spawner.job_status != "" def test_templates(db, io_loop): @@ -181,62 +194,73 @@ def test_templates(db, io_loop): spawner = new_spawner(db=db) # Test when not running - spawner.cmd_expectlist = [re.compile('.*RUN'), - ] + spawner.cmd_expectlist = [ + re.compile(".*RUN"), + ] status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 - assert spawner.job_id == '' + assert spawner.job_id == "" assert spawner.get_state() == {} # Test starting - spawner.cmd_expectlist = [re.compile('.*echo'), - re.compile('.*RUN'), - ] + spawner.cmd_expectlist = [ + re.compile(".*echo"), + re.compile(".*RUN"), + ] io_loop.run_sync(spawner.start, timeout=5) check_ip(spawner, testhost) assert spawner.job_id == testjob # Test poll - running - spawner.cmd_expectlist = [re.compile('.*RUN'), - ] + spawner.cmd_expectlist = [ + re.compile(".*RUN"), + ] status = io_loop.run_sync(spawner.poll, timeout=5) assert status is None # Test stopping - spawner.batch_query_cmd = 'echo NOPE' - spawner.cmd_expectlist = [re.compile('.*STOP'), - re.compile('.*NOPE'), - ] + spawner.batch_query_cmd = "echo NOPE" + spawner.cmd_expectlist = [ + re.compile(".*STOP"), + re.compile(".*NOPE"), + ] io_loop.run_sync(spawner.stop, timeout=5) status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 assert spawner.get_state() == {} + def test_batch_script(db, io_loop): """Test that the batch script substitutes {cmd}""" + class BatchDummyTestScript(BatchDummy): @gen.coroutine def _get_batch_script(self, **subvars): script = yield super()._get_batch_script(**subvars) - assert 'singleuser_command' in script + assert "singleuser_command" in script return script + spawner = new_spawner(db=db, spawner_class=BatchDummyTestScript) - #status = io_loop.run_sync(spawner.poll, timeout=5) + # status = io_loop.run_sync(spawner.poll, timeout=5) io_loop.run_sync(spawner.start, timeout=5) - #status = io_loop.run_sync(spawner.poll, timeout=5) - #io_loop.run_sync(spawner.stop, timeout=5) + # status = io_loop.run_sync(spawner.poll, timeout=5) + # io_loop.run_sync(spawner.stop, timeout=5) + def test_exec_prefix(db, io_loop): """Test that all run_commands have exec_prefix""" + class BatchDummyTestScript(BatchDummy): - exec_prefix = 'PREFIX' + exec_prefix = "PREFIX" + @gen.coroutine def run_command(self, cmd, *args, **kwargs): - assert cmd.startswith('PREFIX ') + assert cmd.startswith("PREFIX ") cmd = cmd[7:] print(cmd) out = yield super().run_command(cmd, *args, **kwargs) return out + spawner = new_spawner(db=db, spawner_class=BatchDummyTestScript) # Not running status = io_loop.run_sync(spawner.poll, timeout=5) @@ -248,13 +272,15 @@ def run_command(self, cmd, *args, **kwargs): status = io_loop.run_sync(spawner.poll, timeout=5) assert status is None # Stop - spawner.batch_query_cmd = 'echo NOPE' + spawner.batch_query_cmd = "echo NOPE" io_loop.run_sync(spawner.stop, timeout=5) status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 -def run_spawner_script(db, io_loop, spawner, script, - batch_script_re_list=None, spawner_kwargs={}): + +def run_spawner_script( + db, io_loop, spawner, script, batch_script_re_list=None, spawner_kwargs={} +): """Run a spawner script and test that the output and behavior is as expected. db: same as in this module @@ -275,23 +301,24 @@ def run_command(self, cmd, input=None, env=None): run_re = cmd_expectlist.pop(0) if run_re: print('run: "{}" [{}]'.format(cmd, run_re)) - assert run_re.search(cmd) is not None, \ - "Failed test: re={0} cmd={1}".format(run_re, cmd) + assert ( + run_re.search(cmd) is not None + ), "Failed test: re={0} cmd={1}".format(run_re, cmd) # Test the stdin - will only be the batch script. For # each regular expression in batch_script_re_list, assert that # each re in that list matches the batch script. if batch_script_re_list and input: batch_script = input for match_re in batch_script_re_list: - assert match_re.search(batch_script) is not None, \ - "Batch script does not match {}".format(match_re) + assert ( + match_re.search(batch_script) is not None + ), "Batch script does not match {}".format(match_re) # Return expected output. out = out_list.pop(0) - print(' --> '+out) + print(" --> " + out) return out - spawner = new_spawner(db=db, spawner_class=BatchDummyTestScript, - **spawner_kwargs) + spawner = new_spawner(db=db, spawner_class=BatchDummyTestScript, **spawner_kwargs) # Not running at beginning (no command run) status = io_loop.run_sync(spawner.poll, timeout=5) assert status == 1 @@ -311,145 +338,212 @@ def run_command(self, cmd, input=None, env=None): assert status == 1 - def test_torque(db, io_loop): spawner_kwargs = { - 'req_nprocs': '5', - 'req_memory': '5678', - 'req_options': 'some_option_asdf', - 'req_prologue': 'PROLOGUE', - 'req_epilogue': 'EPILOGUE', - } + "req_nprocs": "5", + "req_memory": "5678", + "req_options": "some_option_asdf", + "req_prologue": "PROLOGUE", + "req_epilogue": "EPILOGUE", + } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), - re.compile(r'mem=5678'), - re.compile(r'ppn=5'), - re.compile(r'^#PBS some_option_asdf', re.M), - ] + re.compile( + r"^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE", + re.S | re.M, + ), + re.compile(r"mem=5678"), + re.compile(r"ppn=5"), + re.compile(r"^#PBS some_option_asdf", re.M), + ] script = [ - (re.compile(r'sudo.*qsub'), str(testjob)), - (re.compile(r'sudo.*qstat'), 'Q'.format(testhost)), # pending - (re.compile(r'sudo.*qstat'), 'R{}/1'.format(testhost)), # running - (re.compile(r'sudo.*qstat'), 'R{}/1'.format(testhost)), # running - (re.compile(r'sudo.*qdel'), 'STOP'), - (re.compile(r'sudo.*qstat'), ''), - ] + (re.compile(r"sudo.*qsub"), str(testjob)), + ( + re.compile(r"sudo.*qstat"), + "Q", + ), # pending + ( + re.compile(r"sudo.*qstat"), + "R{}/1".format(testhost), + ), # running + ( + re.compile(r"sudo.*qstat"), + "R{}/1".format(testhost), + ), # running + (re.compile(r"sudo.*qdel"), "STOP"), + (re.compile(r"sudo.*qstat"), ""), + ] from .. import TorqueSpawner - run_spawner_script(db, io_loop, TorqueSpawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + TorqueSpawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) def test_moab(db, io_loop): spawner_kwargs = { - 'req_nprocs': '5', - 'req_memory': '5678', - 'req_options': 'some_option_asdf', - 'req_prologue': 'PROLOGUE', - 'req_epilogue': 'EPILOGUE', - } + "req_nprocs": "5", + "req_memory": "5678", + "req_options": "some_option_asdf", + "req_prologue": "PROLOGUE", + "req_epilogue": "EPILOGUE", + } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), - re.compile(r'mem=5678'), - re.compile(r'ppn=5'), - re.compile(r'^#PBS some_option_asdf', re.M), - ] + re.compile( + r"^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE", + re.S | re.M, + ), + re.compile(r"mem=5678"), + re.compile(r"ppn=5"), + re.compile(r"^#PBS some_option_asdf", re.M), + ] script = [ - (re.compile(r'sudo.*msub'), str(testjob)), - (re.compile(r'sudo.*mdiag'), 'State="Idle"'.format(testhost)), # pending - (re.compile(r'sudo.*mdiag'), 'State="Running" AllocNodeList="{}"'.format(testhost)), # running - (re.compile(r'sudo.*mdiag'), 'State="Running" AllocNodeList="{}"'.format(testhost)), # running - (re.compile(r'sudo.*mjobctl.*-c'), 'STOP'), - (re.compile(r'sudo.*mdiag'), ''), - ] + (re.compile(r"sudo.*msub"), str(testjob)), + (re.compile(r"sudo.*mdiag"), 'State="Idle"'), # pending + ( + re.compile(r"sudo.*mdiag"), + 'State="Running" AllocNodeList="{}"'.format(testhost), + ), # running + ( + re.compile(r"sudo.*mdiag"), + 'State="Running" AllocNodeList="{}"'.format(testhost), + ), # running + (re.compile(r"sudo.*mjobctl.*-c"), "STOP"), + (re.compile(r"sudo.*mdiag"), ""), + ] from .. import MoabSpawner - run_spawner_script(db, io_loop, MoabSpawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + MoabSpawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) def test_pbs(db, io_loop): spawner_kwargs = { - 'req_nprocs': '4', - 'req_memory': '10256', - 'req_options': 'some_option_asdf', - 'req_host': 'some_pbs_admin_node', - 'req_runtime': '08:00:00', - } + "req_nprocs": "4", + "req_memory": "10256", + "req_options": "some_option_asdf", + "req_host": "some_pbs_admin_node", + "req_runtime": "08:00:00", + } batch_script_re_list = [ - re.compile(r'singleuser_command'), - re.compile(r'select=1'), - re.compile(r'ncpus=4'), - re.compile(r'mem=10256'), - re.compile(r'walltime=08:00:00'), - re.compile(r'@some_pbs_admin_node'), - re.compile(r'^#PBS some_option_asdf', re.M), - ] + re.compile(r"singleuser_command"), + re.compile(r"select=1"), + re.compile(r"ncpus=4"), + re.compile(r"mem=10256"), + re.compile(r"walltime=08:00:00"), + re.compile(r"@some_pbs_admin_node"), + re.compile(r"^#PBS some_option_asdf", re.M), + ] script = [ - (re.compile(r'sudo.*qsub'), str(testjob)), - (re.compile(r'sudo.*qstat'), 'job_state = Q'.format(testhost)), # pending - (re.compile(r'sudo.*qstat'), 'job_state = R\nexec_host = {}/2*1'.format(testhost)), # running - (re.compile(r'sudo.*qstat'), 'job_state = R\nexec_host = {}/2*1'.format(testhost)), # running - (re.compile(r'sudo.*qdel'), 'STOP'), - (re.compile(r'sudo.*qstat'), ''), - ] + (re.compile(r"sudo.*qsub"), str(testjob)), + (re.compile(r"sudo.*qstat"), "job_state = Q"), # pending + ( + re.compile(r"sudo.*qstat"), + "job_state = R\nexec_host = {}/2*1".format(testhost), + ), # running + ( + re.compile(r"sudo.*qstat"), + "job_state = R\nexec_host = {}/2*1".format(testhost), + ), # running + (re.compile(r"sudo.*qdel"), "STOP"), + (re.compile(r"sudo.*qstat"), ""), + ] from .. import PBSSpawner - run_spawner_script(db, io_loop, PBSSpawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + PBSSpawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) def test_slurm(db, io_loop): spawner_kwargs = { - 'req_runtime': '3-05:10:10', - 'req_nprocs': '5', - 'req_memory': '5678', - 'req_options': 'some_option_asdf', - 'req_prologue': 'PROLOGUE', - 'req_epilogue': 'EPILOGUE', - 'req_reservation': 'RES123', - 'req_gres': 'GRES123', - } + "req_runtime": "3-05:10:10", + "req_nprocs": "5", + "req_memory": "5678", + "req_options": "some_option_asdf", + "req_prologue": "PROLOGUE", + "req_epilogue": "EPILOGUE", + "req_reservation": "RES123", + "req_gres": "GRES123", + } batch_script_re_list = [ - re.compile(r'PROLOGUE.*srun batchspawner-singleuser singleuser_command.*EPILOGUE', re.S), - re.compile(r'^#SBATCH \s+ --cpus-per-task=5', re.X|re.M), - re.compile(r'^#SBATCH \s+ --time=3-05:10:10', re.X|re.M), - re.compile(r'^#SBATCH \s+ some_option_asdf', re.X|re.M), - re.compile(r'^#SBATCH \s+ --reservation=RES123', re.X|re.M), - re.compile(r'^#SBATCH \s+ --gres=GRES123', re.X|re.M), - ] + re.compile( + r"PROLOGUE.*srun batchspawner-singleuser singleuser_command.*EPILOGUE", re.S + ), + re.compile(r"^#SBATCH \s+ --cpus-per-task=5", re.X | re.M), + re.compile(r"^#SBATCH \s+ --time=3-05:10:10", re.X | re.M), + re.compile(r"^#SBATCH \s+ some_option_asdf", re.X | re.M), + re.compile(r"^#SBATCH \s+ --reservation=RES123", re.X | re.M), + re.compile(r"^#SBATCH \s+ --gres=GRES123", re.X | re.M), + ] from .. import SlurmSpawner - run_spawner_script(db, io_loop, SlurmSpawner, normal_slurm_script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + SlurmSpawner, + normal_slurm_script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) + + # We tend to use slurm as our typical example job. These allow quick # Slurm examples. normal_slurm_script = [ - (re.compile(r'sudo.*sbatch'), str(testjob)), - (re.compile(r'sudo.*squeue'), 'PENDING '), # pending - (re.compile(r'sudo.*squeue'), 'slurm_load_jobs error: Unable to contact slurm controller'), # unknown - (re.compile(r'sudo.*squeue'), 'RUNNING '+testhost), # running - (re.compile(r'sudo.*squeue'), 'RUNNING '+testhost), - (re.compile(r'sudo.*scancel'), 'STOP'), - (re.compile(r'sudo.*squeue'), ''), - ] + (re.compile(r"sudo.*sbatch"), str(testjob)), + (re.compile(r"sudo.*squeue"), "PENDING "), # pending + ( + re.compile(r"sudo.*squeue"), + "slurm_load_jobs error: Unable to contact slurm controller", + ), # unknown + (re.compile(r"sudo.*squeue"), "RUNNING " + testhost), # running + (re.compile(r"sudo.*squeue"), "RUNNING " + testhost), + (re.compile(r"sudo.*scancel"), "STOP"), + (re.compile(r"sudo.*squeue"), ""), +] from .. import SlurmSpawner -def run_typical_slurm_spawner(db, io_loop, - spawner=SlurmSpawner, - script=normal_slurm_script, - batch_script_re_list=None, - spawner_kwargs={}): + + +def run_typical_slurm_spawner( + db, + io_loop, + spawner=SlurmSpawner, + script=normal_slurm_script, + batch_script_re_list=None, + spawner_kwargs={}, +): """Run a full slurm job with default (overrideable) parameters. This is useful, for example, for changing options and testing effect of batch scripts. """ - return run_spawner_script(db, io_loop, spawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + return run_spawner_script( + db, + io_loop, + spawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) -#def test_gridengine(db, io_loop): +# def test_gridengine(db, io_loop): # spawner_kwargs = { # 'req_options': 'some_option_asdf', # } @@ -473,77 +567,104 @@ def run_typical_slurm_spawner(db, io_loop, def test_condor(db, io_loop): spawner_kwargs = { - 'req_nprocs': '5', - 'req_memory': '5678', - 'req_options': 'some_option_asdf', - } + "req_nprocs": "5", + "req_memory": "5678", + "req_options": "some_option_asdf", + } batch_script_re_list = [ - re.compile(r'exec batchspawner-singleuser singleuser_command'), - re.compile(r'RequestCpus = 5'), - re.compile(r'RequestMemory = 5678'), - re.compile(r'^some_option_asdf', re.M), - ] + re.compile(r"exec batchspawner-singleuser singleuser_command"), + re.compile(r"RequestCpus = 5"), + re.compile(r"RequestMemory = 5678"), + re.compile(r"^some_option_asdf", re.M), + ] script = [ - (re.compile(r'sudo.*condor_submit'), 'submitted to cluster {}'.format(str(testjob))), - (re.compile(r'sudo.*condor_q'), '1,'.format(testhost)), # pending - (re.compile(r'sudo.*condor_q'), '2, @{}'.format(testhost)), # runing - (re.compile(r'sudo.*condor_q'), '2, @{}'.format(testhost)), - (re.compile(r'sudo.*condor_rm'), 'STOP'), - (re.compile(r'sudo.*condor_q'), ''), - ] + ( + re.compile(r"sudo.*condor_submit"), + "submitted to cluster {}".format(str(testjob)), + ), + (re.compile(r"sudo.*condor_q"), "1,"), # pending + (re.compile(r"sudo.*condor_q"), "2, @{}".format(testhost)), # runing + (re.compile(r"sudo.*condor_q"), "2, @{}".format(testhost)), + (re.compile(r"sudo.*condor_rm"), "STOP"), + (re.compile(r"sudo.*condor_q"), ""), + ] from .. import CondorSpawner - run_spawner_script(db, io_loop, CondorSpawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + CondorSpawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) def test_lfs(db, io_loop): spawner_kwargs = { - 'req_nprocs': '5', - 'req_memory': '5678', - 'req_options': 'some_option_asdf', - 'req_queue': 'some_queue', - 'req_prologue': 'PROLOGUE', - 'req_epilogue': 'EPILOGUE', - } + "req_nprocs": "5", + "req_memory": "5678", + "req_options": "some_option_asdf", + "req_queue": "some_queue", + "req_prologue": "PROLOGUE", + "req_epilogue": "EPILOGUE", + } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), - re.compile(r'#BSUB\s+-q\s+some_queue', re.M), - ] + re.compile( + r"^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE", + re.S | re.M, + ), + re.compile(r"#BSUB\s+-q\s+some_queue", re.M), + ] script = [ - (re.compile(r'sudo.*bsub'), 'Job <{}> is submitted to default queue '.format(str(testjob))), - (re.compile(r'sudo.*bjobs'), 'PEND '.format(testhost)), # pending - (re.compile(r'sudo.*bjobs'), 'RUN {}'.format(testhost)), # running - (re.compile(r'sudo.*bjobs'), 'RUN {}'.format(testhost)), - (re.compile(r'sudo.*bkill'), 'STOP'), - (re.compile(r'sudo.*bjobs'), ''), - ] + ( + re.compile(r"sudo.*bsub"), + "Job <{}> is submitted to default queue ".format(str(testjob)), + ), + (re.compile(r"sudo.*bjobs"), "PEND "), # pending + (re.compile(r"sudo.*bjobs"), "RUN {}".format(testhost)), # running + (re.compile(r"sudo.*bjobs"), "RUN {}".format(testhost)), + (re.compile(r"sudo.*bkill"), "STOP"), + (re.compile(r"sudo.*bjobs"), ""), + ] from .. import LsfSpawner - run_spawner_script(db, io_loop, LsfSpawner, script, - batch_script_re_list=batch_script_re_list, - spawner_kwargs=spawner_kwargs) + + run_spawner_script( + db, + io_loop, + LsfSpawner, + script, + batch_script_re_list=batch_script_re_list, + spawner_kwargs=spawner_kwargs, + ) def test_keepvars(db, io_loop): # req_keepvars spawner_kwargs = { - 'req_keepvars': 'ABCDE', - } + "req_keepvars": "ABCDE", + } batch_script_re_list = [ - re.compile(r'--export=ABCDE', re.X|re.M), - ] - run_typical_slurm_spawner(db, io_loop, - spawner_kwargs=spawner_kwargs, - batch_script_re_list=batch_script_re_list) + re.compile(r"--export=ABCDE", re.X | re.M), + ] + run_typical_slurm_spawner( + db, + io_loop, + spawner_kwargs=spawner_kwargs, + batch_script_re_list=batch_script_re_list, + ) # req_keepvars AND req_keepvars together spawner_kwargs = { - 'req_keepvars': 'ABCDE', - 'req_keepvars_extra': 'XYZ', - } + "req_keepvars": "ABCDE", + "req_keepvars_extra": "XYZ", + } batch_script_re_list = [ - re.compile(r'--export=ABCDE,XYZ', re.X|re.M), - ] - run_typical_slurm_spawner(db, io_loop, - spawner_kwargs=spawner_kwargs, - batch_script_re_list=batch_script_re_list) + re.compile(r"--export=ABCDE,XYZ", re.X | re.M), + ] + run_typical_slurm_spawner( + db, + io_loop, + spawner_kwargs=spawner_kwargs, + batch_script_re_list=batch_script_re_list, + ) diff --git a/scripts/batchspawner-singleuser b/scripts/batchspawner-singleuser index 1b91fe47..a3bea6a2 100644 --- a/scripts/batchspawner-singleuser +++ b/scripts/batchspawner-singleuser @@ -3,4 +3,4 @@ from batchspawner.singleuser import main if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/setup.py b/setup.py index ef729a9c..0956ad25 100755 --- a/setup.py +++ b/setup.py @@ -4,9 +4,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Minimal Python version sanity check (from IPython/Jupyterhub) -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- from __future__ import print_function @@ -21,50 +21,50 @@ # Get the current package version. version_ns = {} -with open(pjoin(here, 'version.py')) as f: +with open(pjoin(here, "version.py")) as f: exec(f.read(), {}, version_ns) -with open(pjoin(here, 'README.md'), encoding='utf-8') as f: +with open(pjoin(here, "README.md"), encoding="utf-8") as f: long_desc = f.read() setup_args = dict( - name = 'batchspawner', - scripts = glob(pjoin('scripts', '*')), - packages = ['batchspawner'], - version = version_ns['__version__'], - description = """Batchspawner: A spawner for Jupyterhub to spawn notebooks using batch resource managers.""", - long_description = long_desc, - long_description_content_type = 'text/markdown', - author = "Michael Milligan, Andrea Zonca, Mike Gilbert", - author_email = "milligan@umn.edu", - url = "http://jupyter.org", - license = "BSD", - platforms = "Linux, Mac OS X", - python_requires = '~=3.3', - keywords = ['Interactive', 'Interpreter', 'Shell', 'Web', 'Jupyter'], - classifiers = [ - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', + name="batchspawner", + scripts=glob(pjoin("scripts", "*")), + packages=["batchspawner"], + version=version_ns["__version__"], + description="""Batchspawner: A spawner for Jupyterhub to spawn notebooks using batch resource managers.""", + long_description=long_desc, + long_description_content_type="text/markdown", + author="Michael Milligan, Andrea Zonca, Mike Gilbert", + author_email="milligan@umn.edu", + url="http://jupyter.org", + license="BSD", + platforms="Linux, Mac OS X", + python_requires="~=3.3", + keywords=["Interactive", "Interpreter", "Shell", "Web", "Jupyter"], + classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", ], - project_urls = { - 'Bug Reports': 'https://github.com/jupyterhub/batchspawner/issues', - 'Source': 'https://github.com/jupyterhub/batchspawner/', - 'About Jupyterhub': 'http://jupyterhub.readthedocs.io/en/latest/', - 'Jupyter Project': 'http://jupyter.org', - } + project_urls={ + "Bug Reports": "https://github.com/jupyterhub/batchspawner/issues", + "Source": "https://github.com/jupyterhub/batchspawner/", + "About Jupyterhub": "http://jupyterhub.readthedocs.io/en/latest/", + "Jupyter Project": "http://jupyter.org", + }, ) # setuptools requirements -if 'setuptools' in sys.modules: - setup_args['install_requires'] = install_requires = [] - with open('requirements.txt') as f: +if "setuptools" in sys.modules: + setup_args["install_requires"] = install_requires = [] + with open("requirements.txt") as f: for line in f.readlines(): req = line.strip() - if not req or req.startswith(('-e', '#')): + if not req or req.startswith(("-e", "#")): continue install_requires.append(req) @@ -72,5 +72,6 @@ def main(): setup(**setup_args) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/version.py b/version.py index 78fffb81..50c1d59b 100644 --- a/version.py +++ b/version.py @@ -5,7 +5,6 @@ 1, 1, 1, - 'dev', # comment-out this line for a release + "dev", # comment-out this line for a release ) -__version__ = '.'.join(map(str, version_info)) - +__version__ = ".".join(map(str, version_info))