From 29049b451c59a9ea2c50cd7ac8a7f0528738415d Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 21 Feb 2023 18:10:43 +0200 Subject: [PATCH 1/6] Add flake8 and pre-commit configs --- .flake8 | 13 ++++++++ .pre-commit-config.yaml | 73 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..44960c38 --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +# flake8 is used for linting Python code setup to automatically run with +# pre-commit. +# +# ref: https://flake8.pycqa.org/en/latest/user/configuration.html +# + +[flake8] +# E: style errors +# W: style warnings +# C: complexity +# D: docstring warnings (unused pydocstyle extension) +# F841: local variable assigned but never used +ignore = E, C, W, D, F841 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..f0a00c71 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,73 @@ +# pre-commit is a tool to perform a predefined set of tasks manually and/or +# automatically before git commits are made. +# +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all-files +# - Register git hooks: pre-commit install --install-hooks +# +repos: + # Autoformat: Python code, syntax patterns are modernized + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: + - --py38-plus + + # Autoformat: Python code + - repo: https://github.com/PyCQA/autoflake + rev: v2.0.1 + hooks: + - id: autoflake + # args ref: https://github.com/PyCQA/autoflake#advanced-usage + args: + - --in-place + + # Autoformat: Python code + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + + # Autoformat: Python code + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + + # Autoformat: markdown, yaml + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0-alpha.4 + hooks: + - id: prettier + + # Misc... + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available + hooks: + # Autoformat: Makes sure files end in a newline and only a newline. + - id: end-of-file-fixer + + # Autoformat: Sorts entries in requirements.txt. + - id: requirements-txt-fixer + + # Lint: Check for files with names that would conflict on a + # case-insensitive filesystem like MacOS HFS+ or Windows FAT. + - id: check-case-conflict + + # Lint: Checks that non-binary executables have a proper shebang. + - id: check-executables-have-shebangs + + # Lint: Python code + - repo: https://github.com/PyCQA/flake8 + rev: "6.0.0" + hooks: + - id: flake8 + +# pre-commit.ci config reference: https://pre-commit.ci/#configuration +ci: + autoupdate_schedule: monthly From 212bf14b419891fd86a5cb1722dc26ee56bd2ba4 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 21 Feb 2023 18:23:27 +0200 Subject: [PATCH 2/6] Move pytest.ini, and autoformating tools config to pyproject.toml --- git-hooks/README.md | 12 ------------ git-hooks/install | 6 ------ git-hooks/pre-commit | 11 ----------- pyproject.toml | 40 ++++++++++++++++++++++++++++++++++++++++ pytest.ini | 2 -- 5 files changed, 40 insertions(+), 31 deletions(-) delete mode 100644 git-hooks/README.md delete mode 100755 git-hooks/install delete mode 100755 git-hooks/pre-commit delete mode 100644 pytest.ini diff --git a/git-hooks/README.md b/git-hooks/README.md deleted file mode 100644 index 5503c024..00000000 --- a/git-hooks/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# git hooks - -Here is a git pre-commit hook for your convenience, -which runs the [black](https://github.com/ambv/black) -code formatter to auto-format your code, -so you don't have to worry about code formatting. - -Install it with: - -./git-hooks/install - -from the root of the repo. diff --git a/git-hooks/install b/git-hooks/install deleted file mode 100755 index ae06f930..00000000 --- a/git-hooks/install +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -hooks="$(dirname $0)" -githooks="$hooks/../.git/hooks" - -cp -v $hooks/pre-commit $githooks/pre-commit diff --git a/git-hooks/pre-commit b/git-hooks/pre-commit deleted file mode 100755 index 2d34c99a..00000000 --- a/git-hooks/pre-commit +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e -# ensure lang is defined -export LANG=${LANG:-en_US.UTF-8} - -if ! which black 2>&1 > /dev/null; then - echo "missing black, skipping autoformatting" >&2 - exit 0 -fi -echo "running black code formatter" -black . diff --git a/pyproject.toml b/pyproject.toml index 3680eba1..a8521a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,30 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" +# autoflake is used for autoformatting Python code +# +# ref: https://github.com/PyCQA/autoflake#readme +# +[tool.autoflake] +ignore-init-module-imports = true +remove-all-unused-imports = true +remove-duplicate-keys = true +remove-unused-variables = true + +# black is used for autoformatting Python code +# +# ref: https://black.readthedocs.io/en/stable/ +# [tool.black] +skip-string-normalization = true +# target-version should be all supported versions, see +# https://github.com/psf/black/issues/751#issuecomment-473066811 +target_version = [ + "py38", + "py39", + "py310", + "py311", +] exclude = ''' /( .git @@ -14,3 +37,20 @@ exclude = ''' | docs/sphinxext/autodoc_traits.py ) ''' + +# isort is used for autoformatting Python code +# +# ref: https://pycqa.github.io/isort/x +# +[tool.isort] +profile = "black" + +# pytest is used for running Python based tests +# +# ref: https://docs.pytest.org/en/stable/ +# +[tool.pytest.ini_options] +addopts = "--verbose --color=yes --durations=10" +asyncio_mode = "auto" +# Ignore thousands of tests in dependencies installed in a virtual environment +norecursedirs = "lib lib64" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 2f4c80e3..00000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -asyncio_mode = auto From 274da4d920d9c7b47cef79b4d7ae2c38029790db Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 23 Feb 2023 09:10:25 +0100 Subject: [PATCH 3/6] avoid running pre-commit on vendored files from versioneer --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0a00c71..59d21119 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,8 @@ # - Run on all files: pre-commit run --all-files # - Register git hooks: pre-commit install --install-hooks # +exclude: 'versioneer\.py|_version\.py' + repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade From fec2918e99f673398be60f89f0f542d7ffd1aeeb Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 23 Feb 2023 09:12:08 +0100 Subject: [PATCH 4/6] pre-commit run --all --- .flake8 | 2 +- .github/workflows/release.yml | 40 +-- CONTRIBUTING.md | 1 - README.md | 18 +- dev-requirements.txt | 12 +- docs/Makefile | 2 +- docs/source/changelog.md | 104 +++---- docs/source/conf.py | 34 +- docs/source/consul.md | 293 +++++++++--------- docs/source/etcd.md | 225 +++++++------- docs/source/file.md | 210 ++++++------- docs/source/install.md | 196 ++++++------ examples/README.md | 29 +- jupyterhub_traefik_proxy/consul.py | 72 +++-- jupyterhub_traefik_proxy/etcd.py | 35 ++- jupyterhub_traefik_proxy/fileprovider.py | 48 ++- jupyterhub_traefik_proxy/install.py | 19 +- jupyterhub_traefik_proxy/kv_proxy.py | 16 +- jupyterhub_traefik_proxy/proxy.py | 102 +++--- jupyterhub_traefik_proxy/toml.py | 8 +- jupyterhub_traefik_proxy/traefik_utils.py | 22 +- performance/check_perf.py | 3 +- performance/dummy_http_server.py | 4 +- performance/dummy_ws_server.py | 3 +- performance/perf_utils.py | 12 +- performance/run_benchmark_sequential.sh | 1 - pyproject.toml | 2 +- setup.py | 2 +- tests/config_files/consul_config.json | 16 +- tests/config_files/traefik.toml | 1 - tests/config_files/traefik_consul_config.json | 50 +-- tests/config_files/traefik_etcd_txns.txt | 2 - tests/conftest.py | 142 +++++---- tests/dummy_http_server.py | 3 +- tests/proxytest.py | 30 +- tests/test_deprecations.py | 25 +- tests/test_installer.py | 8 +- tests/test_proxy.py | 2 - tests/test_traefik_api_auth.py | 5 +- tests/test_traefik_utils.py | 7 +- tests/utils.py | 6 +- 41 files changed, 932 insertions(+), 880 deletions(-) diff --git a/.flake8 b/.flake8 index 44960c38..698ad83b 100644 --- a/.flake8 +++ b/.flake8 @@ -10,4 +10,4 @@ # C: complexity # D: docstring warnings (unused pydocstyle extension) # F841: local variable assigned but never used -ignore = E, C, W, D, F841 \ No newline at end of file +ignore = E, C, W, D, F841 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d93420a..fc224462 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,23 +20,23 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - name: Install build package - run: | - python -m pip install --upgrade pip - pip install build - pip freeze - - name: Build release - run: | - python -m build --sdist --wheel . - ls -l dist - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.1 - if: startsWith(github.ref, 'refs/tags/') - with: - user: __token__ - password: ${{ secrets.pypi_password }} + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install build package + run: | + python -m pip install --upgrade pip + pip install build + pip freeze + - name: Build release + run: | + python -m build --sdist --wheel . + ls -l dist + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.4.1 + if: startsWith(github.ref, 'refs/tags/') + with: + user: __token__ + password: ${{ secrets.pypi_password }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2c5db2e..942d3ce4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,6 @@ You can also install the tools we use for testing and development with: python3 -m pip install -r dev-requirements.txt - ### Auto-format with black We are trying out the [black](https://github.com/ambv/black) auto-formatting diff --git a/README.md b/README.md index 8d9be990..65e1a225 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # JupyterHub Traefik Proxy [![Documentation build status](https://img.shields.io/readthedocs/jupyterhub-traefik-proxy?logo=read-the-docs)](https://jupyterhub-traefik-proxy.readthedocs.org/en/latest/) @@ -19,12 +18,12 @@ depending on how traefik store its routing configuration. For **smaller**, single-node deployments: -* TraefikFileProviderProxy +- TraefikFileProviderProxy For **distributed** setups: -* TraefikEtcdProxy -* TraefikConsulProxy +- TraefikEtcdProxy +- TraefikConsulProxy ## Installation @@ -33,14 +32,13 @@ The [documentation](https://jupyterhub-traefik-proxy.readthedocs.io) contains a guide](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/install.html) with examples for the three different implementations. -* [For TraefikFileProviderProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/file.html#example-setup) -* [For TraefikEtcdProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/etcd.html#example-setup) -* [For TraefikConsulProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/consul.html#example-setup) - +- [For TraefikFileProviderProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/file.html#example-setup) +- [For TraefikEtcdProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/etcd.html#example-setup) +- [For TraefikConsulProxy](https://jupyterhub-traefik-proxy.readthedocs.io/en/latest/consul.html#example-setup) ## Running tests -There are some tests that use *etcdctl* command line client for etcd. Make sure +There are some tests that use _etcdctl_ command line client for etcd. Make sure to set environment variable `ETCDCTL_API=3` before running the tests, so that the v3 API to be used, e.g.: @@ -48,7 +46,7 @@ the v3 API to be used, e.g.: $ export ETCDCTL_API=3 ``` -You can then run the all the test suite from the *traefik-proxy* directory with: +You can then run the all the test suite from the _traefik-proxy_ directory with: ``` $ pytest -v ./tests diff --git a/dev-requirements.txt b/dev-requirements.txt index 74a6b2cc..fe83d2ed 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,5 @@ -pytest -pytest-asyncio -pytest-cov -codecov black -notebook>=4.0 -websockets +codecov # etcd3 & python-consul2 are now soft dependencies # Adding them here prevents CI from failing @@ -13,4 +8,9 @@ websockets # but with updated grpcio compatibility # watch this space to pick a winner... etcdpy +notebook>=4.0 +pytest +pytest-asyncio +pytest-cov python-consul2 +websockets diff --git a/docs/Makefile b/docs/Makefile index 69fe55ec..ba501f6f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,4 +16,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/changelog.md b/docs/source/changelog.md index 59fd61cf..ff8d0845 100644 --- a/docs/source/changelog.md +++ b/docs/source/changelog.md @@ -30,22 +30,22 @@ the command line for details. ### Bugs fixed -* fix Escape character _ cannot be a safe character #109 [#110](https://github.com/jupyterhub/traefik-proxy/pull/110) ([@mofanke](https://github.com/mofanke)) +- fix Escape character \_ cannot be a safe character #109 [#110](https://github.com/jupyterhub/traefik-proxy/pull/110) ([@mofanke](https://github.com/mofanke)) ### Maintenance and upkeep improvements -* Switch to pydata-sphinx-theme and myst-parser [#122](https://github.com/jupyterhub/traefik-proxy/pull/122) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Try unpinning deps and use a more up to date python consul client [#115](https://github.com/jupyterhub/traefik-proxy/pull/115) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Switch to pydata-sphinx-theme and myst-parser [#122](https://github.com/jupyterhub/traefik-proxy/pull/122) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Try unpinning deps and use a more up to date python consul client [#115](https://github.com/jupyterhub/traefik-proxy/pull/115) ([@GeorgianaElena](https://github.com/GeorgianaElena)) ### Other merged PRs -* Remove CircleCI docs build since now we're using the RTD CI [#121](https://github.com/jupyterhub/traefik-proxy/pull/121) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Do not freeze requirements [#120](https://github.com/jupyterhub/traefik-proxy/pull/120) ([@minrk](https://github.com/minrk)) -* Update readthedocs config options and version [#119](https://github.com/jupyterhub/traefik-proxy/pull/119) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* pip-compile is actually just pip in dependabots config file [#117](https://github.com/jupyterhub/traefik-proxy/pull/117) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Freeze requirements and setup dependabot [#116](https://github.com/jupyterhub/traefik-proxy/pull/116) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* ci: make pushing tags trigger release workflow properly [#114](https://github.com/jupyterhub/traefik-proxy/pull/114) ([@consideRatio](https://github.com/consideRatio)) -* Travis -> GitHub workflows [#113](https://github.com/jupyterhub/traefik-proxy/pull/113) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Remove CircleCI docs build since now we're using the RTD CI [#121](https://github.com/jupyterhub/traefik-proxy/pull/121) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Do not freeze requirements [#120](https://github.com/jupyterhub/traefik-proxy/pull/120) ([@minrk](https://github.com/minrk)) +- Update readthedocs config options and version [#119](https://github.com/jupyterhub/traefik-proxy/pull/119) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- pip-compile is actually just pip in dependabots config file [#117](https://github.com/jupyterhub/traefik-proxy/pull/117) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Freeze requirements and setup dependabot [#116](https://github.com/jupyterhub/traefik-proxy/pull/116) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- ci: make pushing tags trigger release workflow properly [#114](https://github.com/jupyterhub/traefik-proxy/pull/114) ([@consideRatio](https://github.com/consideRatio)) +- Travis -> GitHub workflows [#113](https://github.com/jupyterhub/traefik-proxy/pull/113) ([@GeorgianaElena](https://github.com/GeorgianaElena)) ### Contributors to this release @@ -53,15 +53,14 @@ the command line for details. [@consideRatio](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3AconsideRatio+updated%3A2020-05-16..2021-02-24&type=Issues) | [@GeorgianaElena](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3AGeorgianaElena+updated%3A2020-05-16..2021-02-24&type=Issues) | [@manics](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Amanics+updated%3A2020-05-16..2021-02-24&type=Issues) | [@minrk](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Aminrk+updated%3A2020-05-16..2021-02-24&type=Issues) | [@mofanke](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Amofanke+updated%3A2020-05-16..2021-02-24&type=Issues) - ## [0.1.6](https://github.com/jupyterhub/traefik-proxy/compare/0.1.5...0.1.6) 2020-05-16 ### Merged PRs -* Fix circular reference error [#107](https://github.com/jupyterhub/traefik-proxy/pull/107) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* fix TypeError: 'NoneType' object is not iterable, in delete_route when route doesn't exist [#104](https://github.com/jupyterhub/traefik-proxy/pull/104) ([@mofanke](https://github.com/mofanke)) -* Update etcd.py [#102](https://github.com/jupyterhub/traefik-proxy/pull/102) ([@mofanke](https://github.com/mofanke)) -* New Proxy config option traefik_api_validate_cert [#98](https://github.com/jupyterhub/traefik-proxy/pull/98) ([@devnull-mr](https://github.com/devnull-mr)) +- Fix circular reference error [#107](https://github.com/jupyterhub/traefik-proxy/pull/107) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- fix TypeError: 'NoneType' object is not iterable, in delete_route when route doesn't exist [#104](https://github.com/jupyterhub/traefik-proxy/pull/104) ([@mofanke](https://github.com/mofanke)) +- Update etcd.py [#102](https://github.com/jupyterhub/traefik-proxy/pull/102) ([@mofanke](https://github.com/mofanke)) +- New Proxy config option traefik_api_validate_cert [#98](https://github.com/jupyterhub/traefik-proxy/pull/98) ([@devnull-mr](https://github.com/devnull-mr)) ### Contributors to this release @@ -69,20 +68,19 @@ the command line for details. [@consideRatio](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3AconsideRatio+updated%3A2020-03-31..2020-05-16&type=Issues) | [@devnull-mr](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Adevnull-mr+updated%3A2020-03-31..2020-05-16&type=Issues) | [@GeorgianaElena](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3AGeorgianaElena+updated%3A2020-03-31..2020-05-16&type=Issues) | [@mofanke](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Amofanke+updated%3A2020-03-31..2020-05-16&type=Issues) - ## [0.1.5](https://github.com/jupyterhub/traefik-proxy/compare/0.1.4...0.1.5) 2020-03-31 ### Merged PRs -* Fix named servers routing [#96](https://github.com/jupyterhub/traefik-proxy/pull/96) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Show a message when no binary is provided to the installer [#95](https://github.com/jupyterhub/traefik-proxy/pull/95) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Update install utility docs [#93](https://github.com/jupyterhub/traefik-proxy/pull/93) ([@jtpio](https://github.com/jtpio)) -* Travis deploy tags to PyPI [#89](https://github.com/jupyterhub/traefik-proxy/pull/89) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Update README [#87](https://github.com/jupyterhub/traefik-proxy/pull/87) ([@consideRatio](https://github.com/consideRatio)) -* Handle ssl [#84](https://github.com/jupyterhub/traefik-proxy/pull/84) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* CONTRIBUTING: use long option in "pip install -e" [#82](https://github.com/jupyterhub/traefik-proxy/pull/82) ([@muxator](https://github.com/muxator)) -* Change traefik default version [#81](https://github.com/jupyterhub/traefik-proxy/pull/81) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Add info about TraefikConsulProxy in readme [#80](https://github.com/jupyterhub/traefik-proxy/pull/80) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Fix named servers routing [#96](https://github.com/jupyterhub/traefik-proxy/pull/96) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Show a message when no binary is provided to the installer [#95](https://github.com/jupyterhub/traefik-proxy/pull/95) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Update install utility docs [#93](https://github.com/jupyterhub/traefik-proxy/pull/93) ([@jtpio](https://github.com/jtpio)) +- Travis deploy tags to PyPI [#89](https://github.com/jupyterhub/traefik-proxy/pull/89) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Update README [#87](https://github.com/jupyterhub/traefik-proxy/pull/87) ([@consideRatio](https://github.com/consideRatio)) +- Handle ssl [#84](https://github.com/jupyterhub/traefik-proxy/pull/84) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- CONTRIBUTING: use long option in "pip install -e" [#82](https://github.com/jupyterhub/traefik-proxy/pull/82) ([@muxator](https://github.com/muxator)) +- Change traefik default version [#81](https://github.com/jupyterhub/traefik-proxy/pull/81) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Add info about TraefikConsulProxy in readme [#80](https://github.com/jupyterhub/traefik-proxy/pull/80) ([@GeorgianaElena](https://github.com/GeorgianaElena)) ### Contributors to this release @@ -94,19 +92,19 @@ the command line for details. ## Merged PRs -* Add info about TraefikConsulProxy in readme [#80](https://github.com/jupyterhub/traefik-proxy/pull/80) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Stop assuming kv_traefik_prefix ends with a slash [#79](https://github.com/jupyterhub/traefik-proxy/pull/79) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Log info about what dynamic config file it's used by the Hub [#77](https://github.com/jupyterhub/traefik-proxy/pull/77) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Install script [#76](https://github.com/jupyterhub/traefik-proxy/pull/76) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Set defaults for traefik api username and password [#75](https://github.com/jupyterhub/traefik-proxy/pull/75) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Allow etcd and consul client ssl settings [#70](https://github.com/jupyterhub/traefik-proxy/pull/70) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Fix format in install script warnings [#69](https://github.com/jupyterhub/traefik-proxy/pull/69) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Create test coverage report [#65](https://github.com/jupyterhub/traefik-proxy/pull/65) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Explicitly close consul client session [#64](https://github.com/jupyterhub/traefik-proxy/pull/64) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Throughput results updated [#62](https://github.com/jupyterhub/traefik-proxy/pull/62) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Make trefik's log level configurable [#61](https://github.com/jupyterhub/traefik-proxy/pull/61) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* TraefikConsulProxy [#57](https://github.com/jupyterhub/traefik-proxy/pull/57) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* WIP Common proxy profiling suite [#54](https://github.com/jupyterhub/traefik-proxy/pull/54) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Add info about TraefikConsulProxy in readme [#80](https://github.com/jupyterhub/traefik-proxy/pull/80) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Stop assuming kv_traefik_prefix ends with a slash [#79](https://github.com/jupyterhub/traefik-proxy/pull/79) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Log info about what dynamic config file it's used by the Hub [#77](https://github.com/jupyterhub/traefik-proxy/pull/77) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Install script [#76](https://github.com/jupyterhub/traefik-proxy/pull/76) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Set defaults for traefik api username and password [#75](https://github.com/jupyterhub/traefik-proxy/pull/75) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Allow etcd and consul client ssl settings [#70](https://github.com/jupyterhub/traefik-proxy/pull/70) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Fix format in install script warnings [#69](https://github.com/jupyterhub/traefik-proxy/pull/69) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Create test coverage report [#65](https://github.com/jupyterhub/traefik-proxy/pull/65) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Explicitly close consul client session [#64](https://github.com/jupyterhub/traefik-proxy/pull/64) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Throughput results updated [#62](https://github.com/jupyterhub/traefik-proxy/pull/62) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Make trefik's log level configurable [#61](https://github.com/jupyterhub/traefik-proxy/pull/61) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- TraefikConsulProxy [#57](https://github.com/jupyterhub/traefik-proxy/pull/57) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- WIP Common proxy profiling suite [#54](https://github.com/jupyterhub/traefik-proxy/pull/54) ([@GeorgianaElena](https://github.com/GeorgianaElena)) ## Contributors to this release @@ -116,12 +114,12 @@ the command line for details. ## [0.1.3](https://github.com/jupyterhub/traefik-proxy/compare/0.1.2...0.1.3) 2019-02-26 -* Load initial routing table from disk in TraefikTomlProxy when resuming from a previous session. +- Load initial routing table from disk in TraefikTomlProxy when resuming from a previous session. ### Merged PRs -* Try to load routes from file if cache is empty [#52](https://github.com/jupyterhub/traefik-proxy/pull/52) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* close temporary file before renaming it [#51](https://github.com/jupyterhub/traefik-proxy/pull/51) ([@minrk](https://github.com/minrk)) +- Try to load routes from file if cache is empty [#52](https://github.com/jupyterhub/traefik-proxy/pull/52) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- close temporary file before renaming it [#51](https://github.com/jupyterhub/traefik-proxy/pull/51) ([@minrk](https://github.com/minrk)) ### Contributors to this release @@ -129,27 +127,25 @@ the command line for details. [@GeorgianaElena](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3AGeorgianaElena+updated%3A2019-02-22..2019-02-26&type=Issues) | [@minrk](https://github.com/search?q=repo%3Ajupyterhub%2Ftraefik-proxy+involves%3Aminrk+updated%3A2019-02-22..2019-02-26&type=Issues) - ## [0.1.2](https://github.com/jupyterhub/traefik-proxy/compare/0.1.1...0.1.2) 2019-02-22 -* Fix possible race in atomic_writing with TraefikTomlProxy +- Fix possible race in atomic_writing with TraefikTomlProxy ## [0.1.1](https://github.com/jupyterhub/traefik-proxy/compare/0.1.0...0.1.1) 2019-02-22 -* make proxytest reusable with any Proxy implementation -* improve documentation -* improve logging and error handling -* make check_route_timeout configurable - +- make proxytest reusable with any Proxy implementation +- improve documentation +- improve logging and error handling +- make check_route_timeout configurable ### Merged PRs -* more logging / error handling [#49](https://github.com/jupyterhub/traefik-proxy/pull/49) ([@minrk](https://github.com/minrk)) -* make check_route_timeout configurable [#48](https://github.com/jupyterhub/traefik-proxy/pull/48) ([@minrk](https://github.com/minrk)) -* Update documentation and readme [#47](https://github.com/jupyterhub/traefik-proxy/pull/47) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* Define only the proxy fixture in test_proxy [#46](https://github.com/jupyterhub/traefik-proxy/pull/46) ([@GeorgianaElena](https://github.com/GeorgianaElena)) -* add mocks so that test_check_routes needs only proxy fixture [#44](https://github.com/jupyterhub/traefik-proxy/pull/44) ([@minrk](https://github.com/minrk)) -* Etcd with credentials [#43](https://github.com/jupyterhub/traefik-proxy/pull/43) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- more logging / error handling [#49](https://github.com/jupyterhub/traefik-proxy/pull/49) ([@minrk](https://github.com/minrk)) +- make check_route_timeout configurable [#48](https://github.com/jupyterhub/traefik-proxy/pull/48) ([@minrk](https://github.com/minrk)) +- Update documentation and readme [#47](https://github.com/jupyterhub/traefik-proxy/pull/47) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- Define only the proxy fixture in test_proxy [#46](https://github.com/jupyterhub/traefik-proxy/pull/46) ([@GeorgianaElena](https://github.com/GeorgianaElena)) +- add mocks so that test_check_routes needs only proxy fixture [#44](https://github.com/jupyterhub/traefik-proxy/pull/44) ([@minrk](https://github.com/minrk)) +- Etcd with credentials [#43](https://github.com/jupyterhub/traefik-proxy/pull/43) ([@GeorgianaElena](https://github.com/GeorgianaElena)) ### Contributors to this release diff --git a/docs/source/conf.py b/docs/source/conf.py index 00568d58..d4f51a73 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. # @@ -129,15 +128,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -147,8 +143,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'JupyterHubTraefikProxy.tex', 'JupyterHub Traefik Proxy Documentation', - 'Jupyter Contributors', 'manual'), + ( + master_doc, + 'JupyterHubTraefikProxy.tex', + 'JupyterHub Traefik Proxy Documentation', + 'Jupyter Contributors', + 'manual', + ), ] @@ -157,8 +158,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'jupyterhubtraefikproxy', 'JupyterHub Traefik Proxy Documentation', - [author], 1) + ( + master_doc, + 'jupyterhubtraefikproxy', + 'JupyterHub Traefik Proxy Documentation', + [author], + 1, + ) ] @@ -168,9 +174,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'JupyterHubTraefikProxy', 'JupyterHub Traefik Proxy Documentation', - author, 'JupyterHubTraefikProxy', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + 'JupyterHubTraefikProxy', + 'JupyterHub Traefik Proxy Documentation', + author, + 'JupyterHubTraefikProxy', + 'One line description of project.', + 'Miscellaneous', + ), ] diff --git a/docs/source/consul.md b/docs/source/consul.md index 92f21e2e..9987616d 100644 --- a/docs/source/consul.md +++ b/docs/source/consul.md @@ -9,11 +9,11 @@ e.g. with multiple traefik-proxy instances. ## How-To install TraefikConsulProxy 3. Install **jupyterhub** -2. Install **jupyterhub-traefik-proxy** -3. Install **traefik** -4. Install **consul** +4. Install **jupyterhub-traefik-proxy** +5. Install **traefik** +6. Install **consul** -* You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) +- You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) ## How-To enable TraefikConsulProxy @@ -22,29 +22,29 @@ using the `proxy_class` configuration option. You can choose to: -* use the `traefik_consul` entrypoint, new in JupyterHub 1.0, e.g.: +- use the `traefik_consul` entrypoint, new in JupyterHub 1.0, e.g.: - ```python - c.JupyterHub.proxy_class = "traefik_consul" - ``` + ```python + c.JupyterHub.proxy_class = "traefik_consul" + ``` -* use the TraefikConsulProxy object, in which case, you have to import the module, e.g.: +- use the TraefikConsulProxy object, in which case, you have to import the module, e.g.: - ```python - from jupyterhub_traefik_proxy import TraefikConsulProxy - c.JupyterHub.proxy_class = TraefikConsulProxy - ``` + ```python + from jupyterhub_traefik_proxy import TraefikConsulProxy + c.JupyterHub.proxy_class = TraefikConsulProxy + ``` ## Consul configuration -1. Depending on the value of the ```should_start``` proxy flag, you can choose whether or not TraefikConsulProxy willl be externally managed. +1. Depending on the value of the `should_start` proxy flag, you can choose whether or not TraefikConsulProxy willl be externally managed. - * When **should_start** is set to **True**, TraefikConsulProxy will auto-generate its static configuration - (using the override values or the defaults) and store it in ```traefik.toml``` file. + - When **should_start** is set to **True**, TraefikConsulProxy will auto-generate its static configuration + (using the override values or the defaults) and store it in `traefik.toml` file. The traefik process will then be launched using this file. - * When **should_start** is set to **False**, prior to starting the traefik process, you must create a *toml* file with the desired + - When **should_start** is set to **False**, prior to starting the traefik process, you must create a _toml_ file with the desired traefik static configuration and pass it to traefik. Keep in mind that in order for the routes to be stored in **consul**, - this *toml* file **must** specify consul as the provider. + this _toml_ file **must** specify consul as the provider. 2. TraefikConsulProxy searches in the consul key-value store the keys starting with the **kv_traefik_prefix** prefix in order to build its static configuration. @@ -54,65 +54,69 @@ You can choose to: If you want to change or add traefik's static configuration options, you can add them to consul under this prefix and traefik will pick them up. ``` - * The **default** values of this configurations options are: - ``` - kv_traefik_prefix = "traefik/" - kv_jupyterhub_prefix = "jupyterhub/" - ``` + - The **default** values of this configurations options are: - * You can **override** the default values of the prefixes by passing their desired values through `jupyterhub_config.py` e.g.: - ``` - c.TraefikConsulProxy.kv_traefik_prefix="some_static_config_prefix/" - c.TraefikConsulProxy.kv_jupyterhub_prefix="some_dynamic_config_prefix/" - ``` + ``` + kv_traefik_prefix = "traefik/" + kv_jupyterhub_prefix = "jupyterhub/" + ``` + + - You can **override** the default values of the prefixes by passing their desired values through `jupyterhub_config.py` e.g.: + ``` + c.TraefikConsulProxy.kv_traefik_prefix="some_static_config_prefix/" + c.TraefikConsulProxy.kv_jupyterhub_prefix="some_dynamic_config_prefix/" + ``` 3. By **default**, TraefikConsulProxy assumes consul accepts client requests on the official **default** consul port `8500` for client requests. - ```python - c.TraefikConsulProxy.consul_url = "http://127.0.0.1:8500" - ``` - - If the consul cluster is deployed differently than using the consul defaults, then you **must** pass the consul url to the proxy using - the `consul_url` option in *jupyterhub_config.py*: - - ```python - c.TraefikConsulProxy.consul_url = "scheme://hostname:port" - ``` - - ```{note} - **TraefikConsulProxy does not manage the consul cluster** and assumes it is up and running before the proxy itself starts. - However, based on how consul is configured and started, TraefikConsulProxy needs to be told about - some consul configuration details, such as: - * consul **address** where it accepts client requests - ```python - c.TraefikConsulProxy.consul_url = "scheme://hostname:port" - ``` - * consul **credentials** (if consul has acl enabled) - ```python - c.TraefikConsulProxy.consul_password = "123" - ``` - - Checkout the [consul documentation](https://learn.hashicorp.com/consul/) - to find out more about possible consul configuration options. - ``` + ```python + c.TraefikConsulProxy.consul_url = "http://127.0.0.1:8500" + ``` + + If the consul cluster is deployed differently than using the consul defaults, then you **must** pass the consul url to the proxy using + the `consul_url` option in _jupyterhub_config.py_: + + ```python + c.TraefikConsulProxy.consul_url = "scheme://hostname:port" + ``` + + ````{note} + **TraefikConsulProxy does not manage the consul cluster** and assumes it is up and running before the proxy itself starts. + However, based on how consul is configured and started, TraefikConsulProxy needs to be told about + some consul configuration details, such as: + * consul **address** where it accepts client requests + ```python + c.TraefikConsulProxy.consul_url = "scheme://hostname:port" + ``` + * consul **credentials** (if consul has acl enabled) + ```python + c.TraefikConsulProxy.consul_password = "123" + ``` + + Checkout the [consul documentation](https://learn.hashicorp.com/consul/) + to find out more about possible consul configuration options. + ```` ## Externally managed TraefikConsulProxy If TraefikConsulProxy is used as an externally managed service, then make sure you follow the steps enumerated below: -1. Let JupyterHub know that the proxy being used is TraefikConsulProxy, using the *proxy_class* configuration option: - ```python - c.JupyterHub.proxy_class = "traefik_consul" - ``` +1. Let JupyterHub know that the proxy being used is TraefikConsulProxy, using the _proxy_class_ configuration option: + + ```python + c.JupyterHub.proxy_class = "traefik_consul" + ``` 2. Configure `TraefikConsulProxy` in **jupyterhub_config.py** - JupyterHub configuration file, *jupyterhub_config.py* must specify at least: - * That the proxy is externally managed - * The traefik api credentials - * The consul credentials (if consul acl is enabled) + JupyterHub configuration file, _jupyterhub_config.py_ must specify at least: + + - That the proxy is externally managed + - The traefik api credentials + - The consul credentials (if consul acl is enabled) Example configuration: + ```python # JupyterHub shouldn't start the proxy, it's already running c.TraefikConsulProxy.should_start = False @@ -128,62 +132,62 @@ If TraefikConsulProxy is used as an externally managed service, then make sure y c.TraefikConsulProxy.consul_password = "456" ``` -3. Create a *toml* file with traefik's desired static configuration +3. Create a _toml_ file with traefik's desired static configuration - Before starting the traefik process, you must create a *toml* file with the desired + Before starting the traefik process, you must create a _toml_ file with the desired traefik static configuration and pass it to traefik when you launch the process. Keep in mind that in order for the routes to be stored in **consul**, - this *toml* file **must** specify consul as the provider/ + this _toml_ file **must** specify consul as the provider/ - * **Keep in mind that the static configuration must configure at least:** - * The default entrypoint - * The api entrypoint (*and authenticate it*) - * The websockets protocol - * The consul endpoint + - **Keep in mind that the static configuration must configure at least:** + - The default entrypoint + - The api entrypoint (_and authenticate it_) + - The websockets protocol + - The consul endpoint - Example: + Example: - ``` - defaultentrypoints = ["http"] - debug = true - logLevel = "ERROR" + ``` + defaultentrypoints = ["http"] + debug = true + logLevel = "ERROR" - [api] - dashboard = true - entrypoint = "auth_api" + [api] + dashboard = true + entrypoint = "auth_api" - [wss] - protocol = "http" + [wss] + protocol = "http" - [entryPoints.http] - address = "127.0.0.1:8000" + [entryPoints.http] + address = "127.0.0.1:8000" - [entryPoints.auth_api] - address = "127.0.0.1:8099" + [entryPoints.auth_api] + address = "127.0.0.1:8099" - [entryPoints.auth_api.auth.basic] - users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] + [entryPoints.auth_api.auth.basic] + users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] - [consul] - endpoint = "127.0.0.1:8500" - prefix = "traefik/" - watch = true - ``` + [consul] + endpoint = "127.0.0.1:8500" + prefix = "traefik/" + watch = true + ``` - ```{note} - If you choose to enable consul Access Control Lists (ACLs) to secure the UI, API, CLI, service communications, and agent communications, you can use this *toml* file to pass the credentials to traefik, e.g.: - ``` - [consul] - password = "admin" - ... - ``` - ``` + ````{note} + If you choose to enable consul Access Control Lists (ACLs) to secure the UI, API, CLI, service communications, and agent communications, you can use this *toml* file to pass the credentials to traefik, e.g.: + ``` + [consul] + password = "admin" + ... + ``` + ```` ## Example setup This is an example setup for using JupyterHub and TraefikConsulProxy managed by another service than JupyterHub. -1. Configure the proxy through the JupyterHub configuration file, *jupyterhub_config.py*, e.g.: +1. Configure the proxy through the JupyterHub configuration file, _jupyterhub_config.py_, e.g.: ```python from jupyterhub_traefik_proxy import TraefikConsulProxy @@ -204,14 +208,15 @@ This is an example setup for using JupyterHub and TraefikConsulProxy managed by c.JupyterHub.proxy_class = TraefikConsulProxy ``` - ```{note} - If you intend to enable consul acl, add the acl token to *jupyterhub_config.py* under *consul_password*: + ```{note} + If you intend to enable consul acl, add the acl token to *jupyterhub_config.py* under *consul_password*: - # consul token - c.TraefikConsulProxy.consul_password = "456" - ``` + # consul token + c.TraefikConsulProxy.consul_password = "456" + ``` 2. Starts the agent in development mode on the default port on localhost. e.g.: + ```bash $ consul agent -dev ``` @@ -220,45 +225,45 @@ This is an example setup for using JupyterHub and TraefikConsulProxy managed by If you intend to enable consul acl, checkout [this guide](https://learn.hashicorp.com/consul/security-networking/production-acls). ``` -3. Create a traefik static configuration file, *traefik.toml*, e.g:. +3. Create a traefik static configuration file, _traefik.toml_, e.g:. - ``` - # the default entrypoint - defaultentrypoints = ["http"] - - # the api entrypoint - [api] - dashboard = true - entrypoint = "auth_api" - - # websockets protocol - [wss] - protocol = "http" - - # the port on localhost where traefik accepts http requests - [entryPoints.http] - address = ":8000" - - # the port on localhost where the traefik api and dashboard can be found - [entryPoints.auth_api] - address = ":8099" - - # authenticate the traefik api entrypoint - [entryPoints.auth_api.auth.basic] - users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] - - [consul] - # the consul acl token (if acl is enabled) - password = "456" - # the consul address - endpoint = "127.0.0.1:8500" - # the prefix to use for the static configuration - prefix = "traefik/" - # watch consul for changes - watch = true + ``` + # the default entrypoint + defaultentrypoints = ["http"] + + # the api entrypoint + [api] + dashboard = true + entrypoint = "auth_api" + + # websockets protocol + [wss] + protocol = "http" + + # the port on localhost where traefik accepts http requests + [entryPoints.http] + address = ":8000" + + # the port on localhost where the traefik api and dashboard can be found + [entryPoints.auth_api] + address = ":8099" + + # authenticate the traefik api entrypoint + [entryPoints.auth_api.auth.basic] + users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] + + [consul] + # the consul acl token (if acl is enabled) + password = "456" + # the consul address + endpoint = "127.0.0.1:8500" + # the prefix to use for the static configuration + prefix = "traefik/" + # watch consul for changes + watch = true ``` 4. Start traefik with the configuration specified above, e.g.: - ```bash - $ traefik -c traefik.toml - ``` + ```bash + $ traefik -c traefik.toml + ``` diff --git a/docs/source/etcd.md b/docs/source/etcd.md index b8952b91..d78b0ef4 100644 --- a/docs/source/etcd.md +++ b/docs/source/etcd.md @@ -9,11 +9,11 @@ e.g. with multiple traefik-proxy instances. ## How-To install TraefikEtcdProxy 3. Install **jupyterhub** -2. Install **jupyterhub-traefik-proxy** -3. Install **traefik** -4. Install **etcd** +4. Install **jupyterhub-traefik-proxy** +5. Install **traefik** +6. Install **etcd** -* You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) +- You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) ## How-To enable TraefikEtcdProxy @@ -22,29 +22,29 @@ using the `proxy_class` configuration option. You can choose to: -* use the `traefik_etcd` entrypoint, new in JupyterHub 1.0, e.g.: +- use the `traefik_etcd` entrypoint, new in JupyterHub 1.0, e.g.: - ```python - c.JupyterHub.proxy_class = "traefik_etcd" - ``` + ```python + c.JupyterHub.proxy_class = "traefik_etcd" + ``` -* use the TraefikEtcdProxy object, in which case, you have to import the module, e.g.: +- use the TraefikEtcdProxy object, in which case, you have to import the module, e.g.: - ```python - from jupyterhub_traefik_proxy import TraefikEtcdProxy - c.JupyterHub.proxy_class = TraefikEtcdProxy - ``` + ```python + from jupyterhub_traefik_proxy import TraefikEtcdProxy + c.JupyterHub.proxy_class = TraefikEtcdProxy + ``` ## Etcd configuration -1. Depending on the value of the ```should_start``` proxy flag, you can choose whether or not TraefikEtcdProxy willl be externally managed. +1. Depending on the value of the `should_start` proxy flag, you can choose whether or not TraefikEtcdProxy willl be externally managed. - * When **should_start** is set to **True**, TraefikEtcdProxy will auto-generate its static configuration - (using the override values or the defaults) and store it in ```traefik.toml``` file. + - When **should_start** is set to **True**, TraefikEtcdProxy will auto-generate its static configuration + (using the override values or the defaults) and store it in `traefik.toml` file. The traefik process will then be launched using this file. - * When **should_start** is set to **False**, prior to starting the traefik process, you must create a *toml* file with the desired + - When **should_start** is set to **False**, prior to starting the traefik process, you must create a _toml_ file with the desired traefik static configuration and pass it to traefik. Keep in mind that in order for the routes to be stored in **etcd**, - this *toml* file **must** specify etcd as the provider. + this _toml_ file **must** specify etcd as the provider. 2. TraefikEtcdProxy searches in the etcd key-value store the keys starting with the **kv_traefik_prefix** prefix in order to build its static configuration. @@ -54,32 +54,33 @@ You can choose to: If you want to change or add traefik's static configuration options, you can add them to etcd under this prefix and traefik will pick them up. ``` - * The **default** values of this configurations options are: - ```python - kv_traefik_prefix = "/traefik/" - kv_jupyterhub_prefix = "/jupyterhub/" - ``` + - The **default** values of this configurations options are: - * You can **override** the default values of the prefixes by passing their desired values through `jupyterhub_config.py` e.g.: - ```python - c.TraefikEtcdProxy.kv_traefik_prefix = "/some_static_config_prefix/" - c.TraefikEtcdProxy.kv_jupyterhub_prefix = "/some_dynamic_config_prefix/" - ``` + ```python + kv_traefik_prefix = "/traefik/" + kv_jupyterhub_prefix = "/jupyterhub/" + ``` + + - You can **override** the default values of the prefixes by passing their desired values through `jupyterhub_config.py` e.g.: + ```python + c.TraefikEtcdProxy.kv_traefik_prefix = "/some_static_config_prefix/" + c.TraefikEtcdProxy.kv_jupyterhub_prefix = "/some_dynamic_config_prefix/" + ``` 3. By **default**, TraefikEtcdProxy assumes etcd accepts client requests on the official **default** etcd port `2379` for client requests. - ```python - c.TraefikEtcdProxy.etcd_url = "http://127.0.0.1:2379" - ``` + ```python + c.TraefikEtcdProxy.etcd_url = "http://127.0.0.1:2379" + ``` - If the etcd cluster is deployed differently than using the etcd defaults, then you **must** pass the etcd url to the proxy using - the `etcd_url` option in *jupyterhub_config.py*: + If the etcd cluster is deployed differently than using the etcd defaults, then you **must** pass the etcd url to the proxy using + the `etcd_url` option in _jupyterhub_config.py_: - ```python - c.TraefikEtcdProxy.etcd_url = "scheme://hostname:port" - ``` + ```python + c.TraefikEtcdProxy.etcd_url = "scheme://hostname:port" + ``` -```{note} +````{note} 1. **TraefikEtcdProxy does not manage the etcd cluster** and assumes it is up and running before the proxy itself starts. @@ -100,25 +101,28 @@ because the API V2 won't be supported in the future. Checkout the [etcd documentation](https://coreos.com/etcd/docs/latest/op-guide/configuration.html) to find out more about possible etcd configuration options. -``` +```` ## Externally managed TraefikEtcdProxy If TraefikEtcdProxy is used as an externally managed service, then make sure you follow the steps enumerated below: -1. Let JupyterHub know that the proxy being used is TraefikEtcdProxy, using the *proxy_class* configuration option: - ```python - c.JupyterHub.proxy_class = "traefik_etcd" - ``` +1. Let JupyterHub know that the proxy being used is TraefikEtcdProxy, using the _proxy_class_ configuration option: + + ```python + c.JupyterHub.proxy_class = "traefik_etcd" + ``` 2. Configure `TraefikEtcdProxy` in **jupyterhub_config.py** - JupyterHub configuration file, *jupyterhub_config.py* must specify at least: - * That the proxy is externally managed - * The traefik api credentials - * The etcd credentials (if etcd authentication is enabled) + JupyterHub configuration file, _jupyterhub_config.py_ must specify at least: + + - That the proxy is externally managed + - The traefik api credentials + - The etcd credentials (if etcd authentication is enabled) Example configuration: + ```python # JupyterHub shouldn't start the proxy, it's already running c.TraefikEtcdProxy.should_start = False @@ -135,20 +139,21 @@ If TraefikEtcdProxy is used as an externally managed service, then make sure you c.TraefikEtcdProxy.etcd_password = "456" ``` -3. Create a *toml* file with traefik's desired static configuration +3. Create a _toml_ file with traefik's desired static configuration - Before starting the traefik process, you must create a *toml* file with the desired + Before starting the traefik process, you must create a _toml_ file with the desired traefik static configuration and pass it to traefik when you launch the process. Keep in mind that in order for the routes to be stored in **etcd**, - this *toml* file **must** specify etcd as the provider/ + this _toml_ file **must** specify etcd as the provider/ + + - **Keep in mind that the static configuration must configure at least:** - * **Keep in mind that the static configuration must configure at least:** - * The default entrypoint - * The api entrypoint (*and authenticate it*) - * The websockets protocol - * The etcd endpoint + - The default entrypoint + - The api entrypoint (_and authenticate it_) + - The websockets protocol + - The etcd endpoint - * **Example:** + - **Example:** ``` defaultentrypoints = ["http"] @@ -178,21 +183,21 @@ If TraefikEtcdProxy is used as an externally managed service, then make sure you watch = true ``` - ```{note} - **If you choose to enable the authentication on etcd**, you can use this *toml* file to pass the credentials to traefik, e.g.: + ```{note} + **If you choose to enable the authentication on etcd**, you can use this *toml* file to pass the credentials to traefik, e.g.: - [etcd] - username = "root" - password = "admin" - endpoint = "127.0.0.1:2379" - ... - ``` + [etcd] + username = "root" + password = "admin" + endpoint = "127.0.0.1:2379" + ... + ``` ## Example setup This is an example setup for using JupyterHub and TraefikEtcdProxy managed by another service than JupyterHub. -1. Configure the proxy through the JupyterHub configuration file, *jupyterhub_config.py*, e.g.: +1. Configure the proxy through the JupyterHub configuration file, _jupyterhub_config.py_, e.g.: ```python from jupyterhub_traefik_proxy import TraefikEtcdProxy @@ -211,8 +216,7 @@ This is an example setup for using JupyterHub and TraefikEtcdProxy managed by an # configure JupyterHub to use TraefikEtcdProxy c.JupyterHub.proxy_class = TraefikEtcdProxy - ``` - + ``` ```{note} If you intend to enable authentication on etcd, add the etcd credentials to *jupyterhub_config.py*: @@ -221,9 +225,10 @@ This is an example setup for using JupyterHub and TraefikEtcdProxy managed by an c.TraefikEtcdProxy.etcd_username = "def" # etcd password c.TraefikEtcdProxy.etcd_password = "456" - ``` + ``` 2. Start a single-note etcd cluster on the default port on localhost. e.g.: + ```bash $ etcd ``` @@ -233,49 +238,49 @@ This is an example setup for using JupyterHub and TraefikEtcdProxy managed by an [this guide](https://coreos.com/etcd/docs/latest/op-guide/authentication.html). ``` -3. Create a traefik static configuration file, *traefik.toml*, e.g:. - - ``` - # the default entrypoint - defaultentrypoints = ["http"] - - # the api entrypoint - [api] - dashboard = true - entrypoint = "auth_api" - - # websockets protocol - [wss] - protocol = "http" - - # the port on localhost where traefik accepts http requests - [entryPoints.http] - address = ":8000" - - # the port on localhost where the traefik api and dashboard can be found - [entryPoints.auth_api] - address = ":8099" - - # authenticate the traefik api entrypoint - [entryPoints.auth_api.auth.basic] - users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] - - [etcd] - # the etcd username (if auth is enabled) - username = "def" - # the etcd password (if auth is enabled) - password = "456" - # the etcd address - endpoint = "127.0.0.1:2379" - # the prefix to use for the static configuration - prefix = "/traefik/" - # tell etcd to use the v3 version of the api - useapiv3 = true - # watch etcd for changes - watch = true +3. Create a traefik static configuration file, _traefik.toml_, e.g:. + + ``` + # the default entrypoint + defaultentrypoints = ["http"] + + # the api entrypoint + [api] + dashboard = true + entrypoint = "auth_api" + + # websockets protocol + [wss] + protocol = "http" + + # the port on localhost where traefik accepts http requests + [entryPoints.http] + address = ":8000" + + # the port on localhost where the traefik api and dashboard can be found + [entryPoints.auth_api] + address = ":8099" + + # authenticate the traefik api entrypoint + [entryPoints.auth_api.auth.basic] + users = [ "abc:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] + + [etcd] + # the etcd username (if auth is enabled) + username = "def" + # the etcd password (if auth is enabled) + password = "456" + # the etcd address + endpoint = "127.0.0.1:2379" + # the prefix to use for the static configuration + prefix = "/traefik/" + # tell etcd to use the v3 version of the api + useapiv3 = true + # watch etcd for changes + watch = true ``` 4. Start traefik with the configuration specified above, e.g.: - ``` - $ traefik -c traefik.toml - ``` + ``` + $ traefik -c traefik.toml + ``` diff --git a/docs/source/file.md b/docs/source/file.md index 6d601aae..68bd097d 100644 --- a/docs/source/file.md +++ b/docs/source/file.md @@ -9,7 +9,7 @@ 2. Install **jupyterhub-traefik-proxy** 3. Install **traefik** -* You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) +- You can find the full installation guide and examples in the [Introduction section](install.html#traefik-proxy-installation) ## How-To enable TraefikFileProviderProxy @@ -17,27 +17,26 @@ You can enable JupyterHub to work with `TraefikFileProviderProxy` in jupyterhub_ You can choose to: -* use the `traefik_file` entrypoint, new in JupyterHub 1.0, e.g.: +- use the `traefik_file` entrypoint, new in JupyterHub 1.0, e.g.: - ```python - c.JupyterHub.proxy_class = "traefik_file" - ``` + ```python + c.JupyterHub.proxy_class = "traefik_file" + ``` -* use the TraefikFileProviderProxy object, in which case, you have to import the module, e.g.: - - ```python - from jupyterhub_traefik_proxy.fileprovider import TraefikFileProviderProxy - c.JupyterHub.proxy_class = TraefikFileProviderProxy - ``` +- use the TraefikFileProviderProxy object, in which case, you have to import the module, e.g.: + ```python + from jupyterhub_traefik_proxy.fileprovider import TraefikFileProviderProxy + c.JupyterHub.proxy_class = TraefikFileProviderProxy + ``` ## Traefik configuration Traefik's configuration is divided into two parts: -* The **static** configuration (loaded only at the beginning) -* The **dynamic** configuration (can be hot-reloaded, without restarting the proxy), -where the routing table will be updated continuously. +- The **static** configuration (loaded only at the beginning) +- The **dynamic** configuration (can be hot-reloaded, without restarting the proxy), + where the routing table will be updated continuously. Traefik allows us to have one file for the static configuration file (`traefik.toml` or `traefik.yaml`) and one or several files for the routes, that traefik would watch. @@ -45,12 +44,11 @@ Traefik allows us to have one file for the static configuration file (`traefik.t **TraefikFileProviderProxy**, uses two configuration files: one file for the routes (**rules.toml** or **rules.yaml**), and one for the static configuration (**traefik.toml** or **traefik.yaml**). ``` - By **default**, Traefik will search for `traefik.toml` and `rules.toml` in the following places: -* /etc/traefik/ -* $HOME/.traefik/ -* . the working directory +- /etc/traefik/ +- $HOME/.traefik/ +- . the working directory You can override this in TraefikFileProviderProxy, by modifying the **static_config_file** argument: @@ -84,77 +82,81 @@ or [docker](https://www.docker.com/) will be responsible for starting and stoppi If TraefikFileProviderProxy is used as an externally managed service, then make sure you follow the steps enumerated below: -1. Let JupyterHub know that the proxy being used is TraefikFileProviderProxy, using the *proxy_class* configuration option: - ```python - c.JupyterHub.proxy_class = "traefik_file" - ``` +1. Let JupyterHub know that the proxy being used is TraefikFileProviderProxy, using the _proxy_class_ configuration option: + + ```python + c.JupyterHub.proxy_class = "traefik_file" + ``` 2. Configure `TraefikFileProviderProxy` in **jupyterhub_config.py** - JupyterHub configuration file, *jupyterhub_config.py* must specify at least: - * That the proxy is externally managed - * The traefik api credentials - * The dynamic configuration file, if different from *rules.toml* or if this + JupyterHub configuration file, _jupyterhub_config.py_ must specify at least: + + - That the proxy is externally managed + - The traefik api credentials + - The dynamic configuration file, if different from _rules.toml_ or if this file is located in a place other than traefik's default search directories (etc/traefik/, $HOME/.traefik/, the working directory). traefik must also be able to access the dynamic configuration file. - Example configuration: - ```python - # JupyterHub shouldn't start the proxy, it's already running - c.TraefikFileProviderProxy.should_start = False + Example configuration: - # if not the default: - c.TraefikFileProviderProxy.dynamic_config_file = "/path/to/somefile.toml" + ```python + # JupyterHub shouldn't start the proxy, it's already running + c.TraefikFileProviderProxy.should_start = False - # traefik api credentials - c.TraefikFileProviderProxy.traefik_api_username = "abc" - c.TraefikFileProviderProxy.traefik_api_password = "xxx" + # if not the default: + c.TraefikFileProviderProxy.dynamic_config_file = "/path/to/somefile.toml" - # Validate the certificate on traefik's API? Default = True - # c.TraefikFileProviderProxy.traefik_api_validate_cert = True + # traefik api credentials + c.TraefikFileProviderProxy.traefik_api_username = "abc" + c.TraefikFileProviderProxy.traefik_api_password = "xxx" - # jupyterhub will configure traefik for itself, using this Host name - # (and optional path) on the router rule:- - c.JupyterHub.bind_url = 'https://hub.contoso.com' + # Validate the certificate on traefik's API? Default = True + # c.TraefikFileProviderProxy.traefik_api_validate_cert = True - # jupyterhub will also configure traefik's 'service' url, so this needs - # to be accessible from traefik. By default, jupyterhub will bind to - # 'localhost', but this will bind jupyterhub to its hostname - c.JupyterHub.hub_bind_url = 'http://:8000' + # jupyterhub will configure traefik for itself, using this Host name + # (and optional path) on the router rule:- + c.JupyterHub.bind_url = 'https://hub.contoso.com' - # jupyterhub will only allow path-based routing by default. To stop - # jupyterhub from serving all requests, i.e. it will add a global router - # rule of just PathPrefix(`/`) by default, we must configure jupyterhub as - # a subdomain host. - c.JupyterHub.subdomain_host = "https://hub.contoso.com" + # jupyterhub will also configure traefik's 'service' url, so this needs + # to be accessible from traefik. By default, jupyterhub will bind to + # 'localhost', but this will bind jupyterhub to its hostname + c.JupyterHub.hub_bind_url = 'http://:8000' - # traefik can automatically request certificates from an ACME CA. - # JupyterHub needs to know the name of traefik's certificateResolver - c.TraefikFileProviderProxy.traefik_cert_resolver = "leresolver" + # jupyterhub will only allow path-based routing by default. To stop + # jupyterhub from serving all requests, i.e. it will add a global router + # rule of just PathPrefix(`/`) by default, we must configure jupyterhub as + # a subdomain host. + c.JupyterHub.subdomain_host = "https://hub.contoso.com" - # For jupyterhub to let traefik manage certificates, 'ssl_cert' needs a - # value. (This gets around a validate rule on 'proxy.bind_url', which - # forces the protocol to 'http' unless there is a value in ssl_cert). - c.JupyterHub.ssl_cert = 'externally managed' + # traefik can automatically request certificates from an ACME CA. + # JupyterHub needs to know the name of traefik's certificateResolver + c.TraefikFileProviderProxy.traefik_cert_resolver = "leresolver" - ``` + # For jupyterhub to let traefik manage certificates, 'ssl_cert' needs a + # value. (This gets around a validate rule on 'proxy.bind_url', which + # forces the protocol to 'http' unless there is a value in ssl_cert). + c.JupyterHub.ssl_cert = 'externally managed' + + ``` 3. Ensure **traefik.toml** / **traefik.yaml** - The static configuration file, *traefik.toml* (or **traefik.yaml**) must configure at least: - * The default entrypoint - * The api entrypoint (*and authenticate it in a user-managed dynamic configuration file*) - * The websockets protocol - * The dynamic configuration directory to watch - (*make sure this configuration directory exists, even if empty before the proxy is launched*) - * Check `tests/config_files/traefik.toml` for an example. + The static configuration file, _traefik.toml_ (or **traefik.yaml**) must configure at least: + + - The default entrypoint + - The api entrypoint (_and authenticate it in a user-managed dynamic configuration file_) + - The websockets protocol + - The dynamic configuration directory to watch + (_make sure this configuration directory exists, even if empty before the proxy is launched_) + - Check `tests/config_files/traefik.toml` for an example. ## Example setup This is an example setup for using JupyterHub and TraefikFileProviderProxy managed by another service than JupyterHub. -1. Configure the proxy through the JupyterHub configuration file, *jupyterhub_config.py*, e.g.: +1. Configure the proxy through the JupyterHub configuration file, _jupyterhub_config.py_, e.g.: ```python @@ -172,50 +174,50 @@ This is an example setup for using JupyterHub and TraefikFileProviderProxy manag # configure JupyterHub to use TraefikFileProviderProxy c.JupyterHub.proxy_class = "traefik_file" - ``` - -2. Create a traefik static configuration file, *traefik.toml*, e.g.: - - ``` - # the api entrypoint - [api] - dashboard = true - - # websockets protocol - [wss] - protocol = "http" - - # the port on localhost where traefik accepts http requests - [entryPoints.web] - address = ":8000" + ``` - # the port on localhost where the traefik api and dashboard can be found - [entryPoints.enter_api] - address = ":8099" +2. Create a traefik static configuration file, _traefik.toml_, e.g.: - # the dynamic configuration directory - # This must match the directory provided in Step 1. above. - [providers.file] - directory = "/var/run/traefik" - watch = true + ``` + # the api entrypoint + [api] + dashboard = true + + # websockets protocol + [wss] + protocol = "http" + + # the port on localhost where traefik accepts http requests + [entryPoints.web] + address = ":8000" + + # the port on localhost where the traefik api and dashboard can be found + [entryPoints.enter_api] + address = ":8099" + + # the dynamic configuration directory + # This must match the directory provided in Step 1. above. + [providers.file] + directory = "/var/run/traefik" + watch = true ``` 3. Create a traefik dynamic configuration file in the directory provided in the dynamic configuration above, to provide the api authentication parameters, e.g. - ``` - # Router configuration for the api service - [http.routers.router-api] - rule = "Host(`localhost`) && PathPrefix(`/api`)" - entryPoints = ["enter_api"] - service = "api@internal" - middlewares = ["auth_api"] - - # authenticate the traefik api entrypoint - [http.middlewares.auth_api.basicAuth] - users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] - ``` + ``` + # Router configuration for the api service + [http.routers.router-api] + rule = "Host(`localhost`) && PathPrefix(`/api`)" + entryPoints = ["enter_api"] + service = "api@internal" + middlewares = ["auth_api"] + + # authenticate the traefik api entrypoint + [http.middlewares.auth_api.basicAuth] + users = [ "api_admin:$apr1$eS/j3kum$q/X2khsIEG/bBGsteP.x./",] + ``` 4. Start traefik with the configuration specified above, e.g.: - ```bash - $ traefik --configfile traefik.toml - ``` + ```bash + $ traefik --configfile traefik.toml + ``` diff --git a/docs/source/install.md b/docs/source/install.md index aeb079ac..5c823149 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -1,18 +1,18 @@ # Installation - ## Traefik-proxy installation 1. Install **JupyterHub**: - ``` - $ python3 -m pip install jupyterhub - ``` + + ``` + $ python3 -m pip install jupyterhub + ``` 2. Install **jupyterhub-traefik-proxy**, which is available now as pre-release: - ``` - python3 -m pip install jupyterhub-traefik-proxy - ``` + ``` + python3 -m pip install jupyterhub-traefik-proxy + ``` 3. In order to be able to launch JupyterHub with traefik-proxy or run the tests, **traefik**, must first be installed and added to your `PATH`. @@ -28,15 +28,16 @@ This will install the default versions of traefik, to to `/usr/local/bin` specified through the `--output` option. - If no directory is passed to the installer, a *dependencies* directory will be created in the `traefik-proxy` directory. In this case, you **must** add this directory to `PATH`, e.g. + If no directory is passed to the installer, a _dependencies_ directory will be created in the `traefik-proxy` directory. In this case, you **must** add this directory to `PATH`, e.g. ``` $ export PATH=$PATH:{$PWD}/dependencies ``` If you want to install other versions of traefik in a directory of your choice, just specify it to the installer through the following arguments: - * `--traefik-version` - * `--output` + + - `--traefik-version` + - `--output` Example: @@ -53,9 +54,8 @@ $ python3 -m jupyterhub_traefik_proxy.install --help ``` - 2. From traefik **release page**: - * Install [`traefik`](https://traefik.io/#easy-to-install) - + 2. From traefik **release page**: + - Install [`traefik`](https://traefik.io/#easy-to-install) ## Installing a key-value store @@ -63,9 +63,9 @@ If you want to use a key-value store to mediate configuration (mainly for use in distributed deployments, such as containers), you can get etcd or consul via their respective release pages: -* Install [`etcd`](https://github.com/etcd-io/etcd/releases) +- Install [`etcd`](https://github.com/etcd-io/etcd/releases) -* Install [`consul`](https://github.com/hashicorp/consul/releases) +- Install [`consul`](https://github.com/hashicorp/consul/releases) Or, more likely, select the appropriate container image. You will also need to install a Python client for the Key-Value store of your choice: @@ -75,19 +75,18 @@ You will also need to install a Python client for the Key-Value store of your ch ## Enabling traefik-proxy in JupyterHub +[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/HEAD/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. -[TraefikFileProviderProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/fileprovider.py), [TraefikEtcdProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/etcd.py) and [TraefikConsulProxy](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/jupyterhub_traefik_proxy/consul.py) are custom proxy implementations that subclass [Proxy](https://github.com/jupyterhub/jupyterhub/blob/HEAD/jupyterhub/proxy.py) and can register in JupyterHub config using `c.JupyterHub.proxy_class` entrypoint. - -On startup, JupyterHub will look by default for a configuration file, *jupyterhub_config.py*, in the current working directory. If the configuration file is not in the current working directory, +On startup, JupyterHub will look by default for a configuration file, _jupyterhub_config.py_, in the current working directory. If the configuration file is not in the current working directory, you can load a specific config file and start JupyterHub using: ``` $ jupyterhub -f /path/to/jupyterhub_config.py ``` -There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/examples/jupyterhub_config.py) that configures JupyterHub to run with *TraefikEtcdProxy* as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. +There is an example configuration file [here](https://github.com/jupyterhub/traefik-proxy/blob/HEAD/examples/jupyterhub_config.py) that configures JupyterHub to run with _TraefikEtcdProxy_ as the proxy and uses dummyauthenticator and simplespawner to enable testing without administrative privileges. -In *jupyterhub_config.py*: +In _jupyterhub_config.py_: ``` c.JupyterHub.proxy_class = "traefik_file" @@ -110,97 +109,100 @@ c.JupyterHub.proxy_class = "traefik_consul" 1. **Traefik Dashboard** - Traefik provides a Web UI **dashboard** where you can see the frontends and backends registered, the routing rules, some metrics, but also other configuration elements. Find out more about traefik api's, [here](https://docs.traefik.io/configuration/api/#security). + Traefik provides a Web UI **dashboard** where you can see the frontends and backends registered, the routing rules, some metrics, but also other configuration elements. Find out more about traefik api's, [here](https://docs.traefik.io/configuration/api/#security). + + Because of **security** concerns, in traefik-proxy implementation, traefik api endpoint isn't exposed on the public http endpoint. Instead, it runs on a dedicated **authenticated endpoint** that's on localhost by default. + + The port on which traefik-proxy's api will run, as well as the username and password used for authenticating, can be passed to the proxy through `jupyterhub_config.py`, e.g.: - Because of **security** concerns, in traefik-proxy implementation, traefik api endpoint isn't exposed on the public http endpoint. Instead, it runs on a dedicated **authenticated endpoint** that's on localhost by default. + ``` + c.TraefikFileProviderProxy.traefik_api_url = "http://127.0.0.1:8099" + c.TraefikFileProviderProxy.traefik_api_password = "admin" + c.TraefikFileProviderProxy.traefik_api_username = "admin" + ``` - The port on which traefik-proxy's api will run, as well as the username and password used for authenticating, can be passed to the proxy through `jupyterhub_config.py`, e.g.: + Check out TraefikProxy's **API Reference** for more configuration options. +

- ``` - c.TraefikFileProviderProxy.traefik_api_url = "http://127.0.0.1:8099" - c.TraefikFileProviderProxy.traefik_api_password = "admin" - c.TraefikFileProviderProxy.traefik_api_username = "admin" - ``` - Check out TraefikProxy's **API Reference** for more configuration options. -

2. **TKvProxy class** - TKvProxy is a JupyterHub Proxy implementation that uses traefik and a key-value store. - **TraefikEtcdProxy** and **TraefikConsulProxy** are proxy implementations that sublass `TKvProxy`. - Other custom proxies that wish to implementat a JupyterHub Trafik KV store Proxy can sublass `TKvProxy`. - **TKvProxy** implements JupyterHub's Proxy public API and there is no need to override these public methods. - The methods that **must be implemented** by the proxies that sublass `TKvProxy` are: - * ***_define_kv_specific_static_config()*** - * Define the traefik static configuration that configures - traefik's communication with the key-value store. - * Will be called during startup if should_start is True. - * Subclasses must define this method if the proxy is to be started by the Hub. - * In order to be picked up by the proxy, the static configuration - must be stored into `proxy.static_config` dict under the `kv_name` key. - * ***_kv_atomic_add_route_parts(jupyterhub_routespec, target, data, route_keys, rule)*** - * Add the key-value pairs associated with a route within a key-value store transaction. - * Will be called during add_route. - * When retrieving or deleting a route, the parts of a route are expected to have the following structure: - ``` - [ key: jupyterhub_routespec , value: target ] - [ key: target , value: data ] - [ key: route_keys.backend_url_path , value: target ] - [ key: route_keys.frontend_rule_path , value: rule ] - [ key: route_keys.frontend_backend_path, value: route_keys.backend_alias] - [ key: route_keys.backend_weight_path , value: w(int) ] - # where w is the weight of the backend to be used during load balancing) - ``` - * Returns: - * result (tuple): - * The transaction status (int, 0: failure, positive: success) - * The transaction response(str) - * ***_kv_atomic_delete_route_parts(jupyterhub_routespec, route_keys)*** - * Delete the key-value pairs associated with a route, within a key-value store transaction (if the route exists). - * Will be called during delete_route. - * The keys associated with a route are: - * jupyterhub_routespec - * target - * route_keys.backend_url_path - * route_keys.frontend_rule_path - * route_keys.frontend_backend_path - * route_keys.backend_weight_path - * Returns: - * result (tuple): - * The transaction status (int, 0: failure, positive: success) - * The transaction response (str) - * ***_kv_get_target(jupyterhub_routespec)*** - * Retrive the target from the key-value store. - * The target is the value associated with `jupyterhub_routespec` key. - * Returns: - * The full URL associated with this route (str) - * ***_kv_get_data(target)*** - * Retrive the data associated with the `target` from the key-value store. - * Returns: - * A JSONable dict that holds extra info about the route (dict) - * ***_kv_get_route_parts(kv_entry)*** - * Retrive all the parts that make up a route (i.e. routespec, target, data) from the key-value store given a `kv_entry`. - * A `kv_entry` is a key-value store entry where the key starts with `proxy.jupyterhub_prefix`. It is expected that only the routespecs - will be prefixed with `proxy.jupyterhub_prefix` when added to the kv store. - * Returns: - * routespec: The normalized route specification passed in to add_route ([host]/path/) - * target: The target host for this route (proto://host) - * data: The arbitrary data dict that was passed in by JupyterHub when adding this route. - * ***_kv_get_jupyterhub_prefixed_entries()*** - * Retrive from the kv store all the key-value pairs where the key starts with `proxy.jupyterhub_prefix`. - * It is expected that only the routespecs will be prefixed with `proxy.jupyterhub_prefix` when added to the kv store. - * Returns: - * routes: A list of key-value store entries where the keys start with `proxy.jupyterhub_prefix`. + TKvProxy is a JupyterHub Proxy implementation that uses traefik and a key-value store. + **TraefikEtcdProxy** and **TraefikConsulProxy** are proxy implementations that sublass `TKvProxy`. + Other custom proxies that wish to implementat a JupyterHub Trafik KV store Proxy can sublass `TKvProxy`. + **TKvProxy** implements JupyterHub's Proxy public API and there is no need to override these public methods. + The methods that **must be implemented** by the proxies that sublass `TKvProxy` are: + + - **_\_define_kv_specific_static_config()_** + - Define the traefik static configuration that configures + traefik's communication with the key-value store. + - Will be called during startup if should_start is True. + - Subclasses must define this method if the proxy is to be started by the Hub. + - In order to be picked up by the proxy, the static configuration + must be stored into `proxy.static_config` dict under the `kv_name` key. + - **_\_kv_atomic_add_route_parts(jupyterhub_routespec, target, data, route_keys, rule)_** + - Add the key-value pairs associated with a route within a key-value store transaction. + - Will be called during add_route. + - When retrieving or deleting a route, the parts of a route are expected to have the following structure: + ``` + [ key: jupyterhub_routespec , value: target ] + [ key: target , value: data ] + [ key: route_keys.backend_url_path , value: target ] + [ key: route_keys.frontend_rule_path , value: rule ] + [ key: route_keys.frontend_backend_path, value: route_keys.backend_alias] + [ key: route_keys.backend_weight_path , value: w(int) ] + # where w is the weight of the backend to be used during load balancing) + ``` + - Returns: + - result (tuple): + - The transaction status (int, 0: failure, positive: success) + - The transaction response(str) + - **_\_kv_atomic_delete_route_parts(jupyterhub_routespec, route_keys)_** + - Delete the key-value pairs associated with a route, within a key-value store transaction (if the route exists). + - Will be called during delete_route. + - The keys associated with a route are: + - jupyterhub_routespec + - target + - route_keys.backend_url_path + - route_keys.frontend_rule_path + - route_keys.frontend_backend_path + - route_keys.backend_weight_path + - Returns: + - result (tuple): + - The transaction status (int, 0: failure, positive: success) + - The transaction response (str) + - **_\_kv_get_target(jupyterhub_routespec)_** + - Retrive the target from the key-value store. + - The target is the value associated with `jupyterhub_routespec` key. + - Returns: + - The full URL associated with this route (str) + - **_\_kv_get_data(target)_** + - Retrive the data associated with the `target` from the key-value store. + - Returns: + - A JSONable dict that holds extra info about the route (dict) + - **_\_kv_get_route_parts(kv_entry)_** + - Retrive all the parts that make up a route (i.e. routespec, target, data) from the key-value store given a `kv_entry`. + - A `kv_entry` is a key-value store entry where the key starts with `proxy.jupyterhub_prefix`. It is expected that only the routespecs + will be prefixed with `proxy.jupyterhub_prefix` when added to the kv store. + - Returns: + - routespec: The normalized route specification passed in to add_route ([host]/path/) + - target: The target host for this route (proto://host) + - data: The arbitrary data dict that was passed in by JupyterHub when adding this route. + - **_\_kv_get_jupyterhub_prefixed_entries()_** + - Retrive from the kv store all the key-value pairs where the key starts with `proxy.jupyterhub_prefix`. + - It is expected that only the routespecs will be prefixed with `proxy.jupyterhub_prefix` when added to the kv store. + - Returns: + - routes: A list of key-value store entries where the keys start with `proxy.jupyterhub_prefix`. ## Testing jupyterhub-traefik-proxy -There are some tests that use *etcdctl* command line client for etcd. +There are some tests that use _etcdctl_ command line client for etcd. Make sure to set environment variable ETCDCTL_API=3 before running the tests, so that the v3 API to be used, e.g.: ``` $ export ETCDCTL_API=3 ``` -You can then run the all the test suite from the *traefik-proxy* directory with: +You can then run the all the test suite from the _traefik-proxy_ directory with: ``` $ pytest -v ./tests diff --git a/examples/README.md b/examples/README.md index 01e2e36d..e5b39cc7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,22 +3,25 @@ Steps to follow when using a configuration example: 1. install jupyterhub, e.g.: - ``` - $ python3 -m pip install jupyterhub - ``` + + ``` + $ python3 -m pip install jupyterhub + ``` 2. install traefik and etcd, e.g.: - ``` - $ python3 -m jupyterhub_traefik_proxy.install --output=/usr/local/bin - ``` + + ``` + $ python3 -m jupyterhub_traefik_proxy.install --output=/usr/local/bin + ``` 3. if you're using the configuration example for traefik_etcd, start etcd, e.g.: - ``` - $ etcd - ``` + + ``` + $ etcd + ``` 4. start jupyterhub using a configuration example, e.g.: - ``` - jupyterhub --ip 127.0.0.1 --port=8000 -f ./examples/jupyterhub_config_etcd.py - ``` - Visit http://localhost:8000 in your browser, and sign in using any username and password. + ``` + jupyterhub --ip 127.0.0.1 --port=8000 -f ./examples/jupyterhub_config_etcd.py + ``` + Visit http://localhost:8000 in your browser, and sign in using any username and password. diff --git a/jupyterhub_traefik_proxy/consul.py b/jupyterhub_traefik_proxy/consul.py index 955bfed8..33f18214 100644 --- a/jupyterhub_traefik_proxy/consul.py +++ b/jupyterhub_traefik_proxy/consul.py @@ -18,12 +18,12 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from urllib.parse import urlparse import base64 import string +from urllib.parse import urlparse import escapism -from traitlets import default, Any, Unicode +from traitlets import Any, Unicode, default from .kv_proxy import TKvProxy @@ -81,7 +81,9 @@ def _default_client(self): try: import consul.aio except ImportError: - raise ImportError("Please install python-consul2 package to use traefik-proxy with consul") + raise ImportError( + "Please install python-consul2 package to use traefik-proxy with consul" + ) consul_service = urlparse(self.consul_url) kwargs = { "host": consul_service.hostname, @@ -124,15 +126,19 @@ async def persist_dynamic_config(self): self.dynamic_config, prefix=self.kv_traefik_prefix ) payload = [] + def append_payload(key, val): - payload.append({ - "KV": { - "Verb": "set", - "Key": key, - "Value": base64.b64encode(val.encode()).decode(), + payload.append( + { + "KV": { + "Verb": "set", + "Key": key, + "Value": base64.b64encode(val.encode()).decode(), + } } - }) - for k,v in data.items(): + ) + + for k, v in data.items(): append_payload(k, v) try: @@ -156,14 +162,18 @@ async def _kv_atomic_add_route_parts( ) payload = [] + def append_payload(key, value): - payload.append({ - "KV": { - "Verb": "set", - "Key": key, - "Value": base64.b64encode(value.encode()).decode() + payload.append( + { + "KV": { + "Verb": "set", + "Key": key, + "Value": base64.b64encode(value.encode()).decode(), + } } - }) + ) + append_payload(jupyterhub_routespec, target) append_payload(jupyterhub_target, data) append_payload(route_keys.service_url_path, target) @@ -198,7 +208,6 @@ def append_payload(key, value): return status, response async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): - index, v = await self.consul.kv.get(jupyterhub_routespec) if v is None: self.log.warning( @@ -210,7 +219,7 @@ async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): [self.kv_jupyterhub_prefix, "targets", escapism.escape(target)] ) - payload=[ + payload = [ {"KV": {"Verb": "delete", "Key": jupyterhub_routespec}}, {"KV": {"Verb": "delete", "Key": jupyterhub_target}}, {"KV": {"Verb": "delete", "Key": route_keys.service_url_path}}, @@ -225,11 +234,22 @@ async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): payload.append({"KV": {"Verb": "delete-tree", "Key": tls_path}}) # delete any configured entrypoints - payload.append({"KV": {"Verb": "delete-tree", "Key": - self.kv_separator.join( - ["traefik", "http", "routers", route_keys.router_alias, "entryPoints"] - ) - }}) + payload.append( + { + "KV": { + "Verb": "delete-tree", + "Key": self.kv_separator.join( + [ + "traefik", + "http", + "routers", + route_keys.router_alias, + "entryPoints", + ] + ), + } + } + ) try: status, response = await self.consul.txn.put(payload=payload) @@ -260,9 +280,7 @@ async def _kv_get_route_parts(self, kv_entry): # Strip the "jupyterhub/routes/" prefix from the routespec sep = self.kv_separator - route_prefix = sep.join( - [self.kv_jupyterhub_prefix, "routes"] - ) + route_prefix = sep.join([self.kv_jupyterhub_prefix, "routes"]) routespec = key.replace(route_prefix + sep, "") target = base64.b64decode(value.encode()).decode() @@ -282,7 +300,7 @@ async def _kv_get_jupyterhub_prefixed_entries(self): "Verb": "get-tree", "Key": self.kv_separator.join( [self.kv_jupyterhub_prefix, "routes"] - ) + ), } } ] diff --git a/jupyterhub_traefik_proxy/etcd.py b/jupyterhub_traefik_proxy/etcd.py index 96b504d5..42a469d5 100644 --- a/jupyterhub_traefik_proxy/etcd.py +++ b/jupyterhub_traefik_proxy/etcd.py @@ -23,7 +23,7 @@ import escapism from tornado.concurrent import run_on_executor -from traitlets import Any, default, Bool, List, Unicode +from traitlets import Any, Bool, List, Unicode, default from .kv_proxy import TKvProxy @@ -117,14 +117,16 @@ def _default_client(self): try: import etcd3 except ImportError: - raise ImportError("Please install etcd3 or etcdpy package to use traefik-proxy with etcd3") + raise ImportError( + "Please install etcd3 or etcdpy package to use traefik-proxy with etcd3" + ) kwargs = { 'host': etcd_service.hostname, 'port': etcd_service.port, 'ca_cert': self.etcd_client_ca_cert, 'cert_cert': self.etcd_client_cert_crt, 'cert_key': self.etcd_client_cert_key, - 'grpc_options': self.grpc_options + 'grpc_options': self.grpc_options, } if self.etcd_password: kwargs.update( @@ -159,12 +161,16 @@ def _etcd_get_prefix(self, prefix): def _define_kv_specific_static_config(self): self.log.debug("Setting up the etcd provider in the static config") url = urlparse(self.etcd_url) - self.static_config.update({"providers" : { - "etcd" : { - "endpoints": [url.netloc], - "rootKey": self.kv_traefik_prefix, + self.static_config.update( + { + "providers": { + "etcd": { + "endpoints": [url.netloc], + "rootKey": self.kv_traefik_prefix, + } + } } - } }) + ) if url.scheme == "https": # If etcd is running over TLS, then traefik needs to know tls_conf = {} @@ -214,11 +220,8 @@ async def _kv_atomic_add_route_parts( tls_value = self.traefik_cert_resolver success.append(put(tls_path, tls_value)) - # Specify the entrypoint that jupyterhub's router should bind to - ep_path = self.kv_separator.join( - [router_key, "entryPoints", "0"] - ) + ep_path = self.kv_separator.join([router_key, "entryPoints", "0"]) if not self.traefik_entrypoint: self.traefik_entrypoint = await self._get_traefik_entrypoint() success.append(put(ep_path, self.traefik_entrypoint)) @@ -248,9 +251,7 @@ async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys): delete(route_keys.service_url_path), delete(route_keys.router_service_path), delete(route_keys.router_rule_path), - delete(self.kv_separator.join( - [router_path, "entryPoints", "0"] - )) + delete(self.kv_separator.join([router_path, "entryPoints", "0"])), ] # If it was enabled, delete TLS on the router too if self.is_https: @@ -296,7 +297,9 @@ async def _kv_get_jupyterhub_prefixed_entries(self): return routes async def persist_dynamic_config(self): - data = self.flatten_dict_for_kv(self.dynamic_config, prefix=self.kv_traefik_prefix) + data = self.flatten_dict_for_kv( + self.dynamic_config, prefix=self.kv_traefik_prefix + ) transactions = [] for k, v in data.items(): transactions.append(self.etcd.transactions.put(k, v)) diff --git a/jupyterhub_traefik_proxy/fileprovider.py b/jupyterhub_traefik_proxy/fileprovider.py index a266e655..f7e00f1f 100644 --- a/jupyterhub_traefik_proxy/fileprovider.py +++ b/jupyterhub_traefik_proxy/fileprovider.py @@ -18,11 +18,11 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import os import asyncio -import escapism +import os -from traitlets import Any, default, Unicode, observe +import escapism +from traitlets import Any, Unicode, default, observe from . import traefik_utils from .proxy import TraefikProxy @@ -54,7 +54,9 @@ def _default_handler(self): # If dynamic_config_file is changed, then update the dynamic config file handler @observe("dynamic_config_file") def _set_dynamic_config_file(self, change): - self.dynamic_config_handler = traefik_utils.TraefikConfigFileHandler(self.dynamic_config_file) + self.dynamic_config_handler = traefik_utils.TraefikConfigFileHandler( + self.dynamic_config_file + ) @default("dynamic_config") def _load_dynamic_config(self): @@ -66,8 +68,8 @@ def _load_dynamic_config(self): if not dynamic_config: dynamic_config = { - "http" : {"services": {}, "routers": {}}, - "jupyter": {"routers" : {} } + "http": {"services": {}, "routers": {}}, + "jupyter": {"routers": {}}, } return dynamic_config @@ -84,10 +86,7 @@ async def _setup_traefik_dynamic_config(self): async def _setup_traefik_static_config(self): self.static_config["providers"] = { - "file" : { - "filename": self.dynamic_config_file, - "watch": True - } + "file": {"filename": self.dynamic_config_file, "watch": True} } await super()._setup_traefik_static_config() @@ -111,7 +110,9 @@ def _get_route_unsafe(self, traefik_routespec): # Will this ever cause a KeyError? result["target"] = service_node["loadBalancer"]["servers"][0]["url"] - jupyter_routers = self.dynamic_config["jupyter"]["routers"].get(router_alias, None) + jupyter_routers = self.dynamic_config["jupyter"]["routers"].get( + router_alias, None + ) if jupyter_routers is not None: result["data"] = jupyter_routers["data"] @@ -183,7 +184,7 @@ async def add_route(self, routespec, target, data): self.dynamic_config["http"]["routers"][router_alias] = { "service": service_alias, "rule": rule, - "entryPoints": [self.traefik_entrypoint] + "entryPoints": [self.traefik_entrypoint], } # Enable TLS on this router if globally enabled @@ -192,30 +193,25 @@ async def add_route(self, routespec, target, data): if self.traefik_cert_resolver: tls_config["certResolver"] = self.traefik_cert_resolver - self.dynamic_config["http"]["routers"][router_alias].update({ - "tls": tls_config - }) + self.dynamic_config["http"]["routers"][router_alias].update( + {"tls": tls_config} + ) # Add the data node to a separate top-level node, so traefik doesn't complain. - self.dynamic_config["jupyter"]["routers"][router_alias] = { - "data": data - } + self.dynamic_config["jupyter"]["routers"][router_alias] = {"data": data} if "services" not in self.dynamic_config["http"]: self.dynamic_config["http"]["services"] = {} self.dynamic_config["http"]["services"][service_alias] = { - "loadBalancer": { - "servers": [{"url": target}], - "passHostHeader": True - } + "loadBalancer": {"servers": [{"url": target}], "passHostHeader": True} } self.persist_dynamic_config() if self.should_start: try: # Check if traefik was launched - pid = self.traefik_process.pid + self.traefik_process.pid except AttributeError: self.log.error( "You cannot add routes if the proxy isn't running! Please start the proxy: proxy.start()" @@ -280,9 +276,9 @@ async def get_all_routes(self): escaped_routespec = "".join(router.split("_", 1)[1:]) traefik_routespec = escapism.unescape(escaped_routespec) routespec = self.validate_routespec(traefik_routespec) - all_routes.update({ - routespec : self._get_route_unsafe(traefik_routespec) - }) + all_routes.update( + {routespec: self._get_route_unsafe(traefik_routespec)} + ) return all_routes diff --git a/jupyterhub_traefik_proxy/install.py b/jupyterhub_traefik_proxy/install.py index 0ec54fe5..8ceeaf62 100644 --- a/jupyterhub_traefik_proxy/install.py +++ b/jupyterhub_traefik_proxy/install.py @@ -1,13 +1,13 @@ -import sys +import argparse +import hashlib import os import platform -from urllib.request import urlretrieve +import sys import tarfile -import zipfile -import argparse import textwrap -import hashlib import warnings +import zipfile +from urllib.request import urlretrieve checksums_traefik = { "https://github.com/traefik/traefik/releases/download/v2.4.8/traefik_v2.4.8_linux_arm64.tar.gz": "0931fdd9c855fcafd38eba7568a1d287200fad5afd1aef7d112fb3a48d822fcc", @@ -45,7 +45,9 @@ def install_traefik(prefix, plat, traefik_version): traefik_archive_extension = "tar.gz" traefik_bin = os.path.join(prefix, "traefik") - traefik_archive = "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension + traefik_archive = ( + "traefik_v" + traefik_version + "_" + plat + "." + traefik_archive_extension + ) traefik_archive_path = os.path.join(prefix, traefik_archive) traefik_url = ( @@ -78,7 +80,7 @@ def install_traefik(prefix, plat, traefik_version): if traefik_url in checksums_traefik: if checksum_file(traefik_archive_path) != checksums_traefik[traefik_url]: - raise IOError("Checksum failed") + raise OSError("Checksum failed") else: warnings.warn( f"Traefik {traefik_version} not tested !", @@ -98,10 +100,7 @@ def install_traefik(prefix, plat, traefik_version): print("--- Done ---") - - def main(): - parser = argparse.ArgumentParser( description="Dependencies intaller", epilog=textwrap.dedent( diff --git a/jupyterhub_traefik_proxy/kv_proxy.py b/jupyterhub_traefik_proxy/kv_proxy.py index 17ea52ff..9ec787b7 100644 --- a/jupyterhub_traefik_proxy/kv_proxy.py +++ b/jupyterhub_traefik_proxy/kv_proxy.py @@ -18,12 +18,12 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import escapism import json import os +from collections.abc import MutableMapping +import escapism from traitlets import Unicode -from collections.abc import MutableMapping from . import traefik_utils from .proxy import TraefikProxy @@ -227,7 +227,9 @@ async def add_route(self, routespec, target, data): self.log.debug("Adding route for %s to %s.", routespec, target) routespec = self.validate_routespec(routespec) - route_keys = traefik_utils.generate_route_keys(self, routespec, separator=self.kv_separator) + route_keys = traefik_utils.generate_route_keys( + self, routespec, separator=self.kv_separator + ) # Store the data dict passed in by JupyterHub data = json.dumps(data) @@ -246,7 +248,7 @@ async def add_route(self, routespec, target, data): if self.should_start: try: # Check if traefik was launched - pid = self.traefik_process.pid + self.traefik_process.pid except AttributeError: self.log.error( "You cannot add routes if the proxy isn't running! Please start the proxy: proxy.start()" @@ -277,7 +279,9 @@ async def delete_route(self, routespec): jupyterhub_routespec = self.kv_separator.join( [self.kv_jupyterhub_prefix, "routes", escapism.escape(routespec)] ) - route_keys = traefik_utils.generate_route_keys(self, routespec, separator=self.kv_separator) + route_keys = traefik_utils.generate_route_keys( + self, routespec, separator=self.kv_separator + ) status, response = await self._kv_atomic_delete_route_parts( jupyterhub_routespec, route_keys @@ -404,7 +408,7 @@ def flatten_dict_for_kv(self, data, prefix='traefik'): items.update({new_key: v}) elif isinstance(v, list): for n, item in enumerate(v): - items.update({ f"{new_key}{sep}{n}" : item }) + items.update({f"{new_key}{sep}{n}": item}) else: raise ValueError(f"Cannot upload {v} of type {type(v)} to kv store") return items diff --git a/jupyterhub_traefik_proxy/proxy.py b/jupyterhub_traefik_proxy/proxy.py index 9ad4a9ce..5e871b39 100644 --- a/jupyterhub_traefik_proxy/proxy.py +++ b/jupyterhub_traefik_proxy/proxy.py @@ -24,11 +24,11 @@ from subprocess import Popen, TimeoutExpired from urllib.parse import urlparse -from traitlets import Any, Bool, Dict, Integer, Unicode, default, observe +from jupyterhub.proxy import Proxy +from jupyterhub.utils import exponential_backoff, new_token, url_path_join from tornado.httpclient import AsyncHTTPClient +from traitlets import Any, Bool, Dict, Integer, Unicode, default -from jupyterhub.utils import exponential_backoff, url_path_join, new_token -from jupyterhub.proxy import Proxy from . import traefik_utils @@ -126,14 +126,16 @@ def get_is_https(self): return urlparse(self.public_url).scheme == "https" traefik_cert_resolver = Unicode( - config=True, help="""The traefik certificate Resolver to use for requesting certificates""" + config=True, + help="""The traefik certificate Resolver to use for requesting certificates""", ) # FIXME: How best to enable TLS on routers assigned to only select # entrypoints defined here? traefik_entrypoint = Unicode( help="""The traefik entrypoint names, to which each """ - """jupyterhub-configred Traefik router is assigned""") + """jupyterhub-configred Traefik router is assigned""" + ) async def _get_traefik_entrypoint(self): """Find the traefik entrypoint that matches our :attrib:`self.public_url`""" @@ -143,10 +145,11 @@ async def _get_traefik_entrypoint(self): else: return "web" import re + # FIXME: Adding '_wait_for_static_config' to get through 'external' # tests. Would this be required in the 'real world'? # Adding _wait_for_static_config to the 'external' conftests instead... - #await self._wait_for_static_config() + # await self._wait_for_static_config() resp = await self._traefik_api_request("/api/entrypoints") json_data = json.loads(resp.body) public_url = urlparse(self.public_url) @@ -159,7 +162,9 @@ async def _get_traefik_entrypoint(self): elif public_url.scheme == 'https': hub_port = 443 else: - raise ValueError(f"Cannot discern public_url port from {self.public_url}!") + raise ValueError( + f"Cannot discern public_url port from {self.public_url}!" + ) # Traefik entrypoint format described at:- # https://doc.traefik.io/traefik/routing/entrypoints/#address entrypoint_re = re.compile('([^:]+)?:([0-9]+)/?(tcp|udp)?') @@ -168,7 +173,9 @@ async def _get_traefik_entrypoint(self): if int(port) == hub_port: return entrypoint["name"] entrypoints = [entrypoint["address"] for entrypoint in json_data] - raise ValueError(f"No traefik entrypoint ports ({entrypoints}) match public_url: {self.public_url}!") + raise ValueError( + f"No traefik entrypoint ports ({entrypoints}) match public_url: {self.public_url}!" + ) @default("traefik_api_password") def _warn_empty_password(self): @@ -213,6 +220,7 @@ def _warn_empty_username(self): def _generate_htpassword(self): from passlib.hash import apr_md5_crypt + self.traefik_api_hashed_password = apr_md5_crypt.hash(self.traefik_api_password) async def _check_for_traefik_service(self, routespec, kind): @@ -222,7 +230,9 @@ async def _check_for_traefik_service(self, routespec, kind): from a provider """ # expected e.g. 'service' + '_' + routespec @ file - expected = traefik_utils.generate_alias(routespec, kind) + "@" + self.provider_name + expected = ( + traefik_utils.generate_alias(routespec, kind) + "@" + self.provider_name + ) path = f"/api/http/{kind}s" try: resp = await self._traefik_api_request(path) @@ -244,13 +254,9 @@ async def _wait_for_route(self, routespec): async def _check_traefik_dynamic_conf_ready(): """Check if traefik loaded its dynamic configuration yet""" - if not await self._check_for_traefik_service( - routespec, "service" - ): + if not await self._check_for_traefik_service(routespec, "service"): return False - if not await self._check_for_traefik_service( - routespec, "router" - ): + if not await self._check_for_traefik_service(routespec, "router"): return False return True @@ -327,7 +333,7 @@ def _start_traefik(self): ["traefik", "--configfile", abspath(self.static_config_file)], env=env, ) - except FileNotFoundError as e: + except FileNotFoundError: self.log.error( "Failed to find traefik \n" "The proxy can be downloaded from https://github.com/containous/traefik/releases/download." @@ -348,7 +354,7 @@ async def _setup_traefik_static_config(self): self.log.info("Setting up traefik's static config...") if self.traefik_log_level: - self.static_config["log"] = { "level": self.traefik_log_level } + self.static_config["log"] = {"level": self.traefik_log_level} # FIXME: Do we only create a single entrypoint for jupyterhub? # Why not have an http and https entrypoint? @@ -356,12 +362,12 @@ async def _setup_traefik_static_config(self): self.traefik_entrypoint = await self._get_traefik_entrypoint() entrypoints = { - self.traefik_entrypoint : { + self.traefik_entrypoint: { "address": f":{urlparse(self.public_url).port}", }, - "enter_api" : { + "enter_api": { "address": f":{urlparse(self.traefik_api_url).port}", - } + }, } self.static_config["entryPoints"] = entrypoints @@ -371,7 +377,7 @@ async def _setup_traefik_static_config(self): self.log.debug(f"Persisting the static config: {self.static_config}") handler = traefik_utils.TraefikConfigFileHandler(self.static_config_file) handler.atomic_dump(self.static_config) - except IOError: + except OSError: self.log.exception("Couldn't set up traefik's static config.") raise except: @@ -383,41 +389,41 @@ async def _setup_traefik_dynamic_config(self): self._generate_htpassword() api_url = urlparse(self.traefik_api_url) api_path = api_url.path if api_url.path else '/api' - api_credentials = f"{self.traefik_api_username}:{self.traefik_api_hashed_password}" - self.dynamic_config.update({ - "http": { - "routers": { - "route_api": { - "rule": f"Host(`{api_url.hostname}`) && (PathPrefix(`{api_path}`) || PathPrefix(`/dashboard`))", - "entryPoints": ["enter_api"], - "service": "api@internal", - "middlewares": ["auth_api"] + api_credentials = ( + f"{self.traefik_api_username}:{self.traefik_api_hashed_password}" + ) + self.dynamic_config.update( + { + "http": { + "routers": { + "route_api": { + "rule": f"Host(`{api_url.hostname}`) && (PathPrefix(`{api_path}`) || PathPrefix(`/dashboard`))", + "entryPoints": ["enter_api"], + "service": "api@internal", + "middlewares": ["auth_api"], + }, + }, + "middlewares": { + "auth_api": {"basicAuth": {"users": [api_credentials]}} }, - }, - "middlewares": { - "auth_api": { - "basicAuth": { - "users": [ - api_credentials - ] - } - } } } - }) + ) if self.ssl_cert and self.ssl_key: - self.dynamic_config.update({ - "tls": { - "stores": { - "default": { - "defaultCertificate": { - "certFile": self.ssl_cert, - "keyFile": self.ssl_key + self.dynamic_config.update( + { + "tls": { + "stores": { + "default": { + "defaultCertificate": { + "certFile": self.ssl_cert, + "keyFile": self.ssl_key, + } } } } } - }) + ) def validate_routespec(self, routespec): """Override jupyterhub's default Proxy.validate_routespec method, as traefik diff --git a/jupyterhub_traefik_proxy/toml.py b/jupyterhub_traefik_proxy/toml.py index 802b317c..a9a8901f 100644 --- a/jupyterhub_traefik_proxy/toml.py +++ b/jupyterhub_traefik_proxy/toml.py @@ -1,16 +1,20 @@ - from traitlets import Unicode + from .fileprovider import TraefikFileProviderProxy class TraefikTomlProxy(TraefikFileProviderProxy): """Deprecated alias for file provider""" + toml_dynamic_config_file = Unicode( config=True, ).tag( deprecated_in="0.4", deprecated_for="TraefikFileProvider.dynamic_config_file", ) + def __init__(self, **kwargs): super().__init__(**kwargs) - self.log.warning("TraefikTomlProxy is deprecated in jupyterhub-traefik-proxy 0.4. Use `c.JupyterHub.proxy_class = 'traefik_file'") + self.log.warning( + "TraefikTomlProxy is deprecated in jupyterhub-traefik-proxy 0.4. Use `c.JupyterHub.proxy_class = 'traefik_file'" + ) diff --git a/jupyterhub_traefik_proxy/traefik_utils.py b/jupyterhub_traefik_proxy/traefik_utils.py index 5eff13bf..d7c7fb81 100644 --- a/jupyterhub_traefik_proxy/traefik_utils.py +++ b/jupyterhub_traefik_proxy/traefik_utils.py @@ -1,13 +1,12 @@ import os import string +from collections import namedtuple +from contextlib import contextmanager from tempfile import NamedTemporaryFile -from traitlets import Unicode from urllib.parse import unquote import escapism - -from contextlib import contextmanager -from collections import namedtuple +from traitlets import Unicode class KVStorePrefix(Unicode): @@ -40,7 +39,7 @@ def generate_alias(routespec, server_type=""): return server_type + "_" + escapism.escape(routespec, safe=safe) -def generate_service_entry( proxy, service_alias, separator="/", url=False): +def generate_service_entry(proxy, service_alias, separator="/", url=False): service_entry = separator.join( ["http", "services", service_alias, "loadBalancer", "servers", "server1"] ) @@ -58,9 +57,7 @@ def generate_router_service_entry(proxy, router_alias): def generate_router_rule_entry(proxy, router_alias, separator="/"): - router_rule_entry = separator.join( - ["http", "routers", router_alias] - ) + router_rule_entry = separator.join(["http", "routers", router_alias]) if separator == "/": router_rule_entry = separator.join( [proxy.kv_traefik_prefix, router_rule_entry, "rule"] @@ -138,13 +135,16 @@ def atomic_writing(path): class TraefikConfigFileHandler: """Handles reading and writing Traefik config files. Can operate on both toml and yaml files""" + def __init__(self, file_path): file_ext = file_path.rsplit('.', 1)[-1] if file_ext == 'yaml': try: from ruamel.yaml import YAML except ImportError: - raise ImportError("jupyterhub-traefik-proxy requires ruamel.yaml to use YAML config files") + raise ImportError( + "jupyterhub-traefik-proxy requires ruamel.yaml to use YAML config files" + ) config_handler = YAML(typ="rt") elif file_ext == 'toml': import toml as config_handler @@ -154,13 +154,13 @@ def __init__(self, file_path): self.file_path = file_path # Redefined to either yaml.dump or toml.dump self._dump = config_handler.dump - #self._dumps = config_handler.dumps + # self._dumps = config_handler.dumps # Redefined by __init__, to either yaml.load or toml.load self._load = config_handler.load def load(self): """Depending on self.file_path, call either yaml.load or toml.load""" - with open(self.file_path, "r") as fd: + with open(self.file_path) as fd: return self._load(fd) def dump(self, data): diff --git a/performance/check_perf.py b/performance/check_perf.py index 93e5391a..73b6c2f0 100644 --- a/performance/check_perf.py +++ b/performance/check_perf.py @@ -1,10 +1,9 @@ import asyncio import csv import os -import time -from tornado.httpclient import AsyncHTTPClient, HTTPRequest import websockets +from tornado.httpclient import AsyncHTTPClient, HTTPRequest from . import perf_utils diff --git a/performance/dummy_http_server.py b/performance/dummy_http_server.py index 242065d6..b0d9b4b9 100644 --- a/performance/dummy_http_server.py +++ b/performance/dummy_http_server.py @@ -1,8 +1,6 @@ -import asyncio from http.server import BaseHTTPRequestHandler, HTTPServer + import numpy as np -import sys -import websockets class DummyServer(BaseHTTPRequestHandler): diff --git a/performance/dummy_ws_server.py b/performance/dummy_ws_server.py index 966094ba..8c99fd34 100644 --- a/performance/dummy_ws_server.py +++ b/performance/dummy_ws_server.py @@ -1,4 +1,5 @@ import asyncio + import websockets @@ -9,7 +10,7 @@ async def send_port(websocket, path): if __name__ == "__main__": - from sys import argv + pass asyncio.get_event_loop().run_until_complete( websockets.serve(send_port, "localhost", 9000) diff --git a/performance/perf_utils.py b/performance/perf_utils.py index fa5f62bc..16e529b3 100644 --- a/performance/perf_utils.py +++ b/performance/perf_utils.py @@ -5,12 +5,14 @@ from urllib.parse import urlparse import numpy as np - -from jupyterhub.tests.mocking import MockHub from jupyterhub.proxy import ConfigurableHTTPProxy -from jupyterhub_traefik_proxy import TraefikConsulProxy -from jupyterhub_traefik_proxy import TraefikEtcdProxy -from jupyterhub_traefik_proxy import TraefikFileProviderProxy +from jupyterhub.tests.mocking import MockHub + +from jupyterhub_traefik_proxy import ( + TraefikConsulProxy, + TraefikEtcdProxy, + TraefikFileProviderProxy, +) def configure_argument_parser(): diff --git a/performance/run_benchmark_sequential.sh b/performance/run_benchmark_sequential.sh index 202cedff..76bbba28 100755 --- a/performance/run_benchmark_sequential.sh +++ b/performance/run_benchmark_sequential.sh @@ -10,4 +10,3 @@ python3 -m performance.check_perf --measure=methods --proxy=EtcdProxy --iteratio #stop etcd: pkill etcd -rf default.etcd/ - diff --git a/pyproject.toml b/pyproject.toml index a8521a7e..51cc9266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,4 +53,4 @@ profile = "black" addopts = "--verbose --color=yes --durations=10" asyncio_mode = "auto" # Ignore thousands of tests in dependencies installed in a virtual environment -norecursedirs = "lib lib64" \ No newline at end of file +norecursedirs = "lib lib64" diff --git a/setup.py b/setup.py index eb7e6be3..cbfb764f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import os import sys -from setuptools import setup, find_packages +from setuptools import find_packages, setup from setuptools.command.bdist_egg import bdist_egg # ensure cwd is on sys.path diff --git a/tests/config_files/consul_config.json b/tests/config_files/consul_config.json index c6db17d0..4cb5e2da 100644 --- a/tests/config_files/consul_config.json +++ b/tests/config_files/consul_config.json @@ -1,11 +1,11 @@ { - "server": true, - "acl": { - "enabled": true, - "default_policy": "deny", - "tokens" : { - "initial_management": "secret", - "default" : "secret" - } + "server": true, + "acl": { + "enabled": true, + "default_policy": "deny", + "tokens": { + "initial_management": "secret", + "default": "secret" } + } } diff --git a/tests/config_files/traefik.toml b/tests/config_files/traefik.toml index c50c0948..98e7dfbb 100644 --- a/tests/config_files/traefik.toml +++ b/tests/config_files/traefik.toml @@ -17,4 +17,3 @@ [entryPoints.enter_api] address = "127.0.0.1:8099" - diff --git a/tests/config_files/traefik_consul_config.json b/tests/config_files/traefik_consul_config.json index 7d19f41d..8ea63036 100644 --- a/tests/config_files/traefik_consul_config.json +++ b/tests/config_files/traefik_consul_config.json @@ -1,27 +1,27 @@ [ - { - "key": "traefik/http/middlewares/auth_api/basicAuth/users/0", - "flags": 0, - "value": "YXBpX2FkbWluOiRhcHIxJGVTL2oza3VtJHEvWDJraHNJRUcvYkJHc3RlUC54Li8=" - }, - { - "key": "traefik/http/routers/route_api/entryPoints/0", - "flags": 0, - "value": "ZW50ZXJfYXBp" - }, - { - "key": "traefik/http/routers/route_api/middlewares/0", - "flags": 0, - "value": "YXV0aF9hcGk=" - }, - { - "key": "traefik/http/routers/route_api/rule", - "flags": 0, - "value": "SG9zdChgbG9jYWxob3N0YCkgJiYgUGF0aFByZWZpeChgL2FwaWAp" - }, - { - "key": "traefik/http/routers/route_api/service", - "flags": 0, - "value": "YXBpQGludGVybmFs" - } + { + "key": "traefik/http/middlewares/auth_api/basicAuth/users/0", + "flags": 0, + "value": "YXBpX2FkbWluOiRhcHIxJGVTL2oza3VtJHEvWDJraHNJRUcvYkJHc3RlUC54Li8=" + }, + { + "key": "traefik/http/routers/route_api/entryPoints/0", + "flags": 0, + "value": "ZW50ZXJfYXBp" + }, + { + "key": "traefik/http/routers/route_api/middlewares/0", + "flags": 0, + "value": "YXV0aF9hcGk=" + }, + { + "key": "traefik/http/routers/route_api/rule", + "flags": 0, + "value": "SG9zdChgbG9jYWxob3N0YCkgJiYgUGF0aFByZWZpeChgL2FwaWAp" + }, + { + "key": "traefik/http/routers/route_api/service", + "flags": 0, + "value": "YXBpQGludGVybmFs" + } ] diff --git a/tests/config_files/traefik_etcd_txns.txt b/tests/config_files/traefik_etcd_txns.txt index 61e28e49..f78a18ed 100644 --- a/tests/config_files/traefik_etcd_txns.txt +++ b/tests/config_files/traefik_etcd_txns.txt @@ -4,5 +4,3 @@ put traefik/http/routers/route_api/entryPoints/0 "enter_api" put traefik/http/routers/route_api/middlewares/0 "auth_api" put traefik/http/routers/route_api/rule "Host(`localhost`) && PathPrefix(`/api`)" put traefik/http/routers/route_api/service "api@internal" - - diff --git a/tests/conftest.py b/tests/conftest.py index a5c3b741..1c012ac0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,25 +3,23 @@ import asyncio import logging import os -from pathlib import Path import shutil import subprocess import sys import time +from pathlib import Path from tempfile import TemporaryDirectory import pytest from _pytest.mark import Mark +from consul.aio import Consul +from jupyterhub.utils import exponential_backoff from traitlets.log import get_logger -from jupyterhub_traefik_proxy.etcd import TraefikEtcdProxy from jupyterhub_traefik_proxy.consul import TraefikConsulProxy +from jupyterhub_traefik_proxy.etcd import TraefikEtcdProxy from jupyterhub_traefik_proxy.fileprovider import TraefikFileProviderProxy -from jupyterhub.utils import exponential_backoff - -from consul.aio import Consul - HERE = Path(__file__).parent.resolve() config_files = os.path.join(HERE, "config_files") @@ -56,6 +54,7 @@ class Config: # Putting here, can easily change between http and https public_url = "https://127.0.0.1:8000" + # Define a "slow" test marker so that we can run the slow tests at the end # ref: https://docs.pytest.org/en/6.0.1/example/simple.html#control-skipping-of-tests-according-to-command-line-option # ref: https://stackoverflow.com/questions/61533694/run-slow-pytest-commands-at-the-end-of-the-test-suite @@ -144,11 +143,12 @@ def auth_etcd_proxy(enable_auth_in_etcd, launch_etcd_proxy): """ yield launch_etcd_proxy + @pytest.fixture async def launch_etcd_proxy(): grpc_options = [ ("grpc.ssl_target_name_override", "localhost"), - ("grpc.default_authority", "localhost") + ("grpc.default_authority", "localhost"), ] proxy = TraefikEtcdProxy( public_url=Config.public_url, @@ -188,30 +188,28 @@ def traitlets_log(): @pytest.fixture async def file_proxy_toml(): """Fixture returning a configured TraefikFileProviderProxy""" - dynamic_config_file = os.path.join( - config_files, "dynamic_config", "rules.toml" - ) + dynamic_config_file = os.path.join(config_files, "dynamic_config", "rules.toml") static_config_file = "traefik.toml" - proxy = _file_proxy(dynamic_config_file, - static_config_file=static_config_file, - should_start=True) + proxy = _file_proxy( + dynamic_config_file, static_config_file=static_config_file, should_start=True + ) await proxy.start() yield proxy await proxy.stop() + @pytest.fixture async def file_proxy_yaml(): - dynamic_config_file = os.path.join( - config_files, "dynamic_config", "rules.yaml" - ) + dynamic_config_file = os.path.join(config_files, "dynamic_config", "rules.yaml") static_config_file = "traefik.yaml" - proxy = _file_proxy(dynamic_config_file, - static_config_file=static_config_file, - should_start=True) + proxy = _file_proxy( + dynamic_config_file, static_config_file=static_config_file, should_start=True + ) await proxy.start() yield proxy await proxy.stop() + def _file_proxy(dynamic_config_file, **kwargs): return TraefikFileProviderProxy( public_url=Config.public_url, @@ -225,26 +223,17 @@ def _file_proxy(dynamic_config_file, **kwargs): @pytest.fixture async def external_file_proxy_yaml(launch_traefik_file): - dynamic_config_file = os.path.join( - config_files, "dynamic_config", "rules.yaml" - ) - proxy = _file_proxy( - dynamic_config_file, - should_start=False - ) + dynamic_config_file = os.path.join(config_files, "dynamic_config", "rules.yaml") + proxy = _file_proxy(dynamic_config_file, should_start=False) await proxy._wait_for_static_config() yield proxy os.remove(dynamic_config_file) + @pytest.fixture async def external_file_proxy_toml(launch_traefik_file): - dynamic_config_file = os.path.join( - config_files, "dynamic_config", "rules.toml" - ) - proxy = _file_proxy( - dynamic_config_file, - should_start=False - ) + dynamic_config_file = os.path.join(config_files, "dynamic_config", "rules.toml") + proxy = _file_proxy(dynamic_config_file, should_start=False) await proxy._wait_for_static_config() yield proxy os.remove(dynamic_config_file) @@ -308,6 +297,7 @@ async def auth_external_etcd_proxy( # authentication # ######################################################################### + @pytest.fixture def launch_traefik_file(): args = ("--configfile", os.path.join(config_files, "traefik.toml")) @@ -329,7 +319,7 @@ def launch_traefik_etcd(): def launch_traefik_etcd_auth(configure_etcd_auth): extra_args = ( "--providers.etcd.username=" + Config.etcd_user, - "--providers.etcd.password=" + Config.etcd_password + "--providers.etcd.password=" + Config.etcd_password, ) proc = _launch_traefik_cli(*extra_args, env=Config.etcdctl_env) yield proc @@ -381,28 +371,32 @@ def _launch_traefik(*extra_args, env=None): # Etcd Launchers and configurers # ################################## + @pytest.fixture def configure_etcd(): """Load traefik api rules into the etcd kv store""" yield _config_etcd() + @pytest.fixture def configure_etcd_auth(): """Load traefik api rules into the etcd kv store, with authentication""" - yield _config_etcd( - "--user=" + Config.etcd_user + ":" + Config.etcd_password - ) + yield _config_etcd("--user=" + Config.etcd_user + ":" + Config.etcd_password) + def _config_etcd(*extra_args): data_store_cmd = ("etcdctl", "txn") + extra_args # Load a pre-baked dynamic configuration into the etcd store. # This essentially puts authentication on the traefik api handler. - with open(os.path.join(config_files, "traefik_etcd_txns.txt"), "r") as fd: + with open(os.path.join(config_files, "traefik_etcd_txns.txt")) as fd: txns = fd.read() - proc = subprocess.Popen(data_store_cmd, stdin=subprocess.PIPE, env=Config.etcdctl_env) + proc = subprocess.Popen( + data_store_cmd, stdin=subprocess.PIPE, env=Config.etcdctl_env + ) proc.communicate(txns.encode()) proc.wait() + @pytest.fixture def enable_auth_in_etcd(launch_etcd_auth): user = Config.etcd_user @@ -410,15 +404,19 @@ def enable_auth_in_etcd(launch_etcd_auth): common_args = [ "--insecure-skip-tls-verify=true", "--insecure-transport=false", - "--debug" - ] - subprocess.call(["etcdctl", "user", "add", f"{user}:{pw}"] + common_args, - env=Config.etcdctl_env) - subprocess.call(["etcdctl", "user", "grant-role", user, "root"] + common_args, - env=Config.etcdctl_env) + "--debug", + ] + subprocess.call( + ["etcdctl", "user", "add", f"{user}:{pw}"] + common_args, env=Config.etcdctl_env + ) + subprocess.call( + ["etcdctl", "user", "grant-role", user, "root"] + common_args, + env=Config.etcdctl_env, + ) assert ( - subprocess.check_output(["etcdctl", "auth", "enable"] + common_args, - env=Config.etcdctl_env) + subprocess.check_output( + ["etcdctl", "auth", "enable"] + common_args, env=Config.etcdctl_env + ) .decode(sys.stdout.encoding) .strip() == "Authentication Enabled" @@ -428,38 +426,47 @@ def enable_auth_in_etcd(launch_etcd_auth): assert ( subprocess.check_output( ["etcdctl", "--user", f"{user}:{pw}", "auth", "disable"] + common_args, - env=Config.etcdctl_env - ).decode(sys.stdout.encoding) - .strip() == "Authentication Disabled" + env=Config.etcdctl_env, + ) + .decode(sys.stdout.encoding) + .strip() + == "Authentication Disabled" + ) + subprocess.call( + ["etcdctl", "user", "revoke-role", "root", user] + common_args, + env=Config.etcdctl_env, + ) + subprocess.call( + ["etcdctl", "user", "delete", user] + common_args, env=Config.etcdctl_env ) - subprocess.call(["etcdctl", "user", "revoke-role", "root", user] + common_args, - env=Config.etcdctl_env) - subprocess.call(["etcdctl", "user", "delete", user] + common_args, - env=Config.etcdctl_env) @pytest.fixture async def launch_etcd_auth(): etcd_proc = subprocess.Popen( - ["etcd", "--log-level=debug", "--peer-auto-tls", - f"--cert-file={config_files}/test-cert.crt", - f"--key-file={config_files}/test-key.key", - "--initial-cluster=default=https://localhost:2380", - "--initial-advertise-peer-urls=https://localhost:2380", - "--listen-peer-urls=https://localhost:2380", - "--listen-client-urls=https://localhost:2379", - "--advertise-client-urls=https://localhost:2379", - "--log-level=debug"], + [ + "etcd", + "--log-level=debug", + "--peer-auto-tls", + f"--cert-file={config_files}/test-cert.crt", + f"--key-file={config_files}/test-key.key", + "--initial-cluster=default=https://localhost:2380", + "--initial-advertise-peer-urls=https://localhost:2380", + "--listen-peer-urls=https://localhost:2380", + "--listen-client-urls=https://localhost:2379", + "--advertise-client-urls=https://localhost:2379", + "--log-level=debug", + ], ) try: await _wait_for_etcd( - "--insecure-skip-tls-verify=true", - "--insecure-transport=false", - "--debug") + "--insecure-skip-tls-verify=true", "--insecure-transport=false", "--debug" + ) yield etcd_proc finally: shutdown_etcd(etcd_proc) + @pytest.fixture async def launch_etcd(): with TemporaryDirectory() as etcd_path: @@ -473,6 +480,7 @@ async def launch_etcd(): finally: shutdown_etcd(etcd_proc) + async def _wait_for_etcd(*etcd_args): """Etcd may not be ready if we jump straight into the tests. Make sure it's running before we continue with configuring it or running @@ -648,9 +656,11 @@ def shutdown_etcd(etcd_proc): if os.path.exists(default_etcd): shutil.rmtree(default_etcd) + def shutdown_traefik(traefik_process): terminate_process(traefik_process) + def terminate_process(proc, timeout=5): proc.terminate() try: diff --git a/tests/dummy_http_server.py b/tests/dummy_http_server.py index 2b329963..9e4df806 100644 --- a/tests/dummy_http_server.py +++ b/tests/dummy_http_server.py @@ -1,5 +1,6 @@ -from http.server import BaseHTTPRequestHandler, HTTPServer import asyncio +from http.server import BaseHTTPRequestHandler, HTTPServer + import websockets diff --git a/tests/proxytest.py b/tests/proxytest.py index 9340d25d..0a9c8e6c 100644 --- a/tests/proxytest.py +++ b/tests/proxytest.py @@ -1,37 +1,34 @@ """Tests for the base traefik proxy""" import asyncio -import inspect import copy -import utils +import inspect +import pprint import subprocess import sys - from contextlib import contextmanager -from os.path import dirname, join, abspath +from os.path import abspath, dirname, join from random import randint from unittest.mock import Mock -from urllib.parse import quote -from urllib.parse import urlparse +from urllib.parse import quote, urlparse import pytest +import utils +import websockets from jupyterhub.objects import Hub, Server from jupyterhub.user import User from jupyterhub.utils import exponential_backoff, url_path_join -from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPClientError -import websockets - +from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest -import pprint pp = pprint.PrettyPrinter(indent=2) + class MockApp: def __init__(self): self.hub = Hub(routespec="/") class MockSpawner: - name = "" server = None pending = None @@ -78,6 +75,7 @@ def __init__(self, name): def _new_spawner(self, spawner_name, **kwargs): return MockSpawner(spawner_name, user=self, **kwargs) + def assert_equal(value, expected): try: assert value == expected @@ -86,6 +84,7 @@ def assert_equal(value, expected): pp.pprint({"expected": expected}) raise + @pytest.fixture def launch_backend(): dummy_server_path = abspath(join(dirname(__file__), "dummy_http_server.py")) @@ -203,7 +202,7 @@ async def test_route_exist(spec, backend): if not expect_value_error(spec): try: - del( route["data"]["last_activity"] ) # CHP + del route["data"]["last_activity"] # CHP except TypeError as e: raise TypeError(f"{e}\nRoute got:{route}") except KeyError: @@ -389,7 +388,7 @@ async def test_host_origin_headers(proxy, launch_backend): req_url, method="GET", headers={"Host": expected_host_header, "Origin": expected_origin_header}, - validate_cert=False + validate_cert=False, ) resp = await AsyncHTTPClient().fetch(req) @@ -443,13 +442,14 @@ async def test_check_routes(proxy, username): async def test_websockets(proxy, launch_backend): import ssl + routespec = "/user/username/" target = "http://127.0.0.1:9000" data = {} proxy_url = urlparse(proxy.public_url) - traefik_port = proxy_url.port - traefik_host = proxy_url.hostname + proxy_url.port + proxy_url.hostname default_backend_port = urlparse(target).port launch_backend(default_backend_port, "ws") diff --git a/tests/test_deprecations.py b/tests/test_deprecations.py index 1721125b..07c96cfe 100644 --- a/tests/test_deprecations.py +++ b/tests/test_deprecations.py @@ -1,8 +1,9 @@ from traitlets.config import Config -from jupyterhub_traefik_proxy.toml import TraefikTomlProxy -from jupyterhub_traefik_proxy.etcd import TraefikEtcdProxy from jupyterhub_traefik_proxy.consul import TraefikConsulProxy +from jupyterhub_traefik_proxy.etcd import TraefikEtcdProxy +from jupyterhub_traefik_proxy.toml import TraefikTomlProxy + def test_toml_deprecation(caplog): cfg = Config() @@ -13,13 +14,11 @@ def test_toml_deprecation(caplog): assert p.dynamic_config_file == 'deprecated-dynamic.toml' - log = '\n'.join([ - record.msg - for record in caplog.records - ]) + log = '\n'.join([record.msg for record in caplog.records]) assert 'TraefikFileProvider.dynamic_config_file instead' in log assert 'static_config_file instead' in log + def test_etcd_deprecation(caplog): cfg = Config() cfg.TraefikEtcdProxy.kv_url = "http://1.2.3.4:12345" @@ -27,14 +26,11 @@ def test_etcd_deprecation(caplog): cfg.TraefikEtcdProxy.kv_password = "pass" p = TraefikEtcdProxy(config=cfg) - assert p.etcd_url=="http://1.2.3.4:12345" + assert p.etcd_url == "http://1.2.3.4:12345" assert p.etcd_username == "user" assert p.etcd_password == "pass" - log = '\n'.join([ - record.msg - for record in caplog.records - ]) + log = '\n'.join([record.msg for record in caplog.records]) assert 'TraefikEtcdProxy.etcd_url instead' in log assert 'TraefikEtcdProxy.etcd_username instead' in log assert 'TraefikEtcdProxy.etcd_password instead' in log @@ -47,14 +43,11 @@ def test_consul_deprecation(caplog): cfg.TraefikConsulProxy.kv_password = "pass" p = TraefikConsulProxy(config=cfg) - assert p.consul_url=="http://1.2.3.4:12345" + assert p.consul_url == "http://1.2.3.4:12345" assert p.consul_username == "user" assert p.consul_password == "pass" - log = '\n'.join([ - record.msg - for record in caplog.records - ]) + log = '\n'.join([record.msg for record in caplog.records]) assert 'TraefikConsulProxy.consul_url instead' in log assert 'TraefikConsulProxy.consul_username instead' in log assert 'TraefikConsulProxy.consul_password instead' in log diff --git a/tests/test_installer.py b/tests/test_installer.py index 6bc05360..365480f3 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -1,13 +1,15 @@ -import pytest -import sys -import subprocess import os +import subprocess +import sys + +import pytest installer_module = "jupyterhub_traefik_proxy.install" # Mark all tests in this file as slow pytestmark = pytest.mark.slow + def assert_only_traefik_existence(deps_dir): assert deps_dir.exists() assert os.listdir(deps_dir) == ["traefik"] diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 11d712b1..e341cea0 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -2,8 +2,6 @@ import pytest -from proxytest import * - # Mark all tests in this file as asyncio and slow pytestmark = [pytest.mark.asyncio, pytest.mark.slow] diff --git a/tests/test_traefik_api_auth.py b/tests/test_traefik_api_auth.py index e82a8021..bd2e5519 100644 --- a/tests/test_traefik_api_auth.py +++ b/tests/test_traefik_api_auth.py @@ -1,6 +1,5 @@ """Tests for the authentication to the traefik proxy api (dashboard)""" import pytest - from jupyterhub.utils import exponential_backoff from tornado.httpclient import AsyncHTTPClient @@ -57,9 +56,7 @@ async def cmp_api_login(): else: return False - await exponential_backoff( - cmp_api_login, "Traefik API not reacheable" - ) + await exponential_backoff(cmp_api_login, "Traefik API not reacheable") rc = await api_login() assert rc == expected_rc diff --git a/tests/test_traefik_utils.py b/tests/test_traefik_utils.py index 2fc2c4e6..8a9c9027 100644 --- a/tests/test_traefik_utils.py +++ b/tests/test_traefik_utils.py @@ -1,11 +1,14 @@ -import pytest import json import os + +import pytest + from jupyterhub_traefik_proxy import traefik_utils + # Mark all tests in this file as asyncio def test_roundtrip_routes(): - pytestmark = pytest.mark.asyncio + pytest.mark.asyncio routes = { "backends": { "backend1": { diff --git a/tests/utils.py b/tests/utils.py index 540ec8e7..02650ec6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,8 @@ -import socket import json +import socket from urllib.parse import urlparse -from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPClientError +from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest _ports = {"default_backend": 9000, "first_backend": 9090, "second_backend": 9099} @@ -43,7 +43,7 @@ async def check_host_up_http(url): return False req = HTTPRequest(url, validate_cert=False) try: - resp = await AsyncHTTPClient().fetch(req) + await AsyncHTTPClient().fetch(req) except HTTPClientError as e: if e.code >= 599: # connection error From bf0d31499c408f67e4387e018abd8140eb871669 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 23 Feb 2023 09:13:32 +0100 Subject: [PATCH 5/6] enable lint in config files `c = get_config() # noqa` fixes lint errors --- examples/jupyterhub_config_etcd.py | 2 ++ examples/jupyterhub_config_toml.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/jupyterhub_config_etcd.py b/examples/jupyterhub_config_etcd.py index 42494ae0..e28f6395 100644 --- a/examples/jupyterhub_config_etcd.py +++ b/examples/jupyterhub_config_etcd.py @@ -8,6 +8,8 @@ requires jupyterhub 1.0.dev """ +c = get_config() # noqa + c.JupyterHub.proxy_class = "traefik_etcd" c.TraefikEtcdProxy.traefik_api_username = "admin" c.TraefikEtcdProxy.traefik_api_password = "admin" diff --git a/examples/jupyterhub_config_toml.py b/examples/jupyterhub_config_toml.py index 927ea54d..30207bcf 100644 --- a/examples/jupyterhub_config_toml.py +++ b/examples/jupyterhub_config_toml.py @@ -8,6 +8,8 @@ requires jupyterhub 1.0.dev """ +c = get_config() # noqa + c.JupyterHub.proxy_class = "traefik_file" c.TraefikFileProviderProxy.traefik_api_username = "admin" c.TraefikFileProviderProxy.traefik_api_password = "admin" From f45dc97121a725e48bc38eece98aca1fd278d1a3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 23 Feb 2023 09:16:34 +0100 Subject: [PATCH 6/6] note switch to pre-commit in CONTRIBUTING --- CONTRIBUTING.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 942d3ce4..d32757a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Make sure to also follow [Project Jupyter's Code of Conduct](https://github.com/ ## Setting up for local development -This package requires Python >= 3.5. +This package requires Python >= 3.6. As a Python package, you can set up a development environment by cloning this repo and running: @@ -18,23 +18,18 @@ You can also install the tools we use for testing and development with: python3 -m pip install -r dev-requirements.txt -### Auto-format with black +### Auto-format with pre-commit -We are trying out the [black](https://github.com/ambv/black) auto-formatting -tool on this repo. +We use the [pre-commit](https://pre-commit.com) tool for autoformatting. -You can run `black` manually on the repo with: +You can install and enable it with: - black . - -in the root of the repo. You can also enable this automatically on each commit -by installing a pre-commit hook: - - ./git-hooks/install + pip install pre-commit + pre-commit install After doing this, every time you make a commit, -the `black` autoformatter will run, -ensuring consistent style without you having to worry too much about style. +`pre-commit` will run autoformatting. +If it makes any changes, it'll let you know and you can make the commit again with the autoformatting changes. ## Running the tests