From 6acb0833101c0bdcbab3b2f7ac036948d1033b05 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Tue, 26 Nov 2024 14:31:18 +0100 Subject: [PATCH] updated INVALID config --- .build/Dockerfile | 45 +++++++++++++++++------------------- .build/docker-stacks | 2 +- .build/docker_healthcheck.py | 4 ++-- .build/start.sh | 14 +++++++---- custom/gpulibs.Dockerfile | 13 ++++++----- generate-Dockerfile.sh | 2 +- 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.build/Dockerfile b/.build/Dockerfile index dd07a47..f8fd1d0 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -41,7 +41,7 @@ USER root # Install all OS dependencies for the Server that starts # but lacks all features (e.g., download as all possible file formats) -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --yes && \ # - `apt-get upgrade` is run to patch known vulnerabilities in system packages # as the Ubuntu base image is rebuilt too seldom sometimes (less than once a month) @@ -51,6 +51,10 @@ RUN apt-get update --yes && \ bzip2 \ ca-certificates \ locales \ + # - `netbase` provides /etc/{protocols,rpc,services}, part of POSIX + # and required by various C functions like getservbyname and getprotobyname + # https://github.com/jupyter/docker-stacks/pull/2129 + netbase \ sudo \ # - `tini` is installed as a helpful container entrypoint, # that reaps zombie processes and such of the actual executable we want to start @@ -85,7 +89,7 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html echo 'eval "$(conda shell.bash hook)"' >> /etc/skel/.bashrc -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group +# Create "${NB_USER}" user (`jovyan` by default) with UID="${NB_UID}" (`1000` by default) and in the 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ @@ -134,9 +138,9 @@ RUN set -x && \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ --yes \ - "${PYTHON_SPECIFIER}" \ + 'jupyter_core' \ 'mamba' \ - 'jupyter_core' && \ + "${PYTHON_SPECIFIER}" && \ rm -rf /tmp/bin/ && \ # Pin major.minor version of python # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning @@ -199,7 +203,7 @@ RUN apt-get update --yes && \ USER ${NB_UID} -# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Install JupyterHub, JupyterLab, NBClassic and Jupyter Notebook # Generate a Jupyter Server config # Cleanup temporary files # Correct permissions @@ -207,10 +211,10 @@ USER ${NB_UID} # files across image layers when the permissions change WORKDIR /tmp RUN mamba install --yes \ - 'jupyterlab' \ - 'notebook' \ 'jupyterhub' \ - 'nbclassic' && \ + 'jupyterlab' \ + 'nbclassic' \ + 'notebook' && \ jupyter server --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ @@ -339,7 +343,7 @@ RUN mamba install --yes \ 'dask' \ 'dill' \ 'h5py' \ - 'ipympl'\ + 'ipympl' \ 'ipywidgets' \ 'jupyterlab-git' \ 'matplotlib-base' \ @@ -357,7 +361,7 @@ RUN mamba install --yes \ 'sqlalchemy' \ 'statsmodels' \ 'sympy' \ - 'widgetsnbextension'\ + 'widgetsnbextension' \ 'xlrd' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ @@ -399,7 +403,7 @@ RUN mamba install --quiet --yes \ # using device_lib.list_local_devices() the cudNN version is shown, adapt version to tested compat USER ${NB_UID} RUN pip install --upgrade pip && \ - pip install --no-cache-dir tensorflow==2.16.1 keras==3.1.1 && \ + pip install --no-cache-dir tensorflow[and-cuda]==2.17.0 keras==3.6.0 && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -410,11 +414,11 @@ RUN pip install --upgrade pip && \ # && torchviz==0.0.2 --extra-index-url https://download.pytorch.org/whl/cu121 RUN set -ex \ && buildDeps=' \ - torch==2.2.2 \ - torchvision==0.17.2 \ - torchaudio==2.2.2 \ + torch==2.5.1 \ + torchvision==0.20.1 \ + torchaudio==2.5.1 \ ' \ - && pip install --no-cache-dir $buildDeps --extra-index-url https://download.pytorch.org/whl/cu121 \ + && pip install --no-cache-dir $buildDeps --extra-index-url https://download.pytorch.org/whl/cu124 \ && fix-permissions "${CONDA_DIR}" \ && fix-permissions "/home/${NB_USER}" @@ -426,14 +430,6 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends cmake libncurses5-dev libncursesw5-dev git && \ apt-get clean && rm -rf /var/lib/apt/lists/* -USER $NB_UID -# These need to be two separate pip install commands, otherwise it will throw an error -# attempting to resolve the nvidia-cuda-nvcc package at the same time as nvidia-pyindex -RUN pip install --no-cache-dir nvidia-pyindex && \ - pip install --no-cache-dir nvidia-cuda-nvcc && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - # reinstall nvcc with cuda-nvcc to install ptax USER $NB_UID # These need to be two separate pip install commands, otherwise it will throw an error @@ -443,7 +439,8 @@ RUN pip install --no-cache-dir nvidia-pyindex && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install cuda-nvcc with sepecific version, see here: https://anaconda.org/nvidia/cuda-nvcc/labels +# Install cuda-nvcc with sepecific version, see here: +# https://anaconda.org/nvidia/cuda-nvcc/labels RUN mamba install -c nvidia cuda-nvcc=12.3.107 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ diff --git a/.build/docker-stacks b/.build/docker-stacks index e838ff3..0098788 160000 --- a/.build/docker-stacks +++ b/.build/docker-stacks @@ -1 +1 @@ -Subproject commit e838ff397a2d9c2ad0faae051ef0ec4f20732320 +Subproject commit 00987883e58d139b5ed01f803f95e639c59bf340 diff --git a/.build/docker_healthcheck.py b/.build/docker_healthcheck.py index 7dd3de0..b0db1a8 100755 --- a/.build/docker_healthcheck.py +++ b/.build/docker_healthcheck.py @@ -9,9 +9,9 @@ import requests # Several operations below deliberately don't check for possible errors -# As this is a healthcheck, it should succeed or raise an exception on error +# As this is a health check, it should succeed or raise an exception on error -# Docker runs healtchecks using an exec +# Docker runs health checks using an exec # It uses the default user configured when running the image: root for the case of a custom NB_USER or jovyan for the case of the default image user. # We manually change HOME to make `jupyter --runtime-dir` report a correct path # More information: diff --git a/.build/start.sh b/.build/start.sh index 33d12d8..295ee26 100755 --- a/.build/start.sh +++ b/.build/start.sh @@ -155,11 +155,14 @@ if [ "$(id -u)" == 0 ]; then unset_explicit_env_vars _log "Running as ${NB_USER}:" "${cmd[@]}" - exec sudo --preserve-env --set-home --user "${NB_USER}" \ - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" \ - PATH="${PATH}" \ - PYTHONPATH="${PYTHONPATH:-}" \ - "${cmd[@]}" + if [ "${NB_USER}" = "root" ] && [ "${NB_UID}" = "$(id -u "${NB_USER}")" ] && [ "${NB_GID}" = "$(id -g "${NB_USER}")" ]; then + HOME="/home/root" exec "${cmd[@]}" + else + exec sudo --preserve-env --set-home --user "${NB_USER}" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" \ + PATH="${PATH}" \ + PYTHONPATH="${PYTHONPATH:-}" \ + "${cmd[@]}" # Notes on how we ensure that the environment that this container is started # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when # we transition from running as root to running as NB_USER. @@ -187,6 +190,7 @@ if [ "$(id -u)" == 0 ]; then # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant # for resolving paths of any subprocesses spawned by `${cmd[@]}`. + fi # The container didn't start as the root user, so we will have to act as the # user we started as. diff --git a/custom/gpulibs.Dockerfile b/custom/gpulibs.Dockerfile index e789019..371ffbb 100644 --- a/custom/gpulibs.Dockerfile +++ b/custom/gpulibs.Dockerfile @@ -14,7 +14,7 @@ RUN mamba install --quiet --yes \ # using device_lib.list_local_devices() the cudNN version is shown, adapt version to tested compat USER ${NB_UID} RUN pip install --upgrade pip && \ - pip install --no-cache-dir tensorflow==2.16.1 keras==3.1.1 && \ + pip install --no-cache-dir tensorflow[and-cuda]==2.17.0 keras==3.6.0 && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -25,11 +25,11 @@ RUN pip install --upgrade pip && \ # && torchviz==0.0.2 --extra-index-url https://download.pytorch.org/whl/cu121 RUN set -ex \ && buildDeps=' \ - torch==2.2.2 \ - torchvision==0.17.2 \ - torchaudio==2.2.2 \ + torch==2.5.1 \ + torchvision==0.20.1 \ + torchaudio==2.5.1 \ ' \ - && pip install --no-cache-dir $buildDeps --extra-index-url https://download.pytorch.org/whl/cu121 \ + && pip install --no-cache-dir $buildDeps --extra-index-url https://download.pytorch.org/whl/cu124 \ && fix-permissions "${CONDA_DIR}" \ && fix-permissions "/home/${NB_USER}" @@ -50,7 +50,8 @@ RUN pip install --no-cache-dir nvidia-pyindex && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install cuda-nvcc with sepecific version, see here: https://anaconda.org/nvidia/cuda-nvcc/labels +# Install cuda-nvcc with sepecific version, see here: +# https://anaconda.org/nvidia/cuda-nvcc/labels RUN mamba install -c nvidia cuda-nvcc=12.3.107 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index a4ef0d6..4d77962 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -5,7 +5,7 @@ cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) export DOCKERFILE=".build/Dockerfile" export STACKS_DIR=".build/docker-stacks" # please test the build of the commit in https://github.com/jupyter/docker-stacks/commits/main in advance -export HEAD_COMMIT="e838ff397a2d9c2ad0faae051ef0ec4f20732320" +export HEAD_COMMIT="00987883e58d139b5ed01f803f95e639c59bf340" while [[ "$#" -gt 0 ]]; do case $1 in -p|--pw|--password) PASSWORD="$2" && USE_PASSWORD=1; shift;;