From 69e783642bec9a2949553284bb66e7325f77c773 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Mon, 10 Jun 2019 11:39:53 -0700 Subject: [PATCH 01/19] experimental --- nvidia/Dockerfile | 69 ++++++++++++++++++++++++++++++++++ nvidia/rstudio.sh | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 nvidia/Dockerfile create mode 100644 nvidia/rstudio.sh diff --git a/nvidia/Dockerfile b/nvidia/Dockerfile new file mode 100644 index 0000000..3cf7505 --- /dev/null +++ b/nvidia/Dockerfile @@ -0,0 +1,69 @@ +FROM nvidia/cuda:10.1-cudnn7-runtime + +ARG DEBIAN_FRONTEND=noninteractive + +RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ + apt-get update && apt-get -y install r-base + + +####### Set up env variables in R ####### +ENV R_HOME=/usr/lib/R +ENV CUDA_HOME=/usr/local/cuda +ENV CUDA_PATH=/usr/local/cuda +ENV PATH=$CUDA_HOME/bin:$PATH +ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$LD_LIBRARY_PATH + +#RUN echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf \ +RUN echo "\n\ + \nCUDA_HOME=$CUDA_HOME \ + \nCUDA_PATH=$CUDA_PATH \ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron + +######### Set up Python3 Virtual Environment to play nice with R ### +#ENV WORKON_HOME /opt/virtualenvs +#ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow + +## Set up a user modifyable python3 environment +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpython3-dev \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +#RUN python3 -m venv ${PYTHON_VENV_PATH} + +#ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} +## And set ENV for R! It doesn't read from the environment... +#RUN echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ +# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ +# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron + + +## reticulate will want virtualenv lib installed as well +#USER rstudio +RUN pip3 install --no-cache-dir virtualenv +#USER root + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip + +RUN R -e "install.packages(c('keras'))" +RUN R -e "install.packages('remotes')" +RUN R -e "remotes::install_github('greta-dev/greta')" + + +## Because reticulate hardwires these PATHs... +#RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ +# ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv + +# RUN chown -R rstudio:rstudio ${WORKON_HOME} + diff --git a/nvidia/rstudio.sh b/nvidia/rstudio.sh new file mode 100644 index 0000000..3a88605 --- /dev/null +++ b/nvidia/rstudio.sh @@ -0,0 +1,94 @@ +S6_VERSION=${S6_VERSION:-v1.21.7.0} +S6_BEHAVIOUR_IF_STAGE2_FAILS=2 +export PATH=/usr/lib/rstudio-server/bin:$PATH +PANDOC_TEMPLATES_VERSION=${PANDOC_TEMPLATES_VERSION:-2.6} + +## Download and install RStudio server & dependencies +## Attempts to get detect latest version, otherwise falls back to version given in $VER +## Symlink pandoc, pandoc-citeproc so they are available system-wide +apt-get update \ + && apt-get install -y --no-install-recommends \ + file \ + git \ + libapparmor1 \ + libcurl4-openssl-dev \ + libedit2 \ + libssl-dev \ + lsb-release \ + psmisc \ + procps \ + python-setuptools \ + sudo \ + wget \ + libclang-dev \ + libclang-3.8-dev \ + libobjc-6-dev \ + libclang1-3.8 \ + libclang-common-3.8-dev \ + libllvm3.8 \ + libobjc4 \ + libgc1c2 \ + && if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/debian9_64/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/debian9/x86_64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ + && wget -q $RSTUDIO_URL \ + && dpkg -i rstudio-server-*-amd64.deb \ + && rm rstudio-server-*-amd64.deb \ + ## Symlink pandoc & standard pandoc templates for use system-wide + && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ + && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc-citeproc /usr/local/bin \ + && git clone --recursive --branch ${PANDOC_TEMPLATES_VERSION} https://github.com/jgm/pandoc-templates \ + && mkdir -p /opt/pandoc/templates \ + && cp -r pandoc-templates*/* /opt/pandoc/templates && rm -rf pandoc-templates* \ + && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/ \ + ## RStudio wants an /etc/R, will populate from $R_HOME/etc + && mkdir -p /etc/R \ + ## Write config files in $R_HOME/etc + && echo '\n\ + \n# Configure httr to perform out-of-band authentication if HTTR_LOCALHOST \ + \n# is not set since a redirect to localhost may not work depending upon \ + \n# where this Docker container is running. \ + \nif(is.na(Sys.getenv("HTTR_LOCALHOST", unset=NA))) { \ + \n options(httr_oob_default = TRUE) \ + \n}' >> /usr/local/lib/R/etc/Rprofile.site \ + && echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron \ + ## Need to configure non-root user for RStudio + && useradd rstudio \ + && echo "rstudio:rstudio" | chpasswd \ + && mkdir /home/rstudio \ + && chown rstudio:rstudio /home/rstudio \ + && addgroup rstudio staff \ + ## Prevent rstudio from deciding to use /usr/bin/R if a user apt-get installs a package + && echo 'rsession-which-r=/usr/local/bin/R' >> /etc/rstudio/rserver.conf \ + ## use more robust file locking to avoid errors when using shared volumes: + && echo 'lock-type=advisory' >> /etc/rstudio/file-locks \ + ## configure git not to request password each time + && git config --system credential.helper 'cache --timeout=3600' \ + && git config --system push.default simple \ + ## Set up S6 init system + && wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ + && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ + && mkdir -p /etc/services.d/rstudio \ + && echo '#!/usr/bin/with-contenv bash \ + \n## load /etc/environment vars first: \ + \n for line in $( cat /etc/environment ) ; do export $line ; done \ + \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0' \ + > /etc/services.d/rstudio/run \ + && echo '#!/bin/bash \ + \n rstudio-server stop' \ + > /etc/services.d/rstudio/finish \ + && mkdir -p /home/rstudio/.rstudio/monitored/user-settings \ + && echo 'alwaysSaveHistory="0" \ + \nloadRData="0" \ + \nsaveAction="0"' \ + > /home/rstudio/.rstudio/monitored/user-settings/user-settings \ + && chown -R rstudio:rstudio /home/rstudio/.rstudio + + + +#COPY userconf.sh /etc/cont-init.d/userconf +#COPY disable_auth_rserver.conf /etc/rstudio/disable_auth_rserver.conf +#COPY pam-helper.sh /usr/lib/rstudio-server/bin/pam-helper + +# EXPOSE 8787 +# CMD ["/init"] From 26bd89be9b22868bfc3072b8614e65fad5008ba5 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Mon, 10 Jun 2019 11:54:39 -0700 Subject: [PATCH 02/19] reference case --- nvidia/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nvidia/Dockerfile b/nvidia/Dockerfile index 3cf7505..7ecb529 100644 --- a/nvidia/Dockerfile +++ b/nvidia/Dockerfile @@ -22,8 +22,8 @@ RUN echo "\n\ \nPATH=$PATH" >> ${R_HOME}/etc/Renviron ######### Set up Python3 Virtual Environment to play nice with R ### -#ENV WORKON_HOME /opt/virtualenvs -#ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow +ENV WORKON_HOME /opt/virtualenvs +ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow ## Set up a user modifyable python3 environment RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -32,9 +32,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-venv && \ rm -rf /var/lib/apt/lists/* -#RUN python3 -m venv ${PYTHON_VENV_PATH} +RUN python3 -m venv ${PYTHON_VENV_PATH} -#ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} +ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} ## And set ENV for R! It doesn't read from the environment... #RUN echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ # echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ @@ -60,6 +60,7 @@ RUN R -e "install.packages(c('keras'))" RUN R -e "install.packages('remotes')" RUN R -e "remotes::install_github('greta-dev/greta')" +RUN pip3 install --no-cache-dir tensorflow-gpu tensorflow-probability keras h5py pyyaml requests Pillow ## Because reticulate hardwires these PATHs... #RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ From f375b27bb7e7476e47a9799a4588aa9a829aaa45 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Mon, 10 Jun 2019 12:17:47 -0700 Subject: [PATCH 03/19] symlink virtualenv so py_install works --- nvidia/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nvidia/Dockerfile b/nvidia/Dockerfile index 7ecb529..da5178a 100644 --- a/nvidia/Dockerfile +++ b/nvidia/Dockerfile @@ -63,8 +63,8 @@ RUN R -e "remotes::install_github('greta-dev/greta')" RUN pip3 install --no-cache-dir tensorflow-gpu tensorflow-probability keras h5py pyyaml requests Pillow ## Because reticulate hardwires these PATHs... -#RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ -# ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv +RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ + ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv # RUN chown -R rstudio:rstudio ${WORKON_HOME} From 2830b6ce5e0878ccb46f1b08670efdb1b6d37b5a Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Tue, 11 Jun 2019 11:46:58 -0700 Subject: [PATCH 04/19] separate 9.0 and 10.0 versions --- nvidia/{ => 10.0}/Dockerfile | 2 +- nvidia/9.0/Dockerfile | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) rename nvidia/{ => 10.0}/Dockerfile (98%) create mode 100644 nvidia/9.0/Dockerfile diff --git a/nvidia/Dockerfile b/nvidia/10.0/Dockerfile similarity index 98% rename from nvidia/Dockerfile rename to nvidia/10.0/Dockerfile index da5178a..379a8dc 100644 --- a/nvidia/Dockerfile +++ b/nvidia/10.0/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:10.1-cudnn7-runtime +FROM nvidia/cuda:10.0-cudnn7-devel ARG DEBIAN_FRONTEND=noninteractive diff --git a/nvidia/9.0/Dockerfile b/nvidia/9.0/Dockerfile new file mode 100644 index 0000000..b8a01ba --- /dev/null +++ b/nvidia/9.0/Dockerfile @@ -0,0 +1,70 @@ +FROM nvidia/cuda:9.0-cudnn7-devel + +ARG DEBIAN_FRONTEND=noninteractive + +RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu xenial-cran35/' >> /etc/apt/sources.list && \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ + apt-get update && apt-get -y install r-base + + +####### Set up env variables in R ####### +ENV R_HOME=/usr/lib/R +ENV CUDA_HOME=/usr/local/cuda +ENV CUDA_PATH=/usr/local/cuda +ENV PATH=$CUDA_HOME/bin:$PATH +ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$LD_LIBRARY_PATH + +#RUN echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf \ +RUN echo "\n\ + \nCUDA_HOME=$CUDA_HOME \ + \nCUDA_PATH=$CUDA_PATH \ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron + +######### Set up Python3 Virtual Environment to play nice with R ### +ENV WORKON_HOME /opt/virtualenvs +ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow + +## Set up a user modifyable python3 environment +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpython3-dev \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv ${PYTHON_VENV_PATH} + +ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} +## And set ENV for R! It doesn't read from the environment... +#RUN echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ +# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ +# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron + + +## reticulate will want virtualenv lib installed as well +#USER rstudio +RUN pip3 install --no-cache-dir virtualenv +#USER root + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip + +RUN R -e "install.packages(c('keras'))" +RUN R -e "install.packages('remotes')" +RUN R -e "remotes::install_github('greta-dev/greta')" + +RUN pip3 install --no-cache-dir tensorflow-gpu==1.12.0 tensorflow-probability==0.5.0 + +## Because reticulate hardwires these PATHs... +RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ + ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv + +# RUN chown -R rstudio:rstudio ${WORKON_HOME} + From 3fdea82b2468d2bd35cd6f92ffb1bf84c23288fc Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Wed, 12 Jun 2019 21:31:31 -0700 Subject: [PATCH 05/19] refactor attempt --- nvidia/10.0/Dockerfile | 75 ++++++++----------------- nvidia/10.0/R.sh | 9 +++ nvidia/10.0/python.sh | 22 ++++++++ nvidia/{ => 10.0}/rstudio.sh | 77 +++++++++++++------------ nvidia/10.0/tensorflow-gpu.sh | 27 +++++++++ nvidia/10.0/userconf.sh | 103 ++++++++++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 86 deletions(-) create mode 100644 nvidia/10.0/R.sh create mode 100644 nvidia/10.0/python.sh rename nvidia/{ => 10.0}/rstudio.sh (57%) create mode 100644 nvidia/10.0/tensorflow-gpu.sh create mode 100644 nvidia/10.0/userconf.sh diff --git a/nvidia/10.0/Dockerfile b/nvidia/10.0/Dockerfile index 379a8dc..49f5d31 100644 --- a/nvidia/10.0/Dockerfile +++ b/nvidia/10.0/Dockerfile @@ -2,69 +2,42 @@ FROM nvidia/cuda:10.0-cudnn7-devel ARG DEBIAN_FRONTEND=noninteractive -RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \ - gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ - apt-get update && apt-get -y install r-base - -####### Set up env variables in R ####### ENV R_HOME=/usr/lib/R +COPY R.sh /tmp/install_R.sh +RUN . /tmp/install_R.sh + + + +COPY rstudio.sh /tmp/install_rstudio.sh +RUN . /tmp/install_rstudio.sh +COPY userconf.sh /etc/cont-init.d/userconf +EXPOSE 8787 +CMD ["/init"] + + + ENV CUDA_HOME=/usr/local/cuda ENV CUDA_PATH=/usr/local/cuda ENV PATH=$CUDA_HOME/bin:$PATH ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -#RUN echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf \ RUN echo "\n\ - \nCUDA_HOME=$CUDA_HOME \ - \nCUDA_PATH=$CUDA_PATH \ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron + \nCUDA_HOME=$CUDA_HOME \ + \nCUDA_PATH=$CUDA_PATH \ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron +RUN if [ -f /etc/rstudio ]; then \ + echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf; fi + -######### Set up Python3 Virtual Environment to play nice with R ### ENV WORKON_HOME /opt/virtualenvs ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow - -## Set up a user modifyable python3 environment -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpython3-dev \ - python3-pip \ - python3-venv && \ - rm -rf /var/lib/apt/lists/* - -RUN python3 -m venv ${PYTHON_VENV_PATH} - +#ENV PYTHON_VENV_PATH /opt/venv ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} -## And set ENV for R! It doesn't read from the environment... -#RUN echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ -# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ -# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron - - -## reticulate will want virtualenv lib installed as well -#USER rstudio -RUN pip3 install --no-cache-dir virtualenv -#USER root - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - pkg-config \ - software-properties-common \ - unzip - -RUN R -e "install.packages(c('keras'))" -RUN R -e "install.packages('remotes')" -RUN R -e "remotes::install_github('greta-dev/greta')" +COPY python.sh /tmp/install_python.sh +RUN . /tmp/install_python.sh -RUN pip3 install --no-cache-dir tensorflow-gpu tensorflow-probability keras h5py pyyaml requests Pillow -## Because reticulate hardwires these PATHs... -RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ - ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv -# RUN chown -R rstudio:rstudio ${WORKON_HOME} +COPY tensorflow-gpu.sh /tmp/install_tensorflow-gpu.sh +RUN . /tmp/install_tensorflow-gpu.sh diff --git a/nvidia/10.0/R.sh b/nvidia/10.0/R.sh new file mode 100644 index 0000000..08c4428 --- /dev/null +++ b/nvidia/10.0/R.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ + apt-get update && apt-get -y install r-base && \ + rm -rf /var/lib/apt/lists/* + + diff --git a/nvidia/10.0/python.sh b/nvidia/10.0/python.sh new file mode 100644 index 0000000..b655162 --- /dev/null +++ b/nvidia/10.0/python.sh @@ -0,0 +1,22 @@ +#/bin/sh + +## Python3 +apt-get update && apt-get install -y --no-install-recommends \ + libpython3-dev \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +python3 -m venv ${PYTHON_VENV_PATH} + +## And set ENV for R; it doesn't read from the environment... +# echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ +# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ +# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron + +## Reticulate needs this too? +pip3 install --no-cache-dir virtualenv + + + + diff --git a/nvidia/rstudio.sh b/nvidia/10.0/rstudio.sh similarity index 57% rename from nvidia/rstudio.sh rename to nvidia/10.0/rstudio.sh index 3a88605..52c7dae 100644 --- a/nvidia/rstudio.sh +++ b/nvidia/10.0/rstudio.sh @@ -1,3 +1,5 @@ +#!/bin/sh + S6_VERSION=${S6_VERSION:-v1.21.7.0} S6_BEHAVIOUR_IF_STAGE2_FAILS=2 export PATH=/usr/lib/rstudio-server/bin:$PATH @@ -21,52 +23,60 @@ apt-get update \ sudo \ wget \ libclang-dev \ - libclang-3.8-dev \ libobjc-6-dev \ - libclang1-3.8 \ - libclang-common-3.8-dev \ - libllvm3.8 \ libobjc4 \ - libgc1c2 \ - && if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/debian9_64/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/debian9/x86_64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ + libgc1c2 +apt-get clean && rm -rf /var/lib/apt/lists/ + +if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/debian9_64/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/debian9/x86_64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ && wget -q $RSTUDIO_URL \ && dpkg -i rstudio-server-*-amd64.deb \ - && rm rstudio-server-*-amd64.deb \ - ## Symlink pandoc & standard pandoc templates for use system-wide - && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ + && rm rstudio-server-*-amd64.deb + +## Symlink pandoc & standard pandoc templates for use system-wide +ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc-citeproc /usr/local/bin \ && git clone --recursive --branch ${PANDOC_TEMPLATES_VERSION} https://github.com/jgm/pandoc-templates \ && mkdir -p /opt/pandoc/templates \ && cp -r pandoc-templates*/* /opt/pandoc/templates && rm -rf pandoc-templates* \ - && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/ \ - ## RStudio wants an /etc/R, will populate from $R_HOME/etc - && mkdir -p /etc/R \ - ## Write config files in $R_HOME/etc - && echo '\n\ + && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates + + +## RStudio wants an /etc/R, will populate from $R_HOME/etc +mkdir -p /etc/R \ + && echo "\n\ \n# Configure httr to perform out-of-band authentication if HTTR_LOCALHOST \ \n# is not set since a redirect to localhost may not work depending upon \ \n# where this Docker container is running. \ - \nif(is.na(Sys.getenv("HTTR_LOCALHOST", unset=NA))) { \ + \nif(is.na(Sys.getenv('HTTR_LOCALHOST', unset=NA))) { \ \n options(httr_oob_default = TRUE) \ - \n}' >> /usr/local/lib/R/etc/Rprofile.site \ - && echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron \ - ## Need to configure non-root user for RStudio - && useradd rstudio \ + \n}" >> ${R_HOME}/etc/Rprofile.site \ + && echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + + +## Need to configure non-root user for RStudio +useradd rstudio \ && echo "rstudio:rstudio" | chpasswd \ - && mkdir /home/rstudio \ - && chown rstudio:rstudio /home/rstudio \ - && addgroup rstudio staff \ + && mkdir /home/rstudio \ + && chown rstudio:rstudio /home/rstudio \ + && addgroup rstudio staff ## Prevent rstudio from deciding to use /usr/bin/R if a user apt-get installs a package - && echo 'rsession-which-r=/usr/local/bin/R' >> /etc/rstudio/rserver.conf \ - ## use more robust file locking to avoid errors when using shared volumes: - && echo 'lock-type=advisory' >> /etc/rstudio/file-locks \ - ## configure git not to request password each time - && git config --system credential.helper 'cache --timeout=3600' \ - && git config --system push.default simple \ - ## Set up S6 init system - && wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ + +R_BIN=`which R` +echo "rsession-which-r=${R_BIN}" >> /etc/rstudio/rserver.conf +## use more robust file locking to avoid errors when using shared volumes: +echo 'lock-type=advisory' >> /etc/rstudio/file-locks + +## Optional configuration file to disable authentication +cp /etc/rstudio/rserver.conf /etc/rstudio/disable_auth_rserver.conf +echo "auth-none=1" >> /etc/rstudio/disable_auth_rserver.conf + +## configure git not to request password each time +git config --system credential.helper 'cache --timeout=3600' \ + && git config --system push.default simple + +## Set up S6 init system +wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ && mkdir -p /etc/services.d/rstudio \ && echo '#!/usr/bin/with-contenv bash \ @@ -87,8 +97,5 @@ apt-get update \ #COPY userconf.sh /etc/cont-init.d/userconf -#COPY disable_auth_rserver.conf /etc/rstudio/disable_auth_rserver.conf -#COPY pam-helper.sh /usr/lib/rstudio-server/bin/pam-helper - # EXPOSE 8787 # CMD ["/init"] diff --git a/nvidia/10.0/tensorflow-gpu.sh b/nvidia/10.0/tensorflow-gpu.sh new file mode 100644 index 0000000..10a1082 --- /dev/null +++ b/nvidia/10.0/tensorflow-gpu.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip + + +## symlink these because reticulate hardwires these PATHs... +ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip +ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv + +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} +echo $PYTHON_VENV_PATH + +## FIXME set versions Consider renv +#R -e "renv::restore()" +pip3 install --no-cache-dir tensorflow-gpu tensorflow-probability keras h5py pyyaml requests Pillow + +R -e "install.packages('keras')" +R -e "install.packages('remotes'); remotes::install_github('greta-dev/greta')" + diff --git a/nvidia/10.0/userconf.sh b/nvidia/10.0/userconf.sh new file mode 100644 index 0000000..c0751c7 --- /dev/null +++ b/nvidia/10.0/userconf.sh @@ -0,0 +1,103 @@ +#!/usr/bin/with-contenv bash + +## Set defaults for environmental variables in case they are undefined +USER=${USER:=rstudio} +PASSWORD=${PASSWORD:=rstudio} +USERID=${USERID:=1000} +GROUPID=${GROUPID:=1000} +ROOT=${ROOT:=FALSE} +UMASK=${UMASK:=022} + +## Make sure RStudio inherits the full path +echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron + +bold=$(tput bold) +normal=$(tput sgr0) + + +if [[ ${DISABLE_AUTH,,} == "true" ]] +then + mv /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf + echo "USER=$USER" >> /etc/environment +fi + + + +if grep --quiet "auth-none=1" /etc/rstudio/rserver.conf +then + echo "Skipping authentication as requested" +elif [ "$PASSWORD" == "rstudio" ] +then + printf "\n\n" + tput bold + printf "\e[31mERROR\e[39m: You must set a unique PASSWORD (not 'rstudio') first! e.g. run with:\n" + printf "docker run -e PASSWORD=\e[92m\e[39m -p 8787:8787 rocker/rstudio\n" + tput sgr0 + printf "\n\n" + exit 1 +fi + +if [ "$USERID" -lt 1000 ] +# Probably a macOS user, https://github.com/rocker-org/rocker/issues/205 + then + echo "$USERID is less than 1000" + check_user_id=$(grep -F "auth-minimum-user-id" /etc/rstudio/rserver.conf) + if [[ ! -z $check_user_id ]] + then + echo "minumum authorised user already exists in /etc/rstudio/rserver.conf: $check_user_id" + else + echo "setting minumum authorised user to 499" + echo auth-minimum-user-id=499 >> /etc/rstudio/rserver.conf + fi +fi + +if [ "$USERID" -ne 1000 ] +## Configure user with a different USERID if requested. + then + echo "deleting user rstudio" + userdel rstudio + echo "creating new $USER with UID $USERID" + useradd -m $USER -u $USERID + mkdir /home/$USER + chown -R $USER /home/$USER + usermod -a -G staff $USER +elif [ "$USER" != "rstudio" ] + then + ## cannot move home folder when it's a shared volume, have to copy and change permissions instead + cp -r /home/rstudio /home/$USER + ## RENAME the user + usermod -l $USER -d /home/$USER rstudio + groupmod -n $USER rstudio + usermod -a -G staff $USER + chown -R $USER:$USER /home/$USER + echo "USER is now $USER" +fi + +if [ "$GROUPID" -ne 1000 ] +## Configure the primary GID (whether rstudio or $USER) with a different GROUPID if requested. + then + echo "Modifying primary group $(id $USER -g -n)" + groupmod -g $GROUPID $(id $USER -g -n) + echo "Primary group ID is now custom_group $GROUPID" +fi + +## Add a password to user +echo "$USER:$PASSWORD" | chpasswd + +# Use Env flag to know if user should be added to sudoers +if [[ ${ROOT,,} == "true" ]] + then + adduser $USER sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + echo "$USER added to sudoers" +fi + +## Change Umask value if desired +if [ "$UMASK" -ne 022 ] + then + echo "server-set-umask=false" >> /etc/rstudio/rserver.conf + echo "Sys.umask(mode=$UMASK)" >> /home/$USER/.Rprofile +fi + +## add these to the global environment so they are avialable to the RStudio user +echo "HTTR_LOCALHOST=$HTTR_LOCALHOST" >> /etc/R/Renviron.site +echo "HTTR_PORT=$HTTR_PORT" >> /etc/R/Renviron.site From 24a6f18b116eb882cde3deb89f3fa37250364b74 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Wed, 12 Jun 2019 21:48:54 -0700 Subject: [PATCH 06/19] draft version working --- nvidia/10.0/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nvidia/10.0/Dockerfile b/nvidia/10.0/Dockerfile index 49f5d31..2d756f9 100644 --- a/nvidia/10.0/Dockerfile +++ b/nvidia/10.0/Dockerfile @@ -29,10 +29,13 @@ RUN if [ -f /etc/rstudio ]; then \ echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf; fi -ENV WORKON_HOME /opt/virtualenvs -ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow -#ENV PYTHON_VENV_PATH /opt/venv -ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} +## Note: forcing PYTHON_RETICULATE_ENV may preclude defining alternative venvs +## Setting WORKON_HOME is more flexible +## ENV WORKON_HOME /opt/virtualenvs +## ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow +ENV PYTHON_VENV_PATH=/opt/venv +ENV RETICULATE_PYTHON_ENV=$PYTHON_VENV_PATH +ENV PATH=${PYTHON_VENV_PATH}/bin:${PATH} COPY python.sh /tmp/install_python.sh RUN . /tmp/install_python.sh From 522efb787302831220b77f091781433dfd171bee Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Wed, 12 Jun 2019 22:39:05 -0700 Subject: [PATCH 07/19] fix rstudio --- nvidia/10.0/Dockerfile | 3 ++- nvidia/10.0/rstudio.sh | 21 ++++++++++----------- nvidia/10.0/userconf.sh | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nvidia/10.0/Dockerfile b/nvidia/10.0/Dockerfile index 2d756f9..ea7aafd 100644 --- a/nvidia/10.0/Dockerfile +++ b/nvidia/10.0/Dockerfile @@ -30,9 +30,10 @@ RUN if [ -f /etc/rstudio ]; then \ ## Note: forcing PYTHON_RETICULATE_ENV may preclude defining alternative venvs -## Setting WORKON_HOME is more flexible +## Setting WORKON_HOME is more flexible, but may result in reticulate and tensorflow installing to diff locations ## ENV WORKON_HOME /opt/virtualenvs ## ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow + ENV PYTHON_VENV_PATH=/opt/venv ENV RETICULATE_PYTHON_ENV=$PYTHON_VENV_PATH ENV PATH=${PYTHON_VENV_PATH}/bin:${PATH} diff --git a/nvidia/10.0/rstudio.sh b/nvidia/10.0/rstudio.sh index 52c7dae..3a0d167 100644 --- a/nvidia/10.0/rstudio.sh +++ b/nvidia/10.0/rstudio.sh @@ -28,7 +28,7 @@ apt-get update \ libgc1c2 apt-get clean && rm -rf /var/lib/apt/lists/ -if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/debian9_64/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/debian9/x86_64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ +if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/bionic/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/bionic/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ && wget -q $RSTUDIO_URL \ && dpkg -i rstudio-server-*-amd64.deb \ && rm rstudio-server-*-amd64.deb @@ -65,7 +65,7 @@ useradd rstudio \ R_BIN=`which R` echo "rsession-which-r=${R_BIN}" >> /etc/rstudio/rserver.conf ## use more robust file locking to avoid errors when using shared volumes: -echo 'lock-type=advisory' >> /etc/rstudio/file-locks +echo "lock-type=advisory" >> /etc/rstudio/file-locks ## Optional configuration file to disable authentication cp /etc/rstudio/rserver.conf /etc/rstudio/disable_auth_rserver.conf @@ -79,23 +79,22 @@ git config --system credential.helper 'cache --timeout=3600' \ wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ && mkdir -p /etc/services.d/rstudio \ - && echo '#!/usr/bin/with-contenv bash \ + && echo "#!/usr/bin/with-contenv bash \ \n## load /etc/environment vars first: \ - \n for line in $( cat /etc/environment ) ; do export $line ; done \ - \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0' \ + \n for line in $( cat /etc/environment ) ; do export $line ; done \ + \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0" \ > /etc/services.d/rstudio/run \ - && echo '#!/bin/bash \ - \n rstudio-server stop' \ + && echo "#!/bin/bash \ + \n rstudio-server stop" \ > /etc/services.d/rstudio/finish \ && mkdir -p /home/rstudio/.rstudio/monitored/user-settings \ - && echo 'alwaysSaveHistory="0" \ - \nloadRData="0" \ - \nsaveAction="0"' \ + && echo "alwaysSaveHistory='0' \ + \nloadRData='0' \ + \nsaveAction='0'" \ > /home/rstudio/.rstudio/monitored/user-settings/user-settings \ && chown -R rstudio:rstudio /home/rstudio/.rstudio - #COPY userconf.sh /etc/cont-init.d/userconf # EXPOSE 8787 # CMD ["/init"] diff --git a/nvidia/10.0/userconf.sh b/nvidia/10.0/userconf.sh index c0751c7..d74a2f4 100644 --- a/nvidia/10.0/userconf.sh +++ b/nvidia/10.0/userconf.sh @@ -9,7 +9,7 @@ ROOT=${ROOT:=FALSE} UMASK=${UMASK:=022} ## Make sure RStudio inherits the full path -echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron +echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron bold=$(tput bold) normal=$(tput sgr0) From 0c89c9d675b5788b2a1633e66728506aa212ad3c Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 11:00:19 -0700 Subject: [PATCH 08/19] do not echo env vars to log --- nvidia/10.0/rstudio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvidia/10.0/rstudio.sh b/nvidia/10.0/rstudio.sh index 3a0d167..f025595 100644 --- a/nvidia/10.0/rstudio.sh +++ b/nvidia/10.0/rstudio.sh @@ -81,7 +81,7 @@ wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${ && mkdir -p /etc/services.d/rstudio \ && echo "#!/usr/bin/with-contenv bash \ \n## load /etc/environment vars first: \ - \n for line in $( cat /etc/environment ) ; do export $line ; done \ + \n for line in $( cat /etc/environment ) ; do export $line > /dev/null; done \ \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0" \ > /etc/services.d/rstudio/run \ && echo "#!/bin/bash \ From 4f907aad56d2afed10147040bad1017f1233cea2 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 11:14:22 -0700 Subject: [PATCH 09/19] rename --- {nvidia => ubuntu}/10.0/Dockerfile | 0 {nvidia => ubuntu}/10.0/R.sh | 0 {nvidia => ubuntu}/10.0/python.sh | 0 {nvidia => ubuntu}/10.0/rstudio.sh | 0 {nvidia => ubuntu}/10.0/tensorflow-gpu.sh | 0 {nvidia => ubuntu}/10.0/userconf.sh | 0 {nvidia => ubuntu}/9.0/Dockerfile | 0 ubuntu/README.md | 4 ++++ 8 files changed, 4 insertions(+) rename {nvidia => ubuntu}/10.0/Dockerfile (100%) rename {nvidia => ubuntu}/10.0/R.sh (100%) rename {nvidia => ubuntu}/10.0/python.sh (100%) rename {nvidia => ubuntu}/10.0/rstudio.sh (100%) rename {nvidia => ubuntu}/10.0/tensorflow-gpu.sh (100%) rename {nvidia => ubuntu}/10.0/userconf.sh (100%) rename {nvidia => ubuntu}/9.0/Dockerfile (100%) create mode 100644 ubuntu/README.md diff --git a/nvidia/10.0/Dockerfile b/ubuntu/10.0/Dockerfile similarity index 100% rename from nvidia/10.0/Dockerfile rename to ubuntu/10.0/Dockerfile diff --git a/nvidia/10.0/R.sh b/ubuntu/10.0/R.sh similarity index 100% rename from nvidia/10.0/R.sh rename to ubuntu/10.0/R.sh diff --git a/nvidia/10.0/python.sh b/ubuntu/10.0/python.sh similarity index 100% rename from nvidia/10.0/python.sh rename to ubuntu/10.0/python.sh diff --git a/nvidia/10.0/rstudio.sh b/ubuntu/10.0/rstudio.sh similarity index 100% rename from nvidia/10.0/rstudio.sh rename to ubuntu/10.0/rstudio.sh diff --git a/nvidia/10.0/tensorflow-gpu.sh b/ubuntu/10.0/tensorflow-gpu.sh similarity index 100% rename from nvidia/10.0/tensorflow-gpu.sh rename to ubuntu/10.0/tensorflow-gpu.sh diff --git a/nvidia/10.0/userconf.sh b/ubuntu/10.0/userconf.sh similarity index 100% rename from nvidia/10.0/userconf.sh rename to ubuntu/10.0/userconf.sh diff --git a/nvidia/9.0/Dockerfile b/ubuntu/9.0/Dockerfile similarity index 100% rename from nvidia/9.0/Dockerfile rename to ubuntu/9.0/Dockerfile diff --git a/ubuntu/README.md b/ubuntu/README.md new file mode 100644 index 0000000..d65f333 --- /dev/null +++ b/ubuntu/README.md @@ -0,0 +1,4 @@ + + +CUDA 9.0 is built on ubuntu:16.04 +CUDA 10.0 is built on ubuntu:18.04 From 9167dfefe5ff40483069986f5083794a48e86f5d Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 12:17:52 -0700 Subject: [PATCH 10/19] rename, fix 9.0 rstudio url --- ubuntu/10.0/Dockerfile | 28 ++--- ubuntu/10.0/{R.sh => install_R.sh} | 3 +- ubuntu/10.0/install_python.sh | 10 ++ .../10.0/{rstudio.sh => install_rstudio.sh} | 12 +- ...rflow-gpu.sh => install_tensorflow-gpu.sh} | 0 ubuntu/9.0/Dockerfile | 101 +++++++--------- ubuntu/9.0/install_R.sh | 10 ++ .../{10.0/python.sh => 9.0/install_python.sh} | 5 - ubuntu/9.0/install_rstudio.sh | 108 ++++++++++++++++++ ubuntu/9.0/install_tensorflow-gpu.sh | 25 ++++ ubuntu/9.0/test/Dockerfile | 22 ++++ ubuntu/9.0/test/install_R.sh | 10 ++ ubuntu/9.0/test/install_rstudio.sh | 104 +++++++++++++++++ ubuntu/9.0/test/userconf.sh | 103 +++++++++++++++++ ubuntu/9.0/userconf.sh | 103 +++++++++++++++++ 15 files changed, 558 insertions(+), 86 deletions(-) rename ubuntu/10.0/{R.sh => install_R.sh} (71%) create mode 100644 ubuntu/10.0/install_python.sh rename ubuntu/10.0/{rstudio.sh => install_rstudio.sh} (91%) rename ubuntu/10.0/{tensorflow-gpu.sh => install_tensorflow-gpu.sh} (100%) create mode 100644 ubuntu/9.0/install_R.sh rename ubuntu/{10.0/python.sh => 9.0/install_python.sh} (52%) create mode 100644 ubuntu/9.0/install_rstudio.sh create mode 100644 ubuntu/9.0/install_tensorflow-gpu.sh create mode 100644 ubuntu/9.0/test/Dockerfile create mode 100644 ubuntu/9.0/test/install_R.sh create mode 100644 ubuntu/9.0/test/install_rstudio.sh create mode 100644 ubuntu/9.0/test/userconf.sh create mode 100644 ubuntu/9.0/userconf.sh diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index ea7aafd..c3747ff 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -1,33 +1,33 @@ FROM nvidia/cuda:10.0-cudnn7-devel +## cuda:9.0 is xenial, cuda:10.0 is bionic +ARG UBUNTU_VERSION=bionic +ARG R_HOME=/usr/lib/R ARG DEBIAN_FRONTEND=noninteractive + ENV R_HOME=/usr/lib/R -COPY R.sh /tmp/install_R.sh +COPY install_R.sh /tmp/install_R.sh RUN . /tmp/install_R.sh -COPY rstudio.sh /tmp/install_rstudio.sh +COPY install_rstudio.sh /tmp/install_rstudio.sh RUN . /tmp/install_rstudio.sh COPY userconf.sh /etc/cont-init.d/userconf EXPOSE 8787 CMD ["/init"] - +## FROM https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables ENV CUDA_HOME=/usr/local/cuda -ENV CUDA_PATH=/usr/local/cuda -ENV PATH=$CUDA_HOME/bin:$PATH -ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV PATH=$PATH:$CUDA_HOME/bin +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64 RUN echo "\n\ - \nCUDA_HOME=$CUDA_HOME \ - \nCUDA_PATH=$CUDA_PATH \ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron -RUN if [ -f /etc/rstudio ]; then \ - echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf; fi - + \nCUDA_HOME=$CUDA_HOME\ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ + echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf ## Note: forcing PYTHON_RETICULATE_ENV may preclude defining alternative venvs ## Setting WORKON_HOME is more flexible, but may result in reticulate and tensorflow installing to diff locations @@ -37,11 +37,11 @@ RUN if [ -f /etc/rstudio ]; then \ ENV PYTHON_VENV_PATH=/opt/venv ENV RETICULATE_PYTHON_ENV=$PYTHON_VENV_PATH ENV PATH=${PYTHON_VENV_PATH}/bin:${PATH} -COPY python.sh /tmp/install_python.sh +COPY install_python.sh /tmp/install_python.sh RUN . /tmp/install_python.sh -COPY tensorflow-gpu.sh /tmp/install_tensorflow-gpu.sh +COPY install_tensorflow-gpu.sh /tmp/install_tensorflow-gpu.sh RUN . /tmp/install_tensorflow-gpu.sh diff --git a/ubuntu/10.0/R.sh b/ubuntu/10.0/install_R.sh similarity index 71% rename from ubuntu/10.0/R.sh rename to ubuntu/10.0/install_R.sh index 08c4428..d3a773b 100644 --- a/ubuntu/10.0/R.sh +++ b/ubuntu/10.0/install_R.sh @@ -1,6 +1,7 @@ #!/bin/sh -echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \ + +echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ apt-get update && apt-get -y install r-base && \ diff --git a/ubuntu/10.0/install_python.sh b/ubuntu/10.0/install_python.sh new file mode 100644 index 0000000..5ed2fc3 --- /dev/null +++ b/ubuntu/10.0/install_python.sh @@ -0,0 +1,10 @@ +#/bin/sh + +apt-get update && apt-get install -y --no-install-recommends \ + libpython3-dev \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* +python3 -m venv ${PYTHON_VENV_PATH} +pip3 install --no-cache-dir virtualenv + diff --git a/ubuntu/10.0/rstudio.sh b/ubuntu/10.0/install_rstudio.sh similarity index 91% rename from ubuntu/10.0/rstudio.sh rename to ubuntu/10.0/install_rstudio.sh index f025595..ff2d602 100644 --- a/ubuntu/10.0/rstudio.sh +++ b/ubuntu/10.0/install_rstudio.sh @@ -25,11 +25,15 @@ apt-get update \ libclang-dev \ libobjc-6-dev \ libobjc4 \ - libgc1c2 -apt-get clean && rm -rf /var/lib/apt/lists/ + libgc1c2 \ + && rm -rf /var/lib/apt/lists/* -if [ -z "$RSTUDIO_VERSION" ]; then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/bionic/rstudio-server-latest-amd64.deb"; else RSTUDIO_URL="http://download2.rstudio.org/server/bionic/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; fi \ - && wget -q $RSTUDIO_URL \ + +if [ -z "$RSTUDIO_VERSION" ]; + then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/${UBUNTU_VERSION}/rstudio-server-latest-amd64.deb"; + else RSTUDIO_URL="http://download2.rstudio.org/server/${UBUNTU_VERSION}/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; +fi +wget -q $RSTUDIO_URL \ && dpkg -i rstudio-server-*-amd64.deb \ && rm rstudio-server-*-amd64.deb diff --git a/ubuntu/10.0/tensorflow-gpu.sh b/ubuntu/10.0/install_tensorflow-gpu.sh similarity index 100% rename from ubuntu/10.0/tensorflow-gpu.sh rename to ubuntu/10.0/install_tensorflow-gpu.sh diff --git a/ubuntu/9.0/Dockerfile b/ubuntu/9.0/Dockerfile index b8a01ba..b4ce10c 100644 --- a/ubuntu/9.0/Dockerfile +++ b/ubuntu/9.0/Dockerfile @@ -1,70 +1,47 @@ FROM nvidia/cuda:9.0-cudnn7-devel +## cuda:9.0 is xenial, cuda:10.0 is bionic +ARG UBUNTU_VERSION=xenial +ARG R_HOME=/usr/lib/R ARG DEBIAN_FRONTEND=noninteractive -RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu xenial-cran35/' >> /etc/apt/sources.list && \ - gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ - apt-get update && apt-get -y install r-base - -####### Set up env variables in R ####### -ENV R_HOME=/usr/lib/R -ENV CUDA_HOME=/usr/local/cuda -ENV CUDA_PATH=/usr/local/cuda -ENV PATH=$CUDA_HOME/bin:$PATH -ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$LD_LIBRARY_PATH -#RUN echo "rsession-ld-library-path=$LD_LIBRARY_PATH" | tee -a /etc/rstudio/rserver.conf \ +COPY install_R.sh /tmp/install_R.sh +RUN . /tmp/install_R.sh + + + +COPY install_rstudio.sh /tmp/install_rstudio.sh +RUN . /tmp/install_rstudio.sh +COPY userconf.sh /etc/cont-init.d/userconf +EXPOSE 8787 +CMD ["/init"] + + + +# CUDA environmental variables for R & RSTUDIO +## FROM https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables +ENV CUDA_HOME=/usr/local/cuda +ENV PATH=$PATH:$CUDA_HOME/bin +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64 RUN echo "\n\ - \nCUDA_HOME=$CUDA_HOME \ - \nCUDA_PATH=$CUDA_PATH \ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron - -######### Set up Python3 Virtual Environment to play nice with R ### -ENV WORKON_HOME /opt/virtualenvs -ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow - -## Set up a user modifyable python3 environment -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpython3-dev \ - python3-pip \ - python3-venv && \ - rm -rf /var/lib/apt/lists/* - -RUN python3 -m venv ${PYTHON_VENV_PATH} - -ENV PATH ${PYTHON_VENV_PATH}/bin:${PATH} -## And set ENV for R! It doesn't read from the environment... -#RUN echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ -# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ -# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron - - -## reticulate will want virtualenv lib installed as well -#USER rstudio -RUN pip3 install --no-cache-dir virtualenv -#USER root - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - pkg-config \ - software-properties-common \ - unzip - -RUN R -e "install.packages(c('keras'))" -RUN R -e "install.packages('remotes')" -RUN R -e "remotes::install_github('greta-dev/greta')" - -RUN pip3 install --no-cache-dir tensorflow-gpu==1.12.0 tensorflow-probability==0.5.0 - -## Because reticulate hardwires these PATHs... -RUN ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip && \ - ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv - -# RUN chown -R rstudio:rstudio ${WORKON_HOME} + \nCUDA_HOME=$CUDA_HOME\ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron +RUN if [ -f /etc/rstudio ]; then \ + echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf; fi + + + +ENV PYTHON_VENV_PATH=/opt/venv +ENV RETICULATE_PYTHON_ENV=$PYTHON_VENV_PATH +ENV PATH=${PYTHON_VENV_PATH}/bin:${PATH} +COPY install_python.sh /tmp/install_python.sh +RUN . /tmp/install_python.sh + + + +COPY install_tensorflow-gpu.sh /tmp/install_tensorflow-gpu.sh +RUN . /tmp/install_tensorflow-gpu.sh + diff --git a/ubuntu/9.0/install_R.sh b/ubuntu/9.0/install_R.sh new file mode 100644 index 0000000..d3a773b --- /dev/null +++ b/ubuntu/9.0/install_R.sh @@ -0,0 +1,10 @@ +#!/bin/sh + + +echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ + apt-get update && apt-get -y install r-base && \ + rm -rf /var/lib/apt/lists/* + + diff --git a/ubuntu/10.0/python.sh b/ubuntu/9.0/install_python.sh similarity index 52% rename from ubuntu/10.0/python.sh rename to ubuntu/9.0/install_python.sh index b655162..a9f3ecc 100644 --- a/ubuntu/10.0/python.sh +++ b/ubuntu/9.0/install_python.sh @@ -9,11 +9,6 @@ apt-get update && apt-get install -y --no-install-recommends \ python3 -m venv ${PYTHON_VENV_PATH} -## And set ENV for R; it doesn't read from the environment... -# echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron && \ -# echo "WORKON_HOME=${WORKON_HOME}" >> ${R_HOME}/etc/Renviron && \ -# echo "RETICULATE_PYTHON_ENV=${PYTHON_VENV_PATH}" >> ${R_HOME}/etc/Renviron - ## Reticulate needs this too? pip3 install --no-cache-dir virtualenv diff --git a/ubuntu/9.0/install_rstudio.sh b/ubuntu/9.0/install_rstudio.sh new file mode 100644 index 0000000..a554aa3 --- /dev/null +++ b/ubuntu/9.0/install_rstudio.sh @@ -0,0 +1,108 @@ +#!/bin/sh +set -e + +RSTUDIO_VERSION=1.2.1335 +S6_VERSION=${S6_VERSION:-v1.21.7.0} +S6_BEHAVIOUR_IF_STAGE2_FAILS=2 +PANDOC_TEMPLATES_VERSION=${PANDOC_TEMPLATES_VERSION:-2.6} +export PATH=/usr/lib/rstudio-server/bin:$PATH + + +## Download and install RStudio server & dependencies +## Attempts to get detect latest version, otherwise falls back to version given in $VER +## Symlink pandoc, pandoc-citeproc so they are available system-wide +apt-get update \ + && apt-get install -y --no-install-recommends \ + file \ + git \ + libapparmor1 \ + libcurl4-openssl-dev \ + libedit2 \ + libssl-dev \ + lsb-release \ + psmisc \ + procps \ + python-setuptools \ + sudo \ + wget \ + libclang-dev \ + libobjc-5-dev \ + libgc1c2 \ + && rm -rf /var/lib/apt/lists/* + +## NOTE: xenial also uses 'trusty' URL here for latest version. +## for latest version, make sure RSTUDIO_VERSION is unset or it will fail. +if [ -z "$RSTUDIO_VERSION" ]; + then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/trusty/rstudio-server-latest-amd64.deb"; + else RSTUDIO_URL="http://download2.rstudio.org/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; +fi + +wget -q $RSTUDIO_URL \ + && dpkg -i rstudio-server-*-amd64.deb \ + && rm rstudio-server-*-amd64.deb + +## Symlink pandoc & standard pandoc templates for use system-wide +ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ + && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc-citeproc /usr/local/bin \ + && git clone --recursive --branch ${PANDOC_TEMPLATES_VERSION} https://github.com/jgm/pandoc-templates \ + && mkdir -p /opt/pandoc/templates \ + && cp -r pandoc-templates*/* /opt/pandoc/templates && rm -rf pandoc-templates* \ + && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates + + +## RStudio wants an /etc/R, will populate from $R_HOME/etc +mkdir -p /etc/R \ + && echo "\n\ + \n# Configure httr to perform out-of-band authentication if HTTR_LOCALHOST \ + \n# is not set since a redirect to localhost may not work depending upon \ + \n# where this Docker container is running. \ + \nif(is.na(Sys.getenv('HTTR_LOCALHOST', unset=NA))) { \ + \n options(httr_oob_default = TRUE) \ + \n}" >> ${R_HOME}/etc/Rprofile.site \ + && echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + + +## Need to configure non-root user for RStudio +useradd rstudio \ + && echo "rstudio:rstudio" | chpasswd \ + && mkdir /home/rstudio \ + && chown rstudio:rstudio /home/rstudio \ + && addgroup rstudio staff + ## Prevent rstudio from deciding to use /usr/bin/R if a user apt-get installs a package + +R_BIN=`which R` +echo "rsession-which-r=${R_BIN}" >> /etc/rstudio/rserver.conf +## use more robust file locking to avoid errors when using shared volumes: +echo "lock-type=advisory" >> /etc/rstudio/file-locks + +## Optional configuration file to disable authentication +cp /etc/rstudio/rserver.conf /etc/rstudio/disable_auth_rserver.conf +echo "auth-none=1" >> /etc/rstudio/disable_auth_rserver.conf + +## configure git not to request password each time +git config --system credential.helper 'cache --timeout=3600' \ + && git config --system push.default simple + +## Set up S6 init system +wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ + && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ + && mkdir -p /etc/services.d/rstudio \ + && echo "#!/usr/bin/with-contenv bash \ + \n## load /etc/environment vars first: \ + \n for line in $( cat /etc/environment ) ; do export $line > /dev/null; done \ + \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0" \ + > /etc/services.d/rstudio/run \ + && echo "#!/bin/bash \ + \n rstudio-server stop" \ + > /etc/services.d/rstudio/finish \ + && mkdir -p /home/rstudio/.rstudio/monitored/user-settings \ + && echo "alwaysSaveHistory='0' \ + \nloadRData='0' \ + \nsaveAction='0'" \ + > /home/rstudio/.rstudio/monitored/user-settings/user-settings \ + && chown -R rstudio:rstudio /home/rstudio/.rstudio + + +#COPY userconf.sh /etc/cont-init.d/userconf +# EXPOSE 8787 +# CMD ["/init"] diff --git a/ubuntu/9.0/install_tensorflow-gpu.sh b/ubuntu/9.0/install_tensorflow-gpu.sh new file mode 100644 index 0000000..578cb9b --- /dev/null +++ b/ubuntu/9.0/install_tensorflow-gpu.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + software-properties-common \ + unzip + + +## symlink these because reticulate hardwires these PATHs... +ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip +ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv + +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} +echo $PYTHON_VENV_PATH + +## FIXME set versions Consider renv +#R -e "renv::restore()" +pip3 install --no-cache-dir tensorflow-gpu==1.11.0 keras h5py pyyaml requests Pillow +R -e "install.packages(c('keras', 'greta'))" + diff --git a/ubuntu/9.0/test/Dockerfile b/ubuntu/9.0/test/Dockerfile new file mode 100644 index 0000000..5ebc1ee --- /dev/null +++ b/ubuntu/9.0/test/Dockerfile @@ -0,0 +1,22 @@ +FROM nvidia/cuda:9.0-cudnn7-devel + +## cuda:9.0 is xenial, cuda:10.0 is bionic +ARG UBUNTU_VERSION=xenial +ARG R_HOME=/usr/lib/R +ARG DEBIAN_FRONTEND=noninteractive + + + +COPY install_R.sh /tmp/install_R.sh +RUN . /tmp/install_R.sh + + + +COPY install_rstudio.sh /tmp/install_rstudio.sh +RUN . /tmp/install_rstudio.sh +COPY userconf.sh /etc/cont-init.d/userconf +EXPOSE 8787 +CMD ["/init"] + + + diff --git a/ubuntu/9.0/test/install_R.sh b/ubuntu/9.0/test/install_R.sh new file mode 100644 index 0000000..d3a773b --- /dev/null +++ b/ubuntu/9.0/test/install_R.sh @@ -0,0 +1,10 @@ +#!/bin/sh + + +echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ + apt-get update && apt-get -y install r-base && \ + rm -rf /var/lib/apt/lists/* + + diff --git a/ubuntu/9.0/test/install_rstudio.sh b/ubuntu/9.0/test/install_rstudio.sh new file mode 100644 index 0000000..a818a60 --- /dev/null +++ b/ubuntu/9.0/test/install_rstudio.sh @@ -0,0 +1,104 @@ +#!/bin/sh +set -e + +RSTUDIO_VERSION=1.2.1335 +S6_VERSION=${S6_VERSION:-v1.21.7.0} +S6_BEHAVIOUR_IF_STAGE2_FAILS=2 +PANDOC_TEMPLATES_VERSION=${PANDOC_TEMPLATES_VERSION:-2.6} +export PATH=/usr/lib/rstudio-server/bin:$PATH + + +## Download and install RStudio server & dependencies +## Attempts to get detect latest version, otherwise falls back to version given in $VER +## Symlink pandoc, pandoc-citeproc so they are available system-wide +apt-get update \ + && apt-get install -y --no-install-recommends \ + file \ + git \ + libapparmor1 \ + libcurl4-openssl-dev \ + libedit2 \ + libssl-dev \ + lsb-release \ + psmisc \ + procps \ + python-setuptools \ + sudo \ + wget \ + libclang-dev \ + libobjc-5-dev \ + libgc1c2 \ + && rm -rf /var/lib/apt/lists/* + +## NOTE: xenial also uses 'trusty' URL here for latest version. +RSTUDIO_URL="https://download2.rstudio.org/server/trusty/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" + +wget -q $RSTUDIO_URL \ + && dpkg -i rstudio-server-*-amd64.deb \ + && rm rstudio-server-*-amd64.deb + +## Symlink pandoc & standard pandoc templates for use system-wide +ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ + && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc-citeproc /usr/local/bin \ + && git clone --recursive --branch ${PANDOC_TEMPLATES_VERSION} https://github.com/jgm/pandoc-templates \ + && mkdir -p /opt/pandoc/templates \ + && cp -r pandoc-templates*/* /opt/pandoc/templates && rm -rf pandoc-templates* \ + && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates + + +## RStudio wants an /etc/R, will populate from $R_HOME/etc +mkdir -p /etc/R \ + && echo "\n\ + \n# Configure httr to perform out-of-band authentication if HTTR_LOCALHOST \ + \n# is not set since a redirect to localhost may not work depending upon \ + \n# where this Docker container is running. \ + \nif(is.na(Sys.getenv('HTTR_LOCALHOST', unset=NA))) { \ + \n options(httr_oob_default = TRUE) \ + \n}" >> ${R_HOME}/etc/Rprofile.site \ + && echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + + +## Need to configure non-root user for RStudio +useradd rstudio \ + && echo "rstudio:rstudio" | chpasswd \ + && mkdir /home/rstudio \ + && chown rstudio:rstudio /home/rstudio \ + && addgroup rstudio staff + ## Prevent rstudio from deciding to use /usr/bin/R if a user apt-get installs a package + +R_BIN=`which R` +echo "rsession-which-r=${R_BIN}" >> /etc/rstudio/rserver.conf +## use more robust file locking to avoid errors when using shared volumes: +echo "lock-type=advisory" >> /etc/rstudio/file-locks + +## Optional configuration file to disable authentication +cp /etc/rstudio/rserver.conf /etc/rstudio/disable_auth_rserver.conf +echo "auth-none=1" >> /etc/rstudio/disable_auth_rserver.conf + +## configure git not to request password each time +git config --system credential.helper 'cache --timeout=3600' \ + && git config --system push.default simple + +## Set up S6 init system +wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ + && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ + && mkdir -p /etc/services.d/rstudio \ + && echo "#!/usr/bin/with-contenv bash \ + \n## load /etc/environment vars first: \ + \n for line in $( cat /etc/environment ) ; do export $line > /dev/null; done \ + \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0" \ + > /etc/services.d/rstudio/run \ + && echo "#!/bin/bash \ + \n rstudio-server stop" \ + > /etc/services.d/rstudio/finish \ + && mkdir -p /home/rstudio/.rstudio/monitored/user-settings \ + && echo "alwaysSaveHistory='0' \ + \nloadRData='0' \ + \nsaveAction='0'" \ + > /home/rstudio/.rstudio/monitored/user-settings/user-settings \ + && chown -R rstudio:rstudio /home/rstudio/.rstudio + + +#COPY userconf.sh /etc/cont-init.d/userconf +# EXPOSE 8787 +# CMD ["/init"] diff --git a/ubuntu/9.0/test/userconf.sh b/ubuntu/9.0/test/userconf.sh new file mode 100644 index 0000000..d74a2f4 --- /dev/null +++ b/ubuntu/9.0/test/userconf.sh @@ -0,0 +1,103 @@ +#!/usr/bin/with-contenv bash + +## Set defaults for environmental variables in case they are undefined +USER=${USER:=rstudio} +PASSWORD=${PASSWORD:=rstudio} +USERID=${USERID:=1000} +GROUPID=${GROUPID:=1000} +ROOT=${ROOT:=FALSE} +UMASK=${UMASK:=022} + +## Make sure RStudio inherits the full path +echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + +bold=$(tput bold) +normal=$(tput sgr0) + + +if [[ ${DISABLE_AUTH,,} == "true" ]] +then + mv /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf + echo "USER=$USER" >> /etc/environment +fi + + + +if grep --quiet "auth-none=1" /etc/rstudio/rserver.conf +then + echo "Skipping authentication as requested" +elif [ "$PASSWORD" == "rstudio" ] +then + printf "\n\n" + tput bold + printf "\e[31mERROR\e[39m: You must set a unique PASSWORD (not 'rstudio') first! e.g. run with:\n" + printf "docker run -e PASSWORD=\e[92m\e[39m -p 8787:8787 rocker/rstudio\n" + tput sgr0 + printf "\n\n" + exit 1 +fi + +if [ "$USERID" -lt 1000 ] +# Probably a macOS user, https://github.com/rocker-org/rocker/issues/205 + then + echo "$USERID is less than 1000" + check_user_id=$(grep -F "auth-minimum-user-id" /etc/rstudio/rserver.conf) + if [[ ! -z $check_user_id ]] + then + echo "minumum authorised user already exists in /etc/rstudio/rserver.conf: $check_user_id" + else + echo "setting minumum authorised user to 499" + echo auth-minimum-user-id=499 >> /etc/rstudio/rserver.conf + fi +fi + +if [ "$USERID" -ne 1000 ] +## Configure user with a different USERID if requested. + then + echo "deleting user rstudio" + userdel rstudio + echo "creating new $USER with UID $USERID" + useradd -m $USER -u $USERID + mkdir /home/$USER + chown -R $USER /home/$USER + usermod -a -G staff $USER +elif [ "$USER" != "rstudio" ] + then + ## cannot move home folder when it's a shared volume, have to copy and change permissions instead + cp -r /home/rstudio /home/$USER + ## RENAME the user + usermod -l $USER -d /home/$USER rstudio + groupmod -n $USER rstudio + usermod -a -G staff $USER + chown -R $USER:$USER /home/$USER + echo "USER is now $USER" +fi + +if [ "$GROUPID" -ne 1000 ] +## Configure the primary GID (whether rstudio or $USER) with a different GROUPID if requested. + then + echo "Modifying primary group $(id $USER -g -n)" + groupmod -g $GROUPID $(id $USER -g -n) + echo "Primary group ID is now custom_group $GROUPID" +fi + +## Add a password to user +echo "$USER:$PASSWORD" | chpasswd + +# Use Env flag to know if user should be added to sudoers +if [[ ${ROOT,,} == "true" ]] + then + adduser $USER sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + echo "$USER added to sudoers" +fi + +## Change Umask value if desired +if [ "$UMASK" -ne 022 ] + then + echo "server-set-umask=false" >> /etc/rstudio/rserver.conf + echo "Sys.umask(mode=$UMASK)" >> /home/$USER/.Rprofile +fi + +## add these to the global environment so they are avialable to the RStudio user +echo "HTTR_LOCALHOST=$HTTR_LOCALHOST" >> /etc/R/Renviron.site +echo "HTTR_PORT=$HTTR_PORT" >> /etc/R/Renviron.site diff --git a/ubuntu/9.0/userconf.sh b/ubuntu/9.0/userconf.sh new file mode 100644 index 0000000..d74a2f4 --- /dev/null +++ b/ubuntu/9.0/userconf.sh @@ -0,0 +1,103 @@ +#!/usr/bin/with-contenv bash + +## Set defaults for environmental variables in case they are undefined +USER=${USER:=rstudio} +PASSWORD=${PASSWORD:=rstudio} +USERID=${USERID:=1000} +GROUPID=${GROUPID:=1000} +ROOT=${ROOT:=FALSE} +UMASK=${UMASK:=022} + +## Make sure RStudio inherits the full path +echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + +bold=$(tput bold) +normal=$(tput sgr0) + + +if [[ ${DISABLE_AUTH,,} == "true" ]] +then + mv /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf + echo "USER=$USER" >> /etc/environment +fi + + + +if grep --quiet "auth-none=1" /etc/rstudio/rserver.conf +then + echo "Skipping authentication as requested" +elif [ "$PASSWORD" == "rstudio" ] +then + printf "\n\n" + tput bold + printf "\e[31mERROR\e[39m: You must set a unique PASSWORD (not 'rstudio') first! e.g. run with:\n" + printf "docker run -e PASSWORD=\e[92m\e[39m -p 8787:8787 rocker/rstudio\n" + tput sgr0 + printf "\n\n" + exit 1 +fi + +if [ "$USERID" -lt 1000 ] +# Probably a macOS user, https://github.com/rocker-org/rocker/issues/205 + then + echo "$USERID is less than 1000" + check_user_id=$(grep -F "auth-minimum-user-id" /etc/rstudio/rserver.conf) + if [[ ! -z $check_user_id ]] + then + echo "minumum authorised user already exists in /etc/rstudio/rserver.conf: $check_user_id" + else + echo "setting minumum authorised user to 499" + echo auth-minimum-user-id=499 >> /etc/rstudio/rserver.conf + fi +fi + +if [ "$USERID" -ne 1000 ] +## Configure user with a different USERID if requested. + then + echo "deleting user rstudio" + userdel rstudio + echo "creating new $USER with UID $USERID" + useradd -m $USER -u $USERID + mkdir /home/$USER + chown -R $USER /home/$USER + usermod -a -G staff $USER +elif [ "$USER" != "rstudio" ] + then + ## cannot move home folder when it's a shared volume, have to copy and change permissions instead + cp -r /home/rstudio /home/$USER + ## RENAME the user + usermod -l $USER -d /home/$USER rstudio + groupmod -n $USER rstudio + usermod -a -G staff $USER + chown -R $USER:$USER /home/$USER + echo "USER is now $USER" +fi + +if [ "$GROUPID" -ne 1000 ] +## Configure the primary GID (whether rstudio or $USER) with a different GROUPID if requested. + then + echo "Modifying primary group $(id $USER -g -n)" + groupmod -g $GROUPID $(id $USER -g -n) + echo "Primary group ID is now custom_group $GROUPID" +fi + +## Add a password to user +echo "$USER:$PASSWORD" | chpasswd + +# Use Env flag to know if user should be added to sudoers +if [[ ${ROOT,,} == "true" ]] + then + adduser $USER sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + echo "$USER added to sudoers" +fi + +## Change Umask value if desired +if [ "$UMASK" -ne 022 ] + then + echo "server-set-umask=false" >> /etc/rstudio/rserver.conf + echo "Sys.umask(mode=$UMASK)" >> /home/$USER/.Rprofile +fi + +## add these to the global environment so they are avialable to the RStudio user +echo "HTTR_LOCALHOST=$HTTR_LOCALHOST" >> /etc/R/Renviron.site +echo "HTTR_PORT=$HTTR_PORT" >> /etc/R/Renviron.site From 3caecabf44edcb95390d5c3d2c24c3b0c492f1c6 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 12:18:28 -0700 Subject: [PATCH 11/19] drop test --- ubuntu/9.0/install_rstudio.sh | 6 +- ubuntu/9.0/test/Dockerfile | 22 ------ ubuntu/9.0/test/install_R.sh | 10 --- ubuntu/9.0/test/install_rstudio.sh | 104 ----------------------------- ubuntu/9.0/test/userconf.sh | 103 ---------------------------- 5 files changed, 1 insertion(+), 244 deletions(-) delete mode 100644 ubuntu/9.0/test/Dockerfile delete mode 100644 ubuntu/9.0/test/install_R.sh delete mode 100644 ubuntu/9.0/test/install_rstudio.sh delete mode 100644 ubuntu/9.0/test/userconf.sh diff --git a/ubuntu/9.0/install_rstudio.sh b/ubuntu/9.0/install_rstudio.sh index a554aa3..a818a60 100644 --- a/ubuntu/9.0/install_rstudio.sh +++ b/ubuntu/9.0/install_rstudio.sh @@ -31,11 +31,7 @@ apt-get update \ && rm -rf /var/lib/apt/lists/* ## NOTE: xenial also uses 'trusty' URL here for latest version. -## for latest version, make sure RSTUDIO_VERSION is unset or it will fail. -if [ -z "$RSTUDIO_VERSION" ]; - then RSTUDIO_URL="https://www.rstudio.org/download/latest/stable/server/trusty/rstudio-server-latest-amd64.deb"; - else RSTUDIO_URL="http://download2.rstudio.org/rstudio-server-${RSTUDIO_VERSION}-amd64.deb"; -fi +RSTUDIO_URL="https://download2.rstudio.org/server/trusty/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" wget -q $RSTUDIO_URL \ && dpkg -i rstudio-server-*-amd64.deb \ diff --git a/ubuntu/9.0/test/Dockerfile b/ubuntu/9.0/test/Dockerfile deleted file mode 100644 index 5ebc1ee..0000000 --- a/ubuntu/9.0/test/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM nvidia/cuda:9.0-cudnn7-devel - -## cuda:9.0 is xenial, cuda:10.0 is bionic -ARG UBUNTU_VERSION=xenial -ARG R_HOME=/usr/lib/R -ARG DEBIAN_FRONTEND=noninteractive - - - -COPY install_R.sh /tmp/install_R.sh -RUN . /tmp/install_R.sh - - - -COPY install_rstudio.sh /tmp/install_rstudio.sh -RUN . /tmp/install_rstudio.sh -COPY userconf.sh /etc/cont-init.d/userconf -EXPOSE 8787 -CMD ["/init"] - - - diff --git a/ubuntu/9.0/test/install_R.sh b/ubuntu/9.0/test/install_R.sh deleted file mode 100644 index d3a773b..0000000 --- a/ubuntu/9.0/test/install_R.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - - -echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ - gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ - apt-get update && apt-get -y install r-base && \ - rm -rf /var/lib/apt/lists/* - - diff --git a/ubuntu/9.0/test/install_rstudio.sh b/ubuntu/9.0/test/install_rstudio.sh deleted file mode 100644 index a818a60..0000000 --- a/ubuntu/9.0/test/install_rstudio.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/sh -set -e - -RSTUDIO_VERSION=1.2.1335 -S6_VERSION=${S6_VERSION:-v1.21.7.0} -S6_BEHAVIOUR_IF_STAGE2_FAILS=2 -PANDOC_TEMPLATES_VERSION=${PANDOC_TEMPLATES_VERSION:-2.6} -export PATH=/usr/lib/rstudio-server/bin:$PATH - - -## Download and install RStudio server & dependencies -## Attempts to get detect latest version, otherwise falls back to version given in $VER -## Symlink pandoc, pandoc-citeproc so they are available system-wide -apt-get update \ - && apt-get install -y --no-install-recommends \ - file \ - git \ - libapparmor1 \ - libcurl4-openssl-dev \ - libedit2 \ - libssl-dev \ - lsb-release \ - psmisc \ - procps \ - python-setuptools \ - sudo \ - wget \ - libclang-dev \ - libobjc-5-dev \ - libgc1c2 \ - && rm -rf /var/lib/apt/lists/* - -## NOTE: xenial also uses 'trusty' URL here for latest version. -RSTUDIO_URL="https://download2.rstudio.org/server/trusty/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" - -wget -q $RSTUDIO_URL \ - && dpkg -i rstudio-server-*-amd64.deb \ - && rm rstudio-server-*-amd64.deb - -## Symlink pandoc & standard pandoc templates for use system-wide -ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc /usr/local/bin \ - && ln -s /usr/lib/rstudio-server/bin/pandoc/pandoc-citeproc /usr/local/bin \ - && git clone --recursive --branch ${PANDOC_TEMPLATES_VERSION} https://github.com/jgm/pandoc-templates \ - && mkdir -p /opt/pandoc/templates \ - && cp -r pandoc-templates*/* /opt/pandoc/templates && rm -rf pandoc-templates* \ - && mkdir /root/.pandoc && ln -s /opt/pandoc/templates /root/.pandoc/templates - - -## RStudio wants an /etc/R, will populate from $R_HOME/etc -mkdir -p /etc/R \ - && echo "\n\ - \n# Configure httr to perform out-of-band authentication if HTTR_LOCALHOST \ - \n# is not set since a redirect to localhost may not work depending upon \ - \n# where this Docker container is running. \ - \nif(is.na(Sys.getenv('HTTR_LOCALHOST', unset=NA))) { \ - \n options(httr_oob_default = TRUE) \ - \n}" >> ${R_HOME}/etc/Rprofile.site \ - && echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron - - -## Need to configure non-root user for RStudio -useradd rstudio \ - && echo "rstudio:rstudio" | chpasswd \ - && mkdir /home/rstudio \ - && chown rstudio:rstudio /home/rstudio \ - && addgroup rstudio staff - ## Prevent rstudio from deciding to use /usr/bin/R if a user apt-get installs a package - -R_BIN=`which R` -echo "rsession-which-r=${R_BIN}" >> /etc/rstudio/rserver.conf -## use more robust file locking to avoid errors when using shared volumes: -echo "lock-type=advisory" >> /etc/rstudio/file-locks - -## Optional configuration file to disable authentication -cp /etc/rstudio/rserver.conf /etc/rstudio/disable_auth_rserver.conf -echo "auth-none=1" >> /etc/rstudio/disable_auth_rserver.conf - -## configure git not to request password each time -git config --system credential.helper 'cache --timeout=3600' \ - && git config --system push.default simple - -## Set up S6 init system -wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-amd64.tar.gz \ - && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \ - && mkdir -p /etc/services.d/rstudio \ - && echo "#!/usr/bin/with-contenv bash \ - \n## load /etc/environment vars first: \ - \n for line in $( cat /etc/environment ) ; do export $line > /dev/null; done \ - \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0" \ - > /etc/services.d/rstudio/run \ - && echo "#!/bin/bash \ - \n rstudio-server stop" \ - > /etc/services.d/rstudio/finish \ - && mkdir -p /home/rstudio/.rstudio/monitored/user-settings \ - && echo "alwaysSaveHistory='0' \ - \nloadRData='0' \ - \nsaveAction='0'" \ - > /home/rstudio/.rstudio/monitored/user-settings/user-settings \ - && chown -R rstudio:rstudio /home/rstudio/.rstudio - - -#COPY userconf.sh /etc/cont-init.d/userconf -# EXPOSE 8787 -# CMD ["/init"] diff --git a/ubuntu/9.0/test/userconf.sh b/ubuntu/9.0/test/userconf.sh deleted file mode 100644 index d74a2f4..0000000 --- a/ubuntu/9.0/test/userconf.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/with-contenv bash - -## Set defaults for environmental variables in case they are undefined -USER=${USER:=rstudio} -PASSWORD=${PASSWORD:=rstudio} -USERID=${USERID:=1000} -GROUPID=${GROUPID:=1000} -ROOT=${ROOT:=FALSE} -UMASK=${UMASK:=022} - -## Make sure RStudio inherits the full path -echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron - -bold=$(tput bold) -normal=$(tput sgr0) - - -if [[ ${DISABLE_AUTH,,} == "true" ]] -then - mv /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf - echo "USER=$USER" >> /etc/environment -fi - - - -if grep --quiet "auth-none=1" /etc/rstudio/rserver.conf -then - echo "Skipping authentication as requested" -elif [ "$PASSWORD" == "rstudio" ] -then - printf "\n\n" - tput bold - printf "\e[31mERROR\e[39m: You must set a unique PASSWORD (not 'rstudio') first! e.g. run with:\n" - printf "docker run -e PASSWORD=\e[92m\e[39m -p 8787:8787 rocker/rstudio\n" - tput sgr0 - printf "\n\n" - exit 1 -fi - -if [ "$USERID" -lt 1000 ] -# Probably a macOS user, https://github.com/rocker-org/rocker/issues/205 - then - echo "$USERID is less than 1000" - check_user_id=$(grep -F "auth-minimum-user-id" /etc/rstudio/rserver.conf) - if [[ ! -z $check_user_id ]] - then - echo "minumum authorised user already exists in /etc/rstudio/rserver.conf: $check_user_id" - else - echo "setting minumum authorised user to 499" - echo auth-minimum-user-id=499 >> /etc/rstudio/rserver.conf - fi -fi - -if [ "$USERID" -ne 1000 ] -## Configure user with a different USERID if requested. - then - echo "deleting user rstudio" - userdel rstudio - echo "creating new $USER with UID $USERID" - useradd -m $USER -u $USERID - mkdir /home/$USER - chown -R $USER /home/$USER - usermod -a -G staff $USER -elif [ "$USER" != "rstudio" ] - then - ## cannot move home folder when it's a shared volume, have to copy and change permissions instead - cp -r /home/rstudio /home/$USER - ## RENAME the user - usermod -l $USER -d /home/$USER rstudio - groupmod -n $USER rstudio - usermod -a -G staff $USER - chown -R $USER:$USER /home/$USER - echo "USER is now $USER" -fi - -if [ "$GROUPID" -ne 1000 ] -## Configure the primary GID (whether rstudio or $USER) with a different GROUPID if requested. - then - echo "Modifying primary group $(id $USER -g -n)" - groupmod -g $GROUPID $(id $USER -g -n) - echo "Primary group ID is now custom_group $GROUPID" -fi - -## Add a password to user -echo "$USER:$PASSWORD" | chpasswd - -# Use Env flag to know if user should be added to sudoers -if [[ ${ROOT,,} == "true" ]] - then - adduser $USER sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - echo "$USER added to sudoers" -fi - -## Change Umask value if desired -if [ "$UMASK" -ne 022 ] - then - echo "server-set-umask=false" >> /etc/rstudio/rserver.conf - echo "Sys.umask(mode=$UMASK)" >> /home/$USER/.Rprofile -fi - -## add these to the global environment so they are avialable to the RStudio user -echo "HTTR_LOCALHOST=$HTTR_LOCALHOST" >> /etc/R/Renviron.site -echo "HTTR_PORT=$HTTR_PORT" >> /etc/R/Renviron.site From 02d619e9498cc83420eca22fc852f6f5800e5c3b Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 12:37:24 -0700 Subject: [PATCH 12/19] locale --- ubuntu/10.0/Dockerfile | 3 ++- ubuntu/10.0/install_R.sh | 15 +++++++++++++-- ubuntu/9.0/Dockerfile | 3 ++- ubuntu/9.0/install_R.sh | 15 +++++++++++++-- ubuntu/Makefile | 10 ++++++++++ 5 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 ubuntu/Makefile diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index c3747ff..29a7d9f 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -7,7 +7,8 @@ ARG DEBIAN_FRONTEND=noninteractive -ENV R_HOME=/usr/lib/R +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 COPY install_R.sh /tmp/install_R.sh RUN . /tmp/install_R.sh diff --git a/ubuntu/10.0/install_R.sh b/ubuntu/10.0/install_R.sh index d3a773b..672ea92 100644 --- a/ubuntu/10.0/install_R.sh +++ b/ubuntu/10.0/install_R.sh @@ -1,10 +1,21 @@ #!/bin/sh - +set -e echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ - apt-get update && apt-get -y install r-base && \ + apt-get update && \ + apt-get -y install --no-install-recommends \ + ca-certificates \ + less \ + locales \ + r-base \ + vim-tiny \ + wget && \ rm -rf /var/lib/apt/lists/* +echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + locale-gen en_US.utf8 && \ + /usr/sbin/update-locale LANG=${LANG} + diff --git a/ubuntu/9.0/Dockerfile b/ubuntu/9.0/Dockerfile index b4ce10c..0e653d7 100644 --- a/ubuntu/9.0/Dockerfile +++ b/ubuntu/9.0/Dockerfile @@ -6,7 +6,8 @@ ARG R_HOME=/usr/lib/R ARG DEBIAN_FRONTEND=noninteractive - +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 COPY install_R.sh /tmp/install_R.sh RUN . /tmp/install_R.sh diff --git a/ubuntu/9.0/install_R.sh b/ubuntu/9.0/install_R.sh index d3a773b..672ea92 100644 --- a/ubuntu/9.0/install_R.sh +++ b/ubuntu/9.0/install_R.sh @@ -1,10 +1,21 @@ #!/bin/sh - +set -e echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/" >> /etc/apt/sources.list && \ gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ gpg -a --export E298A3A825C0D65DFD57CBB651716619E084DAB9 | apt-key add - && \ - apt-get update && apt-get -y install r-base && \ + apt-get update && \ + apt-get -y install --no-install-recommends \ + ca-certificates \ + less \ + locales \ + r-base \ + vim-tiny \ + wget && \ rm -rf /var/lib/apt/lists/* +echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + locale-gen en_US.utf8 && \ + /usr/sbin/update-locale LANG=${LANG} + diff --git a/ubuntu/Makefile b/ubuntu/Makefile new file mode 100644 index 0000000..fe1cea5 --- /dev/null +++ b/ubuntu/Makefile @@ -0,0 +1,10 @@ +all: + make cuda-9.0 + make cuda-10.0 + +cuda-10.0: 10.0/Dockerfile + docker build -t rocker/cuda:10.0 10.0 +cuda-9.0: 9.0/Dockerfile + docker build -t rocker/cuda:9.0 9.0 + + From eec2689e18272ccb79f7ebbca51ffdca159e537d Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 12:55:28 -0700 Subject: [PATCH 13/19] scripts for tidyverse / verse --- ubuntu/10.0/Dockerfile | 18 +++++++-- ubuntu/10.0/install_R.sh | 4 ++ ubuntu/10.0/install_python.sh | 2 + ubuntu/10.0/install_tensorflow-gpu.sh | 1 - ubuntu/10.0/install_tidyverse.sh | 26 +++++++++++++ ubuntu/10.0/install_verse.sh | 54 +++++++++++++++++++++++++++ ubuntu/9.0/install_python.sh | 7 +--- ubuntu/9.0/install_tensorflow-gpu.sh | 1 - ubuntu/README.md | 16 ++++++++ 9 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 ubuntu/10.0/install_tidyverse.sh create mode 100644 ubuntu/10.0/install_verse.sh diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index 29a7d9f..65618dd 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -21,6 +21,18 @@ EXPOSE 8787 CMD ["/init"] + +COPY install_tidyverse.sh /tmp/install_tidyverse.sh +RUN . /tmp/install_tidyverse.sh + + + +COPY install_verse.sh /tmp/install_verse.sh +RUN . /tmp/install_verse.sh + + + + ## FROM https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables ENV CUDA_HOME=/usr/local/cuda ENV PATH=$PATH:$CUDA_HOME/bin @@ -30,10 +42,8 @@ RUN echo "\n\ \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf -## Note: forcing PYTHON_RETICULATE_ENV may preclude defining alternative venvs -## Setting WORKON_HOME is more flexible, but may result in reticulate and tensorflow installing to diff locations -## ENV WORKON_HOME /opt/virtualenvs -## ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow + + ENV PYTHON_VENV_PATH=/opt/venv ENV RETICULATE_PYTHON_ENV=$PYTHON_VENV_PATH diff --git a/ubuntu/10.0/install_R.sh b/ubuntu/10.0/install_R.sh index 672ea92..849d5a6 100644 --- a/ubuntu/10.0/install_R.sh +++ b/ubuntu/10.0/install_R.sh @@ -9,6 +9,7 @@ echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/ ca-certificates \ less \ locales \ + littler \ r-base \ vim-tiny \ wget && \ @@ -18,4 +19,7 @@ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ locale-gen en_US.utf8 && \ /usr/sbin/update-locale LANG=${LANG} +ln -s /usr/lib/R/site-library/littler/examples/install.r /usr/local/bin/install.r \ + && ln -s /usr/lib/R/site-library/littler/examples/install2.r /usr/local/bin/install2.r \ + && ln -s /usr/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r diff --git a/ubuntu/10.0/install_python.sh b/ubuntu/10.0/install_python.sh index 5ed2fc3..cedd558 100644 --- a/ubuntu/10.0/install_python.sh +++ b/ubuntu/10.0/install_python.sh @@ -8,3 +8,5 @@ apt-get update && apt-get install -y --no-install-recommends \ python3 -m venv ${PYTHON_VENV_PATH} pip3 install --no-cache-dir virtualenv +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} + diff --git a/ubuntu/10.0/install_tensorflow-gpu.sh b/ubuntu/10.0/install_tensorflow-gpu.sh index 10a1082..23d3e06 100644 --- a/ubuntu/10.0/install_tensorflow-gpu.sh +++ b/ubuntu/10.0/install_tensorflow-gpu.sh @@ -16,7 +16,6 @@ ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv chown -R rstudio:rstudio ${PYTHON_VENV_PATH} -echo $PYTHON_VENV_PATH ## FIXME set versions Consider renv #R -e "renv::restore()" diff --git a/ubuntu/10.0/install_tidyverse.sh b/ubuntu/10.0/install_tidyverse.sh new file mode 100644 index 0000000..49bce4e --- /dev/null +++ b/ubuntu/10.0/install_tidyverse.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -e + +apt-get update -qq && apt-get -y --no-install-recommends install \ + libxml2-dev \ + libcairo2-dev \ + libsqlite3-dev \ + libmariadbd-dev \ + libmariadb-client-lgpl-dev \ + libpq-dev \ + libssh2-1-dev \ + unixodbc-dev \ + libsasl2-dev && \ + rm -rf /var/lib/apt/lists/* + +install2.r --error \ + --deps TRUE \ + tidyverse \ + dplyr \ + devtools \ + formatR \ + remotes \ + selectr \ + caTools \ + BiocManager + diff --git a/ubuntu/10.0/install_verse.sh b/ubuntu/10.0/install_verse.sh new file mode 100644 index 0000000..672d9e2 --- /dev/null +++ b/ubuntu/10.0/install_verse.sh @@ -0,0 +1,54 @@ +#!/bin/sh + +set -e + +export PATH=$PATH:/opt/TinyTeX/bin/x86_64-linux/ + +## Add LaTeX, rticles and bookdown support +RUN wget "https://travis-bin.yihui.name/texlive-local.deb" \ + && dpkg -i texlive-local.deb \ + && rm texlive-local.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + cmake \ + default-jdk \ + fonts-roboto \ + ghostscript \ + libbz2-dev \ + libicu-dev \ + liblzma-dev \ + libhunspell-dev \ + libmagick++-dev \ + librdf0-dev \ + libv8-dev \ + qpdf \ + texinfo \ + ssh \ + less \ + vim \ + libzmq3-dev \ + libopenmpi-dev \ + && rm -rf /var/lib/apt/lists/* + +install2.r --error tinytex + +## Admin-based install of TinyTeX: +wget -qO- \ + "https://github.com/yihui/tinytex/raw/master/tools/install-unx.sh" | \ + sh -s - --admin --no-path \ + && mv ~/.TinyTeX /opt/TinyTeX \ + && /opt/TinyTeX/bin/*/tlmgr path add \ + && tlmgr install metafont mfware inconsolata tex ae parskip listings \ + && tlmgr path add \ + && Rscript -e "tinytex::r_texmf()" \ + && chown -R root:staff /opt/TinyTeX \ + && chown -R root:staff ${R_HOME}/site-library \ + && chmod -R g+w /opt/TinyTeX \ + && chmod -R g+wx /opt/TinyTeX/bin \ + && echo "PATH=${PATH}" >> ${R_HOME}/etc/Renviron + + +install2.r --error --deps TRUE \ + bookdown rticles rmdshower rJava + + diff --git a/ubuntu/9.0/install_python.sh b/ubuntu/9.0/install_python.sh index a9f3ecc..cedd558 100644 --- a/ubuntu/9.0/install_python.sh +++ b/ubuntu/9.0/install_python.sh @@ -1,17 +1,12 @@ #/bin/sh -## Python3 apt-get update && apt-get install -y --no-install-recommends \ libpython3-dev \ python3-pip \ python3-venv && \ rm -rf /var/lib/apt/lists/* - python3 -m venv ${PYTHON_VENV_PATH} - -## Reticulate needs this too? pip3 install --no-cache-dir virtualenv - - +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} diff --git a/ubuntu/9.0/install_tensorflow-gpu.sh b/ubuntu/9.0/install_tensorflow-gpu.sh index 578cb9b..ed7dea5 100644 --- a/ubuntu/9.0/install_tensorflow-gpu.sh +++ b/ubuntu/9.0/install_tensorflow-gpu.sh @@ -16,7 +16,6 @@ ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv chown -R rstudio:rstudio ${PYTHON_VENV_PATH} -echo $PYTHON_VENV_PATH ## FIXME set versions Consider renv #R -e "renv::restore()" diff --git a/ubuntu/README.md b/ubuntu/README.md index d65f333..bc3ad15 100644 --- a/ubuntu/README.md +++ b/ubuntu/README.md @@ -2,3 +2,19 @@ CUDA 9.0 is built on ubuntu:16.04 CUDA 10.0 is built on ubuntu:18.04 + + + +## Developer notes + + +Currently, we set global env var `PYTHON_RETICULATE_ENV`, which forces `reticulate` and friends to use the virtualenv we set up at that location (`opt/venv`). This may make it dificult to provide alternative venvs (since running `Sys.unsetenv()` from the R console is not sufficient). + +Setting `WORKON_HOME` is more flexible, e.g.: + +``` +ENV WORKON_HOME /opt/virtualenvs +ENV PYTHON_VENV_PATH $WORKON_HOME/r-tensorflow +``` + +which allows `reticulate` to still find `python` in the venv out-of-the-box, but can create additional virtualenvs in `WORKON_HOME` (users should be given write permissions to this). From c5c0b413138f15426ff5e0039afb6d9ebc88c6fd Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 14:18:06 -0700 Subject: [PATCH 14/19] WIP --- ubuntu/10.0/Dockerfile | 9 +- ubuntu/10.0/install_R.sh | 6 +- ubuntu/10.0/install_python.sh | 8 ++ ubuntu/10.0/install_tensorflow-gpu.sh | 9 +- ubuntu/10.0/install_verse.sh | 2 +- ubuntu/10.0/source_R.sh | 124 ++++++++++++++++++++++++++ ubuntu/9.0/Dockerfile | 3 +- ubuntu/9.0/install_python.sh | 7 ++ ubuntu/9.0/install_tensorflow-gpu.sh | 12 ++- 9 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 ubuntu/10.0/source_R.sh diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index 65618dd..a9d8d30 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -1,12 +1,19 @@ FROM nvidia/cuda:10.0-cudnn7-devel +LABEL org.label-schema.license="GPL-2.0" \ + org.label-schema.vcs-url="https://github.com/rocker-org/rocker-versioned" \ + org.label-schema.vendor="Rocker Project" \ + maintainer="Carl Boettiger " + +ENV TERM=xterm + ## cuda:9.0 is xenial, cuda:10.0 is bionic ARG UBUNTU_VERSION=bionic ARG R_HOME=/usr/lib/R ARG DEBIAN_FRONTEND=noninteractive - +#ENV R_VERSION=${R_VERSION:-3.6.0} ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 COPY install_R.sh /tmp/install_R.sh diff --git a/ubuntu/10.0/install_R.sh b/ubuntu/10.0/install_R.sh index 849d5a6..0659c7e 100644 --- a/ubuntu/10.0/install_R.sh +++ b/ubuntu/10.0/install_R.sh @@ -8,9 +8,9 @@ echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/ apt-get -y install --no-install-recommends \ ca-certificates \ less \ - locales \ littler \ - r-base \ + locales \ + r-base-dev \ vim-tiny \ wget && \ rm -rf /var/lib/apt/lists/* @@ -19,7 +19,9 @@ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ locale-gen en_US.utf8 && \ /usr/sbin/update-locale LANG=${LANG} +Rscript -e "install.packages('docopt')" ln -s /usr/lib/R/site-library/littler/examples/install.r /usr/local/bin/install.r \ && ln -s /usr/lib/R/site-library/littler/examples/install2.r /usr/local/bin/install2.r \ && ln -s /usr/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r + diff --git a/ubuntu/10.0/install_python.sh b/ubuntu/10.0/install_python.sh index cedd558..c49fd60 100644 --- a/ubuntu/10.0/install_python.sh +++ b/ubuntu/10.0/install_python.sh @@ -1,4 +1,5 @@ #/bin/sh +set -e apt-get update && apt-get install -y --no-install-recommends \ libpython3-dev \ @@ -8,5 +9,12 @@ apt-get update && apt-get install -y --no-install-recommends \ python3 -m venv ${PYTHON_VENV_PATH} pip3 install --no-cache-dir virtualenv +## Ensure RStudio inherits this env var +echo "\nRETICULATE_PYTHON_ENV=${RETICULATE_PYTHON_ENV}" >> ${R_HOME}/etc/Renviron + +## symlink these because reticulate hardwires these PATHs... +ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip +ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv + chown -R rstudio:rstudio ${PYTHON_VENV_PATH} diff --git a/ubuntu/10.0/install_tensorflow-gpu.sh b/ubuntu/10.0/install_tensorflow-gpu.sh index 23d3e06..123c274 100644 --- a/ubuntu/10.0/install_tensorflow-gpu.sh +++ b/ubuntu/10.0/install_tensorflow-gpu.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -10,16 +11,10 @@ apt-get update && apt-get install -y --no-install-recommends \ software-properties-common \ unzip - -## symlink these because reticulate hardwires these PATHs... -ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip -ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv - -chown -R rstudio:rstudio ${PYTHON_VENV_PATH} - ## FIXME set versions Consider renv #R -e "renv::restore()" pip3 install --no-cache-dir tensorflow-gpu tensorflow-probability keras h5py pyyaml requests Pillow +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} R -e "install.packages('keras')" R -e "install.packages('remotes'); remotes::install_github('greta-dev/greta')" diff --git a/ubuntu/10.0/install_verse.sh b/ubuntu/10.0/install_verse.sh index 672d9e2..5e520b5 100644 --- a/ubuntu/10.0/install_verse.sh +++ b/ubuntu/10.0/install_verse.sh @@ -5,7 +5,7 @@ set -e export PATH=$PATH:/opt/TinyTeX/bin/x86_64-linux/ ## Add LaTeX, rticles and bookdown support -RUN wget "https://travis-bin.yihui.name/texlive-local.deb" \ +wget "https://travis-bin.yihui.name/texlive-local.deb" \ && dpkg -i texlive-local.deb \ && rm texlive-local.deb \ && apt-get update \ diff --git a/ubuntu/10.0/source_R.sh b/ubuntu/10.0/source_R.sh new file mode 100644 index 0000000..d5bfa88 --- /dev/null +++ b/ubuntu/10.0/source_R.sh @@ -0,0 +1,124 @@ +apt-get update \ + && apt-get install -y --no-install-recommends \ + bash-completion \ + ca-certificates \ + file \ + fonts-texgyre \ + g++ \ + gfortran \ + gsfonts \ + libblas-dev \ + libbz2-1.0 \ + libcurl3 \ + libicu57 \ + libjpeg62-turbo \ + libopenblas-dev \ + libpangocairo-1.0-0 \ + libpcre3 \ + libpng16-16 \ + libreadline7 \ + libtiff5 \ + liblzma5 \ + locales \ + make \ + unzip \ + zip \ + zlib1g \ + && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ + && locale-gen en_US.utf8 \ + && /usr/sbin/update-locale LANG=en_US.UTF-8 \ + && BUILDDEPS="curl \ + default-jdk \ + libbz2-dev \ + libcairo2-dev \ + libcurl4-openssl-dev \ + libpango1.0-dev \ + libjpeg-dev \ + libicu-dev \ + libpcre3-dev \ + libpng-dev \ + libreadline-dev \ + libtiff5-dev \ + liblzma-dev \ + libx11-dev \ + libxt-dev \ + perl \ + tcl8.6-dev \ + tk8.6-dev \ + texinfo \ + texlive-extra-utils \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + texlive-latex-recommended \ + x11proto-core-dev \ + xauth \ + xfonts-base \ + xvfb \ + zlib1g-dev" + +apt-get install -y --no-install-recommends $BUILDDEPS + +cd tmp/ \ + && curl -O https://cran.r-project.org/src/base/R-3/R-${R_VERSION}.tar.gz \ + && tar -xf R-${R_VERSION}.tar.gz \ + && cd R-${R_VERSION} \ + && R_PAPERSIZE=letter \ + R_BATCHSAVE="--no-save --no-restore" \ + R_BROWSER=xdg-open \ + PAGER=/usr/bin/pager \ + PERL=/usr/bin/perl \ + R_UNZIPCMD=/usr/bin/unzip \ + R_ZIPCMD=/usr/bin/zip \ + R_PRINTCMD=/usr/bin/lpr \ + LIBnn=lib \ + AWK=/usr/bin/awk \ + CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -g" \ + CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -g" \ + ./configure --enable-R-shlib \ + --enable-memory-profiling \ + --with-readline \ + --with-blas \ + --with-tcltk \ + --disable-nls \ + --with-recommended-packages \ + && make \ + && make install + +## Add a default CRAN mirror +# && echo "options(repos = c(CRAN = 'https://cran.rstudio.com/'), download.file.method = 'libcurl')" >> /usr/local/lib/R/etc/Rprofile.site \ + + +## Add a library directory (for user-installed packages) +mkdir -p /usr/local/lib/R/site-library \ + && chown root:staff /usr/local/lib/R/site-library \ + && chmod g+wx /usr/local/lib/R/site-library \ + +## Fix library path + echo "R_LIBS_USER='/usr/local/lib/R/site-library'" >> /usr/local/lib/R/etc/Renviron \ + && echo "R_LIBS=\${R_LIBS-'/usr/local/lib/R/site-library:/usr/local/lib/R/library:/usr/lib/R/library'}" >> /usr/local/lib/R/etc/Renviron + + +## install packages from date-locked MRAN snapshot of CRAN + [ -z "$BUILD_DATE" ] && BUILD_DATE=$(TZ="America/Los_Angeles" date -I) || true \ + && MRAN=https://mran.microsoft.com/snapshot/${BUILD_DATE} \ + && echo MRAN=$MRAN >> /etc/environment \ + && export MRAN=$MRAN \ + && echo "options(repos = c(CRAN='$MRAN'), download.file.method = 'libcurl')" >> /usr/local/lib/R/etc/Rprofile.site + +## Use littler installation scripts +Rscript -e "install.packages(c('littler', 'docopt'), repo = '$MRAN')" \ + && ln -s /usr/local/lib/R/site-library/littler/examples/install2.r /usr/local/bin/install2.r \ + && ln -s /usr/local/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r \ + && ln -s /usr/local/lib/R/site-library/littler/bin/r /usr/local/bin/r + + +## Clean up from R source install +cd / \ + && rm -rf /tmp/* \ + && apt-get remove --purge -y $BUILDDEPS \ + && apt-get autoremove -y \ + && apt-get autoclean -y \ + && rm -rf /var/lib/apt/lists/* + +#CMD ["R"] + diff --git a/ubuntu/9.0/Dockerfile b/ubuntu/9.0/Dockerfile index 0e653d7..39d6bc6 100644 --- a/ubuntu/9.0/Dockerfile +++ b/ubuntu/9.0/Dockerfile @@ -29,8 +29,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/li RUN echo "\n\ \nCUDA_HOME=$CUDA_HOME\ \nPATH=$PATH" >> ${R_HOME}/etc/Renviron -RUN if [ -f /etc/rstudio ]; then \ - echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf; fi +RUN echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf diff --git a/ubuntu/9.0/install_python.sh b/ubuntu/9.0/install_python.sh index cedd558..937d757 100644 --- a/ubuntu/9.0/install_python.sh +++ b/ubuntu/9.0/install_python.sh @@ -1,4 +1,5 @@ #/bin/sh +set -e apt-get update && apt-get install -y --no-install-recommends \ libpython3-dev \ @@ -8,5 +9,11 @@ apt-get update && apt-get install -y --no-install-recommends \ python3 -m venv ${PYTHON_VENV_PATH} pip3 install --no-cache-dir virtualenv +## Ensure RStudio inherits this env var +echo "\nRETICULATE_PYTHON_ENV=${RETICULATE_PYTHON_ENV}" >> ${R_HOME}/etc/Renviron + +## symlink these because reticulate hardwires these PATHs... +ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip +ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv chown -R rstudio:rstudio ${PYTHON_VENV_PATH} diff --git a/ubuntu/9.0/install_tensorflow-gpu.sh b/ubuntu/9.0/install_tensorflow-gpu.sh index ed7dea5..a5e8c34 100644 --- a/ubuntu/9.0/install_tensorflow-gpu.sh +++ b/ubuntu/9.0/install_tensorflow-gpu.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -11,14 +12,11 @@ apt-get update && apt-get install -y --no-install-recommends \ unzip -## symlink these because reticulate hardwires these PATHs... -ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip -ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv - -chown -R rstudio:rstudio ${PYTHON_VENV_PATH} - ## FIXME set versions Consider renv #R -e "renv::restore()" -pip3 install --no-cache-dir tensorflow-gpu==1.11.0 keras h5py pyyaml requests Pillow +pip3 install --no-cache-dir tensorflow-gpu==1.11.0 tensorflow-probability==0.5.0 +chown -R rstudio:rstudio ${PYTHON_VENV_PATH} + +## greta==0.3.0 R -e "install.packages(c('keras', 'greta'))" From 9c96dafadcd34ef9b5b874c4cd6ee850eb618b0a Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 14:24:49 -0700 Subject: [PATCH 15/19] correct versions for 9.0 to work with R tensorflow and greta --- ubuntu/9.0/install_tensorflow-gpu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntu/9.0/install_tensorflow-gpu.sh b/ubuntu/9.0/install_tensorflow-gpu.sh index a5e8c34..26af2a6 100644 --- a/ubuntu/9.0/install_tensorflow-gpu.sh +++ b/ubuntu/9.0/install_tensorflow-gpu.sh @@ -14,7 +14,7 @@ apt-get update && apt-get install -y --no-install-recommends \ ## FIXME set versions Consider renv #R -e "renv::restore()" -pip3 install --no-cache-dir tensorflow-gpu==1.11.0 tensorflow-probability==0.5.0 +pip3 install --no-cache-dir tensorflow-gpu==1.12.0 tensorflow-probability==0.5.0 chown -R rstudio:rstudio ${PYTHON_VENV_PATH} ## greta==0.3.0 From cb7b761e52649fbdce174174e1f074778b3d57ac Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Thu, 13 Jun 2019 15:01:59 -0700 Subject: [PATCH 16/19] update readme --- ubuntu/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ubuntu/README.md b/ubuntu/README.md index bc3ad15..fe907f4 100644 --- a/ubuntu/README.md +++ b/ubuntu/README.md @@ -1,12 +1,16 @@ +## Developer notes -CUDA 9.0 is built on ubuntu:16.04 -CUDA 10.0 is built on ubuntu:18.04 +- CUDA 9.0 is built on ubuntu:16.04, with tensorflow 1.12.0 +- CUDA 10.0 is built on ubuntu:18.04 with tensorflow 1.13.1 +Note that one cannot easily mix and match versions of CUDA, tensorflow, and the R packages. Pre-compiled tensorflow binaries work only with CUDA 9.0 and 10.0. + +For example, the current dev version of `greta` (0.3.0.9002) needs `tensorflow-gpu==1.13.1`, CUDA 10.0 (and probably python 3.6). More recent or older versions will likely break things. The current stable version, (0.3.0) cannot run in this environment, and needs CUDA 9.0 and tensorflow==`1.12.0`. -## Developer notes +## Managing the Python Virtualenv Currently, we set global env var `PYTHON_RETICULATE_ENV`, which forces `reticulate` and friends to use the virtualenv we set up at that location (`opt/venv`). This may make it dificult to provide alternative venvs (since running `Sys.unsetenv()` from the R console is not sufficient). From 647ed57f2c2b39c00a5d19940c9eeddf97475202 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Tue, 18 Jun 2019 09:24:03 -0700 Subject: [PATCH 17/19] WIP --- ubuntu/10.0/Dockerfile | 14 +++++------- ubuntu/10.0/config_R_cuda.sh | 43 ++++++++++++++++++++++++++++++++++++ ubuntu/10.0/install_R.sh | 6 +++-- ubuntu/9.0/install_R.sh | 9 +++++++- ubuntu/Makefile | 4 ++-- 5 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 ubuntu/10.0/config_R_cuda.sh diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index a9d8d30..3d336db 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -39,17 +39,13 @@ RUN . /tmp/install_verse.sh - -## FROM https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables +## Use Configure CUDA for R, use NVidia libs for BLAS ENV CUDA_HOME=/usr/local/cuda ENV PATH=$PATH:$CUDA_HOME/bin -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64 -RUN echo "\n\ - \nCUDA_HOME=$CUDA_HOME\ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ - echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf - - +ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64/libnvblas.so:$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64 +ENV NVBLAS_CONFIG_FILE=/etc/nvblas.conf +COPY config_R_cuda.sh /tmp/config_R_cuda.sh +RUN . /tmp/config_R_cuda.sh ENV PYTHON_VENV_PATH=/opt/venv diff --git a/ubuntu/10.0/config_R_cuda.sh b/ubuntu/10.0/config_R_cuda.sh new file mode 100644 index 0000000..04b3478 --- /dev/null +++ b/ubuntu/10.0/config_R_cuda.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -e + +## CUDA environmental variables configuration for RStudio + +## cli R inherits these, but RStudio needs to have these set in as follows: +## (From https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables) +echo "\n\ + \nCUDA_HOME=$CUDA_HOME\ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ + echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf + +## Configure R & RStudio to use drop-in CUDA blas +## Allow R to use CUDA for BLAS, with fallback on openblas +echo "NVBLAS_LOGFILE nvblas.log \ + \nNVBLAS_CPU_BLAS_LIB /usr/lib/libopenblas.so \ + \nNVBLAS_GPU_LIST ALL" > /etc/nvblas.conf + + +## We don't want to set LD_PRELOAD globally +##ENV LD_PRELOAD=/usr/local/cuda/lib64/libnvblas.so +# +### Instead, we will set it before calling R, Rscript, or RStudio: +#mv /usr/bin/R /usr/bin/R_ +#mv /usr/bin/Rscript /usr/bin/Rscript_ +# +#echo '\#!/bin/sh \ +# \n LD_PRELOAD=/usr/local/cuda/lib64/libnvblas.so /usr/bin/R_ "$@"' \ +# > /usr/bin/R && \ +# chmod +x /usr/bin/R && \ +# echo '#!/bin/sh \ +# \n LD_PRELOAD=/usr/local/cuda/lib64/libnvblas.so /usr/bin/Rscript_ "$@"' \ +# > /usr/bin/Rscript && \ +# chmod +x /usr/bin/Rscript +# +#echo '#!/usr/bin/with-contenv bash \ +# \n## load /etc/environment vars first: \ +# \n for line in \$( cat /etc/environment ) ; do export $line ; done \ +# \n export LD_PRELOAD=/usr/local/cuda/lib64/libnvblas.so \ +# \n exec /usr/lib/rstudio-server/bin/rserver --server-daemonize 0' \ +# > /etc/services.d/rstudio/run +# diff --git a/ubuntu/10.0/install_R.sh b/ubuntu/10.0/install_R.sh index 0659c7e..cfc594a 100644 --- a/ubuntu/10.0/install_R.sh +++ b/ubuntu/10.0/install_R.sh @@ -9,6 +9,7 @@ echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/ ca-certificates \ less \ littler \ +# libopenblas-base \ locales \ r-base-dev \ vim-tiny \ @@ -19,9 +20,10 @@ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ locale-gen en_US.utf8 && \ /usr/sbin/update-locale LANG=${LANG} -Rscript -e "install.packages('docopt')" +Rscript -e "install.packages(c('littler', 'docopt'))" ln -s /usr/lib/R/site-library/littler/examples/install.r /usr/local/bin/install.r \ && ln -s /usr/lib/R/site-library/littler/examples/install2.r /usr/local/bin/install2.r \ - && ln -s /usr/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r + && ln -s /usr/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r \ + && ln -s /usr/local/lib/R/site-library/littler/bin/r /usr/local/bin/r diff --git a/ubuntu/9.0/install_R.sh b/ubuntu/9.0/install_R.sh index 672ea92..76559c8 100644 --- a/ubuntu/9.0/install_R.sh +++ b/ubuntu/9.0/install_R.sh @@ -8,8 +8,9 @@ echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/ apt-get -y install --no-install-recommends \ ca-certificates \ less \ + libopenblas-base \ locales \ - r-base \ + r-base-dev \ vim-tiny \ wget && \ rm -rf /var/lib/apt/lists/* @@ -18,4 +19,10 @@ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ locale-gen en_US.utf8 && \ /usr/sbin/update-locale LANG=${LANG} +## littler configuration +Rscript -e "install.packages(c('littler', 'docopt'))" +ln -s /usr/lib/R/site-library/littler/examples/install.r /usr/local/bin/install.r \ + && ln -s /usr/lib/R/site-library/littler/examples/install2.r /usr/local/bin/install2.r \ + && ln -s /usr/lib/R/site-library/littler/examples/installGithub.r /usr/local/bin/installGithub.r \ + && ln -s /usr/local/lib/R/site-library/littler/bin/r /usr/local/bin/r diff --git a/ubuntu/Makefile b/ubuntu/Makefile index fe1cea5..90fda63 100644 --- a/ubuntu/Makefile +++ b/ubuntu/Makefile @@ -3,8 +3,8 @@ all: make cuda-10.0 cuda-10.0: 10.0/Dockerfile - docker build -t rocker/cuda:10.0 10.0 + docker build -t rocker/cuda-10.0 10.0 cuda-9.0: 9.0/Dockerfile - docker build -t rocker/cuda:9.0 9.0 + docker build -t rocker/cuda-9.0 9.0 From cc40ba7b39a8b83b3f12c7d2a8e840ced764af2c Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Wed, 31 Jul 2019 13:50:10 -0700 Subject: [PATCH 18/19] minor tweaks --- ubuntu/10.0/config_R_cuda.sh | 14 ++++++++------ ubuntu/10.0/install_R.sh | 2 +- ubuntu/10.0/install_nvtop.sh | 10 ++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 ubuntu/10.0/install_nvtop.sh diff --git a/ubuntu/10.0/config_R_cuda.sh b/ubuntu/10.0/config_R_cuda.sh index 04b3478..a0b3a0a 100644 --- a/ubuntu/10.0/config_R_cuda.sh +++ b/ubuntu/10.0/config_R_cuda.sh @@ -7,15 +7,17 @@ set -e ## cli R inherits these, but RStudio needs to have these set in as follows: ## (From https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables) echo "\n\ - \nCUDA_HOME=$CUDA_HOME\ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ - echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf + \nCUDA_HOME=$CUDA_HOME\ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ + echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf ## Configure R & RStudio to use drop-in CUDA blas ## Allow R to use CUDA for BLAS, with fallback on openblas -echo "NVBLAS_LOGFILE nvblas.log \ - \nNVBLAS_CPU_BLAS_LIB /usr/lib/libopenblas.so \ - \nNVBLAS_GPU_LIST ALL" > /etc/nvblas.conf +echo "\nNVBLAS_LOGFILE /var/log/nvblas.log \ + \nNVBLAS_CPU_BLAS_LIB /usr/lib/libopenblas.so \ + \nNVBLAS_GPU_LIST ALL" > /etc/nvblas.conf + +echo "NVBLAS_CONFIG_FILE=$NVBLAS_CONFIG_FILE" >> ${R_HOME}/etc/Renviron ## We don't want to set LD_PRELOAD globally diff --git a/ubuntu/10.0/install_R.sh b/ubuntu/10.0/install_R.sh index cfc594a..9fa2a50 100644 --- a/ubuntu/10.0/install_R.sh +++ b/ubuntu/10.0/install_R.sh @@ -9,7 +9,7 @@ echo "deb https://cloud.r-project.org/bin/linux/ubuntu ${UBUNTU_VERSION}-cran35/ ca-certificates \ less \ littler \ -# libopenblas-base \ + libopenblas-base \ locales \ r-base-dev \ vim-tiny \ diff --git a/ubuntu/10.0/install_nvtop.sh b/ubuntu/10.0/install_nvtop.sh new file mode 100644 index 0000000..67f174d --- /dev/null +++ b/ubuntu/10.0/install_nvtop.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +apt-get update && apt-get -y install cmake libncurses5-dev libncursesw5-dev git +git clone https://github.com/Syllo/nvtop.git +mkdir -p nvtop/build && cd nvtop/build +cmake .. -DNVML_RETRIEVE_HEADER_ONLINE=True +make +make install + From a08d32dec53a66a85a6ebc5adb23bc1fd8a1b926 Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Wed, 31 Jul 2019 15:50:35 -0700 Subject: [PATCH 19/19] Need to explicitly link correct openblas lib in nvconf Note that this path differs in Debian vs Ubuntu-based images --- ubuntu/10.0/Dockerfile | 10 +++++----- ubuntu/10.0/config_R_cuda.sh | 9 +++++++-- ubuntu/10.0/install_python.sh | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ubuntu/10.0/Dockerfile b/ubuntu/10.0/Dockerfile index 3d336db..daba156 100644 --- a/ubuntu/10.0/Dockerfile +++ b/ubuntu/10.0/Dockerfile @@ -22,20 +22,20 @@ RUN . /tmp/install_R.sh COPY install_rstudio.sh /tmp/install_rstudio.sh -RUN . /tmp/install_rstudio.sh +#RUN . /tmp/install_rstudio.sh COPY userconf.sh /etc/cont-init.d/userconf EXPOSE 8787 -CMD ["/init"] +#CMD ["/init"] COPY install_tidyverse.sh /tmp/install_tidyverse.sh -RUN . /tmp/install_tidyverse.sh +#RUN . /tmp/install_tidyverse.sh COPY install_verse.sh /tmp/install_verse.sh -RUN . /tmp/install_verse.sh +#RUN . /tmp/install_verse.sh @@ -57,5 +57,5 @@ RUN . /tmp/install_python.sh COPY install_tensorflow-gpu.sh /tmp/install_tensorflow-gpu.sh -RUN . /tmp/install_tensorflow-gpu.sh +#RUN . /tmp/install_tensorflow-gpu.sh diff --git a/ubuntu/10.0/config_R_cuda.sh b/ubuntu/10.0/config_R_cuda.sh index a0b3a0a..ebfd742 100644 --- a/ubuntu/10.0/config_R_cuda.sh +++ b/ubuntu/10.0/config_R_cuda.sh @@ -8,13 +8,18 @@ set -e ## (From https://tensorflow.rstudio.com/tools/local_gpu.html#environment-variables) echo "\n\ \nCUDA_HOME=$CUDA_HOME\ - \nPATH=$PATH" >> ${R_HOME}/etc/Renviron && \ + \nPATH=$PATH" >> ${R_HOME}/etc/Renviron + +if test -f /etc/rstudio/rserver.conf; then echo "rsession-ld-library-path=$LD_LIBRARY_PATH" >> /etc/rstudio/rserver.conf +fi + +touch /var/log/nvblas.log && chown :staff /var/log/nvblas.log ## Configure R & RStudio to use drop-in CUDA blas ## Allow R to use CUDA for BLAS, with fallback on openblas echo "\nNVBLAS_LOGFILE /var/log/nvblas.log \ - \nNVBLAS_CPU_BLAS_LIB /usr/lib/libopenblas.so \ + \nNVBLAS_CPU_BLAS_LIB /usr/lib/x86_64-linux-gnu/openblas/libblas.so.3 \ \nNVBLAS_GPU_LIST ALL" > /etc/nvblas.conf echo "NVBLAS_CONFIG_FILE=$NVBLAS_CONFIG_FILE" >> ${R_HOME}/etc/Renviron diff --git a/ubuntu/10.0/install_python.sh b/ubuntu/10.0/install_python.sh index c49fd60..c84123f 100644 --- a/ubuntu/10.0/install_python.sh +++ b/ubuntu/10.0/install_python.sh @@ -16,5 +16,5 @@ echo "\nRETICULATE_PYTHON_ENV=${RETICULATE_PYTHON_ENV}" >> ${R_HOME}/etc/Renviro ln -s ${PYTHON_VENV_PATH}/bin/pip /usr/local/bin/pip ln -s ${PYTHON_VENV_PATH}/bin/virtualenv /usr/local/bin/virtualenv -chown -R rstudio:rstudio ${PYTHON_VENV_PATH} +chown :staff ${PYTHON_VENV_PATH}