Merge pull request #5950 from berkeley-dsep-infra/staging

merging 5915/5935/5936/5937/5938/5939/5940/5941/5942/5944/5945/5948/5951/5952/5953 to prod

deployments/biology/image/Dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:focal-scm
+FROM buildpack-deps:jammy-scm
 
 ENV TZ=America/Los_Angeles
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@@ -29,22 +29,11 @@ RUN apt-get update -qq --yes > /dev/null && \
     apt-get install --yes -qq \
         libpython2.7 > /dev/null
 
-## libraries required for mothur
-## libreadline6 required
-#RUN apt-get update -qq --yes > /dev/null && \
-#    apt-get install --yes -qq \
-#    libreadline6-dev > /dev/null
-
 ## library required for fast-PCA & https://github.com/DReichLab/EIG
 RUN apt-get update -qq --yes && \
     apt-get install --yes --no-install-recommends -qq \
         libgsl-dev >/dev/null
 
-## library required for running ccb293 package qiime
-#RUN apt-get update -qq --yes > /dev/null && \
-#    apt-get install --yes -qq \
-#    tzdata > /dev/null
-
 # Install these without 'recommended' packages to keep image smaller.
 # Useful utils that folks sort of take for granted
 RUN apt-get update -qq --yes && \
@@ -64,25 +53,48 @@ RUN apt-get update -qq --yes && \
 RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \
     locale-gen
 
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
-RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" > /etc/apt/sources.list.d/cran.list
+# Needed by RStudio
+RUN apt-get update -qq --yes && \
+    apt-get install --yes --no-install-recommends -qq \
+        psmisc \
+        sudo \
+        libapparmor1 \
+        lsb-release \
+        libclang-dev \
+        libpq5 > /dev/null
 
+# Needed by many R libraries
+# Picked up from https://github.com/rocker-org/rocker/blob/9dc3e458d4e92a8f41ccd75687cd7e316e657cc0/r-rspm/focal/Dockerfile
+RUN apt-get update && \
+	apt-get install -y --no-install-recommends \
+                   libgdal-dev \
+                   libgeos3.10.2 \
+                   libproj22 \
+                   libudunits2-0 \
+                   libxml2 > /dev/null
 
-# Install R packages
-# Our pre-built R packages from rspm are built against system libs in focal
-# rstan takes forever to compile from source, and needs libnodejs
-# So we install older (10.x) nodejs from apt rather than newer from conda
-ENV R_VERSION=4.1.2-1.2004.0
+# Install R.
+# These packages must be installed into the base stage since they are in system
+# paths rather than /srv.
+# Pre-built R packages from rspm are built against system libs in jammy.
+ENV R_VERSION=4.4.1-1.2204.0
+ENV LITTLER_VERSION=0.3.19-1.2204.0
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
-RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" > /etc/apt/sources.list.d/cran.list
+RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" > /etc/apt/sources.list.d/cran.list
+RUN curl --silent --location --fail https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc > /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
 RUN apt-get update -qq --yes > /dev/null && \
     apt-get install --yes -qq \
-    r-base-core=${R_VERSION} \
-    r-base-dev=${R_VERSION} \
-    r-cran-littler=0.3.14-1.2004.0 \
-    libglpk-dev \
-    libzmq5 \
-    nodejs npm > /dev/null
+        r-base-core=${R_VERSION} \
+        r-base-dev=${R_VERSION} \
+        littler=${LITTLER_VERSION} \
+        libglpk-dev \
+        libzmq5 \
+        nodejs npm > /dev/null
+
+ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2024.04.2-764-amd64.deb
+RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \
+    apt install --no-install-recommends --yes /tmp/rstudio.deb && \
+    rm /tmp/rstudio.deb
 
 # Install desktop packages
 RUN apt-get update -qq --yes > /dev/null && \
@@ -123,30 +135,6 @@ RUN apt-get update -qq --yes > /dev/null && \
 
 WORKDIR /home/jovyan
 
-# Needed by RStudio
-RUN apt-get update -qq --yes && \
-    apt-get install --yes --no-install-recommends -qq \
-        psmisc \
-        sudo \
-        libapparmor1 \
-        lsb-release \
-        libclang-dev \
-        libpq5 > /dev/null
-
-ENV RSTUDIO_URL https://download2.rstudio.org/server/bionic/amd64/rstudio-server-2021.09.1-372-amd64.deb
-RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \
-    dpkg -i /tmp/rstudio.deb && \
-    rm /tmp/rstudio.deb
-
-# Needed by many R libraries
-# Picked up from https://github.com/rocker-org/rocker/blob/9dc3e458d4e92a8f41ccd75687cd7e316e657cc0/r-rspm/focal/Dockerfile
-RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-                   libgdal26 \
-                   libgeos-3.8.0 \
-                   libproj15 \
-                   libudunits2-0 \
-                   libxml2 > /dev/null
 # R_LIBS_USER is set by default in /etc/R/Renviron, which RStudio loads.
 # We uncomment the default, and set what we wanna - so it picks up
 # the packages we install. Without this, RStudio doesn't see the packages

deployments/biology/image/Rprofile.site

@@ -1,10 +1,10 @@
 # Use RStudio's CRAN mirror to get binary packages
 # 'latest' just means it has all available versions.
 # We can specify version numbers in devtools::install_version
-options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/focal/latest"))
+options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/jammy/latest"))
 
 # RStudio's CRAN mirror needs this to figure out which binary package to serve.
 # If not set properly, it will just serve up source packages
 # Quite hilarious, IMO.
 # See https://docs.rstudio.com/rspm/admin/binaries.html
-options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version$platform, R.version$arch, R.version$os)))
+options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version$platform, R.version$arch, R.version$os)))

deployments/biology/image/rsession.conf

@@ -1,2 +1,2 @@
 # Use binary packages!
-r-cran-repos=https://packagemanager.rstudio.com/all/__linux__/focal/latest
+r-cran-repos=https://packagemanager.rstudio.com/all/__linux__/jammy/latest

deployments/datahub/config/common.yaml

@@ -64,6 +64,20 @@ jupyterhub:
           - course::1535811::enrollment_type::teacher
           - course::1535811::enrollment_type::ta
 
+      # MBA 247, Fall 2024, https://github.com/berkeley-dsep-infra/datahub/issues/5923
+      course-staff-1536576:
+      #  description: Enable course staff to view and access servers.
+      #  # this role provides permissions to...
+        scopes:
+          - admin-ui
+          - list:users!group=course::1536576
+          - admin:servers!group=course::1536576
+          - access:servers!group=course::1536576
+      #  # this role will be assigned to...
+        groups:
+          - course::1536576::enrollment_type::teacher
+          - course::1536576::enrollment_type::ta
+
     nodeSelector:
       hub.jupyter.org/pool-name: core-pool-2024-05-08
     initContainers:
@@ -200,3 +214,7 @@ jupyterhub:
       course::1535590: # Data 6, https://github.com/berkeley-dsep-infra/datahub/issues/5847
         mem_limit: 2048M
         mem_guarantee: 2048M
+
+      course::1537045: # ESPM 157, https://github.com/berkeley-dsep-infra/datahub/issues/5914
+        mem_limit: 4096M
+        mem_guarantee: 4096M

deployments/datahub/images/default/Dockerfile

@@ -50,16 +50,17 @@ RUN install -d -o ${NB_USER} -g ${NB_USER} ${R_LIBS_USER}
 # These packages must be installed into the base stage since they are in system
 # paths rather than /srv.
 # Pre-built R packages from rspm are built against system libs in jammy.
-ENV R_VERSION=4.3.2-1.2204.0
-ENV LITTLER_VERSION=0.3.18-2.2204.0
+#ENV R_VERSION=4.3.2-1.2204.0
+#ENV LITTLER_VERSION=0.3.18-2.2204.0
+ENV R_VERSION=4.4.1-1.2204.0
+ENV LITTLER_VERSION=0.3.19-1.2204.0
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
 RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" > /etc/apt/sources.list.d/cran.list
 RUN curl --silent --location --fail https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc > /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
 RUN apt-get update --yes > /dev/null && \
     apt-get install --yes -qq r-base-core=${R_VERSION} r-base-dev=${R_VERSION} littler=${LITTLER_VERSION} > /dev/null
 
-#ENV RSTUDIO_URL https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2023.06.0-421-amd64.deb
-ENV RSTUDIO_URL https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2023.12.0-369-amd64.deb
+ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2024.04.2-764-amd64.deb
 RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \
     apt install --no-install-recommends --yes /tmp/rstudio.deb && \
     rm /tmp/rstudio.deb
@@ -188,6 +189,18 @@ RUN /usr/local/sbin/connector-text.bash
 #COPY connectors/2021-fall-phys-188-288.bash /usr/local/sbin/
 #RUN /usr/local/sbin/2021-fall-phys-188-288.bash
 
+#ESPM, FA 24
+# https://github.com/berkeley-dsep-infra/datahub/issues/5827
+ENV VSCODE_EXTENSIONS=${CONDA_DIR}/share/code-server/extensions
+USER root
+RUN mkdir -p ${VSCODE_EXTENSIONS} && \
+    chown -R jovyan:jovyan ${VSCODE_EXTENSIONS}
+USER ${NB_USER}
+# Install Code Server Jupyter extension 
+RUN /srv/conda/bin/code-server --extensions-dir ${VSCODE_EXTENSIONS} --install-extension ms-toolsai.jupyter
+# Install Code Server Python extension
+RUN /srv/conda/bin/code-server --extensions-dir ${VSCODE_EXTENSIONS} --install-extension ms-python.python
+
 # clear out /tmp
 USER root
 RUN rm -rf /tmp/*

deployments/datahub/images/default/Rprofile.site

@@ -16,8 +16,7 @@ options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/jammy
 # below sets it to be binary. This may improve image build times.
 # If it works, it'd be better to dynamically set the R version as above, and
 # also the RStudio Server version if possible.
-options(HTTPUserAgent = "RStudio Server (2023.12.0.369); R (4.3.2 x86_64-pc-linux-gnu x86_64 linux-gnu)")
-
+options(HTTPUserAgent = "RStudio Server (2024.04.2.764); R (4.4.1 x86_64-pc-linux-gnu x86_64 linux-gnu)")
 
 ## Source extra Rprofile files.
 #

deployments/datahub/images/default/environment.yml

@@ -72,13 +72,13 @@ dependencies:
 - music21==8.3.0
 
 # data8; foundation
-- datascience==0.17.6
+#- datascience==0.17.6
 - jupyter-server-proxy==4.2.0
 - jupyter-rsession-proxy==2.2.0
 - folium==0.12.1.post1
 
 # cogsci131; spring 2018
-- nose==1.3.7
+#- nose==1.3.7
 
 # modules
 - beautifulsoup4==4.9.3
@@ -145,38 +145,41 @@ dependencies:
   # - theano-pymc==1.1.2
   # - pymc3==3.11.2
 
-  # eep 153; spring 2019
   - requests==2.31.0
-  - Pint==0.17
+
   # Google spreadsheets, Eric Van Dusen / Keeley Takimoto / Modules
   - gspread-pandas==2.3.0
   - gspread==4.0.1
 
   # eps 109; fall 2019
+  # sknapp 2024-08-12:  keeping this package as i believe it's still useful
   - ffmpeg-python==0.2.0
 
   # issue #875, global 150Q/pacs 190 - fall 2019
+  # sknapp 2024-08-12:  keeping this package as i believe it's still useful
   - wordcloud==1.9.3
 
   # issue #929, SW 282 - fall 2019
+  # sknapp 2024-08-12:  keeping this package as i believe it's still useful
   - pyreadstat==1.2.7
 
   # issue 954, EPS24 - fall 2019
+  # sknapp 2024-08-12:  keeping this package as i believe it's still useful
   - xarray==0.19.0
 
   # issue 1001, Physics 188/288 - fall 2019
-  - umap-learn==0.5.1
-  - hdbscan==0.8.31
+  #- umap-learn==0.5.1
+  #- hdbscan==0.8.31
 
   # espm 125/bio 105; fall 2019
   # see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-  - bitarray==2.3.0
-  - nlmpy==1.0.1
+  #- bitarray==2.3.0
+  #- nlmpy==1.0.1
 
   # physics 188/288 fall, 2019
-  - getdist==1.3.1
-  - tensorflow-hub==0.12.0
-  - tensorflow-probability==0.13.0
+  #- getdist==1.3.1
+  #- tensorflow-hub==0.12.0
+  #- tensorflow-probability==0.13.0
 
   # cs16A/B, spring 2020
   - lcapy==0.96
@@ -228,6 +231,7 @@ dependencies:
   - rtree==1.2.0
 
   # [DH-319] https://github.com/berkeley-dsep-infra/datahub/issues/5827, ESPM 157
-  - ibis-framework[pandas]==9.2.0
+  - ibis-framework[pandas,duckdb]==9.2.0
+  - jupyterlab_myst==2.4.2
 
   # ATTEMPT TO PUT NEW PACKAGES IN THE CONDA LIST ABOVE FIRST, RATHER THAN PIP

deployments/datahub/images/default/r-packages/2023-fall-mba-247.r

@@ -14,6 +14,7 @@ class_libs = c(
   "pROC", "1.18.4",
   "rpart.plot", "3.1.1",
   "randomForest","4.7-1.1",
-  "xgboost","1.7.5.1"
+  "xgboost","1.7.5.1",
+  "factoextra","1.0.7"
 )
 class_libs_install_version(class_name, class_libs)

deployments/dev/images/default/Dockerfile

@@ -34,8 +34,8 @@ RUN apt-get -qq update --yes && \
 # These packages must be installed into the base stage since they are in system
 # paths rather than /srv.
 # Pre-built R packages from rspm are built against system libs in jammy.
-ENV R_VERSION=4.3.2-1.2204.0
-ENV LITTLER_VERSION=0.3.18-2.2204.0
+ENV R_VERSION=4.4.1-1.2204.0
+ENV LITTLER_VERSION=0.3.19-1.2204.0
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
 RUN echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" > /etc/apt/sources.list.d/cran.list
 RUN curl --silent --location --fail https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc > /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
@@ -45,7 +45,7 @@ RUN apt-get update -qq --yes > /dev/null && \
         r-base-dev=${R_VERSION} \
         littler=${LITTLER_VERSION} > /dev/null
 
-ENV RSTUDIO_URL https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2023.06.0-421-amd64.deb
+ENV RSTUDIO_URL=https://download2.rstudio.org/server/jammy/amd64/rstudio-server-2024.04.2-764-amd64.deb
 RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \
     apt install --no-install-recommends --yes /tmp/rstudio.deb && \
     rm /tmp/rstudio.deb

deployments/dev/images/default/Rprofile.site

@@ -17,7 +17,7 @@ options(
 options(
   HTTPUserAgent = sprintf(
     "RStudio Server (%s); R (%s)",
-    "2022.7.1.554",
+    "2024.04.2.764",
     paste(getRversion(), R.version$platform, R.version$arch, R.version$os)
   )
 )

deployments/ischool/image/Dockerfile

@@ -1,4 +1,4 @@
-FROM rocker/geospatial:4.3.1
+FROM rocker/geospatial:4.4.1
 
 ENV NB_USER rstudio
 ENV NB_UID 1000

deployments/publichealth/image/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/rocker-org/geospatial:4.3.2
+FROM ghcr.io/rocker-org/geospatial:4.4.1
 
 ENV NB_USER rstudio
 ENV NB_UID 1000

deployments/shiny/image/Dockerfile

@@ -1,4 +1,4 @@
-FROM rocker/geospatial:4.3.2
+FROM rocker/geospatial:4.4.1
 
 ENV NB_USER rstudio
 ENV NB_UID 1000
@@ -55,7 +55,7 @@ RUN apt-get update > /dev/null && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.21.1012-amd64.deb
+ENV SHINY_SERVER_URL https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-1.5.22.1017-amd64.deb
 RUN curl --silent --location --fail ${SHINY_SERVER_URL} > /tmp/shiny-server.deb && \
     apt install --no-install-recommends --yes /tmp/shiny-server.deb && \
     rm /tmp/shiny-server.deb

deployments/stat20/image/Dockerfile

@@ -1,4 +1,4 @@
-FROM rocker/geospatial:4.3.2
+FROM rocker/geospatial:4.4.1
 # https://github.com/rocker-org/rocker-versioned2/wiki/geospatial_e06f866673fa
 
 ENV NB_USER rstudio