Skip to content

Commit

Permalink
release: v2.3.0 (#542)
Browse files Browse the repository at this point in the history
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Clayton Parnell <[email protected]>
Co-authored-by: SageMaker Distribution Bot <sm-distribution-github-bot[bot]@amazon.com>
  • Loading branch information
4 people authored Jan 28, 2025
1 parent f4db80b commit 61e0068
Show file tree
Hide file tree
Showing 29 changed files with 2,349 additions and 1 deletion.
40 changes: 40 additions & 0 deletions build_artifacts/v2/v2.3/v2.3.0/CHANGELOG-cpu.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Change log: 2.3.0(cpu)

## Upgrades:

Package | Previous Version | Current Version
---|---|---
jinja2|3.1.4|3.1.5
altair|5.4.1|5.5.0
boto3|1.34.162|1.36.3
ipython|8.29.0|8.31.0
langchain|0.2.17|0.3.15
jupyter-ai|2.28.3|2.29.0
amazon-sagemaker-jupyter-ai-q-developer|1.0.14|1.0.16
jupyter-scheduler|2.9.0|2.10.0
amazon-sagemaker-jupyter-scheduler|3.1.7|3.1.8
amazon_sagemaker_sql_editor|0.1.13|0.1.14
scipy|1.14.1|1.15.1
matplotlib-base|3.9.3|3.10.0
autogluon|1.1.1|1.2.0
aws-glue-sessions|1.0.7|1.0.8
libmamba|1.5.11|1.5.12
conda|24.9.2|24.11.3
docker-cli|27.1.1|27.5.1
uvicorn|0.32.1|0.34.0
fastapi|0.115.6|0.115.7
keras|3.6.0|3.8.0
langchain-aws|0.1.18|0.2.10
mlflow|2.17.2|2.20.0
sagemaker-code-editor|1.4.1|1.4.2
sagemaker-jupyterlab-emr-extension|0.3.4|0.3.7
sagemaker-jupyterlab-extension|0.3.2|0.3.3
sagemaker-python-sdk|2.227.0|2.228.0
sagemaker-studio-analytics-extension|0.1.2|0.1.4

## What's new:

Package | Version
---|---
s3fs|2024.10.0
seaborn|0.13.2
42 changes: 42 additions & 0 deletions build_artifacts/v2/v2.3/v2.3.0/CHANGELOG-gpu.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Change log: 2.3.0(gpu)

## Upgrades:

Package | Previous Version | Current Version
---|---|---
jinja2|3.1.4|3.1.5
pytorch|2.3.1|2.4.1
altair|5.4.1|5.5.0
boto3|1.34.162|1.36.3
ipython|8.29.0|8.31.0
langchain|0.2.17|0.3.15
jupyter-ai|2.28.3|2.29.0
amazon-sagemaker-jupyter-ai-q-developer|1.0.14|1.0.16
jupyter-scheduler|2.9.0|2.10.0
amazon-sagemaker-jupyter-scheduler|3.1.7|3.1.8
amazon_sagemaker_sql_editor|0.1.13|0.1.14
scipy|1.14.1|1.15.1
matplotlib-base|3.9.3|3.10.0
torchvision|0.18.1|0.19.1
autogluon|1.1.1|1.2.0
aws-glue-sessions|1.0.7|1.0.8
libmamba|1.5.11|1.5.12
conda|24.9.2|24.11.3
docker-cli|27.1.1|27.5.1
uvicorn|0.32.1|0.34.0
fastapi|0.115.6|0.115.7
keras|3.6.0|3.8.0
langchain-aws|0.1.18|0.2.10
mlflow|2.17.2|2.20.0
sagemaker-code-editor|1.4.1|1.4.2
sagemaker-jupyterlab-emr-extension|0.3.4|0.3.7
sagemaker-jupyterlab-extension|0.3.2|0.3.3
sagemaker-python-sdk|2.227.0|2.228.0
sagemaker-studio-analytics-extension|0.1.2|0.1.4

## What's new:

Package | Version
---|---
s3fs|2024.10.0
seaborn|0.13.2
213 changes: 213 additions & 0 deletions build_artifacts/v2/v2.3/v2.3.0/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
ARG TAG_FOR_BASE_MICROMAMBA_IMAGE
FROM mambaorg/micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE

ARG CUDA_MAJOR_MINOR_VERSION=''
ARG ENV_IN_FILENAME
ARG ARG_BASED_ENV_IN_FILENAME
ARG IMAGE_VERSION
LABEL "org.amazon.sagemaker-distribution.image.version"=$IMAGE_VERSION

ARG AMZN_BASE="/opt/amazon/sagemaker"
ARG DB_ROOT_DIR="/opt/db"
ARG DIRECTORY_TREE_STAGE_DIR="${AMZN_BASE}/dir-staging"

ARG NB_USER="sagemaker-user"
ARG NB_UID=1000
ARG NB_GID=100

# https://www.openssl.org/source/
ARG FIPS_VALIDATED_SSL=3.0.8
ARG MIN_REQUIRED_MICROMAMBA_VERSION=1.5.11

ENV SAGEMAKER_LOGGING_DIR="/var/log/sagemaker/"
ENV STUDIO_LOGGING_DIR="/var/log/studio/"
ENV EDITOR="nano"
ENV IMAGE_VERSION=$IMAGE_VERSION
ENV PINNED_MICROMAMBA_MINOR_VERSION="1.5.*"
ENV SAGEMAKER_RECOVERY_MODE_HOME=/tmp/sagemaker-recovery-mode-home

USER root
# Upgrade micromamba to the latest patch version in the pinned minor version range, if applicable
RUN CURRENT_MICROMAMBA_VERSION=$(micromamba --version) && \
echo "Current micromamba version: $CURRENT_MICROMAMBA_VERSION" && \
if [[ "$CURRENT_MICROMAMBA_VERSION" == $PINNED_MICROMAMBA_MINOR_VERSION ]]; then \
echo "Upgrading micromamba to the latest $PINNED_MICROMAMBA_MINOR_VERSION version..." && \
micromamba self-update -c conda-forge --version "$PINNED_MICROMAMBA_MINOR_VERSION" && \
micromamba clean --all --yes --force-pkgs-dirs; \
else \
echo "Micromamba is already at version $CURRENT_MICROMAMBA_VERSION (outside $PINNED_MICROMAMBA_MINOR_VERSION). No upgrade performed."; \
fi

RUN usermod "--login=${NB_USER}" "--home=/home/${NB_USER}" --move-home "-u ${NB_UID}" "${MAMBA_USER}" && \
groupmod "--new-name=${NB_USER}" --non-unique "-g ${NB_GID}" "${MAMBA_USER}" && \
# Update the expected value of MAMBA_USER for the
# _entrypoint.sh consistency check.
echo "${NB_USER}" > "/etc/arg_mamba_user" && \
:
ENV MAMBA_USER=$NB_USER
ENV USER=$NB_USER

RUN apt-get update && apt-get upgrade -y && \
apt-get install -y --no-install-recommends sudo gettext-base wget curl unzip git rsync build-essential openssh-client nano cron less mandoc && \
# We just install tzdata below but leave default time zone as UTC. This helps packages like Pandas to function correctly.
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata krb5-user libkrb5-dev libsasl2-dev libsasl2-modules && \
chmod g+w /etc/passwd && \
echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
touch /etc/krb5.conf.lock && chown ${NB_USER}:${MAMBA_USER} /etc/krb5.conf* && \
# Note that we do NOT run `rm -rf /var/lib/apt/lists/*` here. If we did, anyone building on top of our images will
# not be able to run any `apt-get install` commands and that would hamper customizability of the images.
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
sudo ./aws/install && \
rm -rf aws awscliv2.zip && \
: && \
echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile && \
# CodeEditor - create server, user data dirs
mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \
&& chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data && \
# create dir to store user data files
mkdir -p /opt/amazon/sagemaker/user-data \
&& chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data && \
# Merge in OS directory tree contents.
mkdir -p ${DIRECTORY_TREE_STAGE_DIR}
COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/
RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \
rm -rf ${DIRECTORY_TREE_STAGE_DIR} && \
# CodeEditor - download the extensions
mkdir -p /etc/code-editor/extensions && \
while IFS= read -r url || [ -n "$url" ]; do \
echo "Downloading extension from ${url}..." && \
wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \
done < /etc/code-editor/extensions.txt

USER $MAMBA_USER
COPY --chown=$MAMBA_USER:$MAMBA_USER $ENV_IN_FILENAME *.in /tmp/
ARG MAMBA_DOCKERFILE_ACTIVATE=1
ARG CONDA_OVERRIDE_CUDA=$CUDA_MAJOR_MINOR_VERSION

# Make sure that $ENV_IN_FILENAME has a newline at the end before the `tee` command runs. Otherwise, nasty things
# will happen.
RUN if [[ -z $ARG_BASED_ENV_IN_FILENAME ]] ; \
then echo 'No ARG_BASED_ENV_IN_FILENAME passed' ; \
else envsubst < /tmp/$ARG_BASED_ENV_IN_FILENAME | tee --append /tmp/$ENV_IN_FILENAME ; \
fi && \
# Enforce dependencies are all installed from conda-forge
micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \
mkdir -p $SAGEMAKER_RECOVERY_MODE_HOME && \
chown $MAMBA_USER:$MAMBA_USER $SAGEMAKER_RECOVERY_MODE_HOME && \
SUPERVISOR_VERSION=$(micromamba list | grep 'supervisor ' | tr -s ' ' | cut -d ' ' -f 3 | head -n 1) && \
JUPYTERLAB_VERSION=$(micromamba list | grep 'jupyterlab ' | tr -s ' ' | cut -d ' ' -f 3 | head -n 1) && \
SAGEMAKER_JUPYTERLAB_VERSION=$(micromamba list | grep 'sagemaker-jupyterlab-extension ' | tr -s ' ' | cut -d ' ' -f 3 | head -n 1) && \
echo "Installing in sagemaker-recovery-mode micromamba environment: jupyterlab $JUPYTERLAB_VERSION sagemaker-jupyterlab-extension $SAGEMAKER_JUPYTERLAB_VERSION" && \
micromamba create -n sagemaker-recovery-mode && \
micromamba install -n sagemaker-recovery-mode -y -c conda-forge \
jupyterlab==$JUPYTERLAB_VERSION \
sagemaker-jupyterlab-extension==$SAGEMAKER_JUPYTERLAB_VERSION \
supervisor==$SUPERVISOR_VERSION && \
micromamba clean --all --yes --force-pkgs-dirs && \
rm -rf /tmp/*.in && \
sudo ln -s $(which python3) /usr/bin/python && \
# Update npm version
npm i -g npm && \
# Enforce to use `conda-forge` as only channel, by removing `defaults`
conda config --remove channels defaults && \
micromamba config append channels conda-forge --env && \
# Configure CodeEditor - Install extensions and set preferences
extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \
# Loop through all vsix files in /etc/code-editor/extensions and install them
&& for ext in /etc/code-editor/extensions/*.vsix; do \
echo "Installing extension ${ext}..."; \
sagemaker-code-editor --install-extension "${ext}" --extensions-dir "${extensionloc}" --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data; \
done \
# Copy the settings
&& cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json && \
cp /etc/code-editor/code_editor_user_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/User/settings.json && \
# Install glue kernels, and move to shared directory
# Also patching base kernel so Studio background code doesn't start session silently
install-glue-kernels && \
SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \
jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \
jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \
mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_pyspark /opt/conda/share/jupyter/kernels && \
mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_spark /opt/conda/share/jupyter/kernels && \
sed -i '/if not store_history and (/i\ if "sm_analytics_runtime_check" in code:\n return await self._complete_cell()\n' \
"$SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_kernel_base/BaseKernel.py" && \
# Install FIPS Provider for OpenSSL, on top of existing OpenSSL installation
# v3.0.8 is latest FIPS validated provider, so this is the one we install
# But we need to run tests against the installed version.
# see https://github.com/openssl/openssl/blob/master/README-FIPS.md https://www.openssl.org/source/
INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 | head -n 1) && \
# download source code for installed, and FIPS validated openssl versions
curl -L https://github.com/openssl/openssl/releases/download/openssl-$FIPS_VALIDATED_SSL/openssl-$FIPS_VALIDATED_SSL.tar.gz > openssl-$FIPS_VALIDATED_SSL.tar.gz && \
curl -L https://github.com/openssl/openssl/releases/download/openssl-$INSTALLED_SSL/openssl-$INSTALLED_SSL.tar.gz > openssl-$INSTALLED_SSL.tar.gz && \
tar -xf openssl-$FIPS_VALIDATED_SSL.tar.gz && tar -xf openssl-$INSTALLED_SSL.tar.gz && cd openssl-$FIPS_VALIDATED_SSL && \
# Configure both versions to enable FIPS and build
./Configure enable-fips --prefix=/opt/conda --openssldir=/opt/conda/ssl && make && \
cd ../openssl-$INSTALLED_SSL && \
./Configure enable-fips --prefix=/opt/conda --openssldir=/opt/conda/ssl && make && \
# Copy validated provider to installed version for testing
cp ../openssl-$FIPS_VALIDATED_SSL/providers/fips.so providers/. && \
cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \
make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \
# After tests pass, install FIPS provider and remove source code
make install_fips && cd .. && rm -rf ./openssl-* && \
# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS
# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf
cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \
sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \
sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf && \
# Install Kerberos.
# Make sure no dependency is added/updated
pip install "krb5>=0.5.1,<0.6" && \
pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' && \
# https://stackoverflow.com/questions/122327
SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \
# Remove SparkRKernel as it's not supported \
jupyter-kernelspec remove -f -y sparkrkernel && \
# Patch Sparkmagic lib to support Custom Certificates \
# https://github.com/jupyter-incubator/sparkmagic/pull/435/files \
cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/configuration.py ${SYSTEM_PYTHON_PATH}/sparkmagic/utils/ && \
cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/reliablehttpclient.py ${SYSTEM_PYTHON_PATH}/sparkmagic/livyclientlib/reliablehttpclient.py && \
sed -i 's= "python"= "/opt/conda/bin/python"=g' /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
sed -i 's="Spark"="SparkMagic Spark"=g' /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
sed -i 's="PySpark"="SparkMagic PySpark"=g' /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json && \
# Configure RTC - disable jupyter_collaboration by default
jupyter labextension disable @jupyter/collaboration-extension

# Patch glue kernels to use kernel wrapper
COPY patch_glue_pyspark.json /opt/conda/share/jupyter/kernels/glue_pyspark/kernel.json
COPY patch_glue_spark.json /opt/conda/share/jupyter/kernels/glue_spark/kernel.json

USER root

# Create logging directories for supervisor
RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \
chmod a+rw $SAGEMAKER_LOGGING_DIR && \
mkdir -p ${STUDIO_LOGGING_DIR} && \
chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} && \
# Clean up CodeEditor artifacts
rm -rf /etc/code-editor && \
# Create supervisord runtime directory
mkdir -p /var/run/supervisord && \
chmod a+rw /var/run/supervisord && \
# Create root directory for DB
# Create logging directories for supervisor
mkdir -p $DB_ROOT_DIR && \
chmod a+rw $DB_ROOT_DIR && \
HOME_DIR="/home/${NB_USER}/licenses" \
&& mkdir -p ${HOME_DIR} \
&& curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
&& unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \
&& cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \
&& chmod +x /usr/local/bin/testOSSCompliance \
&& chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \
&& ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \
&& rm -rf ${HOME_DIR}/oss_compliance*

# Explicitly disable BuildKit for SM Studio Docker functionality
ENV DOCKER_BUILDKIT=0
ENV PATH="/opt/conda/bin:/opt/conda/condabin:$PATH"
WORKDIR "/home/${NB_USER}"
ENV SHELL=/bin/bash
ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/
USER $MAMBA_USER
67 changes: 67 additions & 0 deletions build_artifacts/v2/v2.3/v2.3.0/RELEASE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Release notes: 2.3.0

Package | gpu| cpu
---|---|---
python|3.11.11|3.11.11
numpy|1.26.4|1.26.4
jinja2|3.1.5|3.1.5
pytorch|2.4.1|2.4.1
pandas|2.2.3|2.2.3
altair|5.5.0|5.5.0
boto3|1.36.3|1.36.3
ipython|8.31.0|8.31.0
jupyter-lsp|2.2.5|2.2.5
jupyterlab|4.2.6|4.2.6
amazon-q-developer-jupyterlab-ext|3.4.5|3.4.5
langchain|0.3.15|0.3.15
jupyter-ai|2.29.0|2.29.0
amazon-sagemaker-jupyter-ai-q-developer|1.0.16|1.0.16
jupyter-scheduler|2.10.0|2.10.0
amazon-sagemaker-jupyter-scheduler|3.1.8|3.1.8
amazon-sagemaker-sql-magic|0.1.3|0.1.3
jupyterlab-lsp|5.0.3|5.0.3
amazon_sagemaker_sql_editor|0.1.14|0.1.14
scipy|1.15.1|1.15.1
matplotlib-base|3.10.0|3.10.0
scikit-learn|1.5.2|1.5.2
evaluate|0.4.1|0.4.1
sentencepiece|0.1.99|0.1.99
pip|24.3.1|24.3.1
torchvision|0.19.1|0.19.1
autogluon|1.2.0|1.2.0
ipywidgets|8.1.5|8.1.5
notebook|7.2.2|7.2.2
aws-glue-sessions|1.0.8|1.0.8
libmamba|1.5.12|1.5.12
conda|24.11.3|24.11.3
dash|2.18.1|2.18.1
docker-cli|27.5.1|27.5.1
uvicorn|0.34.0|0.34.0
fastapi|0.115.7|0.115.7
git-remote-codecommit|1.16|1.16
jupyter-activity-monitor-extension|0.3.1|0.3.1
jupyter-collaboration|2.1.5|2.1.5
jupyter-dash|0.4.2|0.4.2
jupyter-server-proxy|4.4.0|4.4.0
jupyterlab-git|0.50.2|0.50.2
keras|3.8.0|3.8.0
langchain-aws|0.2.10|0.2.10
mlflow|2.20.0|2.20.0
py-xgboost-gpu|2.1.3|
pyhive|0.7.0|0.7.0
python-gssapi|1.9.0|1.9.0
python-lsp-server|1.12.0|1.12.0
s3fs|2024.10.0|2024.10.0
sagemaker-code-editor|1.4.2|1.4.2
sagemaker-headless-execution-driver|0.0.13|0.0.13
sagemaker-jupyterlab-emr-extension|0.3.7|0.3.7
sagemaker-jupyterlab-extension|0.3.3|0.3.3
sagemaker-kernel-wrapper|0.0.4|0.0.4
sagemaker-mlflow|0.1.0|0.1.0
sagemaker-python-sdk|2.228.0|2.228.0
sagemaker-studio-analytics-extension|0.1.4|0.1.4
seaborn|0.13.2|0.13.2
supervisor|4.2.5|4.2.5
tensorflow|2.17.0|2.17.0
tf-keras|2.17.0|2.17.0
py-xgboost-cpu| |2.1.3
Loading

0 comments on commit 61e0068

Please sign in to comment.