feat docker: minimize the layers count and add ppa to the base

apolukhin · apolukhin · commit 99bdb671bf52 · 2024-02-21T15:45:46.000+03:00
Before the changes:
```
REPOSITORY                                                TAG       IMAGE ID       CREATED             SIZE
ghcr.io/userver-framework/ubuntu-22.04-userver-base-ci    latest    ff57d3b2fa34   3 hours ago         2.43GB
ghcr.io/userver-framework/ubuntu-22.04-userver-base       latest    adfde06b0ce5   3 hours ago         1.07GB
```
After:
```
REPOSITORY                                                TAG       IMAGE ID       CREATED              SIZE
ghcr.io/userver-framework/ubuntu-22.04-userver-base-ci    latest    4de2e079395e   15 seconds ago       2.41GB
ghcr.io/userver-framework/ubuntu-22.04-userver-base       latest    c44bd11fdcf6   About a minute ago   1.11GB
```

Tests: протестировано локально
0769691c730049dc5622bf92fdc36b8542e2fbcb
diff --git a/.mapping.json b/.mapping.json
@@ -2658,7 +2658,6 @@
   "scripts/docker/base-ubuntu-22.04.dockerfile":"taxi/uservices/userver/scripts/docker/base-ubuntu-22.04.dockerfile",
   "scripts/docker/core-ubuntu-20-gcc8/Dockerfile":"taxi/uservices/userver/scripts/docker/core-ubuntu-20-gcc8/Dockerfile",
   "scripts/docker/core-ubuntu-20-gcc8/prepare_image.sh":"taxi/uservices/userver/scripts/docker/core-ubuntu-20-gcc8/prepare_image.sh",
-  "scripts/docker/install-additional-compilers-ubuntu-22.04.sh":"taxi/uservices/userver/scripts/docker/install-additional-compilers-ubuntu-22.04.sh",
   "scripts/docker/pip-install.sh":"taxi/uservices/userver/scripts/docker/pip-install.sh",
   "scripts/docker/setup-base-ubuntu-22.04-env.sh":"taxi/uservices/userver/scripts/docker/setup-base-ubuntu-22.04-env.sh",
   "scripts/docs/README.md":"taxi/uservices/userver/scripts/docs/README.md",
diff --git a/scripts/docker/add_user.sh b/scripts/docker/add_user.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-# Exit on any error
-set -eo pipefail
+# Exit on any error and treat unset variables as errors
+set -euo pipefail
 
 DIR_UID="$(stat -c '%u' .)"
 if [ "$DIR_UID" = "0" ]; then
diff --git a/scripts/docker/all-ubuntu-22-cl16/Dockerfile b/scripts/docker/all-ubuntu-22-cl16/Dockerfile
@@ -4,48 +4,8 @@ FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base-ci:latest
 # Set UTC timezone
 ENV DEBIAN_FRONTEND noninteractive
 
-# install main packages
-RUN apt-get update && apt-get install -y --allow-unauthenticated \
-    build-essential \
-    binutils-dev \
-    libmongoc-dev \
-    libbson-dev \
-    libpq-dev \
-    postgresql-server-dev-all \
-    libhiredis-dev \
-    clang-format \
-    clang-tidy \
-    postgresql \
-    python3-protobuf \
-    python3-jinja2 \
-    python3-venv \
-    vim \
-    redis-server
-
-RUN apt-get clean all
-
-
-# add clickhouse repositories
-RUN sudo apt-get install -y apt-transport-https ca-certificates dirmngr
-RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
-
-RUN echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
-    /etc/apt/sources.list.d/clickhouse.list
-
-# add mongodb repositories
-RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
-
-RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
-
-RUN echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list
-
-RUN apt-get update && \
-    apt-get -o Dpkg::Options::="--force-confold" upgrade -q -y --force-yes && \
-    apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --force-yes
-
-RUN apt update && apt install libssl1.1
-
-RUN apt-get install -y --allow-unauthenticated \
+RUN apt-get install -y \
+    redis-server \
     mongodb-org \
     mongodb-org-database \
     mongodb-org-server \
@@ -54,31 +14,9 @@ RUN apt-get install -y --allow-unauthenticated \
     mongodb-org-tools \
     postgresql-common \
     clickhouse-server \
-    clickhouse-client
-
-# add mariadb repositories
-RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup
-RUN chmod +x mariadb_repo_setup
-RUN ./mariadb_repo_setup
-
-RUN apt-get install -y libmariadb-dev mariadb-server
+    clickhouse-client \
+    mariadb-server
 
-# convoluted setup of rabbitmq + erlang taken from https://www.rabbitmq.com/install-debian.html#apt-quick-start-packagecloud
-## Team RabbitMQ's main signing key
-RUN curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null
-## Launchpad PPA that provides modern Erlang releases
-RUN curl -1sLf "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xf77f1eda57ebb1cc" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg > /dev/null
-## PackageCloud RabbitMQ repository
-RUN curl -1sLf "https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.packagecloud.rabbitmq.gpg > /dev/null
-## Add apt repositories maintained by Team RabbitMQ
-RUN printf "\
-deb [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
-deb-src [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
-deb [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main \n\
-deb-src [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main\n" \
-    | sudo tee /etc/apt/sources.list.d/rabbitmq.list
-## Update package indices
-RUN sudo apt-get update -y
 ## Install Erlang packages
 RUN sudo apt-get install -y erlang-base \
                 erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
diff --git a/scripts/docker/all-ubuntu-22/Dockerfile b/scripts/docker/all-ubuntu-22/Dockerfile
@@ -4,48 +4,8 @@ FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest
 # Set UTC timezone
 ENV DEBIAN_FRONTEND noninteractive
 
-# install main packages
-RUN apt-get update && apt-get install -y --allow-unauthenticated \
-    build-essential \
-    binutils-dev \
-    libmongoc-dev \
-    libbson-dev \
-    libpq-dev \
-    postgresql-server-dev-all \
-    libhiredis-dev \
-    clang-format \
-    clang-tidy \
-    postgresql \
-    python3-protobuf \
-    python3-jinja2 \
-    python3-venv \
-    vim \
-    redis-server
-
-RUN apt-get clean all
-
-
-# add clickhouse repositories
-RUN sudo apt-get install -y apt-transport-https ca-certificates dirmngr
-RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
-
-RUN echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
-    /etc/apt/sources.list.d/clickhouse.list
-
-# add mongodb repositories
-RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
-
-RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
-
-RUN echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list
-
-RUN apt-get update && \
-    apt-get -o Dpkg::Options::="--force-confold" upgrade -q -y --force-yes && \
-    apt-get -o Dpkg::Options::="--force-confold" dist-upgrade -q -y --force-yes
-
-RUN apt update && apt install libssl1.1
-
-RUN apt-get install -y --allow-unauthenticated \
+RUN apt-get install -y \
+    redis-server \
     mongodb-org \
     mongodb-org-database \
     mongodb-org-server \
@@ -54,31 +14,9 @@ RUN apt-get install -y --allow-unauthenticated \
     mongodb-org-tools \
     postgresql-common \
     clickhouse-server \
-    clickhouse-client
-
-# add mariadb repositories
-RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup
-RUN chmod +x mariadb_repo_setup
-RUN ./mariadb_repo_setup
-
-RUN apt-get install -y libmariadb-dev mariadb-server
+    clickhouse-client \
+    mariadb-server
 
-# convoluted setup of rabbitmq + erlang taken from https://www.rabbitmq.com/install-debian.html#apt-quick-start-packagecloud
-## Team RabbitMQ's main signing key
-RUN curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null
-## Launchpad PPA that provides modern Erlang releases
-RUN curl -1sLf "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xf77f1eda57ebb1cc" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg > /dev/null
-## PackageCloud RabbitMQ repository
-RUN curl -1sLf "https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.packagecloud.rabbitmq.gpg > /dev/null
-## Add apt repositories maintained by Team RabbitMQ
-RUN printf "\
-deb [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
-deb-src [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
-deb [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main \n\
-deb-src [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main\n" \
-    | sudo tee /etc/apt/sources.list.d/rabbitmq.list
-## Update package indices
-RUN sudo apt-get update -y
 ## Install Erlang packages
 RUN sudo apt-get install -y erlang-base \
                 erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
diff --git a/scripts/docker/base-ubuntu-22.04-ci.dockerfile b/scripts/docker/base-ubuntu-22.04-ci.dockerfile
@@ -1,10 +1,16 @@
 FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest
 
-COPY scripts/docker/install-additional-compilers-ubuntu-22.04.sh /userver_tmp/
-RUN (cd /userver_tmp && ./install-additional-compilers-ubuntu-22.04.sh)
-RUN rm -rf /userver_tmp
-
-RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends redis-server postgresql-14
-
-# fix for porto layers
-RUN mkdir -p /place/berkanavt/ && DEBIAN_FRONTEND=noninteractive apt install -y fuse dupload libuv1 libuv1-dev
+# Fix for porto layers and install a bunch of compilers and databases
+RUN mkdir -p /place/berkanavt/ && \
+  DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+    fuse dupload libuv1 libuv1-dev \
+    redis-server \
+    postgresql-14 \
+    clang-16 \
+    lld-16 \
+    clang-14 \
+    lld-14 \
+    g++-11 \
+    gcc-11 \
+    g++-13 \
+    gcc-13
diff --git a/scripts/docker/base-ubuntu-22.04.dockerfile b/scripts/docker/base-ubuntu-22.04.dockerfile
@@ -3,7 +3,6 @@ FROM ubuntu:22.04
 COPY scripts/docs/en/deps/ubuntu-22.04.md /userver_tmp/
 COPY scripts/docker/setup-base-ubuntu-22.04-env.sh /userver_tmp/
 COPY scripts/docker/add_user.sh /userver_tmp/
-RUN (cd /userver_tmp && ./setup-base-ubuntu-22.04-env.sh && ./add_user.sh)
 
 COPY scripts/external_deps/requirements.txt  /userver_tmp/requirements/external-deps.txt
 COPY scripts/grpc/requirements.txt           /userver_tmp/requirements/grpc-userver.txt
@@ -18,9 +17,14 @@ COPY testsuite/requirements-net.txt          /userver_tmp/requirements/net.txt
 COPY testsuite/requirements.txt              /userver_tmp/requirements/testsuite-base.txt
 
 COPY scripts/docker/pip-install.sh           /userver_tmp/
-RUN (cd /userver_tmp && sudo -u user ./pip-install.sh)
 
-RUN rm -rf /userver_tmp
+RUN ( \
+  cd /userver_tmp \
+  && ./setup-base-ubuntu-22.04-env.sh \
+  && ./add_user.sh \
+  && sudo -u user ./pip-install.sh \
+  && rm -rf /userver_tmp \
+)
 
 # add expose ports
 EXPOSE 8080-8100
diff --git a/scripts/docker/install-additional-compilers-ubuntu-22.04.sh b/scripts/docker/install-additional-compilers-ubuntu-22.04.sh
diff --git a/scripts/docker/pip-install.sh b/scripts/docker/pip-install.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-# Exit on any error
-set -eo pipefail
+# Exit on any error and treat unset variables as errors
+set -euo pipefail
 
 for REQUIREMENT in requirements/*.txt; do
   pip3 install -r ${REQUIREMENT}
diff --git a/scripts/docker/setup-base-ubuntu-22.04-env.sh b/scripts/docker/setup-base-ubuntu-22.04-env.sh
@@ -1,7 +1,62 @@
 #!/bin/bash
 
-# Exit on any error
-set -eo pipefail
+# Exit on any error and treat unset variables as errors
+set -euo pipefail
+
+# Preparing to add new repos
+apt update 
+DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+  apt-transport-https ca-certificates dirmngr wget curl software-properties-common \
+  gnupg gnupg2
+
+gpg_retrieve() {
+  # See https://unix.stackexchange.com/questions/682929/migrating-away-from-apt-key-adv
+  # and https://github.com/llvm/llvm-project/issues/55784#issuecomment-1569643347
+  curl -fsSL "$1" | gpg --dearmor -o "/usr/share/keyrings/$2.gpg"
+  chmod a+r "/usr/share/keyrings/$2.gpg"
+}
+
+
+# Adding clang/llvm repos
+gpg_retrieve https://apt.llvm.org/llvm-snapshot.gpg.key llvm-snapshot
+printf "\
+deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main \n\
+deb-src [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main\n" > /etc/apt/sources.list.d/llvm.list
+
+# Adding GCC repos
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
+
+# Adding clickhouse repositories as in https://clickhouse.com/docs/en/install#setup-the-debian-repository
+GNUPGHOME=$(mktemp -d)
+GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring /usr/share/keyrings/clickhouse-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754
+rm -rf "$GNUPGHOME"
+chmod +r /usr/share/keyrings/clickhouse-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main"  \
+    | tee /etc/apt/sources.list.d/clickhouse.list
+
+# Adding mariadb repositories (from https://www.linuxcapable.com/how-to-install-mariadb-on-ubuntu-linux/ )
+gpg_retrieve http://mirror.mariadb.org/PublicKey_v2 mariadb
+echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] http://mirror.mariadb.org/repo/10.11/ubuntu/ $(lsb_release -cs) main" \
+    | tee /etc/apt/sources.list.d/mariadb.list
+
+# convoluted setup of rabbitmq + erlang taken from https://www.rabbitmq.com/install-debian.html#apt-quick-start-packagecloud
+## Team RabbitMQ's main signing key
+gpg_retrieve https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA com.rabbitmq.team
+## Launchpad PPA that provides modern Erlang releases
+gpg_retrieve "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xf77f1eda57ebb1cc" net.launchpad.ppa.rabbitmq.erlang
+## PackageCloud RabbitMQ repository
+gpg_retrieve https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey io.packagecloud.rabbitmq
+## Add apt repositories maintained by Team RabbitMQ
+printf "\
+deb [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
+deb-src [signed-by=/usr/share/keyrings/net.launchpad.ppa.rabbitmq.erlang.gpg] http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu focal main \n\
+deb [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main \n\
+deb-src [signed-by=/usr/share/keyrings/io.packagecloud.rabbitmq.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ focal main\n" \
+    | tee /etc/apt/sources.list.d/rabbitmq.list
+
+gpg_retrieve https://www.mongodb.org/static/pgp/server-6.0.asc mongodb
+echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" \
+    | tee /etc/apt/sources.list.d/mongodb-org-6.0.list
 
 # Install build dependencies
 if [ ! -f ubuntu-22.04.md ]; then
@@ -26,12 +81,7 @@ apt update
 DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
   ${PACKAGES_TO_INSTALL} \
   chrpath \
-  curl \
   sudo \
-  gnupg \
-  gnupg2 \
-  wget \
-  dirmngr \
   python3-pip \
   locales
 
