Change docker and cache

Author: psychomantys

.dockerignore

@@ -3,3 +3,10 @@ sitestatic
 env
 .env
 temba/settings.py
+.git
+.github
+Dockerfile
+.dockerignore
+.gitignore
+usr
+venv

.github/workflows/build-flows-push-tag-shared.yaml

@@ -12,11 +12,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        if: github.event_name != 'pull_request'
         with:
           ref: "${{env.GITHUB_SHA}}"
           token: ${{ secrets.DEVOPS_GITHUB_PERMANENT_TOKEN }}
 
+      #- name: Docker meta
+      #  id: meta
+      #  uses: docker/metadata-action@v5
+      #  with:
+      #    images: push-backend
+      #    tags: |
+      #      type=semver,pattern={{version}}-develop
+      #      type=semver,pattern={{version}}-staging
+      #      type=semver,pattern={{version}}
+
       - name: Set variables
         run: |
             TAG="$( echo "${GITHUB_REF}" | cut -d'/' -f3 )"
@@ -41,8 +52,7 @@ jobs:
             echo "IMAGE_SOURCE_URL=https://github.com/weni-ai/flows" | tee -a "${GITHUB_ENV}"
             echo "MANIFESTS_REPOSITORY=weni-ai/kubernetes-manifests-platform" | tee -a "${GITHUB_ENV}"
             echo "MANIFESTS_APPLICATION=weni-flows/flows" | tee -a "${GITHUB_ENV}"
-            echo "MANIFESTS_PATCH_TARGET=deployment-flows.json" | tee -a "${GITHUB_ENV}"
-            echo "MANIFESTS_PATCH_TARGET1=deployment-flows-image.json" | tee -a "${GITHUB_ENV}"
+            echo "MANIFESTS_PATCH_TARGET=deployment-flows-image.json" | tee -a "${GITHUB_ENV}"
 
             FLOWEDITOR_VERSION=$(jq -r '.dependencies["@nyaruka/flow-editor"]' < package.json)
             echo "FLOWEDITOR_VERSION=${FLOWEDITOR_VERSION}" | tee -a "${GITHUB_ENV}"
@@ -53,56 +63,90 @@ jobs:
             fi
             echo "FLOWEDITOR_BRANCH=$FLOWEDITOR_BRANCH" | tee -a "${GITHUB_ENV}"
 
-
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to ECR
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
         with:
           registry: ${{ secrets.ECR }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID_SHARED }}      
+          username: ${{ secrets.AWS_ACCESS_KEY_ID_SHARED }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY_SHARED }}
 
+      # Cache
+      - name: Cache var-cache-apt
+        uses: actions/cache@v3
+        with:
+          path: var-cache-apt
+          key: var-cache-apt-${{ hashFiles('docker/Dockerfile') }}
+      - name: Cache var-lib-apt
+        uses: actions/cache@v3
+        with:
+          path: var-lib-apt
+          key: var-lib-apt-${{ hashFiles('docker/Dockerfile') }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: cache-pip
+          key: cache-pip-${{ hashFiles('docker/Dockerfile') }}
+      - name: Cache npm
+        uses: actions/cache@v3
+        with:
+          path: cache-npm
+          key: cache-npm-${{ hashFiles('docker/Dockerfile') }}
+
+      # Inject cache
+      - name: inject var-cache-apt into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: var-cache-apt
+          cache-target: /var/cache/apt
+      - name: inject var-lib-apt into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: var-lib-apt
+          cache-target: /var/lib/apt
+      - name: inject pip cache into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: cache-pip
+          cache-target: /pip_cache
+      - name: inject npm cache into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: cache-npm
+          cache-target: /npm_cache
+
       - name: Build and push - Flows Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           context: .
           labels: |
             tag=${{env.TAG}}
             commit=${{env.COMMIT_SHA}}
             repository=${{env.IMAGE_SOURCE_URL}}
           file: docker/Dockerfile
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: "${{env.IMAGE_TAG}}"
-          no-cache: true
+          #tags: ${{ steps.meta.outputs.tags }}
+          #platforms: linux/amd64,linux/arm64,linux/arm/v7,darwin/amd64,linux/arm/v8
+          #no-cache: true
           build-args: |
             FLOWEDITOR_VERSION=${{ env.FLOWEDITOR_VERSION }}
             FLOWEDITOR_REPO=${{ env.FLOWEDITOR_REPO }}
             FLOWEDITOR_BRANCH=${{ env.FLOWEDITOR_BRANCH }}
-          
-      - name: Build and push - Flows Varnish Image
-        uses: docker/build-push-action@v3
-        with:
-          context: ./docker
-          labels: |
-            tag=${{env.TAG}}
-            commit=${{env.COMMIT_SHA}}
-            repository=${{env.IMAGE_SOURCE_URL}}
-          file: docker/Dockerfile.varnish
-          push: true
-          tags: "${{env.IMAGE_TAG}}-varnish"
-          no-cache: true
-          build-args: |
-            BACKEND_ADDRESS=flows:8000
 
       - name: Check out Kubernetes Manifests
         uses: actions/checkout@master
         with:
           ref: main
+          #ref: feature/flows-cache
           repository: "${{ env.MANIFESTS_REPOSITORY }}"
           token: "${{ secrets.DEVOPS_GITHUB_PERMANENT_TOKEN }}"
           path: ./kubernetes-manifests/
@@ -140,20 +184,13 @@ jobs:
                 )
                 echo "Old image version to compare: ${OLD_VERSION}<=${{env.VERSION}}"
                 if verlte "${OLD_VERSION}" "${VERSION}" || [[ ! "${OLD_VERSION}" =~ [0-9]+\.[0-9]+\.[0-9]+ ]] ; then
-                  echo 'New configurations for varnish:'
-                  new_configuration=$(
-                    cat "${e}/${{ env.MANIFESTS_PATCH_TARGET }}" \
-                      | jq '(..|select(.path == "/spec/template/spec/containers/1/image")?) += {value: "'"${{env.IMAGE_TAG}}"'-varnish"}'
-                  )
-                  echo "${new_configuration}"
-                  echo "${new_configuration}" > "${e}/${{ env.MANIFESTS_PATCH_TARGET }}"
                   echo 'New configurations for image flows:'
                   new_configuration=$(
-                    cat "${e}/${{ env.MANIFESTS_PATCH_TARGET1 }}" \
+                    cat "${e}/${{ env.MANIFESTS_PATCH_TARGET }}" \
                       | jq '(..|select(.path == "/spec/template/spec/containers/0/image")?) += {value: "'"${{env.IMAGE_TAG}}"'"}'
                   )
                   echo "${new_configuration}"
-                  echo "${new_configuration}" > "${e}/${{ env.MANIFESTS_PATCH_TARGET1 }}"
+                  echo "${new_configuration}" > "${e}/${{ env.MANIFESTS_PATCH_TARGET }}"
                 else
                   echo "Version in file is greater than build, skipping update yaml"
                 fi
@@ -167,4 +204,5 @@ jobs:
           repository: "${{ env.MANIFESTS_REPOSITORY }}"
           directory: ./kubernetes-manifests/
           branch: main
+          #branch: feature/flows-cache
           message: "From Flows Build (Push Tag ${{ env.MANIFESTS_ENVIRONMENT }})"

docker/Dockerfile

@@ -1,79 +1,125 @@
-FROM python:3.9-slim-bullseye as build-poetry
-
-WORKDIR /app
-
-COPY ./pyproject.toml .
-COPY ./poetry.lock .
-COPY ./docker/pip-requires.txt .
-
-RUN python -m pip install -U poetry==1.1.15 \
- && poetry cache clear -n --all pypi \
- && pip install requests==2.29.0 \
- && poetry add -n --lock $(cat pip-requires.txt) \
- && poetry export --without-hashes --output requirements.txt
-
-FROM python:3.9-slim-bullseye
+# syntax = docker/dockerfile:1
+
+ARG PYTHON_VERSION="3.9"
+ARG DEBIAN_VERSION="bookworm"
+ARG POETRY_VERSION="1.1.15"
+ARG REQUESTS_VERSION="2.29.0"
+
+ARG BUILD_DEPS="\
+  gcc bzip2 git curl libpq-dev gettext \
+  libgdal-dev python3-cffi python3-gdal \
+  python3-dev default-libmysqlclient-dev build-essential \
+  build-essential \
+  git cmake \
+  autoconf pkg-config autoconf libtool automake \
+  libmariadb-dev npm"
+# default-libmysqlclient-dev
+ARG RUNTIME_DEPS="\
+  git \
+  tzdata \
+  postgresql-client \
+  netcat-traditional \
+  curl \
+  gosu \
+  gdal-bin \
+  npm"
+#libmariadb3 \
+
+FROM python:${PYTHON_VERSION}-slim-${DEBIAN_VERSION} as base
+
+ARG POETRY_VERSION
+ARG REQUESTS_VERSION
+ARG NODE_VERSION
+
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1 \
+  DEBIAN_FRONTEND=noninteractive \
+  PROJECT=rapidpro \
+  PROJECT_PATH=/app \
+  PROJECT_USER=rapidpro \
+  PROJECT_GROUP=rapidpro \
+  NPM_CONFIG_PREFIX=/opt/npm-globals \
+  PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  PATH="/opt/npm-globals/bin:/app/node_modules/.bin/:${PATH}:/install/bin"
 
 ARG COMPRESS_ENABLED
 ARG BRANDING_ENABLED
 
 ARG RAPIDPRO_APPS_GIT_URL
 ARG RAPIDPRO_APPS_GIT_BRANCH
 
-ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND  noninteractive
+LABEL app=${VERSION} \
+  os="debian" \
+  os.version="12" \
+  name="Flows" \
+  description="APP image" \
+  maintainer="https://github.com/weni-ai" \
+  org.opencontainers.image.url="https://github.com/weni-ai/flows" \
+  org.opencontainers.image.documentation="https://github.com/weni-ai/flows" \
+  org.opencontainers.image.source="https://github.com/weni-ai/flows" \
+  org.opencontainers.image.title="Flows"
 
-ENV PROJECT           rapidpro
-ENV PROJECT_PATH      /app
-ENV PROJECT_USER      $PROJECT
-ENV PROJECT_GROUP     $PROJECT_USER
-ENV PROJECT_CONF      ${PROJECT_PATH}-conf
-ENV NPM_CONFIG_PREFIX /opt/npm-globals
-ENV PATH              "${NPM_CONFIG_PREFIX}/bin:${PATH}"
+RUN addgroup --gid 1999 "${PROJECT_GROUP}" \
+  && useradd --system -m -d /app -u 1999 -g 1999 "${PROJECT_USER}"
 
-RUN apt-get update \
- && apt-get install --no-install-recommends --no-install-suggests -y apt-utils \
- && apt-get install --no-install-recommends --no-install-suggests -y gcc bzip2 git curl nginx libpq-dev gettext \
-    libgdal-dev python3-cffi python3-gdal vim
+WORKDIR "${PROJECT_PATH}"
 
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
- && apt-get install --no-install-recommends --no-install-suggests -y nodejs
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
-RUN rm -rf /var/lib/apt/lists/*
+FROM base as build-poetry
 
-RUN useradd -Ums /bin/bash $PROJECT_USER \
- && usermod -aG adm $PROJECT_USER
+ARG POETRY_VERSION
+ARG REQUESTS_VERSION
+ARG NODE_VERSION
 
-RUN mkdir $PROJECT_PATH \
- && mkdir $PROJECT_CONF \
- && mkdir $NPM_CONFIG_PREFIX \
- && chown -R $PROJECT_USER:$PROJECT_GROUP $PROJECT_PATH \
- && chown -R $PROJECT_USER:$PROJECT_GROUP $NPM_CONFIG_PREFIX
+COPY pyproject.toml poetry.lock .
 
-RUN rm /etc/nginx/sites-enabled/default
+RUN --mount=type=cache,mode=0755,target=/pip_cache,id=pip pip install --cache-dir /pip_cache -U poetry=="${POETRY_VERSION}" \
+  && poetry cache clear -n --all pypi \
+  && pip install --cache-dir /pip_cache requests=="${REQUESTS_VERSION}" \
+  && poetry export --without-hashes --output requirements.txt
+#  && poetry add -n --lock $(cat pip-requires.txt) \
 
+FROM base as build
+
+ARG BUILD_DEPS
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update \
+  && apt-get install --no-install-recommends --no-install-suggests -y ${BUILD_DEPS}
+ 
 COPY --from=build-poetry /app/requirements.txt /tmp/dep/
-COPY ./docker/pip-freeze.txt /tmp/dep/
-COPY ./docker/nginx.site.conf /tmp/
-RUN pip install --no-cache-dir -r /tmp/dep/requirements.txt
+RUN --mount=type=cache,mode=0755,target=/pip_cache,id=pip pip install --cache-dir /pip_cache --prefix=/install -r /tmp/dep/requirements.txt
+
+FROM base
 
-RUN cp /tmp/nginx.site.conf /etc/nginx/sites-available/$PROJECT.conf \
- && ln -s /etc/nginx/sites-available/$PROJECT.conf /etc/nginx/sites-enabled/$PROJECT.conf
+ARG BUILD_DEPS
+ARG RUNTIME_DEPS
 
-COPY . $PROJECT_PATH
-RUN chown -R $PROJECT_USER:$PROJECT_GROUP $PROJECT_PATH
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update \
+  && SUDO_FORCE_REMOVE=yes apt-get remove --purge -y ${BUILD_DEPS} \
+  && apt-get autoremove -y \
+  && apt-get install -y --no-install-recommends ${RUNTIME_DEPS} \
+  && rm -rf /usr/share/man /usr/share/doc
 
-WORKDIR $PROJECT_PATH
+COPY --chown=${PROJECT_USER}:${PROJECT_GROUP} package.json package-lock.json ${PROJECT_PATH}
 
-RUN su $PROJECT_USER -c "npm install --global \
+RUN --mount=type=cache,id=npm,target=/npm_cache \
+  npm install --global --ignore-scripts --cache /npm_cache \
   coffeescript \
   less \
-  yarn"
+  yarn \
+  && npm install --ignore-scripts --cache /npm_cache
 
-RUN su $PROJECT_USER -c "npm install"
+COPY --from=build /install /usr/local
+COPY --chown=${PROJECT_USER}:${PROJECT_GROUP} . ${PROJECT_PATH}
 
-RUN su $PROJECT_USER -c "ln -s $PROJECT_PATH/temba/settings.py.prod $PROJECT_PATH/temba/settings.py"
+RUN ln -s "${PROJECT_PATH}/temba/settings.py.prod" "${PROJECT_PATH}/temba/settings.py"
 
-EXPOSE 8000
+USER "${PROJECT_USER}:${PROJECT_USER}"
 EXPOSE 8001
-ENTRYPOINT ["sh", "./docker/start"]
+ENTRYPOINT ["bash", "./docker/start"]
+CMD ["start"]