ci.yml

name: Ubuntu

'on':
    pull_request:
    push:
        branches:
          - master
          - develop
          - feature/**

env:
    UBSAN_OPTIONS: print_stacktrace=1
    ASAN_OPTIONS: detect_odr_violation=2
    CCACHE_DIR: /home/runner/.cache/ccache
    CCACHE_NOHASHDIR: true
    CPM_SOURCE_CACHE: /home/runner/.cache/CPM

jobs:
    posix:
        strategy:
            fail-fast: false
            matrix:
                include:
                  - cmake-flags: >-
                        -GNinja
                        -DCMAKE_CXX_COMPILER=clang++-18
                        -DCMAKE_C_COMPILER=clang-18
                        -DCMAKE_BUILD_TYPE=Debug
                        -DUSERVER_USE_STATIC_LIBS=1
                        -DUSERVER_SANITIZE="ub addr"
                        -DUSERVER_NO_WERROR=0
                        -DUSERVER_BUILD_ALL_COMPONENTS=1
                        -DUSERVER_BUILD_SAMPLES=1
                        -DUSERVER_BUILD_TESTS=1
                        -DUSERVER_NAMESPACE=userver_ns
                        -DUSERVER_NAMESPACE_BEGIN="namespace userver_ns { inline namespace
                        v1 {"
                        -DUSERVER_NAMESPACE_END="}}"
                    os: ubuntu-24.04
                    info: clang-18 + debug + sanitize addr+ub
                    id: ubuntu24_clang18_debug_sanitize-addr-ub
                    tests-flags: ''

                  - cmake-flags: >-
                        -GNinja
                        -DCMAKE_CXX_COMPILER=clang++-14
                        -DCMAKE_C_COMPILER=clang-14
                        -DCMAKE_BUILD_TYPE=Debug
                        -DUSERVER_USE_STATIC_LIBS=1
                        -DUSERVER_SANITIZE="ub addr"
                        -DUSERVER_NO_WERROR=0
                        -DUSERVER_BUILD_ALL_COMPONENTS=1
                        -DUSERVER_BUILD_SAMPLES=1
                        -DUSERVER_BUILD_TESTS=1
                        -DUSERVER_NAMESPACE=userver_ns
                        -DUSERVER_NAMESPACE_BEGIN="namespace userver_ns { inline namespace
                        v1 {"
                        -DUSERVER_NAMESPACE_END="}}"
                    os: ubuntu-22.04
                    info: clang-14 + debug + sanitize addr+ub
                    id: ubuntu22_clang14_debug_sanitize-addr-ub
                    tests-flags: ''

                  - cmake-flags: >-
                        -GNinja
                        -DCMAKE_BUILD_TYPE=Release
                        -DUSERVER_USE_LD=gold
                        -DUSERVER_BUILD_ALL_COMPONENTS=1
                        -DUSERVER_BUILD_SAMPLES=1
                        -DUSERVER_BUILD_TESTS=1
                        -DUSERVER_NO_WERROR=1
                        -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0
                        -DUSERVER_FEATURE_REDIS_HI_MALLOC=1
                        -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0
                        -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                        -DUSERVER_MONGODB_USE_CMAKE_CONFIG=0
                    os: ubuntu-20.04
                    info: g++-9 + release
                    id: ubuntu20_gcc9_release
                    tests-flags: '--gtest_filter=-HttpClient.RedirectHeaders:HttpClient.TestUseIPv4v6'

                  # GCC-8 - testing minimal required compiler
                  - cmake-flags: >-
                        -GNinja
                        -DCMAKE_CXX_COMPILER=g++-8
                        -DCMAKE_C_COMPILER=gcc-8
                        -DCMAKE_BUILD_TYPE=Debug
                        -DUSERVER_USE_LD=gold
                        -DUSERVER_BUILD_ALL_COMPONENTS=1
                        -DUSERVER_BUILD_SAMPLES=1
                        -DUSERVER_BUILD_TESTS=1
                        -DUSERVER_NO_WERROR=1
                        -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0
                        -DUSERVER_FEATURE_REDIS_HI_MALLOC=1
                        -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0
                        -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                        -DUSERVER_MONGODB_USE_CMAKE_CONFIG=0
                    os: ubuntu-20.04
                    info: g++-8 + debug
                    id: ubuntu20_gcc8_debug
                    tests-flags: '--gtest_filter=-HttpClient.RedirectHeaders:HttpClient.TestUseIPv4v6'

        name: '${{matrix.os}}: ${{matrix.info}}'
        runs-on: ${{matrix.os}}

        steps:
          - uses: actions/checkout@v4

          - name: Restore cached directories
            id: restore-cache
            uses: actions/cache/restore@v4
            with:
                path: |
                    ${{env.CCACHE_DIR}}
                    ${{env.CPM_SOURCE_CACHE}}
                key: 'ubuntu-cache-dir ${{matrix.id}} ${{github.ref}} run-${{github.run_number}}'
                restore-keys: |
                    ubuntu-cache-dir ${{matrix.id}} ${{github.ref}}
                    ubuntu-cache-dir ${{matrix.id}}

          - name: Setup ramdrive for testsuites
            run: |
                sudo mkdir -p "/mnt/ramdisk/$USER"
                sudo chmod 777 "/mnt/ramdisk/$USER"
                sudo mount -t tmpfs -o size=2048M tmpfs "/mnt/ramdisk/$USER"

          - name: Free disk space
            run: |
                df -h
                # See https://stackoverflow.com/questions/75536771/github-runner-out-of-disk-space-after-building-docker-image
                sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/lib/php* /opt/ghc \
                    /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
                    /opt/hostedtoolcache/CodeQL || true
                sudo docker image prune --all --force
                df -h

          - name: Add confluentinc repositories
            if: matrix.os != 'ubuntu-24.04'
            run: |
                # Adding librdkafka confluent repositories as in https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software
                gpg_retrieve_keyserver() {
                  GNUPGHOME=$(mktemp -d)
                  GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "/usr/share/keyrings/$2.gpg" --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $1
                  rm -rf "$GNUPGHOME"
                  chmod a+r "/usr/share/keyrings/$2.gpg"
                }
                gpg_retrieve_keyserver 8B1DA6120C2BF624 confluent
                printf "\
                deb [arch=amd64 signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/deb/7.6 stable main\n\
                deb [signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/clients/deb $(lsb_release -cs) main\n" \
                  | sudo tee /etc/apt/sources.list.d/confluent.list

          - name: Install clang-format
            if: matrix.os == 'ubuntu-20.04'
            run: |
                sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
                sudo apt update
                wget https://apt.llvm.org/llvm.sh
                chmod +x llvm.sh
                sudo ./llvm.sh 14
                sudo apt install -y clang-format-14
                sudo ln -sf /usr/bin/clang-format-14 /usr/bin/clang-format
                clang-format --version

          - name: Add missing repositories
            run: |
                sudo apt update
                # Instructions from https://clickhouse.com/docs/en/getting-started/install/
                sudo apt install -y apt-transport-https ca-certificates dirmngr
                sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
                echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee /etc/apt/sources.list.d/clickhouse.list

                # Adding mariadb repositories (from https://www.linuxcapable.com/how-to-install-mariadb-on-ubuntu-linux/ )
                curl -fsSL http://mirror.mariadb.org/PublicKey_v2 | sudo gpg --dearmor -o "/usr/share/keyrings/mariadb.gpg"
                sudo chmod a+r "/usr/share/keyrings/mariadb.gpg"
                # Restore the correct URL after https://jira.mariadb.org/browse/MDBF-651
                #echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] https://deb.mariadb.org/10.11/ubuntu $(lsb_release -cs) main" \
                #    | tee /etc/apt/sources.list.d/mariadb.list
                echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] https://mirror.kumi.systems/mariadb/repo/10.11/ubuntu $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/mariadb.list

                # Adding mongodb repositories
                wget -qO- https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/mongodb-server-7.0.gpg >/dev/null
                echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \
                      | sudo tee -a /etc/apt/sources.list.d/mongodb-org-7.0.list
                # Mongo is not officially supported on Ubuntu 24.04, so install it from jammy

                # Install Kafka
                sudo apt install -y default-jre
                curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz
                sudo mkdir -p /etc/kafka
                sudo tar xf kafka.tgz --directory=/etc/kafka
                sudo cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/
                sudo rm -rf /etc/kafka/kafka_2.13-3.8.0
                ls -lha /etc/kafka

          - name: Install common deps
            run: |
                sudo apt update
                sudo apt purge -y libpq5 libpq-dev postgresql-*
                sudo apt install --allow-downgrades -y clickhouse-common-static redis-server postgresql $(cat scripts/docs/en/deps/${{matrix.os}}.md | tr '\n' ' ')

          - name: Install ccache
            if: matrix.os == 'ubuntu-20.04'
            run: |
                sudo apt remove --purge -y ccache
                git clone --depth 1 -b v4.10.2 https://github.com/ccache/ccache.git
                cd ccache
                cmake -S . -B build -GNinja -DENABLE_TESTING=OFF -DSTATIC_LINK=ON -DCMAKE_BUILD_TYPE=Release
                cmake --build build -j$(nproc)
                sudo cmake --install build

          - name: Setup ccache
            run: |
                ccache -M 2.0GB
                ccache -s -v

          - name: Install grpc, protobuf and abseil
            if: matrix.os == 'ubuntu-20.04'
            run: |
                sudo apt install -y ninja-build
                git clone --recursive --depth 1 -b v1.45.0 https://github.com/grpc/grpc new_grpc
                GRPC_BUILD_ENABLE_CCACHE=true \
                  cmake -GNinja -S./new_grpc -B./new_grpc/build \
                  -DABSL_PROPAGATE_CXX_STD=ON -DABSL_ENABLE_INSTALL=ON \
                  -DgRPC_SSL_PROVIDER=package \
                  -DgRPC_CARES_PROVIDER=package \
                  -DgRPC_ZLIB_PROVIDER=package \
                  -DgRPC_INSTALL=ON
                cmake --build new_grpc/build -j$(nproc)
                sudo cmake --install new_grpc/build
                rm -rf new_grpc
                ccache -s -v

          - name: Install g++-8
            if: matrix.info == 'g++-8 + debug'
            run: |
                sudo apt install --allow-downgrades -y g++-8

          - name: Install RabbitMQ packages
            run: |
                ./scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh

          - name: Install gdb and pytest
            run: |
                sudo apt install -y gdb python3-pytest

          - name: Run cmake
            run: |
                cmake -S . -B build_debug ${{matrix.cmake-flags}}

          - name: Run clang-tidy
            if: matrix.info == 'clang-14 + debug + sanitize addr+ub'
            run: |
                [[ -z "${GITHUB_BASE_REF}" ]] && BRANCH_VAR=HEAD~1 || BRANCH_VAR=origin/${GITHUB_BASE_REF}
                echo "${BRANCH_VAR}"
                echo "$(git diff --name-only ${BRANCH_VAR})"
                EXCLUDE_PATTERNS="-e '.pb.cc' -e '.pb.cpp' -e 'benchmark.cpp' -e '/pq-extra/' -e '/tools/' -e 'third_party' "
                LIST_CHECK_FILES=$(git diff --name-only ${BRANCH_VAR} | grep --color=never '.cpp' | grep -v ${EXCLUDE_PATTERNS} | tr '\n' ' ')
                if [[ -n ${LIST_CHECK_FILES} ]]
                then
                  sudo apt install -y clang-tidy-14
                  curl -o tools/run-clang-tidy https://raw.githubusercontent.com/llvm-mirror/clang-tools-extra/release_90/clang-tidy/tool/run-clang-tidy.py
                  python3 ./tools/run-clang-tidy -clang-tidy-binary clang-tidy-14 -p build_debug -q -j$(nproc) ${LIST_CHECK_FILES}
                else
                  echo "Nothing check in clang-tidy"
                fi

          - name: Compile
            run: |
                pwd
                cd build_debug
                cmake --build . -j $(nproc)

          - name: Save cached directories
            uses: actions/cache/save@v4
            with:
                path: |
                    ${{env.CCACHE_DIR}}
                    ${{env.CPM_SOURCE_CACHE}}
                key: ${{ steps.restore-cache.outputs.cache-primary-key }}

          - name: Show cache stats
            run: |
                du -h -d 1 ${{env.CCACHE_DIR}}
                du -h -d 1 ${{env.CPM_SOURCE_CACHE}}
                ccache -s -v


          - name: Run tests (universal)
            run: |
                echo "UBSAN_OPTIONS=${UBSAN_OPTIONS} ASAN_OPTIONS=${ASAN_OPTIONS}"
                cd build_debug
                ./universal/userver-universal-unittest ${{matrix.tests-flags}}

          - name: Run tests (core)
            run: |
                cd build_debug
                ./core/userver-core-unittest ${{matrix.tests-flags}}

          - name: Run tests (kafka)
            run: |
                cd build_debug
                mkdir kafka || :
                cd kafka
                ctest -V

          - name: Run tests (clickhouse)
            run: |
                cd build_debug
                mkdir clickhouse || :
                cd clickhouse
                ctest -V

          - name: Run tests (rabbitmq)
            run: |
                cd build_debug
                mkdir rabbitmq || :
                cd rabbitmq
                ctest -V

          - name: Run tests (postgresql)
            run: |
                cd build_debug
                mkdir postgresql || :
                cd postgresql
                ctest -V

          - name: Run tests (redis)
            run: |
                cd build_debug/redis/
                ctest -V

          - name: Run tests (mongo)
            # libbson unnecessarily overaligns structures and GCC-9 does not support overaligned new
            if: matrix.info != 'g++-9 + release'
            run: |
                sudo apt install -y mongodb-mongosh
                cd build_debug
                mkdir mongo || :
                cd mongo
                if [ -f ./userver-mongo-unittest ]; then ./userver-mongo-unittest ; fi
                # Mongo is not available on Ubuntu 22.04 from default repos
                if [ "${{matrix.os}}" != "ubuntu-22.04" ]; then ctest -V -R userver-mongo-mongotest ; fi


          - name: Run tests (gRPC)
            if: matrix.os != 'ubuntu-24.04'
            run: |
                cd build_debug
                mkdir grpc || :
                cd grpc
                ulimit -n 4096 && ctest -V

          - name: Run tests (mysql)
            # MySQL benchmarks do not work on new Ubuntu
            if: matrix.os != 'ubuntu-24.04'
            run: |
                sudo apt install -y mariadb-server
                cd build_debug
                mkdir mysql || :
                cd mysql
                ctest -V

          - name: Run tests (gdb)
            run: |
                cd build_debug/scripts/gdb/tests
                ctest -V