Merge branch 'master' into issue-4928

Author: guzzijones

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     # 'machine' executor runs Unit tests ~x1.5 faster, comparing to 'docker' executor
     # but the fastest is still ~x1.5-2 slower, comparing to Travis
     machine: true
-    parallelism: 4
+    parallelism: 3
     working_directory: ~/st2
     steps:
       - checkout
@@ -43,8 +43,8 @@ jobs:
   # Run st2 Integration tests
   integration:
     docker:
-      - image: circleci/python:3.6
-      - image: mongo:4.0
+      - image: circleci/python:3.8
+      - image: mongo:4.4
       - image: rabbitmq:3
     working_directory: ~/st2
     steps:
@@ -57,8 +57,9 @@ jobs:
           name: Install Mongo Shell
           command: |
             set -x
-            sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4
-            echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/4.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list
+            sudo apt-get -qq -y install gnupg curl
+            curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/mongodb-server-4.4.gpg
+            echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list
             sudo apt-get -qq -y update
             sudo apt-get -qq -y install mongodb-org-shell
       - run:
@@ -79,8 +80,8 @@ jobs:
   # Run st2 Lint Checks
   lint:
     docker:
-      - image: circleci/python:3.6
-      - image: mongo:4.0
+      - image: circleci/python:3.8
+      - image: mongo:4.4
       - image: rabbitmq:3
     working_directory: ~/st2
     steps:
@@ -107,33 +108,48 @@ jobs:
 
   # Build & Test st2 packages
   packages:
-    parallelism: 4
+    parallelism: 3
     # 4CPUs & 8GB RAM CircleCI machine
     # sadly, it doesn't work with 'setup_remote_docker'
     resource_class: large
     docker:
       # The primary container is an instance of the first list image listed. Your build commands run in this container.
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/st2
     environment:
-      - DISTROS: "bionic focal el7 el8"
+      - DISTROS: "focal el8 el9"
       - ST2_PACKAGES_REPO: https://github.com/StackStorm/st2-packages
       - ST2_PACKAGES: "st2"
       - ST2_CHECKOUT: 0
       - ST2_GITDIR: /tmp/st2
       - BASH_ENV: ~/.buildenv
     steps:
       - checkout
+      - run:
+          name: Install latest Docker Compose V2
+          command: |
+            set -x
+            export CODENAME=$(source /etc/os-release && echo "$VERSION_CODENAME")
+            export DISTRO=$(source /etc/os-release && echo "$ID")
+            export ARCH=$(dpkg --print-architecture)
+            # get gpg key for download.docker
+            curl -fsSL https://download.docker.com/linux/${DISTRO}/gpg | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/download.docker.gpg
+            # set source list
+            sudo tee <<<"deb [arch=${ARCH}] https://download.docker.com/linux/${DISTRO} ${CODENAME} stable" /etc/apt/sources.list.d/download.docker.list
+            # update package list
+            sudo apt update
+            # install docker CLI and Docker Compose v2
+            sudo apt install docker-ce-cli docker-compose-plugin
       - setup_remote_docker:
           reusable: true    # default - false
           exclusive: true   # default - true
-          version: 19.03.14
+          version: docker24
       - run:
           name: Docker version
           command: |
             set -x
             docker --version
-            docker-compose --version
+            docker compose version
       - run:
           name: Download st2-packages repository
           command: |
@@ -167,7 +183,7 @@ jobs:
           name: Copy st2-packages files to build containers
           command: |
             # creating dummy container which will hold a volume with data files
-            docker create -v /root/st2-packages -v ${ST2_GITDIR} -v /root/build -v /var/log/st2 -v /root/.cache/pip -v /tmp/wheelhouse --name st2-packages-vol alpine:3.4 /bin/true
+            docker create -v /root/st2-packages -v ${ST2_GITDIR} -v /root/build -v /var/log/st2 -v /root/.cache/pip -v /tmp/wheelhouse --name st2-packages-vol alpine:3.12 /bin/true
             # copy st2-packages data files into this volume
             docker cp ~/st2-packages st2-packages-vol:/root
             # copy st2 source files into this volume
@@ -219,10 +235,10 @@ jobs:
   deploy:
     docker:
       # The primary container is an instance of the first list image listed. Your build commands run in this container.
-      - image: circleci/ruby:2.4
+      - image: circleci/ruby:2.7
     working_directory: /tmp/deploy
     environment:
-      - DISTROS: "bionic focal el7 el8"
+      - DISTROS: "focal el8 el9"
     steps:
       - attach_workspace:
           at: .

.github/actions/apt-packages/action.yaml

@@ -0,0 +1,27 @@
+---
+name: Cache and Install APT Dependencies
+description:
+  Light wrapper around the actions/cache action and our script
+  to maintain the input vars in only one place for all workflows.
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cache APT Dependencies
+      id: cache-apt-deps
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/apt_cache
+        key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-v8-apt-
+
+    - name: Install APT Depedencies
+      shell: bash
+      env:
+        CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+      run: |
+        # install dev dependencies for Python YAML and LDAP packages
+        # https://github.com/StackStorm/st2-auth-ldap
+        ./scripts/github/install-apt-packages-use-cache.sh

.github/actions/init-pants/action.yaml

@@ -0,0 +1,38 @@
+---
+name: Initialize Pants and its GHA caches
+description:
+  Light wrapper around the pantsbuild/actions/init-pants action
+  to maintain the input vars in only one place for all workflows.
+
+inputs:
+  gha-cache-key:
+    description: Qualify all cache keys with this string. Useful for invalidating everything.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Initialize Pants and its GHA caches
+      uses: pantsbuild/actions/init-pants@v9
+      # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
+      # This action also creates 3 GHA caches (1 is optional).
+      # - `pants-setup` has the bootsrapped pants install
+      # - `pants-named-caches` has pip/wheel and PEX caches
+      # - `pants-lmdb-store` has the fine-grained process cache.
+      #   If we ever use a remote cache, then we can drop this.
+      #   Otherwise, we may need an additional workflow or job to delete old caches
+      #   if they are not expiring fast enough, and we hit the GHA 10GB per repo max.
+      with:
+        base-branch: master
+        # To ignore a bad cache, bump the cache* integer.
+        gha-cache-key: ${{ inputs.gha-cache-key }}
+        # This hash should include all of our lockfiles so that the pip/pex caches
+        # get invalidated on any transitive dependency update.
+        named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
+        # enable the optional lmdb_store cache since we can't rely on remote caching yet.
+        cache-lmdb-store: 'true'
+        # Try the experimental support for using GHA cache as a pants remote cache.
+        experimental-remote-cache-via-gha: 'true'
+        # install whatever version of python we need for our in-repo pants-plugins
+        setup-python-for-plugins: 'true'
+

.github/actions/setup-python/action.yaml

@@ -0,0 +1,34 @@
+---
+name: Install Python and Cache Deps
+description:
+  Light wrapper around the actions/setup-python and actions/cache actions
+  to maintain the input vars in only one place for all workflows.
+
+input:
+  python-version:
+    description: Which version of python to install.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: 'Set up Python (${{ inputs.python-version }})'
+      uses: actions/setup-python@v5
+      with:
+        python-version: '${{ inputs.python-version }}'
+
+    - name: Cache Python Dependencies
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/pip
+          virtualenv
+          ~/virtualenv
+        # TODO: maybe make the virtualenv a partial cache to exclude st2*?
+        # !virtualenv/lib/python*/site-packages/st2*
+        # !virtualenv/bin/st2*
+        key: ${{ runner.os }}-v5-python-${{ inputs.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
+        # Don't use alternative key as if requirements.txt has altered we
+        # don't want to retrieve previous cache
+        #restore-keys: |
+        #  ${{ runner.os }}-v5-python-${{ inputs.python }}-

.github/workflows/checks.yaml

@@ -14,7 +14,7 @@ jobs:
     name: Add CHANGELOG.rst
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - name: Changelog check
         # https://github.com/marketplace/actions/changelog-checker
         uses: Zomzog/[email protected]