NMS-14759: enable (optimized) sonar in Foundation 2022

* (slightly) smarter cache handling for sonar
* add support for the jacoco agent to horizon, minion, and
  sentinel startup scripts (`JACOCO_AGENT_ENABLED=1` env var)
* enable jacoco coverage reports for smoke test runs of horizon
  (minion and sentinel don't allow vm: access for JMX)
* push report-generation to the coverage (sonar.sh) run to
  synthesize integration and smoke test runs into a single report
* update the coverage packing and unpacking scripts to handle
  multiple types of runs without stepping on filenames
* backport some of my previous sonar improvements from develop

Author: Benjamin Reed

.circleci/config.yml

@@ -289,12 +289,15 @@ commands:
       steps:
         - restore_cache:
             keys:
-              - sonar-cache-v2-{{ checksum "pom-version-cache.key" }}
+              - sonar-cache-v3-{{ .Branch }}-{{ checksum "pom-version-cache.key" }}-{{ .Revision }}
+              - sonar-cache-v3-{{ .Branch }}-{{ checksum "pom-version-cache.key" }}-
+              - sonar-cache-v3-{{ .Branch }}-
+              - sonar-cache-v3-
   save-sonar-cache:
       description: "Sonar: Save sonar cache"
       steps:
         - save_cache:
-            key: sonar-cache-v2-{{ checksum "pom-version-cache.key" }}
+            key: sonar-cache-v3-{{ .Branch }}- {{ checksum "pom-version-cache.key" }}
             paths:
               - ~/.sonar
   dockerhub-login:
@@ -500,21 +503,22 @@ commands:
           when: always
           command: |
             mkdir -p ~/generated-tests
-            cp ./surefire_classname* ~/generated-tests/ || :
-            cp ./failsafe_classname* ~/generated-tests/ || :
-            cp /tmp/this_node* ~/generated-tests/       || :
+            cp target/find-tests/* ~/generated-tests/ || :
+            cp /tmp/this_node* ~/generated-tests/     || :
       - when:
           condition: << parameters.run-code-coverage >>
           steps:
             - run:
-                name: Compress Target Directories (Code Coverage)
+                name: Save Code Coverage Data
                 when: always
                 command: |
-                  .circleci/scripts/codecoverage-save.sh
+                  .circleci/scripts/codecoverage-save.sh integration-test
             - persist_to_workspace:
                 root: ~/
                 paths:
                   - code-coverage
+                  - project/target/find-tests
+                  - project/target/structure-graph.json
       - run:
           name: Gather system logs
           when: always
@@ -1583,11 +1587,19 @@ jobs:
           changes-only: false
   code-coverage:
     executor: centos-build-executor
-    resource_class: large
+    resource_class: xlarge
     steps:
       - attach_workspace:
           at: ~/
+      - checkout
+      - cached-download:
+          url: https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.8/org.jacoco.cli-0.8.8-nodeps.jar
+          file: /tmp/jacoco-cli.jar
+      - cached-download:
+          url: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.7.0.2747.zip
+          file: /tmp/sonar-scanner-cli.zip
       - extract-pom-version
+      - restore-maven-cache
       - restore-sonar-cache
       - run:
           name: Restore Target Directories (Code Coverage)
@@ -1598,7 +1610,7 @@ jobs:
           name: Run SonarQube Code Analysis
           when: always
           command: |
-            export MAVEN_OPTS="-Xms3G -Xmx3G"
+            export MAVEN_OPTS="-Xms1g -Xmx12g"
             .circleci/scripts/sonar.sh
       - save-sonar-cache
   smoke-test-core:
@@ -1665,7 +1677,7 @@ jobs:
                 suite: minion
   smoke-test-sentinel:
     executor: smoke-test-executor
-    parallelism: 5
+    parallelism: 4
     # No resource class support for machine executors, we're constrained to use the default
     # medium class which has 2 vCPUs and 8 GB RAM
     #resource_class: large

.circleci/scripts/codecoverage-restore.sh

@@ -1,11 +1,14 @@
-#!/bin/sh -e
-cd ~/project
-for file in ~/code-coverage/target-*.zip; do
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+for file in ~/code-coverage/*.zip; do
     FILENAME="$(basename "$file")"
     NUMBER="$(echo "$FILENAME" | cut -d. -f1 | cut -d- -f2-)"
     echo "* unpacking coverage reports $NUMBER:"
     unzip -qo "$file"
-    find . -type d -name failsafe-reports -o -name surefire-reports | while read DIR; do
+    find . -type d '!' -path './.git/*' \( -name failsafe-reports -o -name surefire-reports -o -name jacoco -o -name coverage \) | while read -r DIR; do
       mkdir -p "${DIR}-${NUMBER}"
       mv "${DIR}"/* "${DIR}-${NUMBER}/"
       rm -rf "${DIR}"

.circleci/scripts/codecoverage-save.sh

@@ -1,4 +1,14 @@
-#!/bin/sh -e
-cd ~/project
+#!/bin/bash
+
+TYPE="$1"; shift
+[ -z "$TYPE" ] && TYPE="coverage"
+
+set -e
+set -o pipefail
+
 mkdir -p ~/code-coverage
-find . -type d | grep -E ".*/target$" | zip ~/code-coverage/target-"$CIRCLE_NODE_INDEX.zip" -9r@ -x \*.gz -x \*.zip -x \*.war -x \*target/dist\* -x \*/node/\* -x \*target/unpacked/\*
+
+find . -type f '!' -path './.git/*' '!' -path '*/node_modules/*' '!' -name '*.jar' '!' -name '*.sh' | grep -E '(surefire-reports|failsafe-reports|jacoco|coverage)' > /tmp/coverage-files.txt
+if [ -s /tmp/coverage-files.txt ]; then
+  zip '-9@' ~/code-coverage/${TYPE}-"${CIRCLE_NODE_INDEX}.zip" < /tmp/coverage-files.txt
+fi

.circleci/scripts/itest.sh

@@ -1,7 +1,9 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 
+FIND_TESTS_DIR="target/find-tests"
+
 # attempt to work around repository flakiness
 retry()
 {
@@ -10,34 +12,38 @@ retry()
 
 find_tests()
 {
+    mkdir -p "${FIND_TESTS_DIR}"
+
+    echo "#### Generate project structure .json"
+    ./compile.pl -s .circleci/scripts/structure-settings.xml --batch-mode --fail-at-end -Prun-expensive-tasks -Pbuild-bamboo org.opennms.maven.plugins:structure-maven-plugin:1.0:structure
+
     # Generate surefire & failsafe test list based on current
     # branch and the list of files changed
     # (The format of the output files contains the canonical class names i.e. org.opennms.core.soa.filter.FilterTest)
-    pyenv local 3.8.5
     python3 .circleci/scripts/find-tests/find-tests.py generate-test-lists \
       --changes-only="${CCI_CHANGES_ONLY:-true}" \
-      --output-unit-test-classes=surefire_classnames \
-      --output-integration-test-classes=failsafe_classnames \
+      --output-unit-test-classes="${FIND_TESTS_DIR}/surefire_classnames" \
+      --output-integration-test-classes="${FIND_TESTS_DIR}/failsafe_classnames" \
       .
 
     # Now determine the tests for this particular container based on the parallelism level and the test timings
-    < surefire_classnames circleci tests split --split-by=timings --timings-type=classname > /tmp/this_node_tests
-    < failsafe_classnames circleci tests split --split-by=timings --timings-type=classname > /tmp/this_node_it_tests
+    < "${FIND_TESTS_DIR}/surefire_classnames" circleci tests split --split-by=timings --timings-type=classname > /tmp/this_node_tests
+    < "${FIND_TESTS_DIR}/failsafe_classnames" circleci tests split --split-by=timings --timings-type=classname > /tmp/this_node_it_tests
 
     # Now determine the Maven modules related to the tests we need to run
     cat /tmp/this_node* | python3 .circleci/scripts/find-tests/find-tests.py generate-test-modules \
       --output=/tmp/this_node_projects \
       .
 }
 
-echo "#### Making sure git is up-to-date"
-git fetch --all
+. ./.circleci/scripts/lib.sh
+ 
+REFERENCE_BRANCH="$(get_reference_branch || echo "develop")"
 
-echo "#### Generate project structure .json"
-./compile.pl -s .circleci/scripts/structure-settings.xml --batch-mode --fail-at-end -Prun-expensive-tasks -Pbuild-bamboo org.opennms.maven.plugins:structure-maven-plugin:1.0:structure
+echo "#### Making sure git is up-to-date"
+git fetch origin "${REFERENCE_BRANCH}"
 
 echo "#### Determining tests to run"
-cd ~/project
 find_tests
 if [ ! -s /tmp/this_node_projects ]; then
   echo "No tests to run."
@@ -51,7 +57,6 @@ echo "#### Allowing non-root ICMP"
 sudo sysctl net.ipv4.ping_group_range='0 429496729'
 
 echo "#### Setting up Postgres"
-cd ~/project
 ./.circleci/scripts/postgres.sh
 
 echo "#### Installing other dependencies"
@@ -96,46 +101,57 @@ export MAVEN_OPTS="$MAVEN_OPTS -Xmx8g -XX:ReservedCodeCacheSize=1g"
 # shellcheck disable=SC3045
 ulimit -n 65536
 
-MAVEN_ARGS="install"
+MAVEN_ARGS=()
 
 case "${CIRCLE_BRANCH}" in
   "master"*|"release-"*|develop)
-    MAVEN_ARGS="-Dbuild.type=production $MAVEN_ARGS"
+    MAVEN_ARGS=("-Dbuild.type=production" "${MAVEN_ARGS[@]}")
   ;;
 esac
 
+# if node tests does not exist or is empty, skip surefire
+if [ ! -s /tmp/this_node_tests ]; then
+  MAVEN_ARGS+=("-DskipSurefire=true")
+fi
+ 
+# if node ITs does not exist or is empty, skip surefire
+if [ ! -s /tmp/this_node_it_tests ]; then
+  MAVEN_ARGS+=("-DskipFailsafe=true")
+fi
+
+MAVEN_COMMANDS=("install")
+
 echo "#### Building Assembly Dependencies"
-./compile.pl $MAVEN_ARGS \
+./compile.pl "${MAVEN_ARGS[@]}" \
            -P'!checkstyle' \
            -P'!production' \
            -Pbuild-bamboo \
            -Dbuild.skip.tarball=true \
            -Dmaven.test.skip.exec=true \
            -DskipTests=true \
            -DskipITs=true \
-           -Dci.instance="${CIRCLE_NODE_INDEX:-0}" \
            --batch-mode \
            "${CCI_FAILURE_OPTION:--fae}" \
            --also-make \
-           --projects "$(< /tmp/this_node_projects paste -s -d, -)"
+           --projects "$(< /tmp/this_node_projects paste -s -d, -)" \
+           install
 
 echo "#### Executing tests"
-./compile.pl $MAVEN_ARGS \
+./compile.pl "${MAVEN_ARGS[@]}" \
            -P'!checkstyle' \
            -P'!production' \
            -Pbuild-bamboo \
+           -Pcoverage \
            -Dbuild.skip.tarball=true \
            -DfailIfNoTests=false \
            -DskipITs=false \
-           -Dci.instance="${CIRCLE_NODE_INDEX:-0}" \
            -Dci.rerunFailingTestsCount="${CCI_RERUN_FAILTEST:-0}" \
-           -Dcode.coverage="${CCI_CODE_COVERAGE:-false}" \
            --batch-mode \
            "${CCI_FAILURE_OPTION:--fae}" \
            -Dorg.opennms.core.test-api.dbCreateThreads=1 \
            -Dorg.opennms.core.test-api.snmp.useMockSnmpStrategy=false \
            -Djava.security.egd=file:/dev/./urandom \
            -Dtest="$(< /tmp/this_node_tests paste -s -d, -)" \
            -Dit.test="$(< /tmp/this_node_it_tests paste -s -d, -)" \
-           --projects "$(< /tmp/this_node_projects paste -s -d, -)"
-
+           --projects "$(< /tmp/this_node_projects paste -s -d, -)" \
+           install

.circleci/scripts/lib.sh

@@ -1,8 +1,37 @@
 #!/bin/bash
 
-# this will return "1" for versions without -SNAPSHOT, and "0" for versions with
-ONMS_MAJOR_REVISION="$(grep '<version>' pom.xml | head -n 1 | grep -c -v -- -SNAPSHOT)"
+OPENNMS_POM_VERSION="$(~/project/.circleci/scripts/pom2version.sh ~/project/pom.xml || echo "0.0.0")"
+OPENNMS_VERSION="$(echo "${OPENNMS_POM_VERSION}" | sed -e 's,^\([0-9\.][0-9\.]*\).*$,\1,g')"
+OPENNMS_SHORT_VERSION="$(echo "${OPENNMS_VERSION}" | cut -d. -f 1-2)"
+OPENNMS_MAJOR_VERSION="$(echo "${OPENNMS_VERSION}" | cut -d. -f1)"
+
+# if $CIRCLE_BRANCH is not set, tag it as a "local" build
+DOCKER_TAGS=("local" "latest")
+
+if [ -n "${CIRCLE_BRANCH}" ]; then
+  # If $CIRCLE_BRANCH _is_ set, dump the local and latest tags,
+  # and instead set tags based on the branch.
+
+  # Always include $CIRCLE_BRANCH with "/" turned into "-".
+  DOCKER_BRANCH_TAG="${CIRCLE_BRANCH//\//-}"
+  DOCKER_TAGS=("${DOCKER_BRANCH_TAG}")
 
+  # In addition, set a few extra tag aliases for convenience.
+  case "${CIRCLE_BRANCH}" in
+    "master-"*)
+      DOCKER_TAGS=("${DOCKER_TAGS[@]}" "${OPENNMS_VERSION}" "${OPENNMS_SHORT_VERSION}-latest" "${OPENNMS_MAJOR_VERSION}-latest" "latest")
+      ;;
+    "release-"*)
+      DOCKER_TAGS=("${DOCKER_TAGS[@]}" "${OPENNMS_POM_VERSION}" "${OPENNMS_VERSION}-rc" "${OPENNMS_SHORT_VERSION}-rc" "${OPENNMS_MAJOR_VERSION}-rc" "release-candidate")
+      ;;
+    "develop")
+      DOCKER_TAGS=("${DOCKER_TAGS[@]}" "${OPENNMS_POM_VERSION}" "${OPENNMS_MAJOR_VERSION}-dev" "bleeding")
+      ;;
+  esac
+fi
+
+# this will return "1" for versions without -SNAPSHOT, and "0" for versions with
+ONMS_MAJOR_REVISION="$(echo "${OPENNMS_POM_VERSION}" | (grep -c -v -- -SNAPSHOT || :))"
 ONMS_MINOR_REVISION="$(date '+%Y%m%d')"
 ONMS_MICRO_REVISION=1
 
@@ -12,7 +41,72 @@ if [ -n "$CIRCLE_BUILD_NUM" ] && [ -n "$CIRCLE_BRANCH" ]; then
   ONMS_MICRO_REVISION="${CIRCLE_BUILD_NUM}"
 fi
 
+# get the PR number from CircleCI environment; caches on success
+__cache_pr_num=""
+get_pr_num() {
+  if [ -z "${__cache_pr_num}" ]; then
+    local _pr_num=0
+    if [ -n "${CIRCLE_PULL_REQUEST}" ]; then
+      _pr_num="$(echo "${CIRCLE_PULL_REQUEST}" | sed -e 's,.*/,,')"
+      if [ -n "${_pr_num}" ] && [ "${_pr_num}" -gt 0 ]; then
+        __cache_pr_num="${_pr_num}"
+      fi
+    fi
+    __cache_pr_num=0
+  fi
+  if [ ! "${__cache_pr_num}" -gt 0 ]; then
+    return 1
+  fi
+  echo "${__cache_pr_num}"
+}
+
+# get the "reference" (merge/pr-parent) branch for this branch; caches on success
+__cache_reference_branch=""
+get_reference_branch() {
+  if [ -z "${__cache_reference_branch}" ]; then
+    local _parent_branch=""
+
+    local _pr_num="$(get_pr_num || echo 0)"
+
+    if [ "${_pr_num}" -gt 0 ] && [ -n "${GITHUB_API_TOKEN}" ]; then
+      local _github_base="$(curl -s -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_API_TOKEN}" "https://api.github.com/repos/OpenNMS/opennms/pulls/${_pr_num}" | jq -r '.base.ref')"
+      if [ -n "${_github_base}" ]; then
+        __cache_reference_branch="${_github_base}"
+      fi
+    fi
+
+    if [ -z "${__cache_reference_branch}" ] && [ -e .nightly ]; then
+      _parent_branch="$(cat .nightly | grep -E '^parent_branch:' | sed -e 's,parent_branch: *,,')"
+      if [ -n "${_parent_branch}" ]; then
+        __cache_reference_branch="${_parent_branch}"
+      fi
+    fi
+  fi
+
+  if [ -n "${__cache_reference_branch}" ]; then
+    echo "${__cache_reference_branch}"
+    return 0
+  fi
+
+  return 1
+}
+
+# retry a command 3 times
+do_with_retries() {
+  for try in 1 2 3; do
+    if "$@"; then return 0; fi
+  done
+  return 1
+}
+
 export \
+  OPENNMS_POM_VERSION \
+  OPENNMS_VERSION \
+  OPENNMS_SHORT_VERSION \
+  OPENNMS_MAJOR_VERSION \
+  \
   ONMS_MAJOR_REVISION \
   ONMS_MINOR_REVISION \
-  ONMS_MICRO_REVISION
+  ONMS_MICRO_REVISION \
+  \
+  DOCKER_TAGS