Merge pull request #57 from hirosystems/feat/ci-on-pr-mutation-testing

Feat: add mutation testing on CI on PRs

.github/workflows/pr-mutants.yml

@@ -0,0 +1,250 @@
+name: PR Differences Mutants
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    paths:
+      - "**.rs"
+  workflow_dispatch:
+
+concurrency:
+  group: pr-differences-${{ github.head_ref || github.ref || github.run_id }}
+  # Always cancel duplicate jobs
+  cancel-in-progress: true
+
+jobs:
+  mutants:
+    name: Mutation Testing
+    runs-on: ubuntu-latest
+    steps:
+      # Cleanup Runner
+      - name: Cleanup Runner
+        id: runner_cleanup
+        uses: stacks-network/actions/cleanup/disk@main
+
+      - name: Checkout repo
+        id: git_checkout
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        with:
+          fetch-depth: 0
+
+      - name: Relative diff
+        id: relative_diff
+        run: |
+          git diff $(git merge-base origin/${{ github.base_ref || 'main' }} HEAD)..HEAD > git.diff
+
+      - name: Install cargo-mutants
+        id: install_cargo_mutants
+        run: |
+          cargo install --version 24.11.2 cargo-mutants --locked # v24.11.2
+
+      - name: Install cargo-nextest
+        id: install_cargo_nextest
+        uses: taiki-e/install-action@2f990e9c484f0590cb76a07296e9677b417493e9 # v2.33.23
+        with:
+          tool: nextest # Latest version
+
+      - name: Update git diff
+        id: update_git_diff
+        run: |
+          input_file="git.diff"
+          temp_file="temp_diff_file.diff"
+
+          # Check if the file exists and is not empty
+          if [ ! -s "$input_file" ]; then
+            echo "Diff file ($input_file) is missing or empty!"
+            exit 1
+          fi
+
+          # Remove all lines related to deleted files including the first 'diff --git' line
+          awk '
+            /^diff --git/ {
+              diff_line = $0
+              getline
+              if ($0 ~ /^deleted file mode/) {
+                in_deleted_file_block = 1
+              } else {
+                if (diff_line != "") {
+                  print diff_line
+                  diff_line = ""
+                }
+                in_deleted_file_block = 0
+              }
+            }
+            !in_deleted_file_block
+          ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file"
+
+          # Remove 'diff --git' lines only when followed by 'similarity index', 'rename from', and 'rename to'
+          awk '
+            /^diff --git/ {
+              diff_line = $0
+              getline
+              if ($0 ~ /^similarity index/) {
+                getline
+                if ($0 ~ /^rename from/) {
+                  getline
+                  if ($0 ~ /^rename to/) {
+                    next
+                  }
+                }
+              }
+              print diff_line  
+            }
+            { print }
+          ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file"
+
+      - name: Run docker-compose
+        uses: hoverkraft-tech/compose-action@f1ca7fefe3627c2dab0ae1db43a106d82740245e # v2.0.2
+        with:
+          compose-file: "./docker/docker-compose.dev.postgres.yml"
+
+      - name: Run mutants
+        id: run_mutants
+        run: |
+          # Disable immediate exit on error
+          set +e
+
+          cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV --in-diff git.diff --output ./ --test-tool=nextest -- --all-targets --test-threads 1
+          exit_code=$?
+
+          # Create the folder only containing the outcomes (.txt files) and make a file containing the exit code of the command
+          mkdir mutants
+          echo "$exit_code" > ./mutants/exit_code.txt
+          mv ./mutants.out/*.txt mutants/
+
+          # Enable immediate exit on error again
+          set -e
+
+      - name: Print mutants
+        id: print_tested_mutants
+        shell: bash
+        run: |
+          # Info for creating the link that paths to the specific mutation tested
+          server_url="${{ github.server_url }}"
+          organisation="${{ github.repository_owner }}"
+          repository="${{ github.event.repository.name }}"
+          commit="${{ github.sha }}"
+
+          # Function to write to github step summary with specific info depending on the mutation category
+          write_section() {
+            local section_title=$1
+            local file_name=$2
+
+            if [ -s "$file_name" ]; then
+              if [[ "$section_title" != "" ]]; then
+                echo "## $section_title" >> "$GITHUB_STEP_SUMMARY"
+              fi
+
+              if [[ "$section_title" == "Missed:" ]]; then
+                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<summary>What are missed mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
+                echo "No test failed with this mutation applied, which seems to indicate a gap in test coverage. Or, it may be that the mutant is undistinguishable from the correct code. You may wish to add a better test, or mark that the function should be skipped." >> "$GITHUB_STEP_SUMMARY"
+                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "" >> "$GITHUB_STEP_SUMMARY"
+              elif [[ "$section_title" == "Timeout:" ]]; then
+                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<summary>What are timeout mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
+                echo "The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped." >> "$GITHUB_STEP_SUMMARY"
+                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "" >> "$GITHUB_STEP_SUMMARY"
+              elif [[ "$section_title" == "Unviable:" ]]; then
+                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<summary>What are unviable mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
+                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
+                echo "The attempted mutation doesn't compile. This is inconclusive about test coverage and no action is needed, unless you wish to test the specific function, in which case you may wish to add a 'Default::default()' implementation for the specific return type." >> "$GITHUB_STEP_SUMMARY"
+                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
+                echo "" >> "$GITHUB_STEP_SUMMARY"
+              fi
+
+              if [[ "$section_title" != "" ]]; then
+                awk -F':' '{gsub("%", "%%"); printf "- [ ] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY"
+              else
+                awk -F':' '{gsub("%", "%%"); printf "- [x] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY"
+              fi
+
+              if [[ "$section_title" == "Missed:" ]]; then
+                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
+                echo "- Modify or add tests including this function." >> "$GITHUB_STEP_SUMMARY"
+                echo "- If you are absolutely certain that this function should not undergo mutation testing, add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
+              elif [[ "$section_title" == "Timeout:" ]]; then
+                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
+                echo "- Modify the tests that include this funcion." >> "$GITHUB_STEP_SUMMARY"
+                echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
+              elif [[ "$section_title" == "Unviable:" ]]; then
+                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
+                echo "- Create 'Default::default()' implementation for the specific structure." >> "$GITHUB_STEP_SUMMARY"
+                echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
+              fi
+
+              echo >> "$GITHUB_STEP_SUMMARY"
+            fi
+          }
+
+          # Print uncaught (missed/timeout/unviable) mutants to summary
+          if [ -s ./mutants/missed.txt -o -s ./mutants/timeout.txt -o -s ./mutants/unviable.txt ]; then
+            echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY"
+            echo "[Documentation - How to treat Mutants Output](https://github.com/stacks-network/actions/tree/main/stacks-core/mutation-testing#how-mutants-output-should-be-treated)" >> "$GITHUB_STEP_SUMMARY"
+            write_section "Missed:" "./mutants/missed.txt"
+            write_section "Timeout:" "./mutants/timeout.txt"
+            write_section "Unviable:" "./mutants/unviable.txt"
+          fi
+
+          # Print caught mutants to summary
+          if [ -s ./mutants/caught.txt ]; then
+            echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY"
+            write_section "" "./mutants/caught.txt"
+          fi
+
+          # Get exit code from the file and match it
+          exit_code=$(<"mutants/exit_code.txt")
+
+          yellow_bold="\033[1;33m"
+          reset="\033[0m"
+          summary_link_message="${yellow_bold}Click here for more information on how to fix:${reset} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#:~:text=Output%20Mutants%20summary"
+
+          case $exit_code in
+            0)
+                if [[ ! -f ./mutants/caught.txt && ! -f ./mutants/missed.txt && ! -f ./mutants/timeout.txt && ! -f ./mutants/unviable.txt ]]; then
+                  echo "No mutants found to test!"    
+                elif [[ -s ./mutants/unviable.txt ]]; then
+                  echo -e "$summary_link_message"
+                  echo "Found unviable mutants!"
+                  exit 5
+                else
+                  echo "All new and updated functions are caught!"  
+                fi
+              ;;
+            1)
+              echo -e "$summary_link_message"
+              echo "Invalid command line arguments!"
+              exit $exit_code
+              ;;
+            2)
+              echo -e "$summary_link_message"
+              echo "Found missed mutants!"
+              exit $exit_code
+              ;;
+            3)
+              echo -e "$summary_link_message"
+              echo "Found timeout mutants!"
+              exit $exit_code
+              ;;
+            4)
+              echo -e "$summary_link_message"
+              echo "Building the packages failed without any mutations!"
+              exit $exit_code
+              ;;
+            *)
+              echo -e "$summary_link_message"
+              echo "Unknown exit code: $exit_code"
+              exit $exit_code
+            ;;
+          esac
+
+          exit 0