pr-mutants.yml

name: PR Differences Mutants

on:
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
    paths:
      - "**.rs"
  workflow_dispatch:

concurrency:
  group: pr-differences-${{ github.head_ref || github.ref || github.run_id }}
  # Always cancel duplicate jobs
  cancel-in-progress: true

jobs:
  mutants:
    name: Mutation Testing
    runs-on: ubuntu-latest
    steps:
      # Cleanup Runner
      - name: Cleanup Runner
        id: runner_cleanup
        uses: stacks-network/actions/cleanup/disk@main

      - name: Checkout repo
        id: git_checkout
        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
        with:
          fetch-depth: 0

      - name: Relative diff
        id: relative_diff
        run: |
          git diff $(git merge-base origin/${{ github.base_ref || 'main' }} HEAD)..HEAD > git.diff

      - name: Install cargo-mutants
        id: install_cargo_mutants
        run: |
          cargo install --version 24.11.2 cargo-mutants --locked # v24.11.2

      - name: Install cargo-nextest
        id: install_cargo_nextest
        uses: taiki-e/install-action@2f990e9c484f0590cb76a07296e9677b417493e9 # v2.33.23
        with:
          tool: nextest # Latest version

      - name: Update git diff
        id: update_git_diff
        run: |
          input_file="git.diff"
          temp_file="temp_diff_file.diff"

          # Check if the file exists and is not empty
          if [ ! -s "$input_file" ]; then
            echo "Diff file ($input_file) is missing or empty!"
            exit 1
          fi

          # Remove all lines related to deleted files including the first 'diff --git' line
          awk '
            /^diff --git/ {
              diff_line = $0
              getline
              if ($0 ~ /^deleted file mode/) {
                in_deleted_file_block = 1
              } else {
                if (diff_line != "") {
                  print diff_line
                  diff_line = ""
                }
                in_deleted_file_block = 0
              }
            }
            !in_deleted_file_block
          ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file"

          # Remove 'diff --git' lines only when followed by 'similarity index', 'rename from', and 'rename to'
          awk '
            /^diff --git/ {
              diff_line = $0
              getline
              if ($0 ~ /^similarity index/) {
                getline
                if ($0 ~ /^rename from/) {
                  getline
                  if ($0 ~ /^rename to/) {
                    next
                  }
                }
              }
              print diff_line  
            }
            { print }
          ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file"

      - name: Run docker-compose
        uses: hoverkraft-tech/compose-action@f1ca7fefe3627c2dab0ae1db43a106d82740245e # v2.0.2
        with:
          compose-file: "./docker/docker-compose.dev.postgres.yml"

      - name: Run mutants
        id: run_mutants
        run: |
          # Disable immediate exit on error
          set +e

          cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV --in-diff git.diff --output ./ --test-tool=nextest -- --all-targets --test-threads 1
          exit_code=$?

          # Create the folder only containing the outcomes (.txt files) and make a file containing the exit code of the command
          mkdir mutants
          echo "$exit_code" > ./mutants/exit_code.txt
          mv ./mutants.out/*.txt mutants/

          # Enable immediate exit on error again
          set -e

      - name: Print mutants
        id: print_tested_mutants
        shell: bash
        run: |
          # Info for creating the link that paths to the specific mutation tested
          server_url="${{ github.server_url }}"
          organisation="${{ github.repository_owner }}"
          repository="${{ github.event.repository.name }}"
          commit="${{ github.sha }}"

          # Function to write to github step summary with specific info depending on the mutation category
          write_section() {
            local section_title=$1
            local file_name=$2

            if [ -s "$file_name" ]; then
              if [[ "$section_title" != "" ]]; then
                echo "## $section_title" >> "$GITHUB_STEP_SUMMARY"
              fi

              if [[ "$section_title" == "Missed:" ]]; then
                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
                echo "<summary>What are missed mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
                echo "No test failed with this mutation applied, which seems to indicate a gap in test coverage. Or, it may be that the mutant is undistinguishable from the correct code. You may wish to add a better test, or mark that the function should be skipped." >> "$GITHUB_STEP_SUMMARY"
                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
                echo "" >> "$GITHUB_STEP_SUMMARY"
              elif [[ "$section_title" == "Timeout:" ]]; then
                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
                echo "<summary>What are timeout mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
                echo "The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped." >> "$GITHUB_STEP_SUMMARY"
                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
                echo "" >> "$GITHUB_STEP_SUMMARY"
              elif [[ "$section_title" == "Unviable:" ]]; then
                echo "<details>" >> "$GITHUB_STEP_SUMMARY"
                echo "<summary>What are unviable mutants?</summary>" >> "$GITHUB_STEP_SUMMARY"
                echo "<br>" >> "$GITHUB_STEP_SUMMARY"
                echo "The attempted mutation doesn't compile. This is inconclusive about test coverage and no action is needed, unless you wish to test the specific function, in which case you may wish to add a 'Default::default()' implementation for the specific return type." >> "$GITHUB_STEP_SUMMARY"
                echo "</details>" >> "$GITHUB_STEP_SUMMARY"
                echo "" >> "$GITHUB_STEP_SUMMARY"
              fi

              if [[ "$section_title" != "" ]]; then
                awk -F':' '{gsub("%", "%%"); printf "- [ ] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY"
              else
                awk -F':' '{gsub("%", "%%"); printf "- [x] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY"
              fi

              if [[ "$section_title" == "Missed:" ]]; then
                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
                echo "- Modify or add tests including this function." >> "$GITHUB_STEP_SUMMARY"
                echo "- If you are absolutely certain that this function should not undergo mutation testing, add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
              elif [[ "$section_title" == "Timeout:" ]]; then
                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
                echo "- Modify the tests that include this funcion." >> "$GITHUB_STEP_SUMMARY"
                echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
              elif [[ "$section_title" == "Unviable:" ]]; then
                echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY"
                echo "- Create 'Default::default()' implementation for the specific structure." >> "$GITHUB_STEP_SUMMARY"
                echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY"
              fi

              echo >> "$GITHUB_STEP_SUMMARY"
            fi
          }

          # Print uncaught (missed/timeout/unviable) mutants to summary
          if [ -s ./mutants/missed.txt -o -s ./mutants/timeout.txt -o -s ./mutants/unviable.txt ]; then
            echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY"
            echo "[Documentation - How to treat Mutants Output](https://github.com/stacks-network/actions/tree/main/stacks-core/mutation-testing#how-mutants-output-should-be-treated)" >> "$GITHUB_STEP_SUMMARY"
            write_section "Missed:" "./mutants/missed.txt"
            write_section "Timeout:" "./mutants/timeout.txt"
            write_section "Unviable:" "./mutants/unviable.txt"
          fi

          # Print caught mutants to summary
          if [ -s ./mutants/caught.txt ]; then
            echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY"
            write_section "" "./mutants/caught.txt"
          fi

          # Get exit code from the file and match it
          exit_code=$(<"mutants/exit_code.txt")

          yellow_bold="\033[1;33m"
          reset="\033[0m"
          summary_link_message="${yellow_bold}Click here for more information on how to fix:${reset} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#:~:text=Output%20Mutants%20summary"

          case $exit_code in
            0)
                if [[ ! -f ./mutants/caught.txt && ! -f ./mutants/missed.txt && ! -f ./mutants/timeout.txt && ! -f ./mutants/unviable.txt ]]; then
                  echo "No mutants found to test!"    
                elif [[ -s ./mutants/unviable.txt ]]; then
                  echo -e "$summary_link_message"
                  echo "Found unviable mutants!"
                  exit 5
                else
                  echo "All new and updated functions are caught!"  
                fi
              ;;
            1)
              echo -e "$summary_link_message"
              echo "Invalid command line arguments!"
              exit $exit_code
              ;;
            2)
              echo -e "$summary_link_message"
              echo "Found missed mutants!"
              exit $exit_code
              ;;
            3)
              echo -e "$summary_link_message"
              echo "Found timeout mutants!"
              exit $exit_code
              ;;
            4)
              echo -e "$summary_link_message"
              echo "Building the packages failed without any mutations!"
              exit $exit_code
              ;;
            *)
              echo -e "$summary_link_message"
              echo "Unknown exit code: $exit_code"
              exit $exit_code
            ;;
          esac

          exit 0