From 4e0e422642fb33c88ec7e24583965fd46b5d49a9 Mon Sep 17 00:00:00 2001
From: Lyle Barner <lyle.barner@jpl.nasa.gov>
Date: Wed, 13 Mar 2024 11:29:00 -0700
Subject: [PATCH 1/3] Updates to fix SonarQube pagination issue

---
 scrub/tools/templates/sonarqube.template | 73 ++++++++++--------------
 1 file changed, 31 insertions(+), 42 deletions(-)

diff --git a/scrub/tools/templates/sonarqube.template b/scrub/tools/templates/sonarqube.template
index e4819e4..3276cf9 100644
--- a/scrub/tools/templates/sonarqube.template
+++ b/scrub/tools/templates/sonarqube.template
@@ -133,55 +133,44 @@ then
     exit 1
 fi
 
-# Retrieve the results from the SonarQube server
-PAGE=1
-MORE_RESULTS=true
-while $MORE_RESULTS; do
+# Retrieve the issues from the SonarQube server
+# Get the first page
+RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_issues_1.json
+curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=500&componentKeys=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+
+# Get the number of remaining pages
+TOTAL_RESULTS=$(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | head -n1)
+TOTAL_PAGES=$(( ( TOTAL_RESULTS / 500) + ( TOTAL_RESULTS % 500 > 0 ) ))
+if (( TOTAL_PAGES > 20 )); then
+    TOTAL_PAGES=20
+fi
+
+# Get the rest of the issues
+for ((CURRENT_PAGE=2; CURRENT_PAGE <= TOTAL_PAGES; CURRENT_PAGE++));
+do
     # Get the page
-    RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_issues_$PAGE.json
-    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=500&componentKeys=${{SONARQUBE_PROJECT}}&p=$PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
-    # Check to see if the file is empty
-    if [ ! -s "$RESULTS_FILE" ]; then
-        exit 1
-    fi
-    # Check the contents, verify file is not empty, and make sure the max page hasn't been reached
-    if grep -q "Can return only the first 10000 results" $RESULTS_FILE; then
-        echo "WARNING: Not all results have been retrieved."
-        MORE_RESULTS=false
-    elif [ $PAGE -gt 20 ]; then
-        MORE_RESULTS=false
-    elif grep -q "\"issues\":\[\]" $RESULTS_FILE; then
-       rm -f $RESULTS_FILE
-       MORE_RESULTS=false
-    else
-       PAGE=$((PAGE+1))
-    fi
+    RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_issues_$CURRENT_PAGE.json
+    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=500&componentKeys=${{SONARQUBE_PROJECT}}&p=$CURRENT_PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
 done
 
 # Retrieve the hotspots from the SonarQube server
-PAGE=1
-MORE_RESULTS=true
-while $MORE_RESULTS; do
+# Get the first page
+RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$PAGE.json
+curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=500&projectKey=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+if (( TOTAL_PAGES > 20 )); then
+    TOTAL_PAGES=20
+fi
+
+# Get the number of remaining pages
+TOTAL_RESULTS=$(( $(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | head -n1) + 500 ))
+TOTAL_PAGES=$(( ( TOTAL_RESULTS / 500) + ( TOTAL_RESULTS % 500 > 0 ) ))
+
+# Get the rest of the hotspots
+for ((CURRENT_PAGE=2; CURRENT_PAGE <= TOTAL_PAGES; CURRENT_PAGE++));
+do
     # Get the page
     RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$PAGE.json
     curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=500&projectKey=${{SONARQUBE_PROJECT}}&p=$PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
-    # Check to see if the file is empty
-    if [ ! -s "$RESULTS_FILE" ]; then
-        exit 1
-    fi
-    # Check the contents, verify file is not empty, and make sure the max page hasn't been reached
-    if grep -q "Can return only the first 10000 results" $RESULTS_FILE; then
-        echo "WARNING: Not all results have been retrieved."
-        rm -f $RESULTS_FILE
-        MORE_RESULTS=false
-    elif [ $PAGE -gt 20 ]; then
-        MORE_RESULTS=false
-    elif grep -q "\"hotspots\":\[\]" $RESULTS_FILE; then
-       rm -f $RESULTS_FILE
-       MORE_RESULTS=false
-    else
-       PAGE=$((PAGE+1))
-    fi
 done
 
 # Parse the results

From 93c384d57cb2a6c2a78158db73947662ab4bc371 Mon Sep 17 00:00:00 2001
From: Lyle Barner <lyle.barner@jpl.nasa.gov>
Date: Wed, 13 Mar 2024 11:54:23 -0700
Subject: [PATCH 2/3] Updates to fix SonarQube pagination issue

---
 scrub/tools/templates/sonarqube.template | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrub/tools/templates/sonarqube.template b/scrub/tools/templates/sonarqube.template
index 3276cf9..1d4611e 100644
--- a/scrub/tools/templates/sonarqube.template
+++ b/scrub/tools/templates/sonarqube.template
@@ -157,13 +157,13 @@ done
 # Get the first page
 RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$PAGE.json
 curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=500&projectKey=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
-if (( TOTAL_PAGES > 20 )); then
-    TOTAL_PAGES=20
-fi
 
 # Get the number of remaining pages
 TOTAL_RESULTS=$(( $(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | head -n1) + 500 ))
 TOTAL_PAGES=$(( ( TOTAL_RESULTS / 500) + ( TOTAL_RESULTS % 500 > 0 ) ))
+if (( TOTAL_PAGES > 20 )); then
+    TOTAL_PAGES=20
+fi
 
 # Get the rest of the hotspots
 for ((CURRENT_PAGE=2; CURRENT_PAGE <= TOTAL_PAGES; CURRENT_PAGE++));

From 45348e5be704126e90845a5232ee52650a06dc35 Mon Sep 17 00:00:00 2001
From: Lyle Barner <lyle.barner@jpl.nasa.gov>
Date: Wed, 13 Mar 2024 15:33:50 -0700
Subject: [PATCH 3/3] Clean up

---
 scrub/tools/templates/sonarqube.template | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/scrub/tools/templates/sonarqube.template b/scrub/tools/templates/sonarqube.template
index 1d4611e..3ffeefa 100644
--- a/scrub/tools/templates/sonarqube.template
+++ b/scrub/tools/templates/sonarqube.template
@@ -134,13 +134,14 @@ then
 fi
 
 # Retrieve the issues from the SonarQube server
+PAGE_SIZE=500
 # Get the first page
 RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_issues_1.json
-curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=500&componentKeys=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=$PAGE_SIZE&componentKeys=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
 
 # Get the number of remaining pages
-TOTAL_RESULTS=$(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | head -n1)
-TOTAL_PAGES=$(( ( TOTAL_RESULTS / 500) + ( TOTAL_RESULTS % 500 > 0 ) ))
+TOTAL_RESULTS=$(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | sed -n 1p)
+TOTAL_PAGES=$(( ( TOTAL_RESULTS / PAGE_SIZE ) + ( TOTAL_RESULTS % PAGE_SIZE > 0 ) ))
 if (( TOTAL_PAGES > 20 )); then
     TOTAL_PAGES=20
 fi
@@ -150,17 +151,17 @@ for ((CURRENT_PAGE=2; CURRENT_PAGE <= TOTAL_PAGES; CURRENT_PAGE++));
 do
     # Get the page
     RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_issues_$CURRENT_PAGE.json
-    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=500&componentKeys=${{SONARQUBE_PROJECT}}&p=$CURRENT_PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/issues/search?ps=$PAGE_SIZE&componentKeys=${{SONARQUBE_PROJECT}}&p=$CURRENT_PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
 done
 
 # Retrieve the hotspots from the SonarQube server
 # Get the first page
-RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$PAGE.json
-curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=500&projectKey=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_1.json
+curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=$PAGE_SIZE&projectKey=${{SONARQUBE_PROJECT}}&p=1&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
 
 # Get the number of remaining pages
-TOTAL_RESULTS=$(( $(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | head -n1) + 500 ))
-TOTAL_PAGES=$(( ( TOTAL_RESULTS / 500) + ( TOTAL_RESULTS % 500 > 0 ) ))
+TOTAL_RESULTS=$(( $(grep -E '[0-9]+' -m 1 -o -a $RESULTS_FILE | sed -n 3p) ))
+TOTAL_PAGES=$(( ( TOTAL_RESULTS / PAGE_SIZE ) + ( TOTAL_RESULTS % PAGE_SIZE > 0 ) ))
 if (( TOTAL_PAGES > 20 )); then
     TOTAL_PAGES=20
 fi
@@ -169,8 +170,8 @@ fi
 for ((CURRENT_PAGE=2; CURRENT_PAGE <= TOTAL_PAGES; CURRENT_PAGE++));
 do
     # Get the page
-    RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$PAGE.json
-    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=500&projectKey=${{SONARQUBE_PROJECT}}&p=$PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
+    RESULTS_FILE=${{TOOL_ANALYSIS_DIR}}/sonarqube_hotspots_$CURRENT_PAGE.json
+    curl -u ${{SONARQUBE_TOKEN}}: "${{SONARQUBE_SERVER}}/api/hotspots/search?ps=$PAGE_SIZE&projectKey=${{SONARQUBE_PROJECT}}&p=$CURRENT_PAGE&${{SONARQUBE_CURL_FLAGS}}" -o $RESULTS_FILE
 done
 
 # Parse the results