Improve stage3 parsing #193

* Some mirrors truncate long stage3 file names for display => parse <a href> element instead * Handle possible url encoded + char
edannenberg · May 4, 2021 · 160ed27 · 160ed27
1 parent 39e0056
commit 160ed27
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 8 deletions.
diff --git a/cmd/update.sh b/cmd/update.sh
@@ -7,7 +7,7 @@
 # 2: builder_path
 function update_builders() {
     __update_builders=
-    local builder_path current_ns current_builder update_status s3date_remote update_count
+    local builder_path current_ns current_builder update_status s3date_remote update_count max_cap
     current_ns="$1"
     builder_path="$2"
     update_count=0
@@ -26,9 +26,10 @@ function update_builders() {
                 get_stage3_archive_regex "${STAGE3_BASE}"
                 # shellcheck disable=SC2154
                 if [[ "${__fetch_stage3_archive_name}" =~ ${__get_stage3_archive_regex} ]]; then
-                    s3date_remote="${BASH_REMATCH[1]}"
+                    max_cap="${#BASH_REMATCH[@]}"
+                    s3date_remote="${BASH_REMATCH[$((max_cap-3))]}"
                     # add time string if captured
-                    [[ -n "${BASH_REMATCH[2]}" ]] && s3date_remote+="${BASH_REMATCH[2]}"
+                    [[ -n "${BASH_REMATCH[$((max_cap-2))]}" ]] && s3date_remote+="${BASH_REMATCH[$((max_cap-2))]}"
                     # shellcheck disable=SC2153
                     if is_newer_stage3_date "${STAGE3_DATE}" "${s3date_remote}"; then
                         sed -E -i'' s/^STAGE3_DATE=\(\"\|\'\)?[0-9]*\(T[0-9]*Z\)?\(\"\|\'\)?/STAGE3_DATE=\'"${s3date_remote}"\'/g \

diff --git a/lib/core.sh b/lib/core.sh
@@ -366,22 +366,25 @@ function get_stage3_archive_regex() {
     __get_stage3_archive_regex=
     local stage3_base
     stage3_base="$1"
-    __get_stage3_archive_regex="${stage3_base//+/\\+}-([0-9]{8})(T[0-9]{6}Z)?\\.tar\\.(bz2|xz)"
+    __get_stage3_archive_regex="${stage3_base//+/(\\+|%2B)?}-([0-9]{8})(T[0-9]{6}Z)?\\.tar\\.(bz2|xz)"
 }
 
 # Fetch latest stage3 archive name/type, returns exit signal 3 if no archive could be found
 function fetch_stage3_archive_name() {
     __fetch_stage3_archive_name=
     ARCH="${ARCH:-amd64}"
     ARCH_URL="${ARCH_URL:-${MIRROR}releases/${ARCH}/autobuilds/current-${STAGE3_BASE}/}"
-    local remote_files remote_line remote_date remote_file_type
+    local remote_files remote_line remote_date remote_file_type max_cap
     readarray -t remote_files <<< "$(wget -qO- "${ARCH_URL}")"
     remote_date=0
     get_stage3_archive_regex "${STAGE3_BASE}"
     for remote_line in "${remote_files[@]}"; do
-        if [[ "${remote_line}" =~ ${__get_stage3_archive_regex}\< ]]; then
-            is_newer_stage3_date "${remote_date}" "${BASH_REMATCH[1]}${BASH_REMATCH[2]}" \
-                && { remote_date="${BASH_REMATCH[1]}${BASH_REMATCH[2]}"; remote_file_type="${BASH_REMATCH[3]}"; }
+        if [[ "${remote_line}" =~ href=\"${__get_stage3_archive_regex}\" ]]; then
+            max_cap="${#BASH_REMATCH[@]}"
+            is_newer_stage3_date "${remote_date}" "${BASH_REMATCH[$((max_cap-3))]}${BASH_REMATCH[$((max_cap-2))]}" \
+                && { remote_date="${BASH_REMATCH[$((max_cap-3))]}${BASH_REMATCH[$((max_cap-2))]}";
+                     remote_file_type="${BASH_REMATCH[$((max_cap-1))]}"; }
+	    break
         fi
     done
     [[ "${remote_date//[!0-9]/}" -eq 0 ]] && return 3