os-autoinst-obs-auto-submit

#!/bin/bash

set -euo pipefail

export LC_ALL=C
src_project=${src_project:-devel:openQA}
dst_project=${dst_project:-${src_project}:tested}
staging_project=${staging_project:-${src_project}:testing}
submit_target="${submit_target:-"openSUSE:Factory"}"
submit_target_extra="${submit_target_extra:-}"
dry_run="${dry_run:-"0"}"
osc_poll_interval="${osc_poll_interval:-2}"
osc_build_start_poll_tries="${osc_build_start_poll_tries:-30}"
throttle_days=${throttle_days:-2}
XMLSTARLET=$(command -v xmlstarlet || true)
[[ -n $XMLSTARLET ]] || (echo "Need xmlstarlet" && exit 1)

# shellcheck source=/dev/null
. "$(dirname "$0")"/_common

IFS=',' read -ra targets <<< "$submit_target,$submit_target_extra"
failed_packages=()

encode_variable() {
    #  https://stackoverflow.com/a/298258
    perl -MURI::Escape -e 'print uri_escape($ARGV[0])' "$1"
}

get_obs_sr_id() {
    local target=$1
    local dst_project=$2
    target=$(encode_variable "$target")
    dst_project=$(encode_variable "$dst_project")
    local package=$3
    # writing xpath in url encoding is not for beginners, so don't stare at it too long :)
    local states='(state%2F%40name%3D%27declined%27%20or%20state%2F%40name%3D%27new%27%20or%20state%2F%40name%3D%27review%27)'
    local actions="(action%2Ftarget%2F%40project%3D%27${target}%27%20and%20action%2Fsource%2F%40project%3D%27${dst_project}%27%20and%20action%2Ftarget%2F%40package%3D%27$package%27)"
    $osc api "/search/request/id?match=$states%20and%20$actions" | grep 'request id' | sed -e 's,.*id=.\([0-9]*\)[^[0-9]*$,\1,' | head -n 1
}

reenable_buildtime_services() {
    sed -i -e 's,mode="buildtime" mode="disabled",mode="buildtime",' _service
}

wait_for_package_build() {
    local target=$1
    target_escaped=$(echo -n "$target" | sed -e 's|\:|_|g') # avoid, e.g. "repoid 'openSUSE:Factory' is illegal"
    local osc_query="https://api.opensuse.org/public/build/$dst_project/_result?repository=$target_escaped&package=$package"
    local wip_states='\(unknown\|blocked\|scheduled\|dispatching\|building\|signing\|finished\)'
    local bad_states='\(failed\|unresolvable\|broken\)'
    local attempts="$osc_build_start_poll_tries"
    while ! curl -sS "$osc_query" | grep -e "$wip_states"; do
        if [[ $attempts -le 0 ]]; then
            echo "warning: Re-build of $package has not been considered in time (or package has been re-built so fast that we've missed it)"
            break
        fi
        echo "Waiting for re-build of $package to be considered (attempts left: $attempts)"
        sleep "$osc_poll_interval"
        attempts=$((attempts - 1))
    done
    while curl -sS "$osc_query" | grep -e "$wip_states"; do
        echo "Waiting while $package is in progress"
        sleep "$osc_poll_interval"
    done
    if curl -sS "$osc_query" | grep -e "$bad_states"; then
        echo "Building $package has failed, skipping submission"
        return 1
    fi
}

update_package() {
    package=$1
    xmlstarlet ed -L -i "/services/service" --type attr -n mode --value 'disabled' _service
    sed -i -e 's,mode="disabled" mode="disabled",mode="disabled",' _service
    reenable_buildtime_services
    # ignore those updates
    rm -f _service:*-test.changes
    cp -v .osc/*-test.changes . 2> /dev/null || :
    for file in _service:*; do
        # shellcheck disable=SC2001
        mv -v "$file" "$(echo "$file" | sed -e 's,.*:,,')"
    done
    version=$(sed -n 's/^Version:\s*//p' ./*.spec || sed -n 's/^version:\s*//p' ./*.obsinfo)
    rm -f ./*rpmlintrc _servicedata node_modules.sums package-lock.json
    $osc addremove
    sed -i '/rpmlintrc/d' ./*.spec || true
    local ci_args=""
    if [[ "$package" == "os-autoinst-distri-opensuse-deps" || "$package" =~ "container" ]]; then
        ci_args="--noservice"
    fi
    # We would get an empty changelog when there are no changes to files contained
    # in this package/container. So add some text.
    sed -i -e 's/^  \* $/  * Update to latest openQA version/' ./*.changes
    $osc ci -m "Offline generation of ${version}" $ci_args
    rc=0
    for target in "${targets[@]}"; do
        wait_for_package_build "$target"
        cmd="$osc sr"
        req=$(get_obs_sr_id "$target" "$dst_project" "$package" || :)
        # TODO: check if it's a different revision than HEAD
        if test -n "$req"; then
            cmd="$cmd -s $req"
        fi
        if ! $cmd -m "Update to ${version}" "$target"; then
            rc=$?
            failed_packages+=("$package:$rc")
        fi
    done
    return $rc
}

last_revision() {
    project=$1
    package=$2
    file=$package.changes
    service=obs_scm
    if [[ $project != "$submit_target" ]]; then
        file=_service:$service:$file
    fi
    local line
    # prevent SIGPIPE triggering osc cat to fail with the command group piping
    # exceeding output to /dev/null
    line=$($osc cat "$project/$package/$file" | {
        grep -m1 'Update to version'
        cat > /dev/null
    })
    # shellcheck disable=SC2001
    echo "$line" | sed -e 's,.*\.\([^.]*\):$,\1,'
}

sync_changesrevision() {
    dir=$1
    package=$2
    target_rev=$3
    path="$dir/$package"
    $prefix xmlstarlet ed -L -u "//param[@name='changesrevision']" -v "$target_rev" "$path"/_servicedata
    if ! diff -u "$path"/.osc/_servicedata "$path"/_servicedata; then
        $osc up "$path"
        $osc cat "$submit_target" "$package" "$package".changes > "$path/$package".changes
        $osc ci -m "sync changesrevision with $submit_target" "$path"
        $osc up --server-side-source-service-files "$path"
    fi
}

get_spec_changes_in_requirements() {
    local package=$1
    local project=$2
    { diff -u "$package-factory.spec" "$project/$package/_service:obs_scm:$package.spec" || :; } | grep '^[+-]Requires'
}

generate_os-autoinst-distri-opensuse-deps_changelog() {
    dir=$1
    package=$2
    $osc cat "$submit_target" "$package" "$package".changes > "$package"-factory.changes
    {
        echo "-------------------------------------------------------------------"
        echo "$(LANG=c date -u) - Dominik Heidler <dheidler@suse.de>"
        echo
        get_spec_changes_in_requirements "$package" "$dir" \
            | sed -e 's/Requires:\s*/dependency /g' -e 's/^-/- Removed /g' -e 's/^+/- Added /g'
        echo
        cat "$package"-factory.changes
    } > "$dir/$package/$package".changes
}

handle_auto_submit() {
    package=$1
    $osc service wait "$src_project" "$package"
    $osc co --server-side-source-service-files "$src_project"/"$package"
    if [[ "$package" == "os-autoinst-distri-opensuse-deps" ]]; then
        $osc cat "$submit_target" "$package" "$package.spec" > "$package-factory.spec"
        if get_spec_changes_in_requirements "$package" "$src_project"; then
            # dependency added or removed
            generate_os-autoinst-distri-opensuse-deps_changelog "$src_project" "$package"
        else
            echo "No dependency changes for $package"
            return
        fi
    else
        target_rev=$(last_revision "$submit_target" "$package")
        sync_changesrevision "$src_project" "$package" "$target_rev"
        if [[ "$target_rev" == "$(last_revision "$src_project" "$package")" ]]; then
            echo "Latest revision already in $submit_target"
            return
        fi
    fi
    $osc service wait "$dst_project" "$package"
    $osc co "$dst_project"/"$package"
    rm "$dst_project"/"$package"/*
    cp -v "$src_project"/"$package"/* "$dst_project"/"$package"/
    (
        cd "$dst_project"/"$package"
        update_package "$package"
    )
}

get_project_packages() {
    echo "${packages:-$($osc ls "$dst_project" | grep -v '\-test$')}"
}

has_pending_submission() {
    # throttle number of submissions to allow only one SR within a certain number of days
    # note: Avoid using `grep --quiet` here to keep consuming input so osc does not run into "BrokenPipeError: [Errno 32] Broken pipe".
    if [[ $throttle_days != 0 ]] && $osc request list --project "$dst_project" --type submit --state new,review --mine --days "$throttle_days" | grep 'Created by' > /dev/null; then
        echo "Skipping submission, there is still a pending SR younger than $throttle_days days."
        return 1
    fi
    return 0
}

is_staging_specified() {
    if [[ -n $staging_project && $staging_project != none && $staging_project != "$src_project" ]]; then
        src_project="$staging_project"
        return 0
    else
        return 1
    fi
}

submit() {
    # submit from staging project if specified
    if [[ -e job_post_skip_submission ]]; then
        echo "Skipping submission, reason: "
        cat job_post_skip_submission
        exit 0
    fi
    has_pending_submission || exit 0
    (
        cd "$TMPDIR"
        rc=0
        auto_submit_packages=$(get_project_packages)
        for package in $auto_submit_packages; do
            handle_auto_submit "$package" || rc=$?
        done
        # delete package from staging project
        is_staging_specified && $osc rdelete -m "Cleaning up $package from $staging_project for next submission" "$staging_project" "$package"
        if [[ ${#failed_packages[@]} -gt 0 ]]; then
            echo "Failed packages:"
            for item in "${failed_packages[@]}"; do
                package=${item%:*}
                exit_status=${item#*:}
                echo "- $package (exit status: $exit_status)"
            done
        fi
        exit "$rc"
    )
}

TMPDIR=${TMPDIR:-$(mktemp -d -t os-autoinst-obs-auto-submit-XXXX)}
trap 'rm -rf "$TMPDIR"' EXIT

prefix="${prefix:-""}"
[ "$dry_run" = "1" ] && prefix="echo"
osc="${osc:-"$prefix retry -e -- osc"}"

caller 0 > /dev/null || submit "$@"