implement retry & tests

cascading-pr-lib.sh

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: MIT
 
+set -o pipefail
+
 declare -A options
 
 PREFIX===============
@@ -16,6 +18,8 @@ DEBUG=false
 
 : ${LOOP_DELAY:=5}
 
+: ${RETRY_DELAYS:=1 1 5 5 15 30}
+
 function dependencies() {
     if ! which jq curl > /dev/null ; then
 	apt-get update -qq
@@ -23,6 +27,28 @@ function dependencies() {
     fi
 }
 
+function retry() {
+    rm -f $TMPDIR/retry.out
+    success=false
+    for delay in $RETRY_DELAYS ; do
+        if "$@" | tee -a $TMPDIR/retry.out > $TMPDIR/retry-attempt.out 2>&1 ; then
+            success=true
+            break
+        fi
+        cat $TMPDIR/retry-attempt.out >&2
+        log_verbose waiting $delay "$@"
+        sleep $delay
+    done
+    if $success ; then
+	cat $TMPDIR/retry-attempt.out
+	return 0
+    else
+	log_error retry failed for "$@"
+        cat $TMPDIR/retry.out
+        return 1
+    fi
+}
+
 function debug() {
     DEBUG=true
     set -x

tests/run.sh

@@ -92,7 +92,56 @@ function finalize_options() {
     options[password]=admin1234
 }
 
-function run() {
+function unit_retry_fail() {
+    local file=$1
+    local value=$(cat $file)
+    expr $value - 1 > $file
+    echo RESULT
+    if test $value -gt 0 ; then
+	return 1
+    else
+	return 0
+    fi
+}
+
+function unit_retry() {
+    local two=$TMPDIR/unit_retry_two
+
+    rm -f $TMPDIR/retry.*
+
+    #
+    # Succeeds after two tries
+    #
+    echo 2 > $TMPDIR/unit_retry_two
+    RETRY_DELAYS='1 1 1 1' retry unit_retry_fail $two > $TMPDIR/retry.result 2> $TMPDIR/retry.log
+    test "$(cat $TMPDIR/retry.result)" = "RESULT"
+    test "$(grep -c 'waiting 1 unit_retry_fail' $TMPDIR/retry.log)" = 2
+
+    #
+    # Succeeds immediately
+    #
+    RETRY_DELAYS='1' retry unit_retry_fail $two > $TMPDIR/retry.result 2> $TMPDIR/retry.log
+    test "$(cat $TMPDIR/retry.result)" = "RESULT"
+    test "$(grep -c 'waiting 1 unit_retry_fail' $TMPDIR/retry.log)" = 0
+
+    #
+    # Verify the output is only the output of the last run and is not polluted by
+    # unrelated output
+    #
+    echo 2 > $TMPDIR/unit_retry_two
+    test "$(RETRY_DELAYS='1 1 1' retry unit_retry_fail $two)" = "RESULT"
+
+    #
+    # Fails after one try
+    #
+    echo 2 > $TMPDIR/unit_retry_two
+    if RETRY_DELAYS='1' retry unit_retry_fail $two |& tee $TMPDIR/retry.log ; then
+	return 1
+    fi
+    grep --quiet 'retry failed' $TMPDIR/retry.log
+}
+
+function integration() {
     user_create user2 user2@example.com
     forgejo-test-helper.sh push tests/destinationrepo http://user2:admin1234@${options[host_port]} user2 destinationrepo
 
@@ -120,6 +169,15 @@ function run() {
     wait_success ${options[url]}/api/v1/repos/user1/originrepo $(cat $TMPDIR/originrepo.sha)
 }
 
+function unit() {
+    unit_retry
+}
+
+function run() {
+    unit
+    integration
+}
+
 function main() {
     local command=run