implement retry & tests

cascading-pr-lib.sh

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: MIT
 
+set -o pipefail
+
 declare -A options
 
 PREFIX===============
@@ -16,6 +18,8 @@ DEBUG=false
 
 : ${LOOP_DELAY:=5}
 
+: ${RETRY_DELAYS:=1 1 5 5 15 30}
+
 function dependencies() {
     if ! which jq curl > /dev/null ; then
 	apt-get update -qq
@@ -23,6 +27,28 @@ function dependencies() {
     fi
 }
 
+function retry() {
+    rm -f $TMPDIR/retry.out
+    success=false
+    for delay in $RETRY_DELAYS ; do
+        if "$@" | tee -a $TMPDIR/retry.out > $TMPDIR/retry-attempt.out 2>&1 ; then
+            success=true
+            break
+        fi
+        cat $TMPDIR/retry-attempt.out >&2
+        log_verbose waiting $delay "$@"
+        sleep $delay
+    done
+    if $success ; then
+	cat $TMPDIR/retry-attempt.out
+	return 0
+    else
+	log_error retry failed for "$@"
+        cat $TMPDIR/retry.out
+        return 1
+    fi
+}
+
 function debug() {
     DEBUG=true
     set -x