1 files changed, 61 insertions, 1 deletions

diff --git a/lib/common-functions b/lib/common-functions
index 319a85e..6e0892d 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -260,7 +260,7 @@ remove_old_package_versions() {
         s|^|rm "|
         s|$|"|
       ' "${tmp_dir}/sftp-removes" | \
-        ${master_mirror_sftp_command}
+        failsafe_sftp
     fi
 
   )
@@ -822,3 +822,63 @@ recompress_gz() {
   done
   rm "${tmp_file}"
 }
+
+# failsafe_sftp
+# execute the commands from stdin on the master mirror, retrying if
+# unsuccessful
+# caveats:
+#   - only checks for success of "rm", "ln", "rename"
+#   - commands must be executable in arbitrary order
+failsafe_sftp() {
+  ( # new shell is intentional
+    temp_dir=$(mktemp -d "tmp.common-functions.sftp_failsave.XXXXXXXXXX")
+    trial_counter=20
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+    cat > "${temp_dir}/input"
+    sed -n '
+      s/^rm "\([^"]\+\)"$/- \1/
+      s/^ln\( [^"]\S*\)* "\([^"]\+\)" "[^"]\+"$/+ \2/
+      s/^rename "\([^"]\+\)" "\([^"]\+\)"$/- \1\n+ \2/
+      T
+      p
+    ' "${temp_dir}/input" > \
+      "${temp_dir}/expectations"
+    if ${master_mirror_sftp_command} < "${temp_dir}/input"; then
+      # we're done
+      exit 0
+    fi
+    # make failing sftp commands non-fatal - maybe some succeeded above?
+    sed -i '
+      s/^[^-]/-\0/
+    ' "${temp_dir}/input"
+    # we stop on exhausted $trial_counter
+    while [ ${trial_counter} -gt 0 ]; do
+      wait_some_time 30
+      # success requires 3 things:
+      # - succeeding sftp executions (commands are non-critical already)
+      # - succeeding sftp command to create listing (e.g. all expected
+      #   files are found)
+      # - no unexpected files were found
+      if ${master_mirror_sftp_command} < \
+        "${temp_dir}/input" && \
+        sed '
+          s/^+ \(\S\+\)$/ls -1 "\1"/
+          s,^- \(\S\+/\)[^/]\+$,ls -1 "\1",
+        ' "${temp_dir}/expectations" | \
+          sort -u | \
+          ${master_mirror_sftp_command} > "${temp_dir}/check" && \
+        ! grep -qxF "$(
+          sed -n '
+            s/^- //
+            T
+            p
+          ' "${temp_dir}/expectations"
+        )" "${temp_dir}/check"; then
+        exit 0
+      fi
+      trial_counter=$((trial_counter-1))
+    done
+    exit 1
+  ) || \
+    return $?
+}