lib/common-functions: failsafe_sftp() new

4 files changed, 64 insertions, 4 deletions

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 8024965..70e26e7 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -189,7 +189,7 @@ if [ -s "${tmp_dir}/to-copy" ]; then
 fi
 
 if [ -s "${tmp_dir}/sftp-command" ]; then
-  ${master_mirror_sftp_command} < \
+  failsafe_sftp < \
     "${tmp_dir}/sftp-command"
 fi
 
diff --git a/bin/db-update b/bin/db-update
index 6bdbf7e..367fb36 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -442,7 +442,7 @@ for source_stability in \
     if ${no_action}; then
       sed 's|^|sftp: |'
     else
-      ${master_mirror_sftp_command}
+      failsafe_sftp
     fi
 
   if ${no_action}; then
diff --git a/bin/delete-packages b/bin/delete-packages
index 6d752d1..50fc853 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -398,4 +398,4 @@ sed '
   s/^/rm "/
   s/$/"/
 ' "${tmp_dir}/package-files" | \
-  ${master_mirror_sftp_command}
+  failsafe_sftp
diff --git a/lib/common-functions b/lib/common-functions
index 319a85e..6e0892d 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -260,7 +260,7 @@ remove_old_package_versions() {
         s|^|rm "|
         s|$|"|
       ' "${tmp_dir}/sftp-removes" | \
-        ${master_mirror_sftp_command}
+        failsafe_sftp
     fi
 
   )
@@ -822,3 +822,63 @@ recompress_gz() {
   done
   rm "${tmp_file}"
 }
+
+# failsafe_sftp
+# execute the commands from stdin on the master mirror, retrying if
+# unsuccessful
+# caveats:
+#   - only checks for success of "rm", "ln", "rename"
+#   - commands must be executable in arbitrary order
+failsafe_sftp() {
+  ( # new shell is intentional
+    temp_dir=$(mktemp -d "tmp.common-functions.sftp_failsave.XXXXXXXXXX")
+    trial_counter=20
+    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
+    cat > "${temp_dir}/input"
+    sed -n '
+      s/^rm "\([^"]\+\)"$/- \1/
+      s/^ln\( [^"]\S*\)* "\([^"]\+\)" "[^"]\+"$/+ \2/
+      s/^rename "\([^"]\+\)" "\([^"]\+\)"$/- \1\n+ \2/
+      T
+      p
+    ' "${temp_dir}/input" > \
+      "${temp_dir}/expectations"
+    if ${master_mirror_sftp_command} < "${temp_dir}/input"; then
+      # we're done
+      exit 0
+    fi
+    # make failing sftp commands non-fatal - maybe some succeeded above?
+    sed -i '
+      s/^[^-]/-\0/
+    ' "${temp_dir}/input"
+    # we stop on exhausted $trial_counter
+    while [ ${trial_counter} -gt 0 ]; do
+      wait_some_time 30
+      # success requires 3 things:
+      # - succeeding sftp executions (commands are non-critical already)
+      # - succeeding sftp command to create listing (e.g. all expected
+      #   files are found)
+      # - no unexpected files were found
+      if ${master_mirror_sftp_command} < \
+        "${temp_dir}/input" && \
+        sed '
+          s/^+ \(\S\+\)$/ls -1 "\1"/
+          s,^- \(\S\+/\)[^/]\+$,ls -1 "\1",
+        ' "${temp_dir}/expectations" | \
+          sort -u | \
+          ${master_mirror_sftp_command} > "${temp_dir}/check" && \
+        ! grep -qxF "$(
+          sed -n '
+            s/^- //
+            T
+            p
+          ' "${temp_dir}/expectations"
+        )" "${temp_dir}/check"; then
+        exit 0
+      fi
+      trial_counter=$((trial_counter-1))
+    done
+    exit 1
+  ) || \
+    return $?
+}