bin/build-packages: try to download sources by hash

3 files changed, 80 insertions, 0 deletions

diff --git a/bin/build-packages b/bin/build-packages
index fc3a205..c207966 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -188,6 +188,16 @@ while [ ${count} -ne 0 ]; do
         fi
 
         if echo "${straw}" | \
+          grep -qF ':mirrored_source_by_hash:'; then
+          # maybe a missing source is/was the problem?
+          # download it from sources.archlinux32.org by its hash
+          if ! download_sources_by_hash "${package}" "${repository}" "${git_revision}" "${mod_git_revision}"; then
+            # we can't improve anything, if no source was downloadable
+            continue
+          fi
+        fi
+
+        if echo "${straw}" | \
           grep -qF ':with_build_support:'; then
           build_command='staging-with-build-support-i686-build'
         else
diff --git a/bin/common-functions b/bin/common-functions
index f721614..1f2d108 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -680,3 +680,69 @@ find_dependencies_on_build_list() {
     uniq -d
 
 }
+
+# download_sources_by_hash $package $repository $git_revision $git_mod_revision
+# try to download all sources by their hash into the current directory
+# returns 0 if any source was downloaded and 1 otherwise
+
+download_sources_by_hash() {
+
+  local package="$1"
+  local repository="$2"
+  local git_revision="$3"
+  local git_mod_revision="$4"
+
+  local return_value=1
+  local tmp_dir=$(mktemp -d)
+  local sum_type
+  local arch_suffix
+
+  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
+    >&2 echo 'download_sources_by_hash: make_source_info failed.'
+    rm -rf --one-file-system "${tmp_dir}"
+    return 1
+  fi
+
+  if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
+    >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
+    rm -rf --one-file-system "${tmp_dir}"
+    return 1
+  fi
+
+  for arch_suffix in '' '_i686'; do
+    for sum_type in 'sha256sum' 'sha512sum'; do
+      grep "^\s*${sum_type}s${arch_suffix} = " "${tmp_dir}/.SRCINFO" | \
+        sed 's|^.* = ||' | \
+        cat -n > \
+        "${tmp_dir}/sums"
+      grep "^\s*source${arch_suffix} = " "${tmp_dir}/.SRCINFO" | \
+        sed '
+          s|^.* = ||
+          s|::.*$||
+          s|.*/||
+        ' | \
+        cat -n > \
+        "${tmp_dir}/urls"
+      if [ $(wc -l < "${tmp_dir}/sums") -eq $(wc -l < "${tmp_dir}/urls") ]; then
+        join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
+          "${tmp_dir}/joined"
+        while read -r sum file; do
+          if echo "${sum}  ${file}" | \
+            ${sum_type} -c > /dev/null 2>&1; then
+            # the correct source is already there
+            continue
+          fi
+          if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
+            mv "${tmp_dir}/transfer" "${file}"
+            return_value=0
+          fi
+        done < \
+          "${tmp_dir}/joined"
+      fi
+    done
+  done
+
+  rm -rf --one-file-system "${tmp_dir}"
+  return ${return_value}
+
+}
diff --git a/conf/default.conf b/conf/default.conf
index bb39d8e..6e98d85 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -38,6 +38,9 @@ master_mirror_rsync_directory='rsync://buildmaster@mirror.archlinux32.org/packag
 # to access the master mirror via sftp
 master_mirror_sftp_commend='sftp -b- user@mirror'
 
+# mirror of sources, identified (solely) by hash
+source_by_hash_mirror='http://sources.archlinux32.org/'
+
 # directory to keep the build log files in
 build_log_directory='/srv/http/build-logs'
 
@@ -46,6 +49,7 @@ straws_that_might_repair_failing_builds=$(
   printf '%s\n' \
     ':' \
     ':clean_chroot:' \
+    ':mirrored_source_by_hash:' \
     ':mirrored_source:' \
     ':with_build_support:' \
     ':with_build_support:clean_chroot:'