bin/db-update: uses intentions-queue, now

1 files changed, 150 insertions, 84 deletions

diff --git a/bin/db-update b/bin/db-update
index e1255d6..e6bb1b0 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -9,8 +9,6 @@
 
 # TODO: make (all) locking less restrictive
 
-# TODO: use the intention-queue
-
 # shellcheck disable=SC2039,SC2119,SC2120
 
 # shellcheck source=../lib/load-configuration
@@ -67,7 +65,7 @@ eval set -- "$(
 
 block_flag='-n'
 ignore_insanity=false
-no_action=false
+no_action_flag=''
 progressive=false
 force_pkgs=''
 only=''
@@ -90,7 +88,7 @@ do
       ignore_insanity=true
     ;;
     -n|--no-action)
-      no_action=true
+      no_action_flag='-n'
     ;;
     -o|--only)
       shift
@@ -167,7 +165,7 @@ if [ -n "${only}" ]; then
   )
 fi
 
-if ! ${no_action}; then
+if [ -z "${no_action_flag}" ]; then
   # Create lock.
   exec 9> "${package_database_lock_file}"
   if ! verbose_flock ${block_flag} 9; then
@@ -513,21 +511,35 @@ export TMPDIR="${tmp_dir}"
       "${tmp_dir}/rm" \
       "${tmp_dir}/rm.id"
 
+    perma_tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.permanent.XXXXXXXXXX")
     # shellcheck disable=SC2086
     for s in "${tmp_dir}/"*; do
-      sort -u "${s}" | \
-        sponge "${s}"
+      sort -u "${s}" \
+      > "${perma_tmp_dir}/${s##*/}"
     done
 
     # receive the repository databases from the master mirror
-    mkdir "${tmp_dir}/dbs"
     while read -r arch repo; do
-      mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
-      # shellcheck disable=SC2086
-      failsafe_rsync \
-        "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
-        "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
-        "${tmp_dir}/dbs/${arch}/${repo}/"
+      {
+        printf 'mkdir -p "%s/dbs/%s/%s"\n' \
+          "${perma_tmp_dir}" \
+          "${arch}" \
+          "${repo}"
+        printf 'failsafe_rsync'
+        for suffix in 'db' 'files'; do
+          printf ' "%s/%s/%s/%s.%s."*' \
+            "${master_mirror_rsync_directory}" \
+            "${arch}" \
+            "${repo}" \
+            "${repo}" \
+            "${suffix}"
+        done
+        printf ' "%s/dbs/%s/%s/"\n' \
+          "${perma_tmp_dir}" \
+          "${arch}" \
+          "${repo}"
+      } \
+      | intent_something "${no_action_flag}"
     done < \
       "${tmp_dir}/repositories"
 
@@ -537,11 +549,21 @@ export TMPDIR="${tmp_dir}"
       "${tmp_dir}/rm" | \
       sort -u | \
       while read -r arch repo; do
-        grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
-          sed '
+        {
+# TODO: sign repositories
+          printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \
+            "${perma_tmp_dir}" \
+            "${arch}" \
+            "${repo}" \
+            "${repo}"
+          grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" \
+          | sed '
             s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
-          ' | \
-          xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
+          ' \
+          | tr '\n' ' '
+          printf '\n'
+        } \
+        | intent_something "${no_action_flag}"
       done
 
     # copy and delete moved packages
@@ -550,89 +572,133 @@ export TMPDIR="${tmp_dir}"
       "${tmp_dir}/mv" | \
       sort -u | \
       while read -r from_arch from_repo to_arch to_repo; do
-        grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
-          sed '
+        {
+          printf '"%s/bin/repo-copy" ' \
+            "${base_dir}"
+          printf '"%s/dbs/%s/%s/%s.db.tar.gz" ' \
+            "${perma_tmp_dir}" "${from_arch}" "${from_repo}" "${from_repo}" \
+            "${perma_tmp_dir}" "${to_arch}" "${to_repo}" "${to_repo}"
+          grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \
+          | sed '
             s/-[^-]\+\( \S\+\)\{4\}$//
-          ' | \
-          xargs -r "${base_dir}/bin/repo-copy" \
-            "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
-            "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
-        grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
-          sed '
+          ' \
+          | tr '\n' ' '
+          printf '\n'
+          printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \
+            "${perma_tmp_dir}" \
+            "${from_arch}" \
+            "${from_repo}" \
+            "${from_repo}"
+          grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \
+          | sed '
             s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
-          ' | \
-          xargs -r repo-remove -q \
-            "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
+          ' \
+          | tr '\n' ' '
+          printf '\n'
+        } \
+        | intent_something "${no_action_flag}"
       done
 
     # create real file names of packages, because
     # mysql_query_and_delete_unneeded_binary_packages does so, too
-    sed -i '
-      s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
-    ' "${tmp_dir}/rm"
+    {
+      printf 'sed -i '"'"'\n'
+      printf '  s,^\\(\\S\\+\) \\(\\S\\+\\) \\(\\S\\+\\)$,\\2/\\3/\\1,\n'
+      printf "'"' "%s/rm"\n' \
+        "${perma_tmp_dir}"
+    } \
+    | intent_something "${no_action_flag}"
 
     # somewhat inaccurate
-    if ! ${no_action}; then
-      {
-        printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
-        printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
-        printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
-          "${tmp_dir}/mv.id" 'moved_bpir' \
-          "${tmp_dir}/rm.id" 'replaced_bpir'
-        printf 'DELETE `binary_packages_in_repositories`'
-        printf ' FROM `binary_packages_in_repositories`'
-        printf ' JOIN `replaced_bpir`'
-        printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
-        mysql_query_and_delete_unneeded_binary_packages
-        printf 'UPDATE `binary_packages_in_repositories`'
-        printf ' JOIN `moved_bpir`'
-        printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
-        printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
-        printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
-        printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
-      } | \
-        mysql_run_query | \
-        sort -u >> \
-        "${tmp_dir}/rm"
-    fi
+    {
+      printf '{\n'
+      printf '  mysql_run_query'
+      printf ' | sort -u'
+      printf ' >> "%s/rm"\n' \
+        "${perma_tmp_dir}"
+      printf '} <<END_OF_MYSQL_QUERY\n'
+
+      printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+      printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+      printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
+        "${perma_tmp_dir}/mv.id" 'moved_bpir' \
+        "${perma_tmp_dir}/rm.id" 'replaced_bpir'
+      printf 'DELETE `binary_packages_in_repositories`'
+      printf ' FROM `binary_packages_in_repositories`'
+      printf ' JOIN `replaced_bpir`'
+      printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
+      mysql_query_and_delete_unneeded_binary_packages
+      printf 'UPDATE `binary_packages_in_repositories`'
+      printf ' JOIN `moved_bpir`'
+      printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
+      printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
+      printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
+      printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
+
+      printf 'END_OF_MYSQL_QUERY\n'
+    } \
+    | sed 's/`/\\`/g' \
+    | intent_something "${no_action_flag}"
 
     # move the packages remotely via sftp
     {
-      sed '
-        s/^/rm "/
-        s/$/"/
-        p
-        s/"$/.sig"/
-      ' "${tmp_dir}/rm"
-      sed '
-        s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
-      ' "${tmp_dir}/mv"
-      echo 'quit'
-    } | \
-      if ${no_action}; then
-        sed 's|^|sftp: |'
-      else
-        failsafe_sftp
-      fi
-
-    if ${no_action}; then
-      continue
-    fi
+      printf '{\n'
+      printf '  sed '"'"'\n'
+      printf '    s/^/rm "/\n'
+      printf '    s/$/"/\n'
+      printf '    p\n'
+      printf '    s/"$/.sig"/\n'
+      printf '  '"'"' "%s/rm"\n' \
+        "${perma_tmp_dir}"
+      printf '  sed '"'"'\n'
+      printf '    s,^\\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\)$,'
+      printf 'rename "\\2/\\3/\\1" "\\4/\\5/\\1"\\n'
+      printf 'rename "\\2/\\3/\\1.sig" "\\4/\\5/\\1.sig",\n'
+      printf '  '"'"' "%s/mv"\n' \
+        "${perma_tmp_dir}"
+      printf '  echo quit\n'
+      printf '} | failsafe_sftp\n'
+    } \
+    | intent_something "${no_action_flag}"
 
     # and push our local *.db.tar.gz via rsync
     while read -r arch repo; do
-      recompress_gz \
-        "${tmp_dir}" \
-        "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
-        "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
-      # shellcheck disable=SC2086
-      failsafe_rsync \
-        "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
-        "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
-        "${master_mirror_rsync_directory}/${arch}/${repo}/"
+      {
+        printf 'recompress_gz "%s"' \
+          "${tmp_dir}"
+        for suffix in '' '.old'; do
+          printf ' "%s/dbs/%s/%s/%s."*".tar.gz%s"' \
+            "${perma_tmp_dir}" \
+            "${arch}" \
+            "${repo}" \
+            "${repo}" \
+            "${suffix}"
+        done
+        printf '\n'
+        printf 'failsafe_rsync '
+        for suffix in 'db' 'files'; do
+          printf '"%s/dbs/%s/%s/%s.%s."* ' \
+            "${perma_tmp_dir}" \
+            "${arch}" \
+            "${repo}" \
+            "${repo}" \
+            "${suffix}"
+        done
+        printf ' "%s/%s/%s/"\n' \
+          "${master_mirror_rsync_directory}" \
+          "${arch}" \
+          "${repo}"
+      } \
+      | intent_something "${no_action_flag}"
     done < \
       "${tmp_dir}/repositories"
 
+    printf 'rm -rf --one-file-system "%s"\n' \
+      "${perma_tmp_dir}" \
+    | intent_something
+
+    execute_all_intentions
+
   done
 
 trigger_mirror_refreshs