Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2020-01-31 10:52:26 +0100
committerErich Eckner <git@eckner.net>2020-01-31 10:52:26 +0100
commite040271149a8ca0a7d8cb9322e60ac91f3fcb21a (patch)
treec728d9da73bcb3c3d083552c9a2ab0d7f1a1ebbb
parentb2554880deb779f50aa82e73b247edf3a63b8837 (diff)
bin/db-update: uses intentions-queue, now
-rwxr-xr-xbin/db-update234
1 files changed, 150 insertions, 84 deletions
diff --git a/bin/db-update b/bin/db-update
index e1255d6..e6bb1b0 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -9,8 +9,6 @@
# TODO: make (all) locking less restrictive
-# TODO: use the intention-queue
-
# shellcheck disable=SC2039,SC2119,SC2120
# shellcheck source=../lib/load-configuration
@@ -67,7 +65,7 @@ eval set -- "$(
block_flag='-n'
ignore_insanity=false
-no_action=false
+no_action_flag=''
progressive=false
force_pkgs=''
only=''
@@ -90,7 +88,7 @@ do
ignore_insanity=true
;;
-n|--no-action)
- no_action=true
+ no_action_flag='-n'
;;
-o|--only)
shift
@@ -167,7 +165,7 @@ if [ -n "${only}" ]; then
)
fi
-if ! ${no_action}; then
+if [ -z "${no_action_flag}" ]; then
# Create lock.
exec 9> "${package_database_lock_file}"
if ! verbose_flock ${block_flag} 9; then
@@ -513,21 +511,35 @@ export TMPDIR="${tmp_dir}"
"${tmp_dir}/rm" \
"${tmp_dir}/rm.id"
+ perma_tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.permanent.XXXXXXXXXX")
# shellcheck disable=SC2086
for s in "${tmp_dir}/"*; do
- sort -u "${s}" | \
- sponge "${s}"
+ sort -u "${s}" \
+ > "${perma_tmp_dir}/${s##*/}"
done
# receive the repository databases from the master mirror
- mkdir "${tmp_dir}/dbs"
while read -r arch repo; do
- mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
- # shellcheck disable=SC2086
- failsafe_rsync \
- "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
- "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
- "${tmp_dir}/dbs/${arch}/${repo}/"
+ {
+ printf 'mkdir -p "%s/dbs/%s/%s"\n' \
+ "${perma_tmp_dir}" \
+ "${arch}" \
+ "${repo}"
+ printf 'failsafe_rsync'
+ for suffix in 'db' 'files'; do
+ printf ' "%s/%s/%s/%s.%s."*' \
+ "${master_mirror_rsync_directory}" \
+ "${arch}" \
+ "${repo}" \
+ "${repo}" \
+ "${suffix}"
+ done
+ printf ' "%s/dbs/%s/%s/"\n' \
+ "${perma_tmp_dir}" \
+ "${arch}" \
+ "${repo}"
+ } \
+ | intent_something "${no_action_flag}"
done < \
"${tmp_dir}/repositories"
@@ -537,11 +549,21 @@ export TMPDIR="${tmp_dir}"
"${tmp_dir}/rm" | \
sort -u | \
while read -r arch repo; do
- grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
- sed '
+ {
+# TODO: sign repositories
+ printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \
+ "${perma_tmp_dir}" \
+ "${arch}" \
+ "${repo}" \
+ "${repo}"
+ grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" \
+ | sed '
s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
- ' | \
- xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
+ ' \
+ | tr '\n' ' '
+ printf '\n'
+ } \
+ | intent_something "${no_action_flag}"
done
# copy and delete moved packages
@@ -550,89 +572,133 @@ export TMPDIR="${tmp_dir}"
"${tmp_dir}/mv" | \
sort -u | \
while read -r from_arch from_repo to_arch to_repo; do
- grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
- sed '
+ {
+ printf '"%s/bin/repo-copy" ' \
+ "${base_dir}"
+ printf '"%s/dbs/%s/%s/%s.db.tar.gz" ' \
+ "${perma_tmp_dir}" "${from_arch}" "${from_repo}" "${from_repo}" \
+ "${perma_tmp_dir}" "${to_arch}" "${to_repo}" "${to_repo}"
+ grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \
+ | sed '
s/-[^-]\+\( \S\+\)\{4\}$//
- ' | \
- xargs -r "${base_dir}/bin/repo-copy" \
- "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
- "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
- grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
- sed '
+ ' \
+ | tr '\n' ' '
+ printf '\n'
+ printf 'repo-remove -q "%s/dbs/%s/%s/%s.db.tar.gz" ' \
+ "${perma_tmp_dir}" \
+ "${from_arch}" \
+ "${from_repo}" \
+ "${from_repo}"
+ grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" \
+ | sed '
s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
- ' | \
- xargs -r repo-remove -q \
- "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
+ ' \
+ | tr '\n' ' '
+ printf '\n'
+ } \
+ | intent_something "${no_action_flag}"
done
# create real file names of packages, because
# mysql_query_and_delete_unneeded_binary_packages does so, too
- sed -i '
- s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
- ' "${tmp_dir}/rm"
+ {
+ printf 'sed -i '"'"'\n'
+ printf ' s,^\\(\\S\\+\) \\(\\S\\+\\) \\(\\S\\+\\)$,\\2/\\3/\\1,\n'
+ printf "'"' "%s/rm"\n' \
+ "${perma_tmp_dir}"
+ } \
+ | intent_something "${no_action_flag}"
# somewhat inaccurate
- if ! ${no_action}; then
- {
- printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
- printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
- "${tmp_dir}/mv.id" 'moved_bpir' \
- "${tmp_dir}/rm.id" 'replaced_bpir'
- printf 'DELETE `binary_packages_in_repositories`'
- printf ' FROM `binary_packages_in_repositories`'
- printf ' JOIN `replaced_bpir`'
- printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
- mysql_query_and_delete_unneeded_binary_packages
- printf 'UPDATE `binary_packages_in_repositories`'
- printf ' JOIN `moved_bpir`'
- printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
- printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
- printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
- printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
- } | \
- mysql_run_query | \
- sort -u >> \
- "${tmp_dir}/rm"
- fi
+ {
+ printf '{\n'
+ printf ' mysql_run_query'
+ printf ' | sort -u'
+ printf ' >> "%s/rm"\n' \
+ "${perma_tmp_dir}"
+ printf '} <<END_OF_MYSQL_QUERY\n'
+
+ printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
+ "${perma_tmp_dir}/mv.id" 'moved_bpir' \
+ "${perma_tmp_dir}/rm.id" 'replaced_bpir'
+ printf 'DELETE `binary_packages_in_repositories`'
+ printf ' FROM `binary_packages_in_repositories`'
+ printf ' JOIN `replaced_bpir`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
+ mysql_query_and_delete_unneeded_binary_packages
+ printf 'UPDATE `binary_packages_in_repositories`'
+ printf ' JOIN `moved_bpir`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
+ printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
+ printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
+ printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
+
+ printf 'END_OF_MYSQL_QUERY\n'
+ } \
+ | sed 's/`/\\`/g' \
+ | intent_something "${no_action_flag}"
# move the packages remotely via sftp
{
- sed '
- s/^/rm "/
- s/$/"/
- p
- s/"$/.sig"/
- ' "${tmp_dir}/rm"
- sed '
- s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
- ' "${tmp_dir}/mv"
- echo 'quit'
- } | \
- if ${no_action}; then
- sed 's|^|sftp: |'
- else
- failsafe_sftp
- fi
-
- if ${no_action}; then
- continue
- fi
+ printf '{\n'
+ printf ' sed '"'"'\n'
+ printf ' s/^/rm "/\n'
+ printf ' s/$/"/\n'
+ printf ' p\n'
+ printf ' s/"$/.sig"/\n'
+ printf ' '"'"' "%s/rm"\n' \
+ "${perma_tmp_dir}"
+ printf ' sed '"'"'\n'
+ printf ' s,^\\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\) \\(\\S\\+\\)$,'
+ printf 'rename "\\2/\\3/\\1" "\\4/\\5/\\1"\\n'
+ printf 'rename "\\2/\\3/\\1.sig" "\\4/\\5/\\1.sig",\n'
+ printf ' '"'"' "%s/mv"\n' \
+ "${perma_tmp_dir}"
+ printf ' echo quit\n'
+ printf '} | failsafe_sftp\n'
+ } \
+ | intent_something "${no_action_flag}"
# and push our local *.db.tar.gz via rsync
while read -r arch repo; do
- recompress_gz \
- "${tmp_dir}" \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
- # shellcheck disable=SC2086
- failsafe_rsync \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
- "${master_mirror_rsync_directory}/${arch}/${repo}/"
+ {
+ printf 'recompress_gz "%s"' \
+ "${tmp_dir}"
+ for suffix in '' '.old'; do
+ printf ' "%s/dbs/%s/%s/%s."*".tar.gz%s"' \
+ "${perma_tmp_dir}" \
+ "${arch}" \
+ "${repo}" \
+ "${repo}" \
+ "${suffix}"
+ done
+ printf '\n'
+ printf 'failsafe_rsync '
+ for suffix in 'db' 'files'; do
+ printf '"%s/dbs/%s/%s/%s.%s."* ' \
+ "${perma_tmp_dir}" \
+ "${arch}" \
+ "${repo}" \
+ "${repo}" \
+ "${suffix}"
+ done
+ printf ' "%s/%s/%s/"\n' \
+ "${master_mirror_rsync_directory}" \
+ "${arch}" \
+ "${repo}"
+ } \
+ | intent_something "${no_action_flag}"
done < \
"${tmp_dir}/repositories"
+ printf 'rm -rf --one-file-system "%s"\n' \
+ "${perma_tmp_dir}" \
+ | intent_something
+
+ execute_all_intentions
+
done
trigger_mirror_refreshs