Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
path: root/bin/db-update
diff options
context:
space:
mode:
Diffstat (limited to 'bin/db-update')
-rwxr-xr-xbin/db-update875
1 files changed, 302 insertions, 573 deletions
diff --git a/bin/db-update b/bin/db-update
index fe45f41..6722f24 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -4,27 +4,13 @@
# additionally tested packages from testing to the respective stable
# repository (if possible [1])
-# 1] Condition for moving a package A from staging (testing) to
-# testing (stable) is that:
-# a) nothing on the build-list run-depends on A and
-# b) no done package B (in a not-more stable repository) which is
-# not being moved run-depends on A
+# The condition [1] is explained in the stored function
+# calculate_maximal_moveable_set which is created in bin/bootsrap-mysql
# TODO: separate locks for staging, testing (and stable)
-# TODO: handle deletion of parts of a split package
-
-# TODO: handle dependencies of parts of a split package separately
-
-# TODO: improve tracking of packages:
-# track packages in stable, too
-# know _exactly_ in which repository the packages are
-
-# TODO: fully integrate database (in other scripts too) - e.g. rely on
-# info in db instead of state-files
-
-# TODO: add possibility to manually clone a (binary) package (e.g.
-# for build-support) - this should also be possible via email
+# TODO: we should delete more packages than just the ones in repositories
+# where we move to (think of [extra] -> [community])
# shellcheck disable=SC2039
# shellcheck source=conf/default.conf
@@ -34,369 +20,40 @@
usage() {
>&2 echo ''
>&2 echo 'db-update [options] [packages]:'
- >&2 echo ' move possible packages from staging to testing.'
>&2 echo ' move tested packages from testing to stable.'
+ >&2 echo ' move possible packages from staging to testing.'
>&2 echo ''
>&2 echo 'possible options:'
- >&2 echo ' -b|--block: If necessary, wait for lock blocking.'
- >&2 echo ' -h|--help: Show this help and exit.'
- >&2 echo ' -n|--no-action: Only print what would be moved.'
- >&2 echo ' -s|--stabilize $package:'
- >&2 echo ' Assume, package $package can be stabilized.'
- >&2 echo ' -u|--unstage $package:'
- >&2 echo ' Assume, package $package can be unstaged.'
+ >&2 echo ' -b|--block:'
+ >&2 echo ' If necessary, wait for lock blocking.'
+ >&2 echo ' -f|--force $package-id:'
+ >&2 echo ' Force movement of Package with given id and move nothing else.'
+ >&2 echo ' -h|--help:'
+ >&2 echo ' Show this help and exit.'
+ >&2 echo ' -n|--no-action:'
+ >&2 echo ' Only print what would be moved.'
+ >&2 echo ' -p|--progressive:'
+ >&2 echo ' Move forward any package which replaces no package whose'
+ >&2 echo ' dependencies are all available somewhere.'
+ >&2 echo ' Note, that this _may_ move _less_ packages.'
[ -z "$1" ] && exit 1 || exit "$1"
}
-# move_packages file with one "$package $from_repository $to_repository" per line
-# the existence of a directory $tmp_dir is assumed
-
-move_packages() {
-
- if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
- >&2 echo 'move_packages: No tmp_dir provided.'
- exit 2
- fi
-
- local package
- local from_repo
- local to_repo
- local from_ending
- local rm_ending
- local to_ending
- local repo
- local part
- local dummynator
- local file
-
- if [ -e "${tmp_dir:?}/tmp" ]; then
- rm -rf --one-file-system "${tmp_dir:?}/tmp"
- fi
- mkdir "${tmp_dir}/tmp"
-
- touch "${tmp_dir}/tmp/repos"
- touch "${tmp_dir}/tmp/packages"
- touch "${tmp_dir}/tmp/master-mirror-listing"
- mkdir "${tmp_dir}/tmp/transit"
-
- if ${no_action}; then
- dummynator='echo'
- else
- dummynator=''
- fi
-
- ls_master_mirror 'i686' | \
- while read -r repo; do
- ls_master_mirror "i686/${repo}" | \
- sed "s|^|i686/${repo}/|" >> \
- "${tmp_dir}/tmp/master-mirror-listing"
- done
-
- while read -r package from_repo to_repo; do
- if [ -z "${package}" ]; then
- continue
- fi
-
- if ${no_action}; then
- printf \
- 'move "%s" from "%s" to "%s"\n' \
- "${package}" \
- "${from_repo}" \
- "${to_repo}"
- fi
-
- echo "${package}" >> \
- "${tmp_dir}/tmp/packages"
-
- if echo "${from_repo}" | \
- grep -q 'staging$' && \
- echo "${to_repo}" | \
- grep -q 'testing$'; then
- from_ending='done'
- rm_ending='test\(ing\|ed\)'
- to_ending='testing'
- elif echo "${from_repo}" | \
- grep -q 'testing$' && \
- ! echo "${to_repo}" | \
- grep -q 'testing$\|staging$'; then
- from_ending='tested'
- rm_ending=''
- to_ending=''
- else
- >&2 printf 'move_packages: Cannot move package "%s" from "%s" to "%s".\n' "${package}" "${from_repo}" "${to_repo}"
- exit 2
- fi
-
- echo "${from_repo}" > \
- "${tmp_dir}/tmp/${package}.from_repo"
- echo "${to_repo}" > \
- "${tmp_dir}/tmp/${package}.to_repo"
- echo "${from_ending}" > \
- "${tmp_dir}/tmp/${package}.from_ending"
- echo "${rm_ending}" > \
- "${tmp_dir}/tmp/${package}.rm_ending"
- echo "${to_ending}" > \
- "${tmp_dir}/tmp/${package}.to_ending"
-
- if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
- >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}"
- exit 2
- fi
-
- cp \
- "${work_dir}/package-states/${package}.${from_ending}" \
- "${tmp_dir}/tmp/${package}.parts"
-
- sed \
- 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
- "${tmp_dir}/tmp/${package}.parts" > \
- "${tmp_dir}/tmp/${package}.parts_names"
-
- sed \
- 'p;s|$|.sig|' \
- "${tmp_dir}/tmp/${package}.parts" > \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- while read -r part; do
- if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then
- >&2 printf \
- 'move_packages: Cannot find file "%s", part of package "%s".\n' \
- "i686/${from_repo}/${part}" \
- "${package}"
- exit 2
- fi
- done < \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- mkdir -p "${tmp_dir}/tmp/${from_repo}"
- mkdir -p "${tmp_dir}/tmp/${to_repo}"
-
- repos=$(
- # shellcheck disable=SC2046
- printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \
- sort -u
- )
- echo "${repos}" > \
- "${tmp_dir}/tmp/repos"
-
- done < \
- "$1"
-
- if ${no_action}; then
- find "${tmp_dir}/tmp" -type f | \
- while read -r file; do
- if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] ||
- [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then
- echo "'${file}'"
- else
- echo "${file}:"
- sed 's|^|<<|;s|$|>>|' "${file}"
- fi
- echo
- done
- fi
-
- # receive the *.db.tar.gz's and *.files.tar.gz's
-
- while read -r repo; do
-
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
- "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
- "${tmp_dir}/tmp/${repo}/"
-
- # add and remove the packages locally
-
- if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then
-
- # shellcheck disable=SC2046
- repo-remove -q \
- "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
- $(
- grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \
- sed '
- s|\.from_repo$|.parts_names|
- ' | \
- xargs -rn1 cat
- )
- fi
-
- if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then
- grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \
- sed '
- s|\.to_repo$||
- ' | \
- while read -r package; do
- while read -r part; do
- ${master_mirror_rsync_command} \
- "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \
- "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \
- "${tmp_dir}/tmp/transit/"
- repo-add -q \
- "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
- "${tmp_dir}/tmp/transit/${part}"
- rm \
- "${tmp_dir}/tmp/transit/${part}" \
- "${tmp_dir}/tmp/transit/${part}.sig"
- done < \
- "${package}.parts"
- done
- fi
-
- done < "${tmp_dir}/tmp/repos"
-
- if ${no_action}; then
- find "${tmp_dir}/tmp" -type f
- fi
-
- # move the packages remotely via sftp
-
- {
- while read -r package; do
-
- if [ -z "${package}" ]; then
- continue
- fi
-
- while read -r part; do
- if [ -z "${part}" ]; then
- continue
- fi
- printf \
- 'rename "%s" "%s"\n' \
- "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \
- "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}"
- done < \
- "${tmp_dir}/tmp/${package}.parts_and_signatures"
-
- done < \
- "${tmp_dir}/tmp/packages"
- echo 'quit'
- } | \
- if ${no_action}; then
- sed 's|^|sftp: |'
- else
- ${master_mirror_sftp_command}
- fi
-
- # and push our local *.db.tar.gz via rsync
-
- while read -r repo; do
-
- # shellcheck disable=SC2086
- ${dummynator} ${master_mirror_rsync_command} \
- "${tmp_dir}/tmp/${repo}/${repo}.db."* \
- "${tmp_dir}/tmp/${repo}/${repo}.files."* \
- "${master_mirror_rsync_directory}/i686/${repo}/"
-
- done < \
- "${tmp_dir}/tmp/repos"
-
- while read -r package; do
-
- # then we can safely remove old versions
-
- while read -r part; do
- ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}"
- done < \
- "${tmp_dir}/tmp/${package}.parts"
-
- # and update the state files
-
- from_ending=$(
- cat "${tmp_dir}/tmp/${package}.from_ending"
- )
- rm_ending=$(
- cat "${tmp_dir}/tmp/${package}.rm_ending"
- )
- to_ending=$(
- cat "${tmp_dir}/tmp/${package}.to_ending"
- )
-
- if [ -z "${to_ending}" ]; then
- ${dummynator} rm \
- "${work_dir}/package-states/${package}.${from_ending}"
- else
- # remove old state files of $package with ending $rm_ending
- find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
- -regex '.*/'"$(str_to_regex "${package%.*.*.*}")"'\(\.[^.]\+\)\{3\}\.'"${rm_ending}" \
- -execdir ${dummynator} rm {} \;
- ${dummynator} mv \
- "${work_dir}/package-states/${package}.${from_ending}" \
- "${work_dir}/package-states/${package}.${to_ending}"
- fi
-
- done < \
- "${tmp_dir}/tmp/packages"
-
- # shellcheck disable=SC2016
- while read -r package; do
- while read -r part; do
- printf 'UPDATE `binary_packages`'
- printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`'
- printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`'
- printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s"))' \
- "$(
- base64_encode_each < \
- "${tmp_dir}/tmp/${package}.to_repo"
- )"
- printf ' WHERE'
- printf ' `repositories`.`name`=from_base64("%s")' \
- "$(
- base64_encode_each < \
- "${tmp_dir}/tmp/${package}.from_repo"
- )"
- printf '%s\n' "${part}" | \
- sed '
- s/\.pkg\.tar\.xz$//
- s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/
- s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/
- s/^\(.\+\)-\([^-:]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/\1\n\2\n\3\n\4\n\5\n\6/
- ' | \
- base64_encode_each | \
- tr '\n' ' ' | \
- sed '
- s,\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) $,'"$(
- printf ' AND `binary_packages`.`%s`=from_base64("%s")' \
- 'pkgname' '\1' \
- 'epoch' '\2' \
- 'pkgver' '\3' \
- 'pkgrel' '\4' \
- 'sub_pkgrel' '\5'
- printf ' AND `architectures`.`name`=from_base64("\\6")'
- )"',
- '
- printf ';\n'
- done < \
- "${tmp_dir}/tmp/${package}.parts"
- done < \
- "${tmp_dir}/tmp/packages" | \
- if ${no_action}; then
- sed 's|^|mysql: |'
- else
- ${mysql_command}
- fi
-
- ${dummynator} trigger_mirror_refreshs
-
- rm -rf --one-file-system "${tmp_dir:?}/tmp"
-
-}
-
eval set -- "$(
- getopt -o bf:hns:u: \
+ getopt -o bf:hnp \
--long block \
- --long from: \
+ --long force \
--long help \
--long no-action \
- --long stabilize: \
- --long unstage: \
+ --long progressive \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
block_flag='-n'
no_action=false
+progressive=false
+force_ids=''
while true
do
@@ -404,19 +61,22 @@ do
-b|--block)
block_flag=''
;;
+ -f|--force)
+ shift
+ force_ids=$(
+ printf '%s' "$1" | \
+ base64 -w0
+ printf '\n%s' "${force_ids}"
+ )
+ ;;
-h|--help)
usage 0
;;
-n|--no-action)
no_action=true
;;
- -s|--stabilze)
- shift
- packages_to_force_stabilize="${packages_to_force_stabilize} $1"
- ;;
- -u|--unstage)
- shift
- packages_to_force_unstage="${packages_to_force_unstage} $1"
+ -p|--progressive)
+ progressive=true
;;
--)
shift
@@ -435,237 +95,306 @@ if [ $# -ne 0 ]; then
usage
fi
+if ${progressive} && \
+ [ -n "${force_ids}" ]; then
+ >&2 echo 'db-update: conflicting arguments'
+ usage
+fi
+
if [ -s "${work_dir}/build-master-sanity" ]; then
>&2 echo 'Build master is not sane.'
exit
fi
-tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-for package in ${packages_to_force_stabilize}; do
- # some sanity checks
- if [ ! -f "${work_dir}/package-states/${package}.tested" ] && \
- [ ! -f "${work_dir}/package-states/${package}.testing" ] && \
- [ ! -f "${work_dir}/package-states/${package}.done" ] && \
- ! tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${package}"; then
- >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' "${package}"
- exit 2
- fi
-done
+# Create tmp_dir, lock and trap.
-for package in ${packages_to_force_unstage}; do
- # some sanity checks
- if [ ! -f "${work_dir}/package-states/${package}.done" ] && \
- ! tr ' ' '.' < \
- "${work_dir}/build-list" | \
- grep -qxF "${package}"; then
- >&2 printf 'Package "%s" is not in staging or on the build list!\n' "${package}"
- exit 2
- fi
-done
-
-# Create a lock file and a trap.
-
-exec 9> "${build_list_lock_file}"
+exec 9> "${package_database_lock_file}"
if ! flock ${block_flag} 9; then
- >&2 echo 'come back (shortly) later - I cannot lock build list.'
- exit 0
-fi
-
-exec 8> "${package_database_lock_file}"
-if ! flock ${block_flag} 8; then
>&2 echo 'come back (shortly) later - I cannot lock package database.'
exit 0
fi
-exec 7> "${sanity_check_lock_file}"
-if ! flock -s ${block_flag} 7; then
+exec 8> "${sanity_check_lock_file}"
+if ! flock -s ${block_flag} 8; then
>&2 echo 'come back (shortly) later - sanity-check currently running.'
exit 0
fi
-clean_up_lock_file() {
- rm -f "${package_database_lock_file}" "${build_list_lock_file}"
- rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
-# sanity check
-
-for ending in 'done' 'tested'; do
- errors=$(
- find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
- sed 's|\(\.[^.]\+\)\{4\}$||' | \
- sort | \
- uniq -d
- )
- if [ -n "${errors}" ]; then
- >&2 echo 'Removing duplicates not yet implemented:'
- >&2 echo "${errors}"
- exit 42
- fi
-done
-
-# packages which are done
+tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
- sed '
- s|\.done$||
- ' | \
- sort -u > \
- "${tmp_dir}/done-packages"
+export TMPDIR="${tmp_dir}"
-# packages still on the build-list
+for source_stability in 'testing' 'staging'; do
+ find "${tmp_dir}" -mindepth 1 -delete
-grep -vxF 'break_loops' "${work_dir}/build-list" | \
- tr ' ' '.' | \
- sort -u > \
- "${tmp_dir}/build-list-packages"
+ # shellcheck disable=SC2016
+ {
+ if [ -n "${force_ids}" ]; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' VALUES'
+ # shellcheck disable=SC2086
+ printf '(from_base64("%s"),NULL),' \
+ ${force_ids} | \
+ sed 's/,$/;\n/'
+ printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_repositories_repository_stabilities
+ printf ' WHERE `repository_stabilities`.`name`!="%s";\n' \
+ "${source_stability}"
+ printf 'UPDATE `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+ printf ' SET `moveable_binary_packages`.`to_repository`=`repository_moves`.`to_repository`;\n'
+ printf 'DELETE FROM `moveable_binary_packages` WHERE `moveable_binary_packages`.`to_repository` IS NULL;\n'
+ printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+ printf ' SELECT `binary_packages`.`id`,`moveable_binary_packages`.`id`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` AS `subst_bp` ON `moveable_binary_packages`.`id`=`subst_bp`.`id`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+ printf ' AND `binary_packages`.`repository`=`moveable_binary_packages`.`to_repository`;\n'
+ elif ${progressive}; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+
+ printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)'
+ printf ' SELECT `binary_packages`.`id`,`subst_bp`.`id`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`to_repository`=`binary_packages`.`repository`'
+ printf ' JOIN `binary_packages` AS `subst_bp`'
+ printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+ printf ' AND `repository_moves`.`from_repository`=`subst_bp`.`repository`'
+ mysql_join_binary_packages_repositories 'subst_bp' 'subst_r'
+ mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs'
+ printf ' AND `subst_rs`.`name`="%s"' \
+ "${source_stability}"
+ mysql_join_binary_packages_dependencies
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `install_target_providers`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ');\n'
+
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' SELECT `replaced_binary_packages`.`replaced_by`,`binary_packages`.`repository`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`'
+ printf ';\n'
-find "${work_dir}/package-infos" -name '*.groups' \
- -exec grep -qxF 'base' {} \; \
- -printf '%f\n' | \
- sed '
- s|\.groups$||
- ' | \
- sort -u > \
- "${tmp_dir}/base-packages"
+ printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)'
+ printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ mysql_join_repositories_repository_stabilities
+ printf ' AND `repository_stabilities`.`name`="%s"' \
+ "${source_stability}"
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT * FROM `binary_packages` AS `repl_bp`'
+ printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ' AND `repl_bp`.`repository`=`repository_moves`.`to_repository`'
+ printf ');\n'
+ else
+ printf 'CALL calculate_maximal_moveable_set("%s");\n' \
+ "${source_stability}"
+ fi
-{
- # shellcheck disable=SC2086
- printf '%s\n' ${packages_to_force_unstage}
- print_list_of_archaic_packages 'build-list'
-} > \
- "${tmp_dir}/force-unstage-packages"
-
->&2 printf 'calculate what packages should be unstaged ...'
-find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-unstage-packages" > \
- "${tmp_dir}/unstage-packages"
->&2 printf ' ok.\n'
-
-# no base packages on the build list anymore?
-if [ -z "$(
- join -j 1 \
- "${tmp_dir}/base-packages" \
- "${tmp_dir}/build-list-packages"
- )" ]; then
-
- >&2 echo 'db-update unstage: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
- for s in "${tmp_dir}/all-builds" "${tmp_dir}/all-depends"; do
- sed '/ base$/d' "${s}" > \
- "${s}.no-base"
- done
+ printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `moveable_binary_packages`.`to_repository`'
+ printf ' FROM `moveable_binary_packages`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages`.`repository`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages`.`repository`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`;\n'
+
+ printf 'SELECT "repositories",`repositories`.`name`'
+ printf ' FROM `repositories`'
+ printf ' JOIN `rps` ON `rps`.`id`=`repositories`.`id`;\n'
+
+ printf 'SELECT "mv.id",`moveable_binary_packages`.`id`,`moveable_binary_packages`.`to_repository`'
+ printf ' FROM `moveable_binary_packages`;\n'
+
+ printf 'SELECT "mv",'
+ mysql_package_name_query
+ printf ',`repositories`.`name`,`new_repo`.`name`'
+ printf ' FROM `moveable_binary_packages`'
+ printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`moveable_binary_packages`.`to_repository`'
+ printf ';\n'
+
+ printf 'SELECT "rm.id",`replaced_binary_packages`.`id`'
+ printf ' FROM `replaced_binary_packages`;\n'
+
+ printf 'SELECT "rm",'
+ mysql_package_name_query
+ printf ',`repositories`.`name`'
+ printf ' FROM `replaced_binary_packages`'
+ printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`'
+ mysql_join_binary_packages_repositories
+ mysql_join_binary_packages_architectures
+ printf ';\n'
+ } | \
+ mysql_run_query | \
+ tr '\t' ' ' | \
+ grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
+ while read -r what content; do
+ printf '%s\n' "${content}" >> \
+ "${tmp_dir}/${what}"
+ done
- >&2 printf 'calculate what packages should be unstaged ...'
- find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-unstage-packages" > \
- "${tmp_dir}/unstage-packages"
- >&2 printf ' ok.\n'
+ if [ ! -s "${tmp_dir}/repositories" ]; then
+ >&2 printf 'Nothing to move from %s.\n' "${source_stability}"
+ continue
+ fi
-fi
+ touch \
+ "${tmp_dir}/mv" \
+ "${tmp_dir}/mv.id" \
+ "${tmp_dir}/rm" \
+ "${tmp_dir}/rm.id"
-{
# shellcheck disable=SC2086
- printf '%s\n' ${packages_to_force_stabilize}
- print_list_of_archaic_packages 'build-list' 'staging' 'testing'
-} > \
- "${tmp_dir}/force-stabilize-packages"
-
-# calculate what packages should be stabilized
-
-{
- cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages"
- find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -printf '%f\n' | \
- sed 's|\.testing$||'
-} | \
- sort -u > \
- "${tmp_dir}/keep-packages"
-
-find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.tested' -printf '%f\n' | \
- sed 's|\.tested$||' > \
- "${tmp_dir}/stabilize-packages"
-
-# no base packages on the build list or in staging anymore?
-if [ -z "$(
- cat \
- "${tmp_dir}/build-list-packages" \
- "${tmp_dir}/done-packages" | \
- sort -u | \
- join -j 1 \
- "${tmp_dir}/base-packages" \
- -
- )" ]; then
-
- >&2 echo 'db-update stabilize: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages'
- >&2 printf 'calculate what packages should be stabilized ...'
- find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-
-else
-
- >&2 printf 'calculate what packages should be stabilized ...'
- find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-
-fi
+ for s in "${tmp_dir}/"*; do
+ sort -u "${s}" | \
+ sponge "${s}"
+ done
-# unlock build list
+ # receive the repository databases from the master mirror
+ mkdir "${tmp_dir}/dbs"
+ while read -r repo; do
+ mkdir "${tmp_dir}/dbs/${repo}"
+ # shellcheck disable=SC2086
+ ${master_mirror_rsync_command} \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
+ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
+ "${tmp_dir}/dbs/${repo}/"
+ tar -Oxzf "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" --wildcards '*/desc' | \
+ sed -n '
+ /^%FILENAME%$/{
+ N
+ s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+ T
+ p
+ }
+ '
+ done < \
+ "${tmp_dir}/repositories" | \
+ while read -r old new; do
+ for file in 'rm' 'mv'; do
+ sed -i '
+ s/\(\s\|^\)'"$(str_to_regex "${old}")"'\(\s\|$\)/\1'"${new}"'\2/
+ ' "${tmp_dir}/${file}"
+ done
+ done
-rm -f "${build_list_lock_file}"
-flock -u 9
+ # remove to-be-deleted packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2 < \
+ "${tmp_dir}/rm" | \
+ sort -u | \
+ while read -r repo; do
+ grep " $(str_to_regex "${repo}")\$" "${tmp_dir}/rm" | \
+ sed '
+ s/\(-[^-]\+\)\{3\} \S\+$//
+ ' | \
+ xargs -r repo-remove -q "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz"
+ done
-clean_up_lock_file() {
- rm -rf --one-file-system "${tmp_dir}"
- rm -f "${package_database_lock_file}"
-}
+ # copy and delete moved packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2,3 < \
+ "${tmp_dir}/mv" | \
+ sort -u | \
+ while read -r from_repo to_repo; do
+ grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/-[^-]\+ \S\+ \S\+$//
+ ' | \
+ xargs -r "${base_dir}/bin/repo-copy" \
+ "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \
+ "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz"
+ grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
+ ' | \
+ xargs -r repo-remove -q \
+ "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz"
+ done
-# testing -> stable
+ # move the packages remotely via sftp
+ {
+ sed '
+ s,^\(\S\+\) \(\S\+\)$,rm "i686/\2/\1"\nrm "i686/\2/\1.sig",
+ ' "${tmp_dir}/rm"
+ sed '
+ s,^\(\S\+\) \(\S\+\) \(\S\+\)$,rename "i686/\2/\1" "i686/\3/\1"\nrename "i686/\2/\1.sig" "i686/\3/\1.sig",
+ ' "${tmp_dir}/mv"
+ echo 'quit'
+ } | \
+ if ${no_action}; then
+ sed 's|^|sftp: |'
+ else
+ ${master_mirror_sftp_command}
+ fi
-while read -r package; do
- if [ -z "${package}" ]; then
- continue
- fi
- printf '%s %s %s\n' \
- "${package}" \
- "$(official_or_community "${package}" 'testing')" \
- "$(repository_of_package "${package}")"
-done < \
- "${tmp_dir}/stabilize-packages" | \
- sponge "${tmp_dir}/stabilize-packages"
-
-# staging -> testing
-
-while read -r package; do
- if [ -z "${package}" ]; then
+ if ${no_action}; then
continue
fi
- printf '%s %s %s\n' \
- "${package}" \
- "$(official_or_community "${package}" 'staging')" \
- "$(official_or_community "${package}" 'testing')"
-done < \
- "${tmp_dir}/unstage-packages" | \
- sponge "${tmp_dir}/unstage-packages"
-
-if [ -s "${tmp_dir}/stabilize-packages" ]; then
- >&2 printf 'move packages from *testing/ to the stable repos ...'
- move_packages "${tmp_dir}/stabilize-packages"
- >&2 printf ' ok.\n'
-else
- >&2 printf 'Nothing to move from *testing to the stable repos.\n'
-fi
-if [ -s "${tmp_dir}/unstage-packages" ]; then
- >&2 printf 'move packages from *staging to *testing ...'
- move_packages "${tmp_dir}/unstage-packages"
- >&2 printf ' ok.\n'
-else
- >&2 printf 'Nothing to move from *staging to *testing.\n'
-fi
+ # and push our local *.db.tar.gz via rsync
+ while read -r repo; do
+ # shellcheck disable=SC2086
+ ${master_mirror_rsync_command} \
+ "${tmp_dir}/dbs/${repo}/${repo}.db."* \
+ "${tmp_dir}/dbs/${repo}/${repo}.files."* \
+ "${master_mirror_rsync_directory}/i686/${repo}/"
+ done < \
+ "${tmp_dir}/repositories"
+
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
+ "${tmp_dir}/mv.id" 'moved_binary_packages' \
+ "${tmp_dir}/rm.id" 'replaced_binary_packages'
+ printf 'DELETE `binary_packages` FROM `binary_packages`'
+ printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n'
+ printf 'UPDATE `binary_packages`'
+ printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`'
+ printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
+ } | \
+ mysql_run_query
+done
-clean_up_lock_file
+trigger_mirror_refreshs