| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/db-update | 707 |
diff --git a/bin/db-update b/bin/db-update index 9e3bfd8..6768518 100755 --- a/bin/db-update +++ b/bin/db-update @@ -162,372 +162,397 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT export TMPDIR="${tmp_dir}" -# shellcheck disable=SC2154 -for source_stability in \ - "${repository_stability_ids__testing}" \ - "${repository_stability_ids__staging}"; do - find "${tmp_dir}" -xdev -mindepth 1 -delete - # shellcheck disable=SC2016 - { - # Note, that we do not need to check that two repositories a and b - # are for the same architecture if we joined them via - # repository_moves, because only repositories with the same - # architectures should be listed there. - if [ -n "${force_pkgs}" ]; then - printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \ - 'moveable' 'replaced' - printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' - printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' - printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' - printf ' SELECT ' - printf '`binary_packages_in_repositories`.`id`,NULL' - printf ' FROM `binary_packages_in_repositories`' - mysql_join_binary_packages_in_repositories_repositories - mysql_join_repositories_architectures - mysql_join_binary_packages_in_repositories_binary_packages - printf ' WHERE CONCAT(' - printf '`architectures`.`name`,"/",' - printf '`repositories`.`name`,"/",' - printf '`binary_packages`.`pkgname`' - printf ') IN (' - # shellcheck disable=SC2086 - printf 'from_base64("%s"),' \ - ${force_pkgs} | \ - sed 's/,$/);\n/' - printf 'DELETE `moveable_bpir`' +# shellcheck disable=SC2016 +{ + printf 'SELECT DISTINCT' + printf ' `repositories`.`architecture`,' + printf '`repositories`.`stability`' + printf ' FROM `repositories`' + printf ' JOIN `repository_moves`' + printf ' ON `repository_moves`.`from_repository`=`repositories`.`id`' + printf ' WHERE `repositories`.`is_on_master_mirror`' + printf ' ORDER BY `repositories`.`stability`,' + printf 'SHA2(' + printf 'CONCAT(' + printf '`repositories`.`architecture`,' + printf '`repositories`.`stability`,' + printf 'NOW()' + printf '),' + printf '256' + printf ')' +} | \ + mysql_run_query | \ + while read -r arch_id source_stability; do + find "${tmp_dir}" -xdev -mindepth 1 -delete + + { + # Note, that we do not need to check that two repositories a and b + # are for the same architecture if we joined them via + # repository_moves, because only repositories with the same + # architectures should be listed there. + if [ -n "${force_pkgs}" ]; then + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \ + 'moveable' 'replaced' + printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' + printf ' SELECT ' + printf '`binary_packages_in_repositories`.`id`,NULL' + printf ' FROM `binary_packages_in_repositories`' + mysql_join_binary_packages_in_repositories_repositories + mysql_join_repositories_architectures + mysql_join_binary_packages_in_repositories_binary_packages + printf ' WHERE CONCAT(' + printf '`architectures`.`name`,"/",' + printf '`repositories`.`name`,"/",' + printf '`binary_packages`.`pkgname`' + printf ') IN (' + # shellcheck disable=SC2086 + printf 'from_base64("%s"),' \ + ${force_pkgs} | \ + sed 's/,$/);\n/' + printf 'DELETE `moveable_bpir`' + printf ' FROM `moveable_bpir`' + printf ' JOIN `binary_packages_in_repositories`' + printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_in_repositories_repositories + printf ' WHERE `repositories`.`stability`!=%s' \ + "${source_stability}" + printf ' OR `repositories`.`architecture`!=%s;\n' \ + "${arch_id}" + printf 'UPDATE `moveable_bpir`' + printf ' JOIN `binary_packages_in_repositories`' + printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`' + printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n' + printf 'DELETE FROM `moveable_bpir`' + printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n' + printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)' + printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`' + printf ' FROM `moveable_bpir`' + printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`' + printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`' + mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp' + printf ' JOIN `binary_packages`' + printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`' + mysql_join_binary_packages_binary_packages_in_repositories + mysql_join_binary_packages_in_repositories_repositories + printf ' JOIN `repositories` AS `m_to_r`' + printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`' + printf ' AND `repositories`.`stability`=`m_to_r`.`stability`' + printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n' + elif ${progressive}; then + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \ + 'moveable' 'replaced' + printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + + # packages that replace broken packages + printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)' + printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`' + printf ' FROM `binary_packages_in_repositories` AS `new_bpir`' + mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp' + mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r' + printf ' AND `from_r`.`stability`=%s' \ + "${source_stability}" + printf ' AND `from_r`.`architecture`=%s' \ + "${arch_id}" + printf ' AND `from_r`.`is_on_master_mirror`' + mysql_join_binary_packages_build_assignments 'new_bp' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`' + printf ' JOIN `repositories` AS `to_r`' + printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`' + printf ' JOIN `binary_packages` AS `old_bp`' + printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`' + mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir' + mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r' + printf ' AND `old_r`.`stability`=`to_r`.`stability`' + printf ' AND `old_r`.`architecture`=`to_r`.`architecture`' + mysql_join_binary_packages_dependencies 'old_bp' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_dependencies_versions '' 'd_v' + printf ' WHERE NOT EXISTS (' + printf 'SELECT 1' + printf ' FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'itp_bp' + mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir' + mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r' + mysql_join_install_target_providers_versions '' 'itp_v' + printf ' JOIN `repository_stability_relations` AS `rsr_a`' + printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND' + mysql_query_ordering_correct \ + '`d_v`.`order`' \ + '`itp_v`.`order`' \ + '`dependencies`.`version_relation`' + printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`' + printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`' + # this is the least stable package of stability at least to_r.stability + printf ' AND NOT EXISTS (' + printf 'SELECT 1' + printf ' FROM `binary_packages` AS `subst_bp`' + mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir' + mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r' + printf ' JOIN `repository_stability_relations` AS `rsr_b`' + printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`' + printf ' JOIN `repository_stability_relations` AS `rsr_c`' + printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`' + printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`' + printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`' + printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`' + printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`' + printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`' + printf ')' + printf ');\n' + + printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' + printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`' + printf ' FROM `replaced_bpir`' + printf ' JOIN `binary_packages_in_repositories`' + printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`' + printf ';\n' + + # packages which replace nothing + printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' + printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`' + printf ' FROM `binary_packages_in_repositories`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_in_repositories_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + printf ' AND `repositories`.`stability`=%s' \ + "${source_stability}" + printf ' AND `repositories`.`architecture`=%s' \ + "${arch_id}" + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`' + printf ' JOIN `repositories` as `to_r`' + printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`' + printf ' WHERE NOT EXISTS (' + printf 'SELECT 1' + printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`' + mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp' + mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r' + printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`' + printf ' AND `repl_r`.`stability`=`to_r`.`stability`' + printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`' + printf ');\n' + else + printf 'CALL calculate_maximal_moveable_set(%s,%s);\n' \ + "${arch_id}" \ + "${source_stability}" + fi + + printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `moveable_bpir`.`to_repository`' + printf ' FROM `moveable_bpir`;\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `binary_packages_in_repositories`.`repository`' printf ' FROM `moveable_bpir`' printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_in_repositories_repositories - printf ' WHERE `repositories`.`stability`!=%s;\n' \ - "${source_stability}" - printf 'UPDATE `moveable_bpir`' + printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `binary_packages_in_repositories`.`repository`' + printf ' FROM `replaced_bpir`' printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - mysql_join_upstream_repositories_repository_moves - printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`' - printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n' - printf 'DELETE FROM `moveable_bpir`' - printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n' - printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)' - printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`' + printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n' + + printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`' + printf ' FROM `rps`' + printf ' JOIN `repositories`' + printf ' ON `rps`.`id`=`repositories`.`id`' + mysql_join_repositories_architectures + printf ';\n' + + printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`' + printf ' FROM `moveable_bpir`;\n' + + printf 'SELECT "mv",' + mysql_package_name_query + printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`' printf ' FROM `moveable_bpir`' - printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`' - printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp' - printf ' JOIN `binary_packages`' - printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`' - mysql_join_binary_packages_binary_packages_in_repositories - mysql_join_binary_packages_in_repositories_repositories - printf ' JOIN `repositories` AS `m_to_r`' - printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`' - printf ' AND `repositories`.`stability`=`m_to_r`.`stability`' - printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n' - elif ${progressive}; then - printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \ - 'moveable' 'replaced' - printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' - printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' - - # packages that replace broken packages - printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)' - printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`' - printf ' FROM `binary_packages_in_repositories` AS `new_bpir`' - mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp' - mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r' - printf ' AND `from_r`.`stability`=%s' \ - "${source_stability}" - printf ' AND `from_r`.`is_on_master_mirror`' - mysql_join_binary_packages_build_assignments 'new_bp' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - mysql_join_upstream_repositories_repository_moves - printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`' - printf ' JOIN `repositories` AS `to_r`' - printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`' - printf ' JOIN `binary_packages` AS `old_bp`' - printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`' - mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir' - mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r' - printf ' AND `old_r`.`stability`=`to_r`.`stability`' - printf ' AND `old_r`.`architecture`=`to_r`.`architecture`' - mysql_join_binary_packages_dependencies 'old_bp' - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_binary_packages`' - mysql_join_dependencies_versions '' 'd_v' - printf ' WHERE NOT EXISTS (' - printf 'SELECT 1' - printf ' FROM `install_target_providers`' - mysql_join_install_target_providers_binary_packages '' 'itp_bp' - mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir' - mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r' - mysql_join_install_target_providers_versions '' 'itp_v' - printf ' JOIN `repository_stability_relations` AS `rsr_a`' - printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`' - printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' AND' - mysql_query_ordering_correct \ - '`d_v`.`order`' \ - '`itp_v`.`order`' \ - '`dependencies`.`version_relation`' - printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`' - printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`' - # this is the least stable package of stability at least to_r.stability - printf ' AND NOT EXISTS (' - printf 'SELECT 1' - printf ' FROM `binary_packages` AS `subst_bp`' - mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir' - mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r' - printf ' JOIN `repository_stability_relations` AS `rsr_b`' - printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`' - printf ' JOIN `repository_stability_relations` AS `rsr_c`' - printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`' - printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`' - printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`' - printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`' - printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`' - printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`' - printf ')' - printf ');\n' - - printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' - printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`' - printf ' FROM `replaced_bpir`' printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`' + printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_in_repositories_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_architectures '' 'r_a' + printf ' JOIN `repositories` AS `new_repo`' + printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`' + mysql_join_repositories_architectures 'new_repo' 'new_r_a' printf ';\n' - # packages which replace nothing - printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)' - printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`' - printf ' FROM `binary_packages_in_repositories`' + printf 'SELECT "rm.id",`replaced_bpir`.`id`' + printf ' FROM `replaced_bpir`;\n' + + printf 'SELECT "rm",' + mysql_package_name_query + printf ',`r_a`.`name`,`repositories`.`name`' + printf ' FROM `replaced_bpir`' + printf ' JOIN `binary_packages_in_repositories`' + printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`' mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_in_repositories_repositories - printf ' AND `repositories`.`is_on_master_mirror`' - printf ' AND `repositories`.`stability`=%s' \ - "${source_stability}" - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - mysql_join_upstream_repositories_repository_moves - printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`' - printf ' JOIN `repositories` as `to_r`' - printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`' - printf ' WHERE NOT EXISTS (' - printf 'SELECT 1' - printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`' - mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp' - mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r' - printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`' - printf ' AND `repl_r`.`stability`=`to_r`.`stability`' - printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`' - printf ');\n' - else - printf 'CALL calculate_maximal_moveable_set(%s);\n' \ + mysql_join_binary_packages_architectures + mysql_join_repositories_architectures '' 'r_a' + printf ';\n' + } | \ + mysql_run_query | \ + tr '\t' ' ' | \ + grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \ + while read -r what content; do + printf '%s\n' "${content}" >> \ + "${tmp_dir}/${what}" + done + + if [ ! -s "${tmp_dir}/repositories" ]; then + >&2 printf 'Nothing to move here (%s,%s).\n' \ + "${arch_id}" \ "${source_stability}" + continue fi - printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n' - printf 'INSERT IGNORE INTO `rps` (`id`)' - printf ' SELECT `moveable_bpir`.`to_repository`' - printf ' FROM `moveable_bpir`;\n' - printf 'INSERT IGNORE INTO `rps` (`id`)' - printf ' SELECT `binary_packages_in_repositories`.`repository`' - printf ' FROM `moveable_bpir`' - printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n' - printf 'INSERT IGNORE INTO `rps` (`id`)' - printf ' SELECT `binary_packages_in_repositories`.`repository`' - printf ' FROM `replaced_bpir`' - printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n' - - printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`' - printf ' FROM `rps`' - printf ' JOIN `repositories`' - printf ' ON `rps`.`id`=`repositories`.`id`' - mysql_join_repositories_architectures - printf ';\n' - - printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`' - printf ' FROM `moveable_bpir`;\n' - - printf 'SELECT "mv",' - mysql_package_name_query - printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`' - printf ' FROM `moveable_bpir`' - printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_in_repositories_repositories - mysql_join_binary_packages_architectures - mysql_join_repositories_architectures '' 'r_a' - printf ' JOIN `repositories` AS `new_repo`' - printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`' - mysql_join_repositories_architectures 'new_repo' 'new_r_a' - printf ';\n' - - printf 'SELECT "rm.id",`replaced_bpir`.`id`' - printf ' FROM `replaced_bpir`;\n' - - printf 'SELECT "rm",' - mysql_package_name_query - printf ',`r_a`.`name`,`repositories`.`name`' - printf ' FROM `replaced_bpir`' - printf ' JOIN `binary_packages_in_repositories`' - printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_in_repositories_repositories - mysql_join_binary_packages_architectures - mysql_join_repositories_architectures '' 'r_a' - printf ';\n' - } | \ - mysql_run_query | \ - tr '\t' ' ' | \ - grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \ - while read -r what content; do - printf '%s\n' "${content}" >> \ - "${tmp_dir}/${what}" - done - - if [ ! -s "${tmp_dir}/repositories" ]; then - >&2 printf 'Nothing to move here (%s).\n' "${source_stability}" - continue - fi - - touch \ - "${tmp_dir}/mv" \ - "${tmp_dir}/mv.id" \ - "${tmp_dir}/rm" \ - "${tmp_dir}/rm.id" - - # shellcheck disable=SC2086 - for s in "${tmp_dir}/"*; do - sort -u "${s}" | \ - sponge "${s}" - done + touch \ + "${tmp_dir}/mv" \ + "${tmp_dir}/mv.id" \ + "${tmp_dir}/rm" \ + "${tmp_dir}/rm.id" - # receive the repository databases from the master mirror - mkdir "${tmp_dir}/dbs" - while read -r arch repo; do - mkdir -p "${tmp_dir}/dbs/${arch}/${repo}" # shellcheck disable=SC2086 - failsafe_rsync \ - "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \ - "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \ - "${tmp_dir}/dbs/${arch}/${repo}/" - done < \ - "${tmp_dir}/repositories" - - # remove to-be-deleted packages - # shellcheck disable=SC2094 - cut -d' ' -f2,3 < \ - "${tmp_dir}/rm" | \ - sort -u | \ - while read -r arch repo; do - grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \ - sed ' - s/\(-[^-]\+\)\{3\} \S\+ \S\+$// - ' | \ - xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz" + for s in "${tmp_dir}/"*; do + sort -u "${s}" | \ + sponge "${s}" done - # copy and delete moved packages - # shellcheck disable=SC2094 - cut -d' ' -f2,3,4,5 < \ - "${tmp_dir}/mv" | \ - sort -u | \ - while read -r from_arch from_repo to_arch to_repo; do - grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ - sed ' - s/-[^-]\+\( \S\+\)\{4\}$// - ' | \ - xargs -r "${base_dir}/bin/repo-copy" \ - "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \ - "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz" - grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ - sed ' - s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$// - ' | \ - xargs -r repo-remove -q \ - "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" - done + # receive the repository databases from the master mirror + mkdir "${tmp_dir}/dbs" + while read -r arch repo; do + mkdir -p "${tmp_dir}/dbs/${arch}/${repo}" + # shellcheck disable=SC2086 + failsafe_rsync \ + "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \ + "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \ + "${tmp_dir}/dbs/${arch}/${repo}/" + done < \ + "${tmp_dir}/repositories" + + # remove to-be-deleted packages + # shellcheck disable=SC2094 + cut -d' ' -f2,3 < \ + "${tmp_dir}/rm" | \ + sort -u | \ + while read -r arch repo; do + grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \ + sed ' + s/\(-[^-]\+\)\{3\} \S\+ \S\+$// + ' | \ + xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz" + done + + # copy and delete moved packages + # shellcheck disable=SC2094 + cut -d' ' -f2,3,4,5 < \ + "${tmp_dir}/mv" | \ + sort -u | \ + while read -r from_arch from_repo to_arch to_repo; do + grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ + sed ' + s/-[^-]\+\( \S\+\)\{4\}$// + ' | \ + xargs -r "${base_dir}/bin/repo-copy" \ + "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \ + "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz" + grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \ + sed ' + s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$// + ' | \ + xargs -r repo-remove -q \ + "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" + done + + # create real file names of packages, because + # mysql_query_and_delete_unneeded_binary_packages does so, too + sed -i ' + s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1, + ' "${tmp_dir}/rm" - # create real file names of packages, because - # mysql_query_and_delete_unneeded_binary_packages does so, too - sed -i ' - s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1, - ' "${tmp_dir}/rm" + # somewhat inaccurate + if ! ${no_action}; then + { + printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \ + "${tmp_dir}/mv.id" 'moved_bpir' \ + "${tmp_dir}/rm.id" 'replaced_bpir' + printf 'DELETE `binary_packages_in_repositories`' + printf ' FROM `binary_packages_in_repositories`' + printf ' JOIN `replaced_bpir`' + printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n' + mysql_query_and_delete_unneeded_binary_packages + printf 'UPDATE `binary_packages_in_repositories`' + printf ' JOIN `moved_bpir`' + printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`' + printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,' + printf '`binary_packages_in_repositories`.`last_moved`=NOW()' + printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n' + } | \ + mysql_run_query | \ + sort -u >> \ + "${tmp_dir}/rm" + fi - # somewhat inaccurate - if ! ${no_action}; then - # shellcheck disable=SC2016 + # move the packages remotely via sftp { - printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n' - printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \ - "${tmp_dir}/mv.id" 'moved_bpir' \ - "${tmp_dir}/rm.id" 'replaced_bpir' - printf 'DELETE `binary_packages_in_repositories`' - printf ' FROM `binary_packages_in_repositories`' - printf ' JOIN `replaced_bpir`' - printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n' - mysql_query_and_delete_unneeded_binary_packages - printf 'UPDATE `binary_packages_in_repositories`' - printf ' JOIN `moved_bpir`' - printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`' - printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,' - printf '`binary_packages_in_repositories`.`last_moved`=NOW()' - printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n' + sed ' + s/^/rm "/ + s/$/"/ + p + s/"$/.sig"/ + ' "${tmp_dir}/rm" + sed ' + s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig", + ' "${tmp_dir}/mv" + echo 'quit' } | \ - mysql_run_query | \ - sort -u >> \ - "${tmp_dir}/rm" - fi + if ${no_action}; then + sed 's|^|sftp: |' + else + failsafe_sftp + fi - # move the packages remotely via sftp - { - sed ' - s/^/rm "/ - s/$/"/ - p - s/"$/.sig"/ - ' "${tmp_dir}/rm" - sed ' - s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig", - ' "${tmp_dir}/mv" - echo 'quit' - } | \ if ${no_action}; then - sed 's|^|sftp: |' - else - failsafe_sftp + continue fi - if ${no_action}; then - continue - fi - - # and push our local *.db.tar.gz via rsync - while read -r arch repo; do - recompress_gz \ - "${tmp_dir}" \ - "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \ - "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old" - # shellcheck disable=SC2086 - failsafe_rsync \ - "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \ - "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \ - "${master_mirror_rsync_directory}/${arch}/${repo}/" - done < \ - "${tmp_dir}/repositories" + # and push our local *.db.tar.gz via rsync + while read -r arch repo; do + recompress_gz \ + "${tmp_dir}" \ + "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \ + "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old" + # shellcheck disable=SC2086 + failsafe_rsync \ + "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \ + "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \ + "${master_mirror_rsync_directory}/${arch}/${repo}/" + done < \ + "${tmp_dir}/repositories" -done + done trigger_mirror_refreshs |