Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

db-update 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
  1. #!/bin/sh
  2. # move binary packages from staging to testing (if possible [1]) and
  3. # additionally tested packages from testing to the respective stable
  4. # repository (if possible [1])
  5. # The condition [1] is explained in the stored function
  6. # calculate_maximal_moveable_set which is created in bin/bootsrap-mysql
  7. # TODO: make (all) locking less restrictive
  8. # shellcheck disable=SC2039,SC2119,SC2120
  9. # shellcheck source=../lib/load-configuration
  10. . "${0%/*}/../lib/load-configuration"
  11. # shellcheck disable=SC2016
  12. usage() {
  13. >&2 echo ''
  14. >&2 echo 'db-update [options] [packages]:'
  15. >&2 echo ' move tested packages from testing to stable.'
  16. >&2 echo ' move possible packages from staging to testing.'
  17. >&2 echo ''
  18. >&2 echo 'possible options:'
  19. >&2 echo ' -f|--force $arch/$repo/$pkgname:'
  20. >&2 echo ' Force movement of given package and move nothing else.'
  21. >&2 echo ' -h|--help:'
  22. >&2 echo ' Show this help and exit.'
  23. >&2 echo ' -i|--ignore-insanity:'
  24. >&2 echo ' Do not abort when insane.'
  25. >&2 echo ' -n|--no-action:'
  26. >&2 echo ' Only print what would be moved.'
  27. >&2 echo ' -p|--progressive:'
  28. >&2 echo ' Move forward any package which replaces no package whose'
  29. >&2 echo ' dependencies are all available somewhere.'
  30. >&2 echo ' Note, that this _may_ move _less_ packages.'
  31. >&2 echo ' -w|--wait:'
  32. >&2 echo ' If necessary, wait for lock blocking.'
  33. [ -z "$1" ] && exit 1 || exit "$1"
  34. }
  35. eval set -- "$(
  36. getopt -o f:hinpw \
  37. --long force \
  38. --long help \
  39. --long ignore-insanity \
  40. --long no-action \
  41. --long progressive \
  42. --long wait \
  43. -n "$(basename "$0")" -- "$@" || \
  44. echo usage
  45. )"
  46. block_flag='-n'
  47. ignore_insanity=false
  48. no_action=false
  49. progressive=false
  50. force_pkgs=''
  51. while true
  52. do
  53. case "$1" in
  54. -f|--force)
  55. shift
  56. force_pkgs=$(
  57. printf '%s' "$1" | \
  58. base64 -w0
  59. printf '\n%s' "${force_pkgs}"
  60. )
  61. ;;
  62. -h|--help)
  63. usage 0
  64. ;;
  65. -i|--ignore-insanity)
  66. ignore_insanity=true
  67. ;;
  68. -n|--no-action)
  69. no_action=true
  70. ;;
  71. -p|--progressive)
  72. progressive=true
  73. ;;
  74. -w|--wait)
  75. block_flag=''
  76. ;;
  77. --)
  78. shift
  79. break
  80. ;;
  81. *)
  82. >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
  83. exit 42
  84. ;;
  85. esac
  86. shift
  87. done
  88. if [ $# -ne 0 ]; then
  89. >&2 echo 'db-update: too many arguments'
  90. usage
  91. fi
  92. if ${progressive} && \
  93. [ -n "${force_pkgs}" ]; then
  94. >&2 echo 'db-update: conflicting arguments'
  95. usage
  96. fi
  97. if [ -s "${work_dir}/build-master-sanity" ]; then
  98. >&2 echo 'Build master is not sane.'
  99. if ! ${ignore_insanity}; then
  100. exit
  101. fi
  102. fi
  103. if ! ${no_action}; then
  104. # Create lock.
  105. exec 9> "${package_database_lock_file}"
  106. if ! verbose_flock ${block_flag} 9; then
  107. >&2 echo 'come back (shortly) later - I cannot lock package database.'
  108. exit 0
  109. fi
  110. exec 8> "${sanity_check_lock_file}"
  111. if ! verbose_flock -s ${block_flag} 8; then
  112. >&2 echo 'come back (shortly) later - sanity-check currently running.'
  113. exit 0
  114. fi
  115. # shellcheck disable=SC2016
  116. {
  117. printf 'UPDATE `binary_packages`'
  118. printf ' JOIN ('
  119. printf 'SELECT `binary_packages_in_repositories`.`package`,'
  120. printf 'MIN(`binary_packages_in_repositories`.`last_moved`) AS `first_last_moved`'
  121. printf ' FROM `binary_packages_in_repositories`'
  122. mysql_join_binary_packages_in_repositories_repositories
  123. # shellcheck disable=SC2154
  124. printf ' WHERE `repositories`.`stability`=%s' \
  125. "${repository_stability_ids__testing}"
  126. printf ' GROUP BY `binary_packages_in_repositories`.`package`'
  127. printf ') AS `binary_packages_in_repositories`'
  128. printf ' ON `binary_packages_in_repositories`.`package`=`binary_packages`.`id`'
  129. printf ' SET `binary_packages`.`is_tested`=1'
  130. printf ' WHERE NOT `binary_packages`.`has_issues`'
  131. printf ' AND NOT `binary_packages`.`is_tested`'
  132. printf ' AND `binary_packages_in_repositories`.`first_last_moved`<SUBDATE(NOW(),from_base64("%s"))' \
  133. "$(
  134. printf -- '%s' "${max_testing_duration}" | \
  135. base64 -w0
  136. )"
  137. } | \
  138. mysql_run_query
  139. fi
  140. # Create tmp_dir and trap.
  141. tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
  142. trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
  143. export TMPDIR="${tmp_dir}"
  144. # shellcheck disable=SC2016
  145. {
  146. printf 'SELECT DISTINCT'
  147. printf ' `repositories`.`architecture`,'
  148. printf '`architectures`.`name`,'
  149. printf '`repositories`.`stability`,'
  150. printf '`repository_stabilities`.`name`'
  151. printf ' FROM `repositories`'
  152. mysql_join_repositories_architectures
  153. mysql_join_repositories_repository_stabilities
  154. printf ' JOIN `repository_moves`'
  155. printf ' ON `repository_moves`.`from_repository`=`repositories`.`id`'
  156. printf ' WHERE `repositories`.`is_on_master_mirror`'
  157. printf ' ORDER BY `repositories`.`stability`,'
  158. printf 'SHA2('
  159. printf 'CONCAT('
  160. printf '`repositories`.`architecture`,'
  161. printf '`repositories`.`stability`,'
  162. printf 'NOW()'
  163. printf '),'
  164. printf '256'
  165. printf ')'
  166. } | \
  167. mysql_run_query | \
  168. while read -r arch_id arch source_stability_id source_stability; do
  169. find "${tmp_dir}" -xdev -mindepth 1 -delete
  170. {
  171. # Note, that we do not need to check that two repositories a and b
  172. # are for the same architecture if we joined them via
  173. # repository_moves, because only repositories with the same
  174. # architectures should be listed there.
  175. if [ -n "${force_pkgs}" ]; then
  176. printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
  177. 'moveable' 'replaced'
  178. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
  179. printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  180. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  181. printf ' SELECT '
  182. printf '`binary_packages_in_repositories`.`id`,NULL'
  183. printf ' FROM `binary_packages_in_repositories`'
  184. mysql_join_binary_packages_in_repositories_repositories
  185. mysql_join_repositories_architectures
  186. mysql_join_binary_packages_in_repositories_binary_packages
  187. printf ' WHERE CONCAT('
  188. printf '`architectures`.`name`,"/",'
  189. printf '`repositories`.`name`,"/",'
  190. printf '`binary_packages`.`pkgname`'
  191. printf ') IN ('
  192. # shellcheck disable=SC2086
  193. printf 'from_base64("%s"),' \
  194. ${force_pkgs} | \
  195. sed 's/,$/);\n/'
  196. printf 'DELETE `moveable_bpir`'
  197. printf ' FROM `moveable_bpir`'
  198. printf ' JOIN `binary_packages_in_repositories`'
  199. printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
  200. mysql_join_binary_packages_in_repositories_binary_packages
  201. mysql_join_binary_packages_in_repositories_repositories
  202. printf ' WHERE `repositories`.`stability`!=%s' \
  203. "${source_stability_id}"
  204. printf ' OR `repositories`.`architecture`!=%s;\n' \
  205. "${arch_id}"
  206. printf 'UPDATE `moveable_bpir`'
  207. printf ' JOIN `binary_packages_in_repositories`'
  208. printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
  209. mysql_join_binary_packages_in_repositories_binary_packages
  210. mysql_join_binary_packages_build_assignments
  211. mysql_join_build_assignments_package_sources
  212. mysql_join_package_sources_upstream_repositories
  213. mysql_join_upstream_repositories_repository_moves
  214. printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
  215. printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n'
  216. printf 'DELETE FROM `moveable_bpir`'
  217. printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n'
  218. printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
  219. printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`'
  220. printf ' FROM `moveable_bpir`'
  221. printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`'
  222. printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`'
  223. mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp'
  224. printf ' JOIN `binary_packages`'
  225. printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
  226. mysql_join_binary_packages_binary_packages_in_repositories
  227. mysql_join_binary_packages_in_repositories_repositories
  228. printf ' JOIN `repositories` AS `m_to_r`'
  229. printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`'
  230. printf ' AND `repositories`.`stability`=`m_to_r`.`stability`'
  231. printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n'
  232. elif ${progressive}; then
  233. printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
  234. 'moveable' 'replaced'
  235. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
  236. printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  237. # packages that replace broken packages
  238. printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
  239. printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`'
  240. printf ' FROM `binary_packages_in_repositories` AS `new_bpir`'
  241. mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp'
  242. mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r'
  243. printf ' AND `from_r`.`stability`=%s' \
  244. "${source_stability_id}"
  245. printf ' AND `from_r`.`architecture`=%s' \
  246. "${arch_id}"
  247. printf ' AND `from_r`.`is_on_master_mirror`'
  248. mysql_join_binary_packages_build_assignments 'new_bp'
  249. mysql_join_build_assignments_package_sources
  250. mysql_join_package_sources_upstream_repositories
  251. mysql_join_upstream_repositories_repository_moves
  252. printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`'
  253. printf ' JOIN `repositories` AS `to_r`'
  254. printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`'
  255. printf ' JOIN `binary_packages` AS `old_bp`'
  256. printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`'
  257. mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir'
  258. mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r'
  259. printf ' AND `old_r`.`stability`=`to_r`.`stability`'
  260. printf ' AND `old_r`.`architecture`=`to_r`.`architecture`'
  261. mysql_join_binary_packages_dependencies 'old_bp'
  262. mysql_join_dependencies_dependency_types
  263. printf ' AND `dependency_types`.`relevant_for_binary_packages`'
  264. mysql_join_dependencies_versions '' 'd_v'
  265. printf ' WHERE NOT EXISTS ('
  266. printf 'SELECT 1'
  267. printf ' FROM `install_target_providers`'
  268. mysql_join_install_target_providers_binary_packages '' 'itp_bp'
  269. mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir'
  270. mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r'
  271. mysql_join_install_target_providers_versions '' 'itp_v'
  272. printf ' JOIN `repository_stability_relations` AS `rsr_a`'
  273. printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`'
  274. printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
  275. printf ' AND'
  276. mysql_query_ordering_correct \
  277. '`d_v`.`order`' \
  278. '`itp_v`.`order`' \
  279. '`dependencies`.`version_relation`'
  280. printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`'
  281. printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`'
  282. # this is the least stable package of stability at least to_r.stability
  283. printf ' AND NOT EXISTS ('
  284. printf 'SELECT 1'
  285. printf ' FROM `binary_packages` AS `subst_bp`'
  286. mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir'
  287. mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r'
  288. printf ' JOIN `repository_stability_relations` AS `rsr_b`'
  289. printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`'
  290. printf ' JOIN `repository_stability_relations` AS `rsr_c`'
  291. printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`'
  292. printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`'
  293. printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`'
  294. printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`'
  295. printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`'
  296. printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`'
  297. printf ')'
  298. printf ');\n'
  299. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  300. printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`'
  301. printf ' FROM `replaced_bpir`'
  302. printf ' JOIN `binary_packages_in_repositories`'
  303. printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`'
  304. printf ';\n'
  305. # packages which replace nothing
  306. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  307. printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`'
  308. printf ' FROM `binary_packages_in_repositories`'
  309. mysql_join_binary_packages_in_repositories_binary_packages
  310. mysql_join_binary_packages_in_repositories_repositories
  311. printf ' AND `repositories`.`is_on_master_mirror`'
  312. printf ' AND `repositories`.`stability`=%s' \
  313. "${source_stability_id}"
  314. printf ' AND `repositories`.`architecture`=%s' \
  315. "${arch_id}"
  316. mysql_join_binary_packages_build_assignments
  317. mysql_join_build_assignments_package_sources
  318. mysql_join_package_sources_upstream_repositories
  319. mysql_join_upstream_repositories_repository_moves
  320. printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
  321. printf ' JOIN `repositories` as `to_r`'
  322. printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`'
  323. printf ' WHERE NOT EXISTS ('
  324. printf 'SELECT 1'
  325. printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`'
  326. mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp'
  327. mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r'
  328. printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
  329. printf ' AND `repl_r`.`stability`=`to_r`.`stability`'
  330. printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`'
  331. printf ');\n'
  332. else
  333. printf 'CALL calculate_maximal_moveable_set(%s,%s);\n' \
  334. "${arch_id}" \
  335. "${source_stability_id}"
  336. fi
  337. printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
  338. printf 'INSERT IGNORE INTO `rps` (`id`)'
  339. printf ' SELECT `moveable_bpir`.`to_repository`'
  340. printf ' FROM `moveable_bpir`;\n'
  341. printf 'INSERT IGNORE INTO `rps` (`id`)'
  342. printf ' SELECT `binary_packages_in_repositories`.`repository`'
  343. printf ' FROM `moveable_bpir`'
  344. printf ' JOIN `binary_packages_in_repositories`'
  345. printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
  346. printf 'INSERT IGNORE INTO `rps` (`id`)'
  347. printf ' SELECT `binary_packages_in_repositories`.`repository`'
  348. printf ' FROM `replaced_bpir`'
  349. printf ' JOIN `binary_packages_in_repositories`'
  350. printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
  351. printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`'
  352. printf ' FROM `rps`'
  353. printf ' JOIN `repositories`'
  354. printf ' ON `rps`.`id`=`repositories`.`id`'
  355. mysql_join_repositories_architectures
  356. printf ';\n'
  357. printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`'
  358. printf ' FROM `moveable_bpir`;\n'
  359. printf 'SELECT "mv",'
  360. mysql_package_name_query
  361. printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`'
  362. printf ' FROM `moveable_bpir`'
  363. printf ' JOIN `binary_packages_in_repositories`'
  364. printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`'
  365. mysql_join_binary_packages_in_repositories_binary_packages
  366. mysql_join_binary_packages_in_repositories_repositories
  367. mysql_join_binary_packages_architectures
  368. mysql_join_repositories_architectures '' 'r_a'
  369. printf ' JOIN `repositories` AS `new_repo`'
  370. printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`'
  371. mysql_join_repositories_architectures 'new_repo' 'new_r_a'
  372. printf ';\n'
  373. printf 'SELECT "rm.id",`replaced_bpir`.`id`'
  374. printf ' FROM `replaced_bpir`;\n'
  375. printf 'SELECT "rm",'
  376. mysql_package_name_query
  377. printf ',`r_a`.`name`,`repositories`.`name`'
  378. printf ' FROM `replaced_bpir`'
  379. printf ' JOIN `binary_packages_in_repositories`'
  380. printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`'
  381. mysql_join_binary_packages_in_repositories_binary_packages
  382. mysql_join_binary_packages_in_repositories_repositories
  383. mysql_join_binary_packages_architectures
  384. mysql_join_repositories_architectures '' 'r_a'
  385. printf ';\n'
  386. } | \
  387. mysql_run_query | \
  388. tr '\t' ' ' | \
  389. grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
  390. while read -r what content; do
  391. printf '%s\n' "${content}" >> \
  392. "${tmp_dir}/${what}"
  393. done
  394. if [ ! -s "${tmp_dir}/repositories" ]; then
  395. >&2 printf 'Nothing to move here (from %s/%s).\n' \
  396. "${arch}" \
  397. "${source_stability}"
  398. continue
  399. fi
  400. touch \
  401. "${tmp_dir}/mv" \
  402. "${tmp_dir}/mv.id" \
  403. "${tmp_dir}/rm" \
  404. "${tmp_dir}/rm.id"
  405. # shellcheck disable=SC2086
  406. for s in "${tmp_dir}/"*; do
  407. sort -u "${s}" | \
  408. sponge "${s}"
  409. done
  410. # receive the repository databases from the master mirror
  411. mkdir "${tmp_dir}/dbs"
  412. while read -r arch repo; do
  413. mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
  414. # shellcheck disable=SC2086
  415. failsafe_rsync \
  416. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
  417. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
  418. "${tmp_dir}/dbs/${arch}/${repo}/"
  419. done < \
  420. "${tmp_dir}/repositories"
  421. # remove to-be-deleted packages
  422. # shellcheck disable=SC2094
  423. cut -d' ' -f2,3 < \
  424. "${tmp_dir}/rm" | \
  425. sort -u | \
  426. while read -r arch repo; do
  427. grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
  428. sed '
  429. s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
  430. ' | \
  431. xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
  432. done
  433. # copy and delete moved packages
  434. # shellcheck disable=SC2094
  435. cut -d' ' -f2,3,4,5 < \
  436. "${tmp_dir}/mv" | \
  437. sort -u | \
  438. while read -r from_arch from_repo to_arch to_repo; do
  439. grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
  440. sed '
  441. s/-[^-]\+\( \S\+\)\{4\}$//
  442. ' | \
  443. xargs -r "${base_dir}/bin/repo-copy" \
  444. "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
  445. "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
  446. grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
  447. sed '
  448. s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
  449. ' | \
  450. xargs -r repo-remove -q \
  451. "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
  452. done
  453. # create real file names of packages, because
  454. # mysql_query_and_delete_unneeded_binary_packages does so, too
  455. sed -i '
  456. s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
  457. ' "${tmp_dir}/rm"
  458. # somewhat inaccurate
  459. if ! ${no_action}; then
  460. {
  461. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  462. printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  463. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
  464. "${tmp_dir}/mv.id" 'moved_bpir' \
  465. "${tmp_dir}/rm.id" 'replaced_bpir'
  466. printf 'DELETE `binary_packages_in_repositories`'
  467. printf ' FROM `binary_packages_in_repositories`'
  468. printf ' JOIN `replaced_bpir`'
  469. printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
  470. mysql_query_and_delete_unneeded_binary_packages
  471. printf 'UPDATE `binary_packages_in_repositories`'
  472. printf ' JOIN `moved_bpir`'
  473. printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
  474. printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
  475. printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
  476. printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
  477. } | \
  478. mysql_run_query | \
  479. sort -u >> \
  480. "${tmp_dir}/rm"
  481. fi
  482. # move the packages remotely via sftp
  483. {
  484. sed '
  485. s/^/rm "/
  486. s/$/"/
  487. p
  488. s/"$/.sig"/
  489. ' "${tmp_dir}/rm"
  490. sed '
  491. s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
  492. ' "${tmp_dir}/mv"
  493. echo 'quit'
  494. } | \
  495. if ${no_action}; then
  496. sed 's|^|sftp: |'
  497. else
  498. failsafe_sftp
  499. fi
  500. if ${no_action}; then
  501. continue
  502. fi
  503. # and push our local *.db.tar.gz via rsync
  504. while read -r arch repo; do
  505. recompress_gz \
  506. "${tmp_dir}" \
  507. "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
  508. "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
  509. # shellcheck disable=SC2086
  510. failsafe_rsync \
  511. "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
  512. "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
  513. "${master_mirror_rsync_directory}/${arch}/${repo}/"
  514. done < \
  515. "${tmp_dir}/repositories"
  516. done
  517. trigger_mirror_refreshs