Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

db-update 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628
  1. #!/bin/sh
  2. # move binary packages from staging to testing (if possible [1]) and
  3. # additionally tested packages from testing to the respective stable
  4. # repository (if possible [1])
  5. # The condition [1] is explained in the stored function
  6. # calculate_maximal_moveable_set which is created in bin/bootsrap-mysql
  7. # TODO: make (all) locking less restrictive
  8. # shellcheck disable=SC2039,SC2119,SC2120
  9. # shellcheck source=../lib/load-configuration
  10. . "${0%/*}/../lib/load-configuration"
  11. # shellcheck disable=SC2016
  12. usage() {
  13. >&2 echo ''
  14. >&2 echo 'db-update [options] [packages]:'
  15. >&2 echo ' move tested packages from testing to stable.'
  16. >&2 echo ' move possible packages from staging to testing.'
  17. >&2 echo ''
  18. >&2 echo 'possible options:'
  19. >&2 echo ' -f|--force $arch/$repo/$pkgname:'
  20. >&2 echo ' Force movement of given package and move nothing else.'
  21. >&2 echo ' -h|--help:'
  22. >&2 echo ' Show this help and exit.'
  23. >&2 echo ' -i|--ignore-insanity:'
  24. >&2 echo ' Do not abort when insane.'
  25. >&2 echo ' -n|--no-action:'
  26. >&2 echo ' Only print what would be moved.'
  27. >&2 echo ' -o|--only $identifier:'
  28. >&2 echo ' Only try to move packages *from* the given'
  29. >&2 echo ' stability/architecture. $identifier must match one of'
  30. >&2 echo ' the regexes'
  31. >&2 echo ' (testing|staging)'
  32. >&2 echo ' !?(i486|i686|pentium4)'
  33. >&2 echo ' !?(i486|i686|pentium4)/(testing|staging)'
  34. >&2 echo ' where preceding exclamation mark negates the selection'
  35. >&2 echo ' of the architecture (but not the stability!).'
  36. >&2 echo ' Can be given multiple times to aggregate the selected'
  37. >&2 echo ' repositories.'
  38. >&2 echo ' -p|--progressive:'
  39. >&2 echo ' Move forward any package which replaces no package whose'
  40. >&2 echo ' dependencies are all available somewhere.'
  41. >&2 echo ' Note, that this _may_ move _less_ packages.'
  42. >&2 echo ' -w|--wait:'
  43. >&2 echo ' If necessary, wait for lock blocking.'
  44. [ -z "$1" ] && exit 1 || exit "$1"
  45. }
  46. eval set -- "$(
  47. getopt -o f:hino:pw \
  48. --long force \
  49. --long help \
  50. --long ignore-insanity \
  51. --long no-action \
  52. --long only: \
  53. --long progressive \
  54. --long wait \
  55. -n "$(basename "$0")" -- "$@" || \
  56. echo usage
  57. )"
  58. block_flag='-n'
  59. ignore_insanity=false
  60. no_action=false
  61. progressive=false
  62. force_pkgs=''
  63. only=''
  64. while true
  65. do
  66. case "$1" in
  67. -f|--force)
  68. shift
  69. force_pkgs=$(
  70. printf '%s' "$1" | \
  71. base64 -w0
  72. printf '\n%s' "${force_pkgs}"
  73. )
  74. ;;
  75. -h|--help)
  76. usage 0
  77. ;;
  78. -i|--ignore-insanity)
  79. ignore_insanity=true
  80. ;;
  81. -n|--no-action)
  82. no_action=true
  83. ;;
  84. -o|--only)
  85. shift
  86. if ! printf '%s\n' "$1" \
  87. | grep -qx '\(testing\|staging\)\|!\?\(i486\|i686\|pentium4\)\|!\?\(i486\|i686\|pentium4\)/\(testing\|staging\)'; then
  88. >&2 printf 'Syntax error, --only parameter "%s" does not match required regex.\n' "$1"
  89. usage
  90. fi
  91. only="$1 ${only}"
  92. ;;
  93. -p|--progressive)
  94. progressive=true
  95. ;;
  96. -w|--wait)
  97. block_flag=''
  98. ;;
  99. --)
  100. shift
  101. break
  102. ;;
  103. *)
  104. >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
  105. exit 42
  106. ;;
  107. esac
  108. shift
  109. done
  110. if [ $# -ne 0 ]; then
  111. >&2 echo 'db-update: too many arguments'
  112. usage
  113. fi
  114. if ${progressive} && \
  115. [ -n "${force_pkgs}" ]; then
  116. >&2 echo 'db-update: conflicting arguments'
  117. usage
  118. fi
  119. if [ -s "${work_dir}/build-master-sanity" ]; then
  120. >&2 echo 'Build master is not sane.'
  121. if ! ${ignore_insanity}; then
  122. exit
  123. fi
  124. fi
  125. if [ -n "${only}" ]; then
  126. only=$(
  127. printf '%s\n' "${only}" \
  128. | tr ' ' '\n'
  129. )
  130. only=$(
  131. printf '%s\n' "${only}" \
  132. | grep '^[^!]' \
  133. || true
  134. printf '%s\n' "${only}" \
  135. | sed "$(
  136. for arch in i486 i686 pentium4; do
  137. printf 's@^!%s\\(/.\\+\\)\\?$@' \
  138. "${arch}"
  139. for arch2 in i486 i686 pentium4; do
  140. if [ "${arch}" == "${arch2}" ]; then
  141. continue
  142. fi
  143. printf '%s\\1\\n' \
  144. "${arch2}"
  145. done \
  146. | sed 's/\\n$//'
  147. printf '@\nt\n'
  148. done
  149. )"'
  150. d
  151. '
  152. )
  153. fi
  154. if ! ${no_action}; then
  155. # Create lock.
  156. exec 9> "${package_database_lock_file}"
  157. if ! verbose_flock ${block_flag} 9; then
  158. >&2 echo 'come back (shortly) later - I cannot lock package database.'
  159. exit 0
  160. fi
  161. exec 8> "${sanity_check_lock_file}"
  162. if ! verbose_flock -s ${block_flag} 8; then
  163. >&2 echo 'come back (shortly) later - sanity-check currently running.'
  164. exit 0
  165. fi
  166. # shellcheck disable=SC2016
  167. {
  168. printf 'UPDATE `binary_packages`'
  169. printf ' JOIN ('
  170. printf 'SELECT `binary_packages_in_repositories`.`package`,'
  171. printf 'MIN(`binary_packages_in_repositories`.`last_moved`) AS `first_last_moved`'
  172. printf ' FROM `binary_packages_in_repositories`'
  173. mysql_join_binary_packages_in_repositories_repositories
  174. # shellcheck disable=SC2154
  175. printf ' WHERE `repositories`.`stability`=%s' \
  176. "${repository_stability_ids__testing}"
  177. printf ' GROUP BY `binary_packages_in_repositories`.`package`'
  178. printf ') AS `binary_packages_in_repositories`'
  179. printf ' ON `binary_packages_in_repositories`.`package`=`binary_packages`.`id`'
  180. printf ' SET `binary_packages`.`is_tested`=1'
  181. printf ' WHERE NOT `binary_packages`.`has_issues`'
  182. printf ' AND NOT `binary_packages`.`is_tested`'
  183. printf ' AND `binary_packages_in_repositories`.`first_last_moved`<SUBDATE(NOW(),from_base64("%s"))' \
  184. "$(
  185. printf -- '%s' "${max_testing_duration}" | \
  186. base64 -w0
  187. )"
  188. } | \
  189. mysql_run_query
  190. fi
  191. # Create tmp_dir and trap.
  192. tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX")
  193. trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
  194. export TMPDIR="${tmp_dir}"
  195. # shellcheck disable=SC2016
  196. {
  197. printf 'SELECT DISTINCT'
  198. printf ' `repositories`.`architecture`,'
  199. printf '`architectures`.`name`,'
  200. printf '`repositories`.`stability`,'
  201. printf '`repository_stabilities`.`name`'
  202. printf ' FROM `repositories`'
  203. mysql_join_repositories_architectures
  204. mysql_join_repositories_repository_stabilities
  205. printf ' JOIN `repository_moves`'
  206. printf ' ON `repository_moves`.`from_repository`=`repositories`.`id`'
  207. printf ' WHERE `repositories`.`is_on_master_mirror`'
  208. printf ' ORDER BY `repositories`.`stability`,'
  209. printf 'SHA2('
  210. printf 'CONCAT('
  211. printf '`repositories`.`architecture`,'
  212. printf '`repositories`.`stability`,'
  213. printf 'NOW()'
  214. printf '),'
  215. printf '256'
  216. printf ')'
  217. } | \
  218. mysql_run_query | \
  219. while read -r arch_id arch source_stability_id source_stability; do
  220. if [ -n "${only}" ]; then
  221. does_match=false
  222. for possible_match in ${only}; do
  223. if [ "${arch}/${source_stability}" = "${possible_match}" ] \
  224. || [ "${source_stability}" = "${possible_match}" ] \
  225. || [ "${arch}" = "${possible_match}" ]; then
  226. does_match=true
  227. break
  228. fi
  229. done
  230. if ! ${does_match}; then
  231. continue
  232. fi
  233. fi
  234. find "${tmp_dir}" -xdev -mindepth 1 -delete
  235. {
  236. # Note, that we do not need to check that two repositories a and b
  237. # are for the same architecture if we joined them via
  238. # repository_moves, because only repositories with the same
  239. # architectures should be listed there.
  240. if [ -n "${force_pkgs}" ]; then
  241. printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
  242. 'moveable' 'replaced'
  243. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
  244. printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  245. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  246. printf ' SELECT '
  247. printf '`binary_packages_in_repositories`.`id`,NULL'
  248. printf ' FROM `binary_packages_in_repositories`'
  249. mysql_join_binary_packages_in_repositories_repositories
  250. mysql_join_repositories_architectures
  251. mysql_join_binary_packages_in_repositories_binary_packages
  252. printf ' WHERE CONCAT('
  253. printf '`architectures`.`name`,"/",'
  254. printf '`repositories`.`name`,"/",'
  255. printf '`binary_packages`.`pkgname`'
  256. printf ') IN ('
  257. # shellcheck disable=SC2086
  258. printf 'from_base64("%s"),' \
  259. ${force_pkgs} | \
  260. sed 's/,$/);\n/'
  261. printf 'DELETE `moveable_bpir`'
  262. printf ' FROM `moveable_bpir`'
  263. printf ' JOIN `binary_packages_in_repositories`'
  264. printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
  265. mysql_join_binary_packages_in_repositories_binary_packages
  266. mysql_join_binary_packages_in_repositories_repositories
  267. printf ' WHERE `repositories`.`stability`!=%s' \
  268. "${source_stability_id}"
  269. printf ' OR `repositories`.`architecture`!=%s;\n' \
  270. "${arch_id}"
  271. printf 'UPDATE `moveable_bpir`'
  272. printf ' JOIN `binary_packages_in_repositories`'
  273. printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
  274. mysql_join_binary_packages_in_repositories_binary_packages
  275. mysql_join_binary_packages_build_assignments
  276. mysql_join_build_assignments_package_sources
  277. mysql_join_package_sources_upstream_repositories
  278. mysql_join_upstream_repositories_repository_moves
  279. printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
  280. printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n'
  281. printf 'DELETE FROM `moveable_bpir`'
  282. printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n'
  283. printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
  284. printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`'
  285. printf ' FROM `moveable_bpir`'
  286. printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`'
  287. printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`'
  288. mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp'
  289. printf ' JOIN `binary_packages`'
  290. printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
  291. mysql_join_binary_packages_binary_packages_in_repositories
  292. mysql_join_binary_packages_in_repositories_repositories
  293. printf ' JOIN `repositories` AS `m_to_r`'
  294. printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`'
  295. printf ' AND `repositories`.`stability`=`m_to_r`.`stability`'
  296. printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n'
  297. elif ${progressive}; then
  298. printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
  299. 'moveable' 'replaced'
  300. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
  301. printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  302. # packages that replace broken packages
  303. printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
  304. printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`'
  305. printf ' FROM `binary_packages_in_repositories` AS `new_bpir`'
  306. mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp'
  307. mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r'
  308. printf ' AND `from_r`.`stability`=%s' \
  309. "${source_stability_id}"
  310. printf ' AND `from_r`.`architecture`=%s' \
  311. "${arch_id}"
  312. printf ' AND `from_r`.`is_on_master_mirror`'
  313. mysql_join_binary_packages_build_assignments 'new_bp'
  314. mysql_join_build_assignments_package_sources
  315. mysql_join_package_sources_upstream_repositories
  316. mysql_join_upstream_repositories_repository_moves
  317. printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`'
  318. printf ' JOIN `repositories` AS `to_r`'
  319. printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`'
  320. printf ' JOIN `binary_packages` AS `old_bp`'
  321. printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`'
  322. mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir'
  323. mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r'
  324. printf ' AND `old_r`.`stability`=`to_r`.`stability`'
  325. printf ' AND `old_r`.`architecture`=`to_r`.`architecture`'
  326. mysql_join_binary_packages_dependencies 'old_bp'
  327. mysql_join_dependencies_dependency_types
  328. printf ' AND `dependency_types`.`relevant_for_binary_packages`'
  329. mysql_join_dependencies_versions '' 'd_v'
  330. printf ' WHERE NOT EXISTS ('
  331. printf 'SELECT 1'
  332. printf ' FROM `install_target_providers`'
  333. mysql_join_install_target_providers_binary_packages '' 'itp_bp'
  334. mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir'
  335. mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r'
  336. mysql_join_install_target_providers_versions '' 'itp_v'
  337. printf ' JOIN `repository_stability_relations` AS `rsr_a`'
  338. printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`'
  339. printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
  340. printf ' AND'
  341. mysql_query_ordering_correct \
  342. '`d_v`.`order`' \
  343. '`itp_v`.`order`' \
  344. '`dependencies`.`version_relation`'
  345. printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`'
  346. printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`'
  347. # this is the least stable package of stability at least to_r.stability
  348. printf ' AND NOT EXISTS ('
  349. printf 'SELECT 1'
  350. printf ' FROM `binary_packages` AS `subst_bp`'
  351. mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir'
  352. mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r'
  353. printf ' JOIN `repository_stability_relations` AS `rsr_b`'
  354. printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`'
  355. printf ' JOIN `repository_stability_relations` AS `rsr_c`'
  356. printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`'
  357. printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`'
  358. printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`'
  359. printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`'
  360. printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`'
  361. printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`'
  362. printf ')'
  363. printf ');\n'
  364. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  365. printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`'
  366. printf ' FROM `replaced_bpir`'
  367. printf ' JOIN `binary_packages_in_repositories`'
  368. printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`'
  369. printf ';\n'
  370. # packages which replace nothing
  371. printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
  372. printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`'
  373. printf ' FROM `binary_packages_in_repositories`'
  374. mysql_join_binary_packages_in_repositories_binary_packages
  375. mysql_join_binary_packages_in_repositories_repositories
  376. printf ' AND `repositories`.`is_on_master_mirror`'
  377. printf ' AND `repositories`.`stability`=%s' \
  378. "${source_stability_id}"
  379. printf ' AND `repositories`.`architecture`=%s' \
  380. "${arch_id}"
  381. mysql_join_binary_packages_build_assignments
  382. mysql_join_build_assignments_package_sources
  383. mysql_join_package_sources_upstream_repositories
  384. mysql_join_upstream_repositories_repository_moves
  385. printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
  386. printf ' JOIN `repositories` as `to_r`'
  387. printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`'
  388. printf ' WHERE NOT EXISTS ('
  389. printf 'SELECT 1'
  390. printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`'
  391. mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp'
  392. mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r'
  393. printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
  394. printf ' AND `repl_r`.`stability`=`to_r`.`stability`'
  395. printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`'
  396. printf ');\n'
  397. else
  398. printf 'CALL calculate_maximal_moveable_set(%s,%s);\n' \
  399. "${arch_id}" \
  400. "${source_stability_id}"
  401. fi
  402. printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
  403. printf 'INSERT IGNORE INTO `rps` (`id`)'
  404. printf ' SELECT `moveable_bpir`.`to_repository`'
  405. printf ' FROM `moveable_bpir`;\n'
  406. printf 'INSERT IGNORE INTO `rps` (`id`)'
  407. printf ' SELECT `binary_packages_in_repositories`.`repository`'
  408. printf ' FROM `moveable_bpir`'
  409. printf ' JOIN `binary_packages_in_repositories`'
  410. printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
  411. printf 'INSERT IGNORE INTO `rps` (`id`)'
  412. printf ' SELECT `binary_packages_in_repositories`.`repository`'
  413. printf ' FROM `replaced_bpir`'
  414. printf ' JOIN `binary_packages_in_repositories`'
  415. printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
  416. printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`'
  417. printf ' FROM `rps`'
  418. printf ' JOIN `repositories`'
  419. printf ' ON `rps`.`id`=`repositories`.`id`'
  420. mysql_join_repositories_architectures
  421. printf ';\n'
  422. printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`'
  423. printf ' FROM `moveable_bpir`;\n'
  424. printf 'SELECT "mv",'
  425. mysql_package_name_query
  426. printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`'
  427. printf ' FROM `moveable_bpir`'
  428. printf ' JOIN `binary_packages_in_repositories`'
  429. printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`'
  430. mysql_join_binary_packages_in_repositories_binary_packages
  431. mysql_join_binary_packages_in_repositories_repositories
  432. mysql_join_binary_packages_architectures
  433. mysql_join_repositories_architectures '' 'r_a'
  434. printf ' JOIN `repositories` AS `new_repo`'
  435. printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`'
  436. mysql_join_repositories_architectures 'new_repo' 'new_r_a'
  437. printf ';\n'
  438. printf 'SELECT "rm.id",`replaced_bpir`.`id`'
  439. printf ' FROM `replaced_bpir`;\n'
  440. printf 'SELECT "rm",'
  441. mysql_package_name_query
  442. printf ',`r_a`.`name`,`repositories`.`name`'
  443. printf ' FROM `replaced_bpir`'
  444. printf ' JOIN `binary_packages_in_repositories`'
  445. printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`'
  446. mysql_join_binary_packages_in_repositories_binary_packages
  447. mysql_join_binary_packages_in_repositories_repositories
  448. mysql_join_binary_packages_architectures
  449. mysql_join_repositories_architectures '' 'r_a'
  450. printf ';\n'
  451. } | \
  452. mysql_run_query | \
  453. tr '\t' ' ' | \
  454. grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
  455. while read -r what content; do
  456. printf '%s\n' "${content}" >> \
  457. "${tmp_dir}/${what}"
  458. done
  459. if [ ! -s "${tmp_dir}/repositories" ]; then
  460. >&2 printf 'Nothing to move here (from %s/%s).\n' \
  461. "${arch}" \
  462. "${source_stability}"
  463. continue
  464. fi
  465. touch \
  466. "${tmp_dir}/mv" \
  467. "${tmp_dir}/mv.id" \
  468. "${tmp_dir}/rm" \
  469. "${tmp_dir}/rm.id"
  470. # shellcheck disable=SC2086
  471. for s in "${tmp_dir}/"*; do
  472. sort -u "${s}" | \
  473. sponge "${s}"
  474. done
  475. # receive the repository databases from the master mirror
  476. mkdir "${tmp_dir}/dbs"
  477. while read -r arch repo; do
  478. mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
  479. # shellcheck disable=SC2086
  480. failsafe_rsync \
  481. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
  482. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
  483. "${tmp_dir}/dbs/${arch}/${repo}/"
  484. done < \
  485. "${tmp_dir}/repositories"
  486. # remove to-be-deleted packages
  487. # shellcheck disable=SC2094
  488. cut -d' ' -f2,3 < \
  489. "${tmp_dir}/rm" | \
  490. sort -u | \
  491. while read -r arch repo; do
  492. grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
  493. sed '
  494. s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
  495. ' | \
  496. xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
  497. done
  498. # copy and delete moved packages
  499. # shellcheck disable=SC2094
  500. cut -d' ' -f2,3,4,5 < \
  501. "${tmp_dir}/mv" | \
  502. sort -u | \
  503. while read -r from_arch from_repo to_arch to_repo; do
  504. grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
  505. sed '
  506. s/-[^-]\+\( \S\+\)\{4\}$//
  507. ' | \
  508. xargs -r "${base_dir}/bin/repo-copy" \
  509. "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
  510. "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
  511. grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
  512. sed '
  513. s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
  514. ' | \
  515. xargs -r repo-remove -q \
  516. "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
  517. done
  518. # create real file names of packages, because
  519. # mysql_query_and_delete_unneeded_binary_packages does so, too
  520. sed -i '
  521. s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
  522. ' "${tmp_dir}/rm"
  523. # somewhat inaccurate
  524. if ! ${no_action}; then
  525. {
  526. printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  527. printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
  528. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
  529. "${tmp_dir}/mv.id" 'moved_bpir' \
  530. "${tmp_dir}/rm.id" 'replaced_bpir'
  531. printf 'DELETE `binary_packages_in_repositories`'
  532. printf ' FROM `binary_packages_in_repositories`'
  533. printf ' JOIN `replaced_bpir`'
  534. printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
  535. mysql_query_and_delete_unneeded_binary_packages
  536. printf 'UPDATE `binary_packages_in_repositories`'
  537. printf ' JOIN `moved_bpir`'
  538. printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
  539. printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
  540. printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
  541. printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
  542. } | \
  543. mysql_run_query | \
  544. sort -u >> \
  545. "${tmp_dir}/rm"
  546. fi
  547. # move the packages remotely via sftp
  548. {
  549. sed '
  550. s/^/rm "/
  551. s/$/"/
  552. p
  553. s/"$/.sig"/
  554. ' "${tmp_dir}/rm"
  555. sed '
  556. s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
  557. ' "${tmp_dir}/mv"
  558. echo 'quit'
  559. } | \
  560. if ${no_action}; then
  561. sed 's|^|sftp: |'
  562. else
  563. failsafe_sftp
  564. fi
  565. if ${no_action}; then
  566. continue
  567. fi
  568. # and push our local *.db.tar.gz via rsync
  569. while read -r arch repo; do
  570. recompress_gz \
  571. "${tmp_dir}" \
  572. "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
  573. "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
  574. # shellcheck disable=SC2086
  575. failsafe_rsync \
  576. "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
  577. "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
  578. "${master_mirror_rsync_directory}/${arch}/${repo}/"
  579. done < \
  580. "${tmp_dir}/repositories"
  581. done
  582. trigger_mirror_refreshs