Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

seed-build-list 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701
  1. #!/bin/sh
  2. # shellcheck disable=SC2119,SC2120
  3. # seed the build list from various sources
  4. #
  5. # This uses the following schedule:
  6. # - find scheduleworthy packages by pkgname/pkgbase (diff to mirror,
  7. # broken dependencies, explicite regex) and their architecture
  8. # - find pkgbases of those packages (ask upstream, if necessary)
  9. # - remove all ignored packages (seed-igonre, explicitely ignored,
  10. # blacklist)
  11. # - run mysql_generate_package_metadata on each of them once (ignoring
  12. # the architecture for now)
  13. # shellcheck source=../lib/load-configuration
  14. . "${0%/*}/../lib/load-configuration"
  15. # TODO: allow to reschedule filtered by architecture
  16. # shellcheck disable=SC2016
  17. usage() {
  18. >&2 echo ''
  19. >&2 echo 'seed-build-list [options]:'
  20. >&2 echo ' seed the build list from different sources'
  21. >&2 echo ''
  22. >&2 echo 'possible options:'
  23. >&2 echo ' -a|--auto:'
  24. >&2 echo ' Automatically reschedule packages which have run-time'
  25. >&2 echo ' dependencies that are not available from any real'
  26. >&2 echo ' repository and which cannot be replaced by another,'
  27. >&2 echo ' less stable version of the same package.'
  28. >&2 echo ' -f|--force'
  29. >&2 echo ' Do not exit if mysql_generate_package_metadata() failed.'
  30. >&2 echo ' -h|--help:'
  31. >&2 echo ' Show this help and exit.'
  32. >&2 echo ' -i|--ignore $pkgbase|$arch/$pkgbase:'
  33. >&2 echo ' Do not update the given package (for the given $arch).'
  34. >&2 echo ' -j|--jostle'
  35. >&2 echo ' Give new build assignments the highest priority.'
  36. >&2 echo ' -m|--mirror $url:'
  37. >&2 echo ' Schedule all packages, that are newer (this includes'
  38. >&2 echo ' packages currently unavailable on archlinux32) on the'
  39. >&2 echo ' given x86_64 mirror - except packages on the blacklist'
  40. >&2 echo ' or in conf/seed-ignore-packages.'
  41. >&2 echo ' -n|--no-action:'
  42. >&2 echo ' Do not actually update build-list, just print it.'
  43. >&2 echo ' -p|--package $pkg_regex:'
  44. >&2 echo ' Reschedule packages with matching pkgname or pkgbase.'
  45. >&2 echo ' Note, that these packages must be known to the database.'
  46. >&2 echo ' -w|--wait:'
  47. >&2 echo ' Wait for lock if necessary.'
  48. [ -z "$1" ] && exit 1 || exit "$1"
  49. }
  50. tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
  51. # shellcheck disable=SC2064
  52. trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
  53. eval set -- "$(
  54. getopt -o afhi:jm:np:w \
  55. --long auto \
  56. --long force \
  57. --long help \
  58. --long ignore: \
  59. --long jostle \
  60. --long mirror: \
  61. --long no-action \
  62. --long package: \
  63. --long wait \
  64. -n "$(basename "$0")" -- "$@" || \
  65. echo usage
  66. )"
  67. auto=false
  68. ignore_mysql_generate_package_metadata_errors=false
  69. jostle=false
  70. update=true
  71. wait_for_lock='-n'
  72. while true
  73. do
  74. case "$1" in
  75. -a|--auto)
  76. auto=true
  77. ;;
  78. -f|--force)
  79. ignore_mysql_generate_package_metadata_errors=true
  80. ;;
  81. -h|--help)
  82. usage 0
  83. ;;
  84. -i|--ignore)
  85. shift
  86. if [ -z "${1%%*/*}" ]; then
  87. printf '%s\n' "$1" \
  88. | tr '/' '\t'
  89. else
  90. printf 'any\t%s\n' "$1"
  91. fi \
  92. >> "${tmp_dir}/ignore-packages"
  93. ;;
  94. -j|--jostle)
  95. jostle=true
  96. ;;
  97. -m|--mirror)
  98. shift
  99. printf '%s\n' "$1" >> \
  100. "${tmp_dir}/mirrors"
  101. ;;
  102. -n|--no-action)
  103. update=false
  104. ;;
  105. -p|--package)
  106. shift
  107. printf '%s\n' "$1" >> \
  108. "${tmp_dir}/package-regexes"
  109. ;;
  110. -w|--wait)
  111. wait_for_lock=''
  112. ;;
  113. --)
  114. shift
  115. break
  116. ;;
  117. *)
  118. >&2 printf 'Whoops, forgot to implement option "%s" internally.\n' \
  119. "$1"
  120. exit 42
  121. ;;
  122. esac
  123. shift
  124. done
  125. if [ $# -ne 0 ]; then
  126. usage 1
  127. fi
  128. if [ ! -s "${tmp_dir}/mirrors" ] && \
  129. [ ! -s "${tmp_dir}/package-regexes" ] && \
  130. ! ${auto}; then
  131. # nothing to do
  132. >&2 echo 'No options given to do anything.'
  133. usage 0
  134. exit 0
  135. fi
  136. # get locks
  137. if ${update}; then
  138. exec 9> "${sanity_check_lock_file}"
  139. if ! verbose_flock -s ${wait_for_lock} 9; then
  140. >&2 echo 'Cannot get sanity-check lock.'
  141. exit 1
  142. fi
  143. exec 8> "${build_list_lock_file}"
  144. if ! verbose_flock ${wait_for_lock} 8; then
  145. >&2 echo 'Cannot get build-list lock.'
  146. exit 1
  147. fi
  148. fi
  149. # shellcheck disable=SC2016
  150. repos=$(
  151. {
  152. printf 'SELECT DISTINCT `repositories`.`name`'
  153. printf ' FROM `repositories`'
  154. # shellcheck disable=SC2154
  155. printf ' WHERE `repositories`.`stability`=%s;\n' \
  156. "${repository_stability_ids__stable}"
  157. } | \
  158. mysql_run_query
  159. printf '%s\n' 'multilib'
  160. )
  161. if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  162. sed '
  163. s@/@\t@
  164. t
  165. s/^/any\t/
  166. ' "${base_dir}/conf/seed-ignore-packages" >> \
  167. "${tmp_dir}/ignore-packages"
  168. fi
  169. # shellcheck disable=SC2154
  170. git -C "${repo_paths__archlinux32}" archive "${repo_heads__archlinux32}" -- blacklist \
  171. | tar -t \
  172. | sed '
  173. s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
  174. t
  175. d
  176. ' \
  177. | tee "${tmp_dir}/blacklist" \
  178. >> "${tmp_dir}/ignore-packages"
  179. for list in 'ignore-packages' 'blacklist'; do
  180. expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \
  181. < "${tmp_dir}/${list}" \
  182. | sponge "${tmp_dir}/${list}"
  183. done
  184. # TODO: translate lib32-* packages instead of ignoring them for -m
  185. # harvest pkgnames from mirror delta
  186. if [ -s "${tmp_dir}/mirrors" ]; then
  187. {
  188. # theirs
  189. while read -r mirror; do
  190. if [ -z "${mirror}" ]; then
  191. continue
  192. fi
  193. for repo in ${repos}; do
  194. curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
  195. tar -Oxz --wildcards '*/desc' | \
  196. sed '
  197. /^%FILENAME%$/!d
  198. N
  199. s/^.*\n//
  200. /^lib32-/d
  201. s/^\(.*-\)x86_64\.pkg\.tar\.xz$/\1i486\n\1i686\n\1pentium4/
  202. s/^\(.*-any\)\.pkg\.tar\.xz$/\1/
  203. ' | \
  204. sed '
  205. s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
  206. '
  207. done
  208. done < \
  209. "${tmp_dir}/mirrors"
  210. # ours
  211. # shellcheck disable=SC2016
  212. {
  213. printf 'SELECT '
  214. mysql_package_name_query
  215. printf ' FROM `binary_packages`'
  216. mysql_join_binary_packages_architectures
  217. mysql_join_binary_packages_binary_packages_in_repositories
  218. mysql_join_binary_packages_in_repositories_repositories
  219. printf ' WHERE `repositories`.`is_on_master_mirror`'
  220. printf ' OR `repositories`.`name`="build-list"'
  221. } | \
  222. mysql_run_query | \
  223. sed '
  224. s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)\.pkg\.tar\.xz$/ours \2 \4 \1/
  225. s/^\(.* \)any\( \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
  226. '
  227. } | \
  228. expand_version 2 | \
  229. sort -k3,4 -k2Vr,2 -k1,1 | \
  230. shrink_version 2 | \
  231. uniq -f2 | \
  232. sed -n '
  233. s/^theirs \(\S\+ \)//
  234. T
  235. p
  236. ' | \
  237. if [ -r "${tmp_dir}/ignore-packages" ]; then
  238. grep -vxFf "${tmp_dir}/ignore-packages"
  239. else
  240. cat
  241. fi | \
  242. sort -u >> \
  243. "${tmp_dir}/pkgnames"
  244. fi
  245. # harvest from the package-regexes
  246. if [ -s "${tmp_dir}/package-regexes" ]; then
  247. # shellcheck disable=SC2016
  248. {
  249. printf 'CREATE TEMPORARY TABLE `names` (`name` VARCHAR(64));\n'
  250. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `names` (`name`);\n' \
  251. "${tmp_dir}/package-regexes"
  252. printf 'SELECT DISTINCT "pkgbase",`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
  253. printf ' FROM `names`'
  254. printf ' JOIN `package_sources`'
  255. printf ' ON `package_sources`.`pkgbase` REGEXP `names`.`name`'
  256. mysql_join_package_sources_upstream_repositories
  257. printf ';\n'
  258. printf 'SELECT DISTINCT "pkgname",`binary_packages`.`pkgname`'
  259. printf ' FROM `names`'
  260. printf ' JOIN `binary_packages`'
  261. printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
  262. } | \
  263. mysql_run_query | \
  264. tr '\t' ' ' | \
  265. sed -n '
  266. /^pkgbase / {
  267. s/^\S\+ /any /
  268. w /dev/stdout
  269. d
  270. }
  271. /^pkgname / {
  272. s/^\S\+ /any /
  273. w /dev/stderr
  274. d
  275. }
  276. ' \
  277. >>"${tmp_dir}/pkgbases" \
  278. 2>>"${tmp_dir}/pkgnames"
  279. fi
  280. # shellcheck disable=SC2016
  281. {
  282. # create pkgbases to given pkgnames
  283. if [ -s "${tmp_dir}/pkgnames" ]; then
  284. printf 'CREATE TEMPORARY TABLE `pkgnames` ('
  285. printf '`architecture` VARCHAR(16),'
  286. printf '`pkgname` VARCHAR(64),'
  287. printf 'UNIQUE KEY `content`(`architecture`,`pkgname`)'
  288. printf ');\n'
  289. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` COLUMNS TERMINATED BY " " (`architecture`,`pkgname`);\n' \
  290. "${tmp_dir}/pkgnames"
  291. printf 'SELECT DISTINCT'
  292. printf ' "pkgbase",'
  293. printf '`pkgnames`.`architecture`,'
  294. printf '`package_sources`.`pkgbase`,'
  295. printf '`upstream_repositories`.`name`'
  296. printf ' FROM `pkgnames`'
  297. printf ' JOIN `binary_packages`'
  298. printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
  299. mysql_join_binary_packages_build_assignments
  300. mysql_join_build_assignments_package_sources
  301. mysql_join_package_sources_upstream_repositories
  302. printf ';\n'
  303. printf 'SELECT DISTINCT'
  304. printf ' "pkgname",'
  305. printf '`pkgnames`.`architecture`,'
  306. printf '`pkgnames`.`pkgname`'
  307. printf ' FROM `package_sources`'
  308. mysql_join_package_sources_upstream_repositories
  309. mysql_join_package_sources_build_assignments
  310. mysql_join_build_assignments_binary_packages
  311. printf ' RIGHT JOIN `pkgnames`'
  312. printf ' ON `pkgnames`.`pkgname`=`binary_packages`.`pkgname`'
  313. printf ' WHERE `package_sources`.`pkgbase` IS NULL;\n'
  314. printf 'DROP TEMPORARY TABLE `pkgnames`;\n'
  315. fi
  316. # auto-detect pkgbases
  317. if ${auto}; then
  318. # schedule any package, that:
  319. # 1) is not on the build-list currently and
  320. # 2) has some dependency which is not provided by any package which
  321. # a) is also on the build-list or
  322. # b) is the least stable package with that name and for that architecture
  323. printf 'CREATE TEMPORARY TABLE `least_stable_bp%s`(`id` BIGINT, `is_on_build_list` BIT(1), PRIMARY KEY `id`(`id`));\n' \
  324. '' '_copy'
  325. # add least-stable built packages
  326. printf 'INSERT IGNORE INTO `least_stable_bp`(`id`,`is_on_build_list`)'
  327. printf ' SELECT `binary_packages`.`id`,'
  328. printf ' `bl_bpir`.`id` IS NOT NULL'
  329. printf ' FROM `binary_packages`'
  330. mysql_join_binary_packages_binary_packages_in_repositories
  331. mysql_join_binary_packages_in_repositories_repositories
  332. printf ' JOIN ('
  333. printf 'SELECT `binary_packages`.`pkgname`,'
  334. printf '`binary_packages`.`architecture`,'
  335. # TODO: this might become wrong if we ever change the order of the
  336. # rows in repository_stabilities -- we might want to refer to
  337. # repository_stability_relations here somehow ...
  338. printf 'MAX(`repositories`.`stability`) AS `stability`'
  339. printf ' FROM `binary_packages`'
  340. mysql_join_binary_packages_binary_packages_in_repositories
  341. mysql_join_binary_packages_in_repositories_repositories
  342. printf ' WHERE `repositories`.`is_on_master_mirror`'
  343. printf ' AND `repositories`.`name`!="build-support"'
  344. printf ' GROUP BY CONCAT('
  345. printf '`binary_packages`.`pkgname`,"-",'
  346. printf '`binary_packages`.`architecture`,"-",'
  347. printf '`repositories`.`architecture`'
  348. printf ')'
  349. printf ') AS `sub_q`'
  350. printf ' ON `sub_q`.`pkgname`=`binary_packages`.`pkgname`'
  351. printf ' AND `sub_q`.`architecture`=`binary_packages`.`architecture`'
  352. printf ' AND `sub_q`.`stability`=`repositories`.`stability`'
  353. printf ' LEFT JOIN ('
  354. printf '`binary_packages` AS `bl_bp`'
  355. mysql_join_binary_packages_binary_packages_in_repositories 'bl_bp' 'bl_bpir'
  356. printf ' AND `bl_bpir`.`repository`=%s' \
  357. "${repository_ids__any_build_list}"
  358. printf ')'
  359. printf ' ON `bl_bp`.`pkgname`=`binary_packages`.`pkgname`'
  360. printf ' AND ('
  361. printf '`bl_bp`.`architecture`=`binary_packages`.`architecture`'
  362. printf ' OR `bl_bp`.`architecture`=%s' \
  363. "${architecture_ids__any}"
  364. printf ' OR `binary_packages`.`architecture`=%s' \
  365. "${architecture_ids__any}"
  366. printf ');\n'
  367. # add all build-list packages
  368. printf 'INSERT IGNORE INTO `least_stable_bp`(`id`,`is_on_build_list`)'
  369. printf ' SELECT `binary_packages_in_repositories`.`package`,1'
  370. printf ' FROM `binary_packages_in_repositories`'
  371. printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
  372. "${repository_ids__any_build_list}"
  373. # this copy shall hold the install-target-providers
  374. printf 'INSERT IGNORE INTO `least_stable_bp_copy`(`id`,`is_on_build_list`)'
  375. printf ' SELECT `least_stable_bp`.`id`,'
  376. printf '`least_stable_bp`.`is_on_build_list`'
  377. printf ' FROM `least_stable_bp`;\n'
  378. # remove packages that should never be rescheduled
  379. printf 'DELETE `least_stable_bp`'
  380. printf ' FROM `least_stable_bp`'
  381. printf ' JOIN `binary_packages`'
  382. printf ' ON `binary_packages`.`id`=`least_stable_bp`.`id`'
  383. # 1)
  384. printf ' WHERE `least_stable_bp`.`is_on_build_list`;'
  385. printf 'SELECT DISTINCT "pkgbase",`ba_a`.`name`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
  386. printf ' FROM `least_stable_bp`'
  387. printf ' JOIN `binary_packages`'
  388. printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`'
  389. mysql_join_binary_packages_dependencies
  390. mysql_join_dependencies_dependency_types
  391. printf ' AND `dependency_types`.`relevant_for_binary_packages`'
  392. mysql_join_dependencies_versions '' 'd_v'
  393. mysql_join_binary_packages_build_assignments
  394. mysql_join_build_assignments_architectures '' 'ba_a'
  395. mysql_join_build_assignments_package_sources
  396. mysql_join_package_sources_upstream_repositories
  397. # some dependencies are not provided by the least stable packages
  398. printf ' WHERE NOT EXISTS ('
  399. printf 'SELECT 1'
  400. printf ' FROM `least_stable_bp_copy`'
  401. printf ' JOIN `binary_packages` AS `itp_bp`'
  402. printf ' ON `least_stable_bp_copy`.`id`=`itp_bp`.`id`'
  403. mysql_join_binary_packages_install_target_providers 'itp_bp'
  404. mysql_join_install_target_providers_versions '' 'itp_v'
  405. # must provide the right install_target
  406. printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
  407. # must provide the right version
  408. printf ' AND ('
  409. mysql_query_ordering_correct \
  410. '`d_v`.`order`' \
  411. '`itp_v`.`order`' \
  412. '`dependencies`.`version_relation`'
  413. # or be on the build-list
  414. printf ' OR `least_stable_bp_copy`.`is_on_build_list`'
  415. # must provide the right architecture
  416. printf ') AND ('
  417. printf '`itp_bp`.`architecture`=`binary_packages`.`architecture`'
  418. printf ' OR `itp_bp`.`architecture`=%s' \
  419. "${architecture_ids__any}"
  420. printf ' OR `binary_packages`.`architecture`=%s' \
  421. "${architecture_ids__any}"
  422. printf ')'
  423. printf ');\n'
  424. printf 'DROP TEMPORARY TABLE `least_stable_bp%s`;\n' \
  425. '' '_copy'
  426. fi
  427. } | \
  428. mysql_run_query | \
  429. tr '\t' ' ' | \
  430. sed -n '
  431. /^pkgbase / {
  432. s/^\S\+ //
  433. w /dev/stderr
  434. d
  435. }
  436. /^pkgname / {
  437. s/^\S\+ //
  438. w /dev/stdout
  439. d
  440. }
  441. ' \
  442. 2>>"${tmp_dir}/pkgbases" \
  443. | sponge "${tmp_dir}/pkgnames"
  444. if [ ! -s "${tmp_dir}/pkgbases" ] && \
  445. [ ! -s "${tmp_dir}/pkgnames" ]; then
  446. >&2 echo 'No packages matched.'
  447. exit
  448. fi
  449. for s in "${tmp_dir}/pkgbases" "${tmp_dir}/pkgnames"; do
  450. if [ -s "${s}" ]; then
  451. sort -u "${s}" | \
  452. sponge "${s}"
  453. fi
  454. done
  455. # pkgnames -> pkgbases (with help from upstream)
  456. if [ -s "${tmp_dir}/pkgnames" ]; then
  457. while read -r architecture pkgname; do
  458. content=$(
  459. curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
  460. tr ',' '\n'
  461. )
  462. repo=$(
  463. printf '%s\n' "${content}" | \
  464. sed -n '
  465. s/^\s*"repo"\s*:\s*"//
  466. T
  467. s/".*$//
  468. T
  469. p
  470. '
  471. )
  472. pkgbase=$(
  473. printf '%s\n' "${content}" | \
  474. sed -n '
  475. s/^\s*"pkgbase"\s*:\s*"//
  476. T
  477. s/".*$//
  478. T
  479. p
  480. '
  481. )
  482. if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
  483. printf '%s %s\n' "${architecture}" "${pkgname}"
  484. continue
  485. fi
  486. printf '%s %s %s\n' \
  487. "${architecture}" \
  488. "${pkgbase}" \
  489. "${repo}" >> \
  490. "${tmp_dir}/pkgbases"
  491. done < \
  492. "${tmp_dir}/pkgnames" | \
  493. sponge "${tmp_dir}/pkgnames"
  494. fi
  495. if [ -s "${tmp_dir}/pkgnames" ]; then
  496. >&2 echo 'Could not find some packages upstream:'
  497. >&2 cat "${tmp_dir}/pkgnames"
  498. exit 2
  499. fi
  500. sort -u "${tmp_dir}/pkgbases" \
  501. | sort -k3,3 \
  502. | sponge "${tmp_dir}/pkgbases"
  503. # get the current HEADs
  504. # shellcheck disable=SC2016
  505. {
  506. printf 'SELECT'
  507. printf ' `upstream_repositories`.`name`,'
  508. printf '`git_repositories`.`head`'
  509. printf ' FROM `upstream_repositories`'
  510. mysql_join_upstream_repositories_git_repositories
  511. } \
  512. | mysql_run_query \
  513. | tr '\t' ' ' \
  514. | sort -k1,1 \
  515. | join -1 3 -2 1 -o 1.1,1.2,1.3,2.2 "${tmp_dir}/pkgbases" - \
  516. | sponge "${tmp_dir}/pkgbases"
  517. if ${update}; then
  518. cut -d' ' -f 2,3,4 "${tmp_dir}/pkgbases" \
  519. | sort -u \
  520. | while read -r pkgbase repo git_rev; do
  521. success=false
  522. # shellcheck disable=SC2154
  523. for gr_r in \
  524. "${git_rev}:${repo}" \
  525. "${repo_heads__packages}:core" \
  526. "${repo_heads__packages}:extra" \
  527. "${repo_heads__community}:community"; do
  528. printf '%s ' "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}" | \
  529. sed 's/ $/\n/'
  530. if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}"; then
  531. success=true
  532. break
  533. fi
  534. done
  535. if ! ${success} && \
  536. ! ${ignore_mysql_generate_package_metadata_errors}; then
  537. exit 2
  538. fi
  539. done
  540. cut -d' ' -f1 < \
  541. "${tmp_dir}/pkgbases" | \
  542. sort -u | \
  543. sponge "${tmp_dir}/pkgbases"
  544. # shellcheck disable=SC2016
  545. {
  546. printf 'CREATE TEMPORARY TABLE `pkgbases` ('
  547. printf '`pkgbase` VARCHAR (64),'
  548. printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
  549. printf ');\n'
  550. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \
  551. "${tmp_dir}/pkgbases"
  552. # TODO: this seems to be broken (deletes too few packages) - still valid
  553. # after implementing the correct blacklisting below?
  554. printf 'DELETE `d_bpir`'
  555. printf ' FROM `pkgbases`'
  556. # the package on the deletion-list ...
  557. printf ' JOIN `package_sources` AS `d_ps`'
  558. printf ' ON `pkgbases`.`pkgbase`=`d_ps`.`pkgbase`'
  559. mysql_join_package_sources_build_assignments 'd_ps' 'd_ba'
  560. mysql_join_build_assignments_binary_packages 'd_ba' 'd_bp'
  561. mysql_join_binary_packages_binary_packages_in_repositories 'd_bp' 'd_bpir'
  562. mysql_join_binary_packages_in_repositories_repositories 'd_bpir' 'd_r'
  563. # shellcheck disable=SC2154
  564. printf ' AND `d_r`.`stability`=%s' \
  565. "${repository_stability_ids__forbidden}"
  566. # ... needs a counter-part on the build-list (which was just created)
  567. printf ' JOIN `package_sources` AS `a_ps`'
  568. printf ' ON `d_ps`.`pkgbase`=`a_ps`.`pkgbase`'
  569. mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
  570. mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
  571. printf ' JOIN `architecture_compatibilities` AS `ac`'
  572. printf ' ON `ac`.`built_for`=`a_bp`.`architecture`'
  573. printf ' AND `ac`.`runs_on`=`d_r`.`architecture`'
  574. printf ' AND `ac`.`fully_compatible`'
  575. mysql_join_binary_packages_binary_packages_in_repositories 'a_bp' 'a_bpir'
  576. printf ' AND `a_bpir`.`repository`=%s;\n' \
  577. "${repository_ids__any_build_list}"
  578. if ${jostle}; then
  579. printf 'SET @`max_priority`=('
  580. printf 'SELECT'
  581. printf ' MAX(`build_assignments`.`priority`)+1'
  582. printf ' FROM `build_assignments`'
  583. printf ');\n'
  584. printf 'UPDATE `build_assignments`'
  585. mysql_join_build_assignments_package_sources
  586. printf ' JOIN `pkgbases`'
  587. printf ' ON `pkgbases`.`pkgbase`=`package_sources`.`pkgbase`'
  588. printf ' SET `build_assignments`.`priority`=@`max_priority`'
  589. printf ';\n'
  590. fi
  591. printf 'CREATE TEMPORARY TABLE `blacklist` ('
  592. printf '`architecture` VARCHAR (16),'
  593. printf '`pkgbase` VARCHAR (64),'
  594. printf 'UNIQUE KEY `content`(`architecture`,`pkgbase`)'
  595. printf ');\n'
  596. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `blacklist` COLUMNS TERMINATED BY " " (`architecture`,`pkgbase`);\n' \
  597. "${tmp_dir}/blacklist"
  598. printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
  599. printf '`package`,'
  600. printf '`repository`,'
  601. printf '`is_to_be_deleted`'
  602. printf ')'
  603. printf ' SELECT'
  604. printf ' `binary_packages`.`id`,'
  605. printf '`dl_r`.`id`,'
  606. printf '1'
  607. printf ' FROM `blacklist`'
  608. printf ' JOIN `package_sources`'
  609. printf ' ON `blacklist`.`pkgbase`=`package_sources`.`pkgbase`'
  610. mysql_join_package_sources_build_assignments
  611. mysql_join_build_assignments_architectures
  612. printf ' AND `architectures`.`name`=`blacklist`.`architecture`'
  613. mysql_join_build_assignments_binary_packages
  614. mysql_join_binary_packages_binary_packages_in_repositories
  615. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  616. "${repository_ids__any_build_list}"
  617. printf ' JOIN `architecture_compatibilities` AS `ac`'
  618. printf ' ON `ac`.`built_for`=`binary_packages`.`architecture`'
  619. printf ' AND `ac`.`fully_compatible`'
  620. printf ' JOIN `repositories` AS `dl_r`'
  621. printf ' ON `ac`.`runs_on`=`dl_r`.`architecture`'
  622. # shellcheck disable=SC2154
  623. printf ' AND `dl_r`.`stability`=%s;\n' \
  624. "${repository_stability_ids__forbidden}"
  625. printf 'DELETE `binary_packages_in_repositories`'
  626. printf ' FROM `blacklist`'
  627. printf ' JOIN `package_sources`'
  628. printf ' ON `blacklist`.`pkgbase`=`package_sources`.`pkgbase`'
  629. mysql_join_package_sources_build_assignments
  630. mysql_join_build_assignments_architectures
  631. printf ' AND `architectures`.`name`=`blacklist`.`architecture`'
  632. mysql_join_build_assignments_binary_packages
  633. mysql_join_binary_packages_binary_packages_in_repositories
  634. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  635. "${repository_ids__any_build_list}"
  636. printf ' AND ('
  637. printf '`binary_packages`.`architecture`=`architectures`.`id`'
  638. printf ' OR `architectures`.`id`=%s' \
  639. "${architecture_ids__any}"
  640. printf ');\n'
  641. } | \
  642. mysql_run_query
  643. mysql_cleanup
  644. mysql_sort_versions
  645. mysql_find_build_assignment_loops
  646. # remove duplicate binary_packages from "build-list"
  647. mysql_query_remove_old_binary_packages_from_build_list | \
  648. mysql_run_query
  649. else
  650. cat "${tmp_dir}/pkgbases"
  651. fi