Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

seed-build-list 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726
  1. #!/bin/sh
  2. # shellcheck disable=SC2119,SC2120
  3. # shellcheck source=../lib/load-configuration
  4. . "${0%/*}/../lib/load-configuration"
  5. # TODO: allow to filter by architecture
  6. # TODO: -a|--auto schedules too much - or is the scheduling really /that/ broken?
  7. # TODO: -u and -m should be merged
  8. # shellcheck disable=SC2016
  9. usage() {
  10. >&2 echo ''
  11. >&2 echo 'seed-build-list [options]:'
  12. >&2 echo ' seed the build list from different sources'
  13. >&2 echo ''
  14. >&2 echo 'possible options:'
  15. >&2 echo ' -a|--auto:'
  16. >&2 echo ' Automatically reschedule packages which have run-time'
  17. >&2 echo ' dependencies that are not available from any real'
  18. >&2 echo ' repository and which cannot be replaced by another,'
  19. >&2 echo ' less stable version of the same package.'
  20. >&2 echo ' -f|--force'
  21. >&2 echo ' Do not exit if mysql_generate_package_metadata() failed.'
  22. >&2 echo ' -h|--help:'
  23. >&2 echo ' Show this help and exit.'
  24. >&2 echo ' -i|--ignore $pkgbase:'
  25. >&2 echo ' Do not update the given package.'
  26. >&2 echo ' -j|--jostle'
  27. >&2 echo ' Give new build assignments the highest priority.'
  28. >&2 echo ' -m|--mirror $url:'
  29. >&2 echo ' Schedule all packages, that are newer on the given'
  30. >&2 echo ' x86_64 mirror - except packages in'
  31. >&2 echo ' conf/seed-ignore-packages.'
  32. >&2 echo ' -n|--no-action:'
  33. >&2 echo ' Do not actually update build-list, just print it.'
  34. >&2 echo ' -p|--package $pkg_regex:'
  35. >&2 echo ' Reschedule packages with matching pkgname or pkgbase.'
  36. >&2 echo ' Note, that these packages must be known to the database.'
  37. >&2 echo ' -u|--undelete $url: '
  38. >&2 echo ' Schedule all former deletion-list packages which do not'
  39. >&2 echo ' belong on the deletion-list anymore (e.g. can be built)'
  40. >&2 echo ' and which are available on the given x86_64 mirror.'
  41. >&2 echo ' -w|--wait:'
  42. >&2 echo ' Wait for lock if necessary.'
  43. [ -z "$1" ] && exit 1 || exit "$1"
  44. }
  45. tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
  46. # shellcheck disable=SC2064
  47. trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
  48. eval set -- "$(
  49. getopt -o afhi:jm:np:u:w \
  50. --long auto \
  51. --long force \
  52. --long help \
  53. --long ignore: \
  54. --long jostle \
  55. --long mirror: \
  56. --long no-action \
  57. --long package: \
  58. --long undelete: \
  59. --long wait \
  60. -n "$(basename "$0")" -- "$@" || \
  61. echo usage
  62. )"
  63. auto=false
  64. ignore_mysql_generate_package_metadata_errors=false
  65. jostle=false
  66. update=true
  67. wait_for_lock='-n'
  68. if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  69. sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \
  70. "${tmp_dir}/ignore-packages"
  71. fi
  72. while true
  73. do
  74. case "$1" in
  75. -a|--auto)
  76. auto=true
  77. ;;
  78. -f|--force)
  79. ignore_mysql_generate_package_metadata_errors=true
  80. ;;
  81. -h|--help)
  82. usage 0
  83. ;;
  84. -i|--ignore)
  85. shift
  86. printf '3\t%s\n' "$1" >> \
  87. "${tmp_dir}/ignore-packages"
  88. ;;
  89. -j|--jostle)
  90. jostle=true
  91. ;;
  92. -m|--mirror)
  93. shift
  94. printf '%s\n' "$1" >> \
  95. "${tmp_dir}/mirrors"
  96. ;;
  97. -n|--no-action)
  98. update=false
  99. ;;
  100. -p|--package)
  101. shift
  102. printf '%s\n' "$1" >> \
  103. "${tmp_dir}/package-regexes"
  104. ;;
  105. -u|--undelete)
  106. shift
  107. printf '%s\n' "$1" >> \
  108. "${tmp_dir}/undelete-mirrors"
  109. ;;
  110. -w|--wait)
  111. wait_for_lock=''
  112. ;;
  113. --)
  114. shift
  115. break
  116. ;;
  117. *)
  118. >&2 printf 'Whoops, forgot to implement option "%s" internally.\n' \
  119. "$1"
  120. exit 42
  121. ;;
  122. esac
  123. shift
  124. done
  125. if [ $# -ne 0 ]; then
  126. usage 1
  127. fi
  128. if [ ! -s "${tmp_dir}/mirrors" ] && \
  129. [ ! -s "${tmp_dir}/package-regexes" ] && \
  130. [ ! -s "${tmp_dir}/undelete-mirrors" ] && \
  131. ! ${auto}; then
  132. # nothing to do
  133. >&2 echo 'No options given to do anything.'
  134. usage 0
  135. exit 0
  136. fi
  137. # get locks
  138. if ${update}; then
  139. exec 9> "${sanity_check_lock_file}"
  140. if ! verbose_flock -s ${wait_for_lock} 9; then
  141. >&2 echo 'Cannot get sanity-check lock.'
  142. exit 1
  143. fi
  144. exec 8> "${build_list_lock_file}"
  145. if ! verbose_flock ${wait_for_lock} 8; then
  146. >&2 echo 'Cannot get build-list lock.'
  147. exit 1
  148. fi
  149. fi
  150. # shellcheck disable=SC2016
  151. repos=$(
  152. {
  153. printf 'SELECT DISTINCT `repositories`.`name`'
  154. printf ' FROM `repositories`'
  155. # shellcheck disable=SC2154
  156. printf ' WHERE `repositories`.`stability`=%s;\n' \
  157. "${repository_stability_ids__stable}"
  158. } | \
  159. mysql_run_query
  160. printf '%s\n' 'multilib'
  161. )
  162. # TODO: translate lib32-* packages instead of ignoring them for -u and -m
  163. # harvest pkgnames from mirror delta
  164. if [ -s "${tmp_dir}/mirrors" ]; then
  165. {
  166. # theirs
  167. while read -r mirror; do
  168. if [ -z "${mirror}" ]; then
  169. continue
  170. fi
  171. for repo in ${repos}; do
  172. curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
  173. tar -Oxz --wildcards '*/desc' | \
  174. sed '
  175. /^%FILENAME%$/!d
  176. N
  177. s/^.*\n//
  178. /^lib32-/d
  179. s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
  180. ' | \
  181. sed '
  182. s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
  183. '
  184. done
  185. done < \
  186. "${tmp_dir}/mirrors"
  187. # ours
  188. # shellcheck disable=SC2016
  189. {
  190. printf 'SELECT '
  191. mysql_package_name_query
  192. printf ' FROM `binary_packages`'
  193. mysql_join_binary_packages_architectures
  194. mysql_join_binary_packages_binary_packages_in_repositories
  195. printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
  196. } | \
  197. mysql_run_query | \
  198. sed '
  199. s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)$/ours \2 \4 \1/
  200. s/^\(.* \)any\(\.\S\+ \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
  201. '
  202. } | \
  203. expand_version 2 | \
  204. sort -k3,4 -k2Vr,2 -k1,1 | \
  205. shrink_version 2 | \
  206. uniq -f2 | \
  207. sed -n '
  208. s/^theirs \(\S\+ \)//
  209. T
  210. s/^\(\S\+ \)//
  211. T
  212. p
  213. ' | \
  214. if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  215. grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
  216. else
  217. cat
  218. fi | \
  219. sed 's/^/1\t/' | \
  220. sort -u >> \
  221. "${tmp_dir}/pkgnames"
  222. fi
  223. # harvest pkgnames from undelete-mirrors
  224. if [ -s "${tmp_dir}/undelete-mirrors" ]; then
  225. while read -r mirror; do
  226. if [ -z "${mirror}" ]; then
  227. continue
  228. fi
  229. for repo in ${repos}; do
  230. curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
  231. tar -Oxz --wildcards '*/desc' | \
  232. sed '
  233. /^%FILENAME%$/!d
  234. N
  235. s/^.*\n//
  236. /^lib32-/d
  237. s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
  238. ' | \
  239. sed '
  240. s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/
  241. '
  242. done
  243. done < \
  244. "${tmp_dir}/undelete-mirrors" | \
  245. if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  246. grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
  247. else
  248. cat
  249. fi | \
  250. sed 's/^/2\t/' | \
  251. sort -u >> \
  252. "${tmp_dir}/pkgnames"
  253. fi
  254. # harvest from the package-regexes
  255. if [ -s "${tmp_dir}/package-regexes" ]; then
  256. # shellcheck disable=SC2016
  257. {
  258. printf 'CREATE TEMPORARY TABLE `names` (`name` VARCHAR(64));\n'
  259. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `names` (`name`);\n' \
  260. "${tmp_dir}/package-regexes"
  261. printf 'SELECT DISTINCT "pkgbase",`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
  262. printf ' FROM `names`'
  263. printf ' JOIN `package_sources`'
  264. printf ' ON `package_sources`.`pkgbase` REGEXP `names`.`name`'
  265. mysql_join_package_sources_upstream_repositories
  266. printf ';\n'
  267. printf 'SELECT DISTINCT "pkgname",`binary_packages`.`pkgname`'
  268. printf ' FROM `names`'
  269. printf ' JOIN `binary_packages`'
  270. printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
  271. } | \
  272. mysql_run_query | \
  273. sed -n '
  274. /^pkgbase\s/ {
  275. s/^\S\+\s/3\t/
  276. w /dev/stdout
  277. d
  278. }
  279. /^pkgname\s/ {
  280. s/^\S\+\s/3\t/
  281. w /dev/stderr
  282. d
  283. }
  284. ' \
  285. >>"${tmp_dir}/pkgbases" \
  286. 2>>"${tmp_dir}/pkgnames"
  287. fi
  288. # shellcheck disable=SC2016
  289. {
  290. # create pkgbases to given pkgnames
  291. if [ -s "${tmp_dir}/pkgnames" ]; then
  292. printf 'CREATE TEMPORARY TABLE `pkgnames` ('
  293. printf '`priority` SMALLINT,'
  294. printf '`pkgname` VARCHAR(64),'
  295. printf 'UNIQUE KEY `pkgname`(`pkgname`)'
  296. printf ');\n'
  297. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \
  298. "${tmp_dir}/pkgnames"
  299. printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
  300. printf ' FROM `pkgnames`'
  301. printf ' JOIN `binary_packages`'
  302. printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
  303. mysql_join_binary_packages_build_assignments
  304. mysql_join_build_assignments_package_sources
  305. mysql_join_package_sources_upstream_repositories
  306. printf ';\n'
  307. printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`'
  308. printf ' FROM `package_sources`'
  309. mysql_join_package_sources_upstream_repositories
  310. mysql_join_package_sources_build_assignments
  311. mysql_join_build_assignments_binary_packages
  312. printf ' RIGHT JOIN `pkgnames`'
  313. printf ' ON `pkgnames`.`pkgname`=`binary_packages`.`pkgname`'
  314. printf ' WHERE `package_sources`.`pkgbase` IS NULL;\n'
  315. printf 'DROP TEMPORARY TABLE `pkgnames`;\n'
  316. fi
  317. # auto-detect pkgbases
  318. if ${auto}; then
  319. # schedule any package, that:
  320. # 1) is not on the build-list currently and
  321. # 2) has some dependency which is not provided by any package which
  322. # a) is also on the build-list or
  323. # b) is the least stable package with that name and for that architecture
  324. printf 'CREATE TEMPORARY TABLE `least_stable_bp%s`(`id` BIGINT, `is_on_build_list` BIT(1), PRIMARY KEY `id`(`id`));\n' \
  325. '' '_copy'
  326. # add least-stable built packages
  327. printf 'INSERT IGNORE INTO `least_stable_bp`(`id`,`is_on_build_list`)'
  328. printf ' SELECT `binary_packages`.`id`,'
  329. printf ' `bl_bpir`.`id` IS NOT NULL'
  330. printf ' FROM `binary_packages`'
  331. mysql_join_binary_packages_binary_packages_in_repositories
  332. mysql_join_binary_packages_in_repositories_repositories
  333. printf ' JOIN ('
  334. printf 'SELECT `binary_packages`.`pkgname`,'
  335. printf '`binary_packages`.`architecture`,'
  336. # TODO: this might become wrong if we ever change the order of the
  337. # rows in repository_stabilities -- we might want to refer to
  338. # repository_stability_relations here somehow ...
  339. printf 'MAX(`repositories`.`stability`) AS `stability`'
  340. printf ' FROM `binary_packages`'
  341. mysql_join_binary_packages_binary_packages_in_repositories
  342. mysql_join_binary_packages_in_repositories_repositories
  343. printf ' WHERE `repositories`.`is_on_master_mirror`'
  344. printf ' AND `repositories`.`name`!="build-support"'
  345. printf ' GROUP BY CONCAT('
  346. printf '`binary_packages`.`pkgname`,"-",`binary_packages`.`architecture`'
  347. printf ')'
  348. printf ') AS `sub_q`'
  349. printf ' ON `sub_q`.`pkgname`=`binary_packages`.`pkgname`'
  350. printf ' AND `sub_q`.`architecture`=`binary_packages`.`architecture`'
  351. printf ' AND `sub_q`.`stability`=`repositories`.`stability`'
  352. printf ' LEFT JOIN ('
  353. printf '`binary_packages` AS `bl_bp`'
  354. mysql_join_binary_packages_binary_packages_in_repositories 'bl_bp' 'bl_bpir'
  355. printf ' AND `bl_bpir`.`repository`=%s' \
  356. "${repository_ids__any_build_list}"
  357. printf ')'
  358. printf ' ON `bl_bp`.`pkgname`=`binary_packages`.`pkgname`'
  359. printf ' AND ('
  360. printf '`bl_bp`.`architecture`=`binary_packages`.`architecture`'
  361. printf ' OR `bl_bp`.`architecture`=%s' \
  362. "${architecture_ids__any}"
  363. printf ' OR `binary_packages`.`architecture`=%s' \
  364. "${architecture_ids__any}"
  365. printf ');\n'
  366. # add all build-list packages
  367. printf 'INSERT IGNORE INTO `least_stable_bp`(`id`,`is_on_build_list`)'
  368. printf ' SELECT `binary_packages_in_repositories`.`package`,1'
  369. printf ' FROM `binary_packages_in_repositories`'
  370. printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
  371. "${repository_ids__any_build_list}"
  372. # this copy shall hold the install-target-providers
  373. printf 'INSERT IGNORE INTO `least_stable_bp_copy`(`id`,`is_on_build_list`)'
  374. printf ' SELECT `least_stable_bp`.`id`,'
  375. printf '`least_stable_bp`.`is_on_build_list`'
  376. printf ' FROM `least_stable_bp`;\n'
  377. # remove packages that should never be rescheduled
  378. printf 'DELETE `least_stable_bp`'
  379. printf ' FROM `least_stable_bp`'
  380. printf ' JOIN `binary_packages`'
  381. printf ' ON `binary_packages`.`id`=`least_stable_bp`.`id`'
  382. # 1)
  383. printf ' WHERE `least_stable_bp`.`is_on_build_list`;'
  384. printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
  385. printf ' FROM `least_stable_bp`'
  386. printf ' JOIN `binary_packages`'
  387. printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`'
  388. mysql_join_binary_packages_dependencies
  389. mysql_join_dependencies_dependency_types
  390. printf ' AND `dependency_types`.`relevant_for_binary_packages`'
  391. mysql_join_dependencies_versions '' 'd_v'
  392. mysql_join_binary_packages_build_assignments
  393. mysql_join_build_assignments_package_sources
  394. mysql_join_package_sources_upstream_repositories
  395. # some dependencies are not provided by the least stable packages
  396. printf ' WHERE NOT EXISTS ('
  397. printf 'SELECT 1'
  398. printf ' FROM `least_stable_bp_copy`'
  399. printf ' JOIN `binary_packages` AS `itp_bp`'
  400. printf ' ON `least_stable_bp_copy`.`id`=`itp_bp`.`id`'
  401. mysql_join_binary_packages_install_target_providers 'itp_bp'
  402. mysql_join_install_target_providers_versions '' 'itp_v'
  403. # must provide the right install_target
  404. printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
  405. # must provide the right version
  406. printf ' AND ('
  407. mysql_query_ordering_correct \
  408. '`d_v`.`order`' \
  409. '`itp_v`.`order`' \
  410. '`dependencies`.`version_relation`'
  411. # or be on the build-list
  412. printf ' OR `least_stable_bp_copy`.`is_on_build_list`'
  413. # must provide the right architecture
  414. printf ') AND ('
  415. printf '`itp_bp`.`architecture`=`binary_packages`.`architecture`'
  416. printf ' OR `itp_bp`.`architecture`=%s' \
  417. "${architecture_ids__any}"
  418. printf ' OR `binary_packages`.`architecture`=%s' \
  419. "${architecture_ids__any}"
  420. printf ')'
  421. printf ');\n'
  422. printf 'DROP TEMPORARY TABLE `least_stable_bp%s`;\n' \
  423. '' '_copy'
  424. fi
  425. } | \
  426. mysql_run_query | \
  427. sed -n '
  428. /^pkgbase\s/ {
  429. s/^\S\+\s//
  430. w /dev/stderr
  431. d
  432. }
  433. /^pkgname\s/ {
  434. s/^\S\+\s//
  435. w /dev/stdout
  436. d
  437. }
  438. ' \
  439. 2>>"${tmp_dir}/pkgbases" \
  440. | sponge "${tmp_dir}/pkgnames"
  441. if [ ! -s "${tmp_dir}/pkgbases" ] && \
  442. [ ! -s "${tmp_dir}/pkgnames" ]; then
  443. >&2 echo 'No packages matched.'
  444. exit
  445. fi
  446. for s in "${tmp_dir}/pkgbases" "${tmp_dir}/pkgnames"; do
  447. if [ -s "${s}" ]; then
  448. sort -u "${s}" | \
  449. sponge "${s}"
  450. fi
  451. done
  452. # pkgnames -> pkgbases (with help from upstream)
  453. if [ -s "${tmp_dir}/pkgnames" ]; then
  454. while read -r priority pkgname; do
  455. content=$(
  456. curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
  457. tr ',' '\n'
  458. )
  459. repo=$(
  460. printf '%s\n' "${content}" | \
  461. sed -n '
  462. s/^\s*"repo"\s*:\s*"//
  463. T
  464. s/".*$//
  465. T
  466. p
  467. '
  468. )
  469. pkgbase=$(
  470. printf '%s\n' "${content}" | \
  471. sed -n '
  472. s/^\s*"pkgbase"\s*:\s*"//
  473. T
  474. s/".*$//
  475. T
  476. p
  477. '
  478. )
  479. if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
  480. printf '%s\t%s\n' "${priority}" "${pkgname}"
  481. continue
  482. fi
  483. printf '%s\t%s\t%s\n' \
  484. "${priority}" \
  485. "${pkgbase}" \
  486. "${repo}" >> \
  487. "${tmp_dir}/pkgbases"
  488. done < \
  489. "${tmp_dir}/pkgnames" | \
  490. sponge "${tmp_dir}/pkgnames"
  491. fi
  492. if [ -s "${tmp_dir}/pkgnames" ]; then
  493. >&2 echo 'Could not find some packages upstream:'
  494. >&2 cat "${tmp_dir}/pkgnames"
  495. exit 2
  496. fi
  497. # now we (re)schedule all pkgbases which are:
  498. # - not explicitely ignored (priority 3) <- explicitely scheduled
  499. # - not explicitely/implicitely ignored, but not currently available
  500. # (priority 2) <- from undelete-mirror-delta
  501. # - neither ignored nor to-be-deleted (priority 1) <- from mirror-delta or auto-rebuild
  502. # shellcheck disable=SC2016
  503. {
  504. printf 'CREATE TEMPORARY TABLE `ignore_packages` ('
  505. printf '`priority` SMALLINT,'
  506. printf '`pkgbase` VARCHAR(64),'
  507. printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)'
  508. printf ');\n'
  509. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \
  510. "${tmp_dir}/ignore-packages"
  511. printf 'CREATE TEMPORARY TABLE `pkgbases` ('
  512. printf '`priority` SMALLINT,'
  513. printf '`pkgbase` VARCHAR(64),'
  514. printf '`repo` VARCHAR(64),'
  515. printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
  516. printf ');\n'
  517. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \
  518. "${tmp_dir}/pkgbases"
  519. printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
  520. printf ' SELECT 2,`sub_q`.`pkgbase`'
  521. printf ' FROM ('
  522. printf 'SELECT'
  523. printf ' `package_sources`.`pkgbase`,'
  524. printf 'GROUP_CONCAT('
  525. printf 'DISTINCT'
  526. printf ' `build_assignments`.`architecture`'
  527. printf ' ORDER BY `build_assignments`.`architecture`'
  528. printf ') AS `architectures`'
  529. printf ' FROM `package_sources`'
  530. mysql_join_package_sources_build_assignments
  531. mysql_join_build_assignments_binary_packages
  532. mysql_join_binary_packages_binary_packages_in_repositories
  533. mysql_join_binary_packages_in_repositories_repositories
  534. printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL'
  535. printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"'
  536. printf ' OR `repositories`.`is_on_master_mirror`'
  537. printf ' OR `binary_packages_in_repositories`.`repository`=%s' \
  538. "${repository_ids__any_build_list}"
  539. printf ' GROUP BY `package_sources`.`pkgbase`'
  540. printf ') AS `sub_q`'
  541. # shellcheck disable=SC2154
  542. printf ' WHERE `sub_q`.`architectures` IN ("%s","%s","%s,%s")' \
  543. "${architecture_ids__any}" \
  544. "${non_any_architecture_ids}" \
  545. "${architecture_ids__any}" \
  546. "${non_any_architecture_ids}"
  547. printf ';\n'
  548. printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
  549. printf ' SELECT 1,`package_sources`.`pkgbase`'
  550. printf ' FROM `package_sources`'
  551. mysql_join_package_sources_build_assignments
  552. mysql_join_build_assignments_binary_packages
  553. mysql_join_binary_packages_binary_packages_in_repositories
  554. printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`'
  555. printf ';\n'
  556. printf 'SELECT '
  557. printf '`pkgbases`.`pkgbase`,'
  558. printf '`git_repositories`.`head`,'
  559. printf '('
  560. printf 'SELECT `al32`.`head`'
  561. printf ' FROM `git_repositories` AS `al32`'
  562. printf ' WHERE `al32`.`name`="archlinux32"'
  563. printf '),'
  564. printf '`pkgbases`.`repo`'
  565. printf ' FROM `pkgbases`'
  566. printf ' JOIN `upstream_repositories`'
  567. printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`'
  568. mysql_join_upstream_repositories_git_repositories
  569. printf ' WHERE NOT EXISTS ('
  570. printf 'SELECT 1'
  571. printf ' FROM `ignore_packages`'
  572. printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`'
  573. printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`'
  574. printf ') AND ('
  575. printf '`pkgbases`.`priority`!=2'
  576. printf ' OR NOT EXISTS ('
  577. printf 'SELECT 1'
  578. printf ' FROM `package_sources`'
  579. mysql_join_package_sources_build_assignments
  580. mysql_join_build_assignments_binary_packages
  581. mysql_join_binary_packages_binary_packages_in_repositories
  582. printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
  583. mysql_join_binary_packages_in_repositories_repositories
  584. printf ' AND ('
  585. printf '`repositories`.`is_on_master_mirror`'
  586. printf ' OR `repositories`.`name`="build-list"'
  587. printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`'
  588. printf ')'
  589. printf ');\n'
  590. } | \
  591. mysql_run_query | \
  592. tr '\t' ' ' | \
  593. sort -u > \
  594. "${tmp_dir}/update-list"
  595. if ${update}; then
  596. while read -r pkgbase git_rev mod_git_rev repo; do
  597. success=false
  598. # shellcheck disable=SC2154
  599. for gr_r in \
  600. "${git_rev}:${repo}" \
  601. "${repo_heads__packages}:core" \
  602. "${repo_heads__packages}:extra" \
  603. "${repo_heads__community}:community"; do
  604. printf '%s ' "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}" | \
  605. sed 's/ $/\n/'
  606. if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}"; then
  607. success=true
  608. break
  609. fi
  610. done
  611. if ! ${success} && \
  612. ! ${ignore_mysql_generate_package_metadata_errors}; then
  613. exit 2
  614. fi
  615. done < \
  616. "${tmp_dir}/update-list"
  617. cut -d' ' -f1 < \
  618. "${tmp_dir}/update-list" | \
  619. sort -u | \
  620. sponge "${tmp_dir}/update-list"
  621. # shellcheck disable=SC2016
  622. {
  623. printf 'CREATE TEMPORARY TABLE `pkgbases` ('
  624. printf '`pkgbase` VARCHAR (64),'
  625. printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
  626. printf ');\n'
  627. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \
  628. "${tmp_dir}/update-list"
  629. printf 'DELETE `d_bpir`'
  630. printf ' FROM `pkgbases`'
  631. # the package on the deletion-list ...
  632. printf ' JOIN `package_sources` AS `d_ps`'
  633. printf ' ON `pkgbases`.`pkgbase`=`d_ps`.`pkgbase`'
  634. mysql_join_package_sources_build_assignments 'd_ps' 'd_ba'
  635. mysql_join_build_assignments_binary_packages 'd_ba' 'd_bp'
  636. mysql_join_binary_packages_binary_packages_in_repositories 'd_bp' 'd_bpir'
  637. mysql_join_binary_packages_in_repositories_repositories 'd_bpir' 'd_r'
  638. # shellcheck disable=SC2154
  639. printf ' AND `d_r`.`stability`=%s' \
  640. "${repository_stability_ids__forbidden}"
  641. # ... needs a counter-part on the build-list (which was just created)
  642. printf ' JOIN `package_sources` AS `a_ps`'
  643. printf ' ON `d_ps`.`pkgbase`=`a_ps`.`pkgbase`'
  644. mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
  645. mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
  646. printf ' JOIN `architecture_compatibilities` AS `ac`'
  647. printf ' ON `ac`.`built_for`=`a_bp`.`architecture`'
  648. printf ' AND `ac`.`runs_on`=`d_r`.`architecture`'
  649. printf ' AND `ac`.`fully_compatible`'
  650. mysql_join_binary_packages_binary_packages_in_repositories 'a_bp' 'a_bpir'
  651. printf ' AND `a_bpir`.`repository`=%s;\n' \
  652. "${repository_ids__any_build_list}"
  653. if ${jostle}; then
  654. printf 'SET @`max_priority`=('
  655. printf 'SELECT'
  656. printf ' MAX(`build_assignments`.`priority`)+1'
  657. printf ' FROM `build_assignments`'
  658. printf ');\n'
  659. printf 'UPDATE `build_assignments`'
  660. mysql_join_build_assignments_package_sources
  661. printf ' JOIN `pkgbases`'
  662. printf ' ON `pkgbases`.`pkgbase`=`package_sources`.`pkgbase`'
  663. printf ' SET `build_assignments`.`priority`=@`max_priority`'
  664. printf ';\n'
  665. fi
  666. } | \
  667. mysql_run_query
  668. mysql_cleanup
  669. mysql_sort_versions
  670. mysql_find_build_assignment_loops
  671. # remove duplicate binary_packages from "build-list"
  672. mysql_query_remove_old_binary_packages_from_build_list | \
  673. mysql_run_query
  674. else
  675. cat "${tmp_dir}/update-list"
  676. fi