Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

return-assignment 39KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242
  1. #!/bin/sh
  2. # report back on a build assignment
  3. # either on success via:
  4. # "$0 $package $revision $mod_revision $repository $arch $sub_pkgrel" and tar'ed packages and logs
  5. # (= a tar of package(s), signature(s) and log(s)) on stdin
  6. # or on failure via:
  7. # "$0 $package $revision $mod_revision $repository $arch ERROR" and tar'ed logs
  8. # or to abort:
  9. # "$0 ABORT"
  10. # exit codes:
  11. # 0: ok
  12. # 1: another instance was already running
  13. # 2: outdated package
  14. # 3: signature error
  15. # 4: package error (e.g. wrong packages sent)
  16. # 5: wrong number of arguments
  17. # TODO: sign database
  18. # shellcheck disable=SC2119,SC2120
  19. # shellcheck source=../lib/load-configuration
  20. . "${0%/*}/../lib/load-configuration"
  21. if [ -s "${work_dir}/build-master-sanity" ]; then
  22. >&2 echo 'Build master is not sane.'
  23. exit 1
  24. fi
  25. # aborting does not need any locks
  26. if [ $# -eq 1 ] && \
  27. [ "$1" = 'ABORT' ]; then
  28. old_pkgbase=$(
  29. # shellcheck disable=SC2016
  30. {
  31. printf 'SELECT `package_sources`.`pkgbase`'
  32. printf ' FROM `build_slaves`'
  33. mysql_join_build_slaves_build_assignments
  34. mysql_join_build_assignments_package_sources
  35. # shellcheck disable=SC2154
  36. printf ' WHERE `build_slaves`.`id`=%s;\n' \
  37. "${slave_id}"
  38. printf 'UPDATE `build_slaves`'
  39. printf ' SET `build_slaves`.`currently_building`=NULL'
  40. # shellcheck disable=SC2154
  41. printf ' WHERE `build_slaves`.`id`=%s;\n' \
  42. "${slave_id}"
  43. } | \
  44. mysql_run_query 'unimportant'
  45. )
  46. if [ -z "${old_pkgbase}" ]; then
  47. >&2 printf 'Umm, nothing to abort for your.\n'
  48. else
  49. >&2 printf 'I aborted your build-assignment (%s).\n' \
  50. "${old_pkgbase}"
  51. fi
  52. exit
  53. fi
  54. if [ $# -ne 6 ]; then
  55. >&2 printf 'return-assignment: Wrong number of arguments (%s)\n' \
  56. "$#"
  57. >&2 printf 'call either:\n'
  58. >&2 printf ' return-assignment ABORT\n'
  59. >&2 printf ' return-assignment pkgbase git_revision mod_git_revision repository arch ERROR\n'
  60. >&2 printf ' return-assignment pkgbase git_revision mod_git_revision repository arch sub-pkgrel\n'
  61. exit 5
  62. fi
  63. # Create a lock file and a trap.
  64. exec 9> "${build_list_lock_file}"
  65. if ! verbose_flock -n 9; then
  66. >&2 echo 'come back (shortly) later - I cannot lock build list.'
  67. exit 1
  68. fi
  69. exec 8> "${sanity_check_lock_file}"
  70. if ! verbose_flock -s -n 8; then
  71. >&2 echo 'come back (shortly) later - sanity-check running.'
  72. exit 1
  73. fi
  74. clean_up_tmp_dir() {
  75. cd "${base_dir}"
  76. rm -rf --one-file-system "${tmp_dir}"
  77. }
  78. tmp_dir=$(mktemp -d "${work_dir}/tmp.return-assignment.XXXXXXXXXX")
  79. trap clean_up_tmp_dir EXIT
  80. if [ "$6" = 'ERROR' ]; then
  81. # the build failed on the build slave
  82. # shellcheck disable=SC2016
  83. infos=$(
  84. {
  85. printf 'SELECT'
  86. printf ' `build_assignments`.`id`,'
  87. printf 'IF(`build_assignments`.`is_broken`,"true","false"),'
  88. printf 'replace(to_base64(%s),"\\n",""),' \
  89. '`package_sources`.`upstream_flag_date`' \
  90. '`binary_packages`.`pkgver`'
  91. printf '`binary_packages`.`epoch`,'
  92. printf '`binary_packages`.`pkgrel`,'
  93. printf '`ba_a`.`name`'
  94. printf ' FROM `build_slaves`'
  95. mysql_join_build_slaves_build_assignments
  96. mysql_join_build_assignments_architectures '' 'ba_a'
  97. mysql_join_build_assignments_package_sources
  98. mysql_join_package_sources_upstream_repositories
  99. mysql_join_build_assignments_binary_packages
  100. mysql_join_binary_packages_binary_packages_in_repositories
  101. printf ' JOIN `architecture_compatibilities`'
  102. printf ' ON `architecture_compatibilities`.`fully_compatible`'
  103. printf ' AND `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
  104. printf ' JOIN `architectures`'
  105. printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
  106. printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \
  107. "$(
  108. # shellcheck disable=SC2154
  109. printf '%s' "${slave_id}" | \
  110. base64 -w0
  111. )"
  112. printf ' AND `package_sources`.`%s`=from_base64("%s")' \
  113. 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \
  114. 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \
  115. 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
  116. printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
  117. "$(printf '%s' "$4" | base64 -w0)"
  118. printf ' AND `architectures`.`name`=from_base64("%s")' \
  119. "$(printf '%s' "$5" | base64 -w0)"
  120. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  121. "${repository_ids__any_build_list}"
  122. printf ' LIMIT 1;\n'
  123. } | \
  124. mysql_run_query | \
  125. tr '\t' ' '
  126. )
  127. if [ -z "${infos}" ]; then
  128. >&2 echo 'You do not build this package (anymore) - move on.'
  129. exit 2
  130. fi
  131. build_assignment_architecture="${infos##* }"
  132. infos="${infos% ${build_assignment_architecture}}"
  133. pkgrel="${infos##* }"
  134. infos="${infos% ${pkgrel}}"
  135. epoch="${infos##* }"
  136. infos="${infos% ${epoch}}"
  137. pkgver="${infos##* }"
  138. infos="${infos% ${pkgver}}"
  139. pkgver=$(
  140. printf '%s' "${pkgver}" \
  141. | base64 -d
  142. )
  143. upstream_flag_date="${infos##* }"
  144. infos="${infos% ${upstream_flag_date}}"
  145. upstream_flag_date=$(
  146. printf '%s' "${upstream_flag_date}" \
  147. | base64 -d
  148. )
  149. was_broken_before="${infos##* }"
  150. build_assignment_id="${infos% ${was_broken_before}}"
  151. # save sent build logs
  152. saved_build_logs=$(
  153. tar -vx \
  154. -C "${build_log_directory}/error" \
  155. --wildcards \
  156. --no-wildcards-match-slash \
  157. --transform="s|^|$1.$2.$3.$4.${build_assignment_architecture}.|" \
  158. '*.build-log.gz'
  159. )
  160. if [ "${upstream_flag_date}" = '0000-00-00 00:00:00' ]; then
  161. upstream_flag_date=$(
  162. curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"$1" \
  163. | sed '
  164. s/{/\0\n/g
  165. s/}/\n\0/g
  166. ' \
  167. | sed '
  168. 1,2 d
  169. $ d
  170. ' \
  171. | grep -F '"pkgname": "'"$1"'"' \
  172. | grep -F '"epoch": '"${epoch}"'' \
  173. | grep -F '"pkgver": "'"${pkgver}"'"' \
  174. | grep -F '"pkgrel": "'"${pkgrel}"'"' \
  175. | sed -n '
  176. s/^.*"flag_date": "\([^"]\+\)",.*$/\1/
  177. T
  178. s/\.[0-9]\+[^.]*$//
  179. T
  180. y/T/ /
  181. /^[0-9]\{4\}\(-[0-9]\{2\}\)\{2\} \([0-9]\{2\}:\)\{2\}[0-9]\{2\}$/ p
  182. '
  183. )
  184. if [ -z "${upstream_flag_date}" ]; then
  185. upstream_flag_date='0000-00-00 00:00:00'
  186. fi
  187. fi
  188. # shellcheck disable=SC2016
  189. {
  190. if [ -n "${saved_build_logs}" ]; then
  191. printf 'CREATE TEMPORARY TABLE `failures` ('
  192. printf '`%s` %s,' \
  193. 'date' 'TIMESTAMP' \
  194. 'reason' 'SMALLINT' \
  195. 'log_file' 'VARCHAR(512)' | \
  196. sed 's/,$//'
  197. printf ');\n'
  198. fail_reason_identifiers=$(
  199. {
  200. printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")'
  201. printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`'
  202. } | \
  203. mysql_run_query
  204. )
  205. for saved_build_log in ${saved_build_logs}; do
  206. printf '%s\n' "${fail_reason_identifiers}" | \
  207. while read -r reason_id identifier; do
  208. if zgrep -qx "\s*$(
  209. printf '%s' "${identifier}" | \
  210. base64 -d
  211. )\s*" \
  212. "${build_log_directory}/error/$1.$2.$3.$4.${build_assignment_architecture}.${saved_build_log}"; then
  213. printf ' (from_base64("%s"),%s,from_base64("%s")),' \
  214. "$(
  215. printf '%s' "${saved_build_log}" | \
  216. sed 's|\.build-log\.gz$||;s|^.*\.||' | \
  217. base64 -w0
  218. )" \
  219. "${reason_id}" \
  220. "$(
  221. printf '%s' "$1.$2.$3.$4.${build_assignment_architecture}.${saved_build_log}" | \
  222. base64 -w0
  223. )"
  224. break
  225. fi
  226. done
  227. done | \
  228. sed '
  229. 1 s/^/INSERT IGNORE INTO `failures` (`date`,`reason`,`log_file`) VALUES /
  230. s/,$/;\n/
  231. '
  232. printf 'INSERT IGNORE INTO `failed_builds` (`build_slave`,`build_assignment`,`date`,`reason`,`log_file`,`log_file_exists`)'
  233. printf ' SELECT '
  234. printf 'from_base64("%s"),' \
  235. "$(printf '%s' "${slave_id}" | base64 -w0)" \
  236. "$(printf '%s' "${build_assignment_id}" | base64 -w0)"
  237. printf '`failures`.`%s`,' \
  238. 'date' 'reason' 'log_file'
  239. printf '1'
  240. printf ' FROM `failures`;\n'
  241. printf 'DROP TEMPORARY TABLE `failures`;\n'
  242. printf 'COMMIT;\n'
  243. fi
  244. printf 'UPDATE `build_assignments`'
  245. mysql_join_build_assignments_package_sources
  246. printf ' SET `build_assignments`.`is_broken`=1,'
  247. printf '`build_assignments`.`priority`=0,'
  248. printf '`package_sources`.`upstream_flag_date`=from_base64("%s")' \
  249. "$(
  250. printf '%s' "${upstream_flag_date}" | \
  251. base64 -w0
  252. )"
  253. printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \
  254. "$(
  255. printf '%s' "${build_assignment_id}" | \
  256. base64 -w0
  257. )"
  258. printf 'COMMIT;\n'
  259. printf 'UPDATE `build_slaves`'
  260. printf ' SET `build_slaves`.`currently_building`=NULL,'
  261. printf ' `build_slaves`.`last_action`=NULL,'
  262. printf ' `build_slaves`.`logged_lines`=NULL,'
  263. printf ' `build_slaves`.`trials`=NULL'
  264. printf ' WHERE `build_slaves`.`id`=from_base64("%s");\n' \
  265. "$(
  266. printf '%s' "${slave_id}" | \
  267. base64 -w0
  268. )"
  269. } | \
  270. mysql_run_query
  271. if ! ${was_broken_before}; then
  272. mysql_load_min_and_max_versions
  273. # this will hold a list of "$build_time $haskell_package $version"
  274. # meaning that the given $haskell_package with version $version did
  275. # not work at $build_time
  276. # note, that $build_time is only considered when $architecture or
  277. # $version are unavailable
  278. find "${build_log_directory}/error" -type f \
  279. -name "$1.$2.$3.$4.${build_assignment_architecture}.*.build-log.gz" \
  280. -exec zgrep -qF "$(
  281. printf '%s\n' \
  282. 'The following packages are broken because other packages they depend on are missing. These broken packages must be rebuilt before they can be used.' \
  283. 'mismatched interface file versions (wanted ' \
  284. )" {} \; \
  285. -printf '%p\n' | \
  286. sed '
  287. s/^.*\.\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\)\.build-log\.gz$/\1 \0/
  288. ' | \
  289. while read -r build_time build_log; do
  290. build_time=$(
  291. date +%s -d"${build_time}"
  292. )
  293. zcat "${build_log}" | \
  294. sed -n '
  295. /^Packages ([0-9]\+) /,/^\s*$/ p
  296. ' | \
  297. tr ' ' '\n' | \
  298. sed '
  299. 1,2 d
  300. /^\s*$/d
  301. p
  302. s/^haskell-//
  303. t
  304. d
  305. ' | \
  306. sort -u | \
  307. sed '
  308. s/-\(\([0-9]\+\):\)\?\([^-:]\+\)-\([^-.]\+\)\(\.\([^-.]\+\)\)\?$/ \2 \3 \4 \6/
  309. s/ $/ 0/
  310. s/ / 0 /g
  311. ' > \
  312. "${tmp_dir}/installed-versions"
  313. zcat "${build_log}" | \
  314. sed '
  315. s/^installed package \(.*\) is broken due to missing package .*$/\1/
  316. t
  317. s@^.*Bad interface file: /usr/lib/ghc-[0-9.]\+/site-local/\([^/]\+\)-[0-9.]\+/.*$@\1@
  318. t
  319. d
  320. ' | \
  321. tr ' ' '\n' | \
  322. tr '[:upper:]' '[:lower:]' | \
  323. sed '
  324. s/^/'"${build_time}"' /
  325. s/-[0-9.]\+$//
  326. ' | \
  327. sort -u | \
  328. sort -k2,2 > \
  329. "${tmp_dir}/broken-packages"
  330. {
  331. cut -d' ' -f1 < \
  332. "${tmp_dir}/installed-versions" | \
  333. sed 'p'
  334. cut -d' ' -f2 < \
  335. "${tmp_dir}/broken-packages"
  336. } | \
  337. sort | \
  338. uniq -u | \
  339. sed '
  340. s/$/ '"${max_version%%:*}"' '"${max_version#*:}"' 0 0/
  341. ' | \
  342. sponge -a "${tmp_dir}/installed-versions"
  343. sort -k1,1 < \
  344. "${tmp_dir}/installed-versions" | \
  345. sponge "${tmp_dir}/installed-versions"
  346. join -1 2 -2 1 \
  347. "${tmp_dir}/broken-packages" \
  348. "${tmp_dir}/installed-versions"
  349. done | \
  350. sort -k2 -k1nr,1 | \
  351. uniq -f1 | \
  352. tr ' ' '\t' > \
  353. "${tmp_dir}/broken-packages-with-version"
  354. # now we look if the broken packages have been rebuilt in the meantime
  355. haskell_rebuild_packages=$(
  356. # shellcheck disable=SC2016
  357. {
  358. printf 'CREATE TEMPORARY TABLE `broken`('
  359. printf '`time_stamp` BIGINT,'
  360. printf '`pkgname` VARCHAR(64),'
  361. printf '`epoch` MEDIUMINT,'
  362. printf '`pkgver` VARCHAR(64),'
  363. printf '`pkgrel` MEDIUMINT,'
  364. printf '`sub_pkgrel` MEDIUMINT,'
  365. printf 'KEY `time_stamp`(`time_stamp`),'
  366. printf 'KEY `pkgname`(`pkgname`),'
  367. printf 'KEY `epoch`(`epoch`),'
  368. printf 'KEY `pkgver`(`pkgver`),'
  369. printf 'KEY `pkgrel`(`pkgrel`),'
  370. printf 'KEY `sub_pkgrel`(`sub_pkgrel`),'
  371. printf 'UNIQUE KEY `pkgfile`(`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`)'
  372. printf ');\n'
  373. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `broken`(' \
  374. "${tmp_dir}/broken-packages-with-version"
  375. printf '`%s`,' \
  376. 'pkgname' \
  377. 'time_stamp' \
  378. 'epoch' \
  379. 'pkgver' \
  380. 'pkgrel' \
  381. 'sub_pkgrel' | \
  382. sed 's/,$//'
  383. printf ');\n'
  384. printf 'SELECT DISTINCT `broken`.`pkgname`'
  385. printf ' FROM `broken`'
  386. printf ' WHERE NOT EXISTS ('
  387. printf 'SELECT 1'
  388. printf ' FROM `binary_packages`'
  389. mysql_join_binary_packages_binary_packages_in_repositories
  390. mysql_join_binary_packages_in_repositories_repositories
  391. printf ' AND ('
  392. printf '`repositories`.`is_on_master_mirror`'
  393. printf ' OR `repositories`.`id`=%s' \
  394. "${repository_ids__any_build_list}"
  395. printf ')'
  396. mysql_join_binary_packages_build_assignments
  397. printf ' JOIN `architecture_compatibilities`'
  398. printf ' ON `architecture_compatibilities`.`fully_compatible`'
  399. printf ' AND `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
  400. printf ' JOIN `architectures`'
  401. printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
  402. printf ' AND `architectures`.`name`=from_base64("%s")' \
  403. "$(
  404. printf '%s' "${build_assignment_architecture}" | \
  405. base64 -w0
  406. )"
  407. printf ' WHERE ('
  408. printf '`binary_packages`.`pkgname`=`broken`.`pkgname`'
  409. printf ' OR `binary_packages`.`pkgname`=CONCAT("haskell-",`broken`.`pkgname`)'
  410. printf ') AND ('
  411. printf '`build_assignments`.`return_date`>FROM_UNIXTIME(`broken`.`time_stamp`)'
  412. printf ' OR ('
  413. printf '`binary_packages`.`epoch`>`broken`.`epoch`'
  414. printf ' OR ('
  415. printf '`binary_packages`.`epoch`=`broken`.`epoch`'
  416. printf ' AND `binary_packages`.`pkgver`>`broken`.`pkgver`'
  417. printf ')'
  418. printf ' OR ('
  419. printf '`binary_packages`.`epoch`=`broken`.`epoch`'
  420. printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
  421. printf ' AND `binary_packages`.`pkgrel`>`broken`.`pkgrel`'
  422. printf ')'
  423. printf ' OR ('
  424. printf '`binary_packages`.`epoch`=`broken`.`epoch`'
  425. printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
  426. printf ' AND `binary_packages`.`pkgrel`=`broken`.`pkgrel`'
  427. printf ' AND `binary_packages`.`sub_pkgrel`>`broken`.`sub_pkgrel`'
  428. printf ')'
  429. printf ')'
  430. printf ' OR `repositories`.`id`=%s' \
  431. "${repository_ids__any_build_list}"
  432. printf ')'
  433. printf ');\n'
  434. } | \
  435. mysql_run_query | \
  436. sed '
  437. s/^/-p ^(haskell-)?/
  438. s/$/$/
  439. '
  440. )
  441. # release lock on build-list - otherwise seed-build-list won't run
  442. flock -u 9
  443. rescheduled_packages=$(
  444. if [ -n "${haskell_rebuild_packages}" ]; then
  445. # shellcheck disable=SC2086
  446. "${base_dir}/bin/seed-build-list" ${haskell_rebuild_packages} | \
  447. sed 's/ .*$//'
  448. fi
  449. )
  450. # prioritize _this_ build assignment iff we rescheduled any broken dependencies
  451. if [ -n "${rescheduled_packages}" ]; then
  452. # shellcheck disable=SC2016
  453. {
  454. printf 'UPDATE `build_assignments`'
  455. printf ' SET `build_assignments`.`priority`=('
  456. printf 'SELECT MAX(`build_assignments`.`priority`)'
  457. printf ' FROM `build_assignments`'
  458. printf ')+1'
  459. printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \
  460. "$(
  461. printf '%s' "${build_assignment_id}" | \
  462. base64 -w0
  463. )"
  464. } | \
  465. mysql_run_query
  466. fi
  467. # shellcheck disable=SC2119
  468. {
  469. printf '%s/%s ' \
  470. "${build_assignment_architecture}" \
  471. "$1"
  472. if [ "${1%s}s" = "$1" ]; then
  473. printf 'are'
  474. else
  475. printf 'is'
  476. fi
  477. # shellcheck disable=SC2154
  478. printf ' broken (says %s)' \
  479. "${slave}"
  480. if [ "$1" = 'electron' ] || [ "$1" = 'electron2' ]; then
  481. printf -- ' - as usual'
  482. fi
  483. if [ -n "${rescheduled_packages}" ]; then
  484. printf -- ' - I rescheduled:'
  485. # shellcheck disable=SC2086
  486. printf ' %s,' ${rescheduled_packages} | \
  487. sed 's/,$//'
  488. fi
  489. if [ "${upstream_flag_date}" != '0000-00-00 00:00:00' ]; then
  490. printf -- ' - already flagged out-of day upstream on %s' \
  491. "${upstream_flag_date%% *}"
  492. fi
  493. printf ': https://archlinux32.org/buildmaster/build-log.php?a=%s&p=%s\n' \
  494. "${build_assignment_architecture}" \
  495. "$1"
  496. } | \
  497. irc_say
  498. fi
  499. exit 0
  500. fi
  501. # the build was successful on the build slave
  502. # so we also need a lock on the package database
  503. exec 7> "${package_database_lock_file}"
  504. if ! verbose_flock -n 7; then
  505. >&2 echo 'come back (shortly) later - I cannot lock package database.'
  506. exit 1
  507. fi
  508. # shellcheck disable=SC2016
  509. build_assignment_id=$(
  510. {
  511. printf 'SELECT DISTINCT `build_assignments`.`id`'
  512. printf ' FROM `build_slaves`'
  513. mysql_join_build_slaves_build_assignments
  514. mysql_join_build_assignments_package_sources
  515. mysql_join_package_sources_upstream_repositories
  516. mysql_join_build_assignments_binary_packages
  517. mysql_join_binary_packages_binary_packages_in_repositories
  518. printf ' JOIN `architecture_compatibilities`'
  519. printf ' ON `build_assignments`.`architecture`=`architecture_compatibilities`.`built_for`'
  520. printf ' AND `architecture_compatibilities`.`fully_compatible`'
  521. printf ' JOIN `architectures`'
  522. printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
  523. printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \
  524. "$(
  525. # shellcheck disable=SC2154
  526. printf '%s' "${slave_id}" | \
  527. base64 -w0
  528. )"
  529. printf ' AND `package_sources`.`%s`=from_base64("%s")' \
  530. 'pkgbase' "$(
  531. printf '%s' "$1" | \
  532. base64 -w0
  533. )" \
  534. 'git_revision' "$(
  535. printf '%s' "$2" | \
  536. base64 -w0
  537. )" \
  538. 'mod_git_revision' "$(
  539. printf '%s' "$3" | \
  540. base64 -w0
  541. )"
  542. printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
  543. "$(
  544. printf '%s' "$4" | \
  545. base64 -w0
  546. )"
  547. printf ' AND `architectures`.`name`=from_base64("%s")' \
  548. "$(
  549. printf '%s' "$5" | \
  550. base64 -w0
  551. )"
  552. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  553. "${repository_ids__any_build_list}"
  554. printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \
  555. "$(
  556. printf '%s' "$6" | \
  557. base64 -w0
  558. )"
  559. } | \
  560. mysql_run_query | \
  561. tr '\t' ' '
  562. )
  563. if [ -z "${build_assignment_id}" ]; then
  564. >&2 echo 'Sorry, the sent package is outdated.'
  565. exit 2
  566. fi
  567. cd "${tmp_dir}"
  568. export TMPDIR="${tmp_dir}"
  569. # extract package(s)
  570. tar -x \
  571. --wildcards \
  572. --no-wildcards-match-slash \
  573. '*.pkg.tar.xz' \
  574. '*.pkg.tar.xz.sig' \
  575. '*.pkg.tar.xz-namcap.log.gz' \
  576. '*.pkg.tar.xz.so.needs.gz' \
  577. '*.pkg.tar.xz.so.provides.gz'
  578. # check if all packages come with:
  579. # - a package file
  580. # - a signature
  581. # - a namcap log
  582. # - a list of needed libraries
  583. # - a list of provided libraries
  584. missing_files=$(
  585. find . -maxdepth 1 \( \
  586. \( \
  587. -name '*.pkg.tar.xz' \
  588. -printf '%f package\n' \
  589. \) -o \
  590. \( \
  591. -name '*.pkg.tar.xz.sig' \
  592. -printf '%f signature\n' \
  593. \) -o \
  594. \( \
  595. -name '*.pkg.tar.xz-namcap.log.gz' \
  596. -printf '%f namcap\n' \
  597. \) -o \
  598. \( \
  599. -name '*.pkg.tar.xz.so.needs.gz' \
  600. -printf '%f needed-libraries\n' \
  601. \) -o \
  602. \( \
  603. -name '*.pkg.tar.xz.so.provides.gz' \
  604. -printf '%f provided-libraries\n' \
  605. \) \
  606. \) | \
  607. sed '
  608. s/\(\.pkg\.tar\.xz\)\(\.sig\|\(-namcap\.log\|\.so\.\(provides\|needs\)\)\.gz\) /\1 /
  609. ' | \
  610. sort -k1,1 -k2,2 | \
  611. sed '
  612. :a
  613. $!N
  614. s/^\(\(\S\+\) [^\n]\+\)\n\2 /\1 /
  615. ta
  616. P
  617. D
  618. ' | \
  619. sed -n '
  620. s/$/ /
  621. / package /!{
  622. h
  623. s/^\(\S\+\) .*$/Package "\1" is missing./
  624. p
  625. g
  626. }
  627. / signature /!{
  628. h
  629. s/^\(\S\+\) .*$/Signature of "\1" is missing./
  630. p
  631. g
  632. }
  633. / namcap /!{
  634. h
  635. s/^\(\S\+\) .*$/Namcap log of "\1" is missing./
  636. p
  637. g
  638. }
  639. / needed-libraries /!{
  640. h
  641. s/^\(\S\+\) .*$/List of libraries needed by "\1" is missing./
  642. p
  643. g
  644. }
  645. / provided-libraries /!{
  646. h
  647. s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./
  648. p
  649. g
  650. }
  651. '
  652. )
  653. if [ -n "${missing_files}" ]; then
  654. >&2 echo 'The following packages lack a signature, namcap log, package file or list of needed/provided libraries:'
  655. printf 'Your buildslave "%s" uploaded some incomplete package(s):\n' \
  656. "${slave}" | \
  657. irc_say "${operator}"
  658. printf '%s\n' "${missing_files}" | \
  659. irc_say "${operator}" 'copy' >&2
  660. exit 3
  661. fi
  662. # check if the signatures are valid
  663. signatures=$(
  664. find . -maxdepth 1 -name '*.pkg.tar.xz' \
  665. -printf 'package file %f\n' \
  666. -exec gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg --verify '{}.sig' '{}' \; 2> /dev/null
  667. )
  668. if [ -z "$(
  669. printf '%s\n' "${signatures}" | \
  670. cut -d' ' -f2 | \
  671. grep -x 'file\|TRUST_FULLY' | \
  672. sort | \
  673. uniq -c | \
  674. awk '{print $1}' | \
  675. uniq -d
  676. )" ]; then
  677. >&2 echo 'Signature(s) is/are not fully trusted:'
  678. printf 'Your buildslave "%s" uploaded a package with a not fully-trusted signature:\n' \
  679. "${slave}" | \
  680. irc_say "${operator}"
  681. printf '%s\n' "${signatures}" | \
  682. irc_say "${operator}" 'copy' >&2
  683. exit 3
  684. fi
  685. # check if the package maintainer is set
  686. errors=$(
  687. find . -maxdepth 1 -name '*.pkg.tar.xz' | \
  688. while read -r pkg; do
  689. tar -OxJf "${pkg}" '.BUILDINFO' 2>/dev/null | \
  690. grep -vxF 'packager = Unknown Packager' | \
  691. grep -q '^packager = ' || \
  692. printf '%s misses a valid packager.\n' \
  693. "${pkg##*/}"
  694. done
  695. )
  696. if [ -n "${errors}" ]; then
  697. >&2 echo 'Packager error(s):'
  698. printf 'Your buildslave "%s" uploaded package(s) with invalid packager:\n' \
  699. "${slave}" | \
  700. irc_say "${operator}"
  701. printf '%s\n' "${errors}" | \
  702. irc_say "${operator}" 'copy' >&2
  703. exit 3
  704. fi
  705. # check if the sent packages are the expected ones
  706. find . -maxdepth 1 -name '*.pkg.tar.xz' -printf '%f\n' > \
  707. "${tmp_dir}/packages"
  708. # shellcheck disable=SC2016
  709. {
  710. printf 'SELECT'
  711. printf ' `binary_packages`.`id`,'
  712. mysql_package_name_query
  713. printf ' FROM `binary_packages`'
  714. mysql_join_binary_packages_architectures
  715. mysql_join_binary_packages_binary_packages_in_repositories
  716. printf ' WHERE `binary_packages`.`build_assignment`=from_base64("%s")' \
  717. "$(
  718. printf '%s' "${build_assignment_id}" | \
  719. base64 -w0
  720. )"
  721. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  722. "${repository_ids__any_build_list}"
  723. printf ';\n'
  724. } | \
  725. mysql_run_query | \
  726. tr '\t' ' ' | \
  727. sort -k2 > \
  728. "${tmp_dir}/package-ids"
  729. package_errors=$(
  730. {
  731. sed '
  732. s|^|was_built: |
  733. ' "${tmp_dir}/packages"
  734. sed '
  735. s|^[0-9]\+ |expected: |
  736. ' "${tmp_dir}/package-ids"
  737. } | \
  738. sort -k2 | \
  739. uniq -u -f1
  740. )
  741. if [ -n "${package_errors}" ]; then
  742. >&2 echo 'The following packages should have been built but are missing or vice versa:'
  743. >&2 printf '%s\n' "${package_errors}"
  744. {
  745. printf 'Your buildslave "%s" uploaded the wrong package(s):\n' \
  746. "${slave}"
  747. printf '%s\n' "${package_errors}"
  748. } | \
  749. irc_say "${operator}"
  750. exit 4
  751. fi
  752. if [ ! -s "${tmp_dir}/package-ids" ]; then
  753. >&2 echo 'No package was expected, no package was built.'
  754. >&2 echo 'That should not happen!'
  755. exit 4
  756. fi
  757. # TODO: maybe, we should not put "any" packages into repositories of all
  758. # architectures at once, but wait until they can actually be installed
  759. # on that architecture? (They might be missing an architecture specific
  760. # dependency)
  761. # shellcheck disable=SC2016
  762. {
  763. printf 'SELECT `binary_packages`.`id`,'
  764. mysql_package_name_query
  765. printf ',`t`.`id`,`t_a`.`name`,`t`.`name`'
  766. printf ' FROM `build_assignments`'
  767. mysql_join_build_assignments_binary_packages
  768. mysql_join_binary_packages_binary_packages_in_repositories
  769. printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
  770. "${repository_ids__any_build_list}"
  771. mysql_join_binary_packages_architectures
  772. mysql_join_build_assignments_package_sources
  773. mysql_join_package_sources_upstream_repositories
  774. mysql_join_upstream_repositories_repository_moves
  775. printf ' JOIN `repositories` as `t`'
  776. printf ' ON `t`.`id`=`repository_moves`.`to_repository`'
  777. mysql_join_repositories_architectures 't' 't_a'
  778. printf ' JOIN `architecture_compatibilities`'
  779. printf ' ON `architecture_compatibilities`.`built_for`=`binary_packages`.`architecture`'
  780. printf ' AND `architecture_compatibilities`.`runs_on`=`t`.`architecture`'
  781. printf ' AND `architecture_compatibilities`.`fully_compatible`'
  782. printf ' WHERE `repository_moves`.`from_repository`=%s' \
  783. "${repository_ids__any_build_list}"
  784. printf ' AND `build_assignments`.`id`=from_base64("%s");\n' \
  785. "$(
  786. printf '%s' "${build_assignment_id}" | \
  787. base64 -w0
  788. )"
  789. } | \
  790. mysql_run_query | \
  791. tr '\t' ' ' | \
  792. sort -u > \
  793. "${tmp_dir}/repository-ids"
  794. errors=$(
  795. {
  796. cut -d' ' -f2 < \
  797. "${tmp_dir}/repository-ids" | \
  798. sort -u | \
  799. sed 's/^/repository-was-found: /'
  800. cut -d' ' -f2 < \
  801. "${tmp_dir}/package-ids" | \
  802. sort -u | \
  803. sed 's/^/package-was-sent: /'
  804. } | \
  805. sort -k2 | \
  806. uniq -uf1
  807. )
  808. if [ -n "${errors}" ]; then
  809. >&2 echo 'I cannot determine, where this package (or some part of it)'
  810. >&2 echo 'should be published:'
  811. >&2 printf '%s\n' "${errors}"
  812. >&2 echo 'This is some internal error and not (necessarily) your fault.'
  813. # We give a temporary error (although resolving this needs manual
  814. # intervention), because there is nothing wrong with the sent package
  815. # whence it does not need to be built again, but can simply be sent again.
  816. exit 1
  817. fi
  818. mysql_load_min_and_max_versions
  819. while read -r package_id package_name; do
  820. # move namcap.logs
  821. mv \
  822. "${tmp_dir}/${package_name}-namcap.log.gz" \
  823. "${build_log_directory}/success/"
  824. # generate checksum
  825. sha512sum "${tmp_dir}/${package_name}" | \
  826. awk '{print "'"${package_id}"'\t" $1}' >> \
  827. "${tmp_dir}/sha512sums"
  828. # generate list of required/provided libraries
  829. for lib in 'provides' 'needs'; do
  830. zcat "${tmp_dir}/${package_name}.so.${lib}.gz" | \
  831. sed '
  832. s/\(=\|<\|<=\|>=\|>\)\([^[:space:]-]\+\)$/\t\1\t\2/
  833. t
  834. h
  835. s/$/\t>=\t'"${min_version}"'/
  836. '"$(
  837. if [ "${lib}" = 'provides' ]; then
  838. printf '%s\n' \
  839. 'p' \
  840. 'g' \
  841. 's/$/\t<=\t'"${max_version}"'/'
  842. fi
  843. )" | \
  844. sed '
  845. s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/
  846. t coda
  847. s/\s\S\+$/\t0\0/
  848. :coda
  849. s/^/'"${package_id}"'\t'"$(
  850. if [ "${lib}" = 'needs' ]; then
  851. printf 'link\\t'
  852. fi
  853. )"'/
  854. ' >> "${tmp_dir}/${lib}"
  855. done
  856. # generate list of make-,check-,rundepends according to .PKGINFO
  857. extract_dependencies_from_package \
  858. "${tmp_dir}/${package_name}" \
  859. | sed 's/^/'"${package_id}"' /' \
  860. | tr ' ' '\t' \
  861. >> "${tmp_dir}/needs"
  862. # TODO: this makes sense for provide= entries in .PKGINFO, too
  863. done < \
  864. "${tmp_dir}/package-ids"
  865. # move packages
  866. cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
  867. sort -u | \
  868. while read -r arch repo; do
  869. mkdir -p "${arch}/${repo}"
  870. failsafe_rsync \
  871. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
  872. "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
  873. "${arch}/${repo}/"
  874. grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/repository-ids" | \
  875. cut -d' ' -f2 | \
  876. xargs -r repo-add "${arch}/${repo}/${repo}.db.tar.gz"
  877. # xargs -r repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz"
  878. done
  879. # upload the packages into /pool
  880. failsafe_rsync -c --copy-dest=/.transfer \
  881. ./*".pkg.tar.xz" \
  882. ./*".pkg.tar.xz.sig" \
  883. "${master_mirror_rsync_directory}/pool/"
  884. # create symlinks
  885. find . \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -printf '%f\n' | \
  886. while read -r file; do
  887. rm "${file}"
  888. ln -s "../../pool/${file}" "${file}"
  889. done
  890. # upload the database and the symlinks into /$arch/$repo
  891. cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
  892. sort -u | \
  893. while read -r arch repo; do
  894. recompress_gz \
  895. "${tmp_dir}" \
  896. "${arch}/${repo}/${repo}."*".tar.gz" \
  897. "${arch}/${repo}/${repo}."*".tar.gz.old"
  898. # shellcheck disable=SC2046
  899. failsafe_rsync \
  900. "${arch}/${repo}/${repo}.db."* \
  901. "${arch}/${repo}/${repo}.files."* \
  902. $(
  903. grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/repository-ids" | \
  904. cut -d' ' -f2 | \
  905. sed '
  906. s,^,./,
  907. p
  908. s/$/.sig/
  909. '
  910. ) \
  911. "${master_mirror_rsync_directory}/${arch}/${repo}/"
  912. done
  913. # shellcheck disable=SC2016
  914. {
  915. # insert checksums into database
  916. printf 'CREATE TEMPORARY TABLE `pkg_hashes` (`pkgid` BIGINT, `sha512sum` VARCHAR(128));\n'
  917. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkg_hashes`;\n' \
  918. "${tmp_dir}/sha512sums"
  919. printf 'UPDATE `binary_packages`'
  920. printf ' JOIN `pkg_hashes`'
  921. printf ' ON `pkg_hashes`.`pkgid`=`binary_packages`.`id`'
  922. printf ' SET `binary_packages`.`sha512sum`=`pkg_hashes`.`sha512sum`;\n'
  923. printf 'COMMIT;\n'
  924. # insert provided/needed libraries into database
  925. for lib_link in 'pl:provides' 'nl:needs'; do
  926. printf 'CREATE TEMPORARY TABLE `%s` (' \
  927. "${lib_link%:*}"
  928. printf '`pkgid` BIGINT,'
  929. if [ "${lib_link}" = 'nl:needs' ]; then
  930. printf '`dep_type` VARCHAR(32),'
  931. fi
  932. printf '`lib` VARCHAR(128),'
  933. printf '`relation` VARCHAR(2),'
  934. printf '`epoch` MEDIUMINT,'
  935. printf '`version` VARCHAR(32)'
  936. printf ');\n'
  937. printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`;\n' \
  938. "${tmp_dir}/${lib_link#*:}" "${lib_link%:*}"
  939. printf 'INSERT IGNORE INTO `install_targets` (`name`)'
  940. printf ' SELECT DISTINCT `%s`.`lib`' \
  941. "${lib_link%:*}"
  942. printf ' FROM `%s`;\n' \
  943. "${lib_link%:*}"
  944. printf 'COMMIT;\n'
  945. # TODO: possibly remove install_target_providers with less restrictive
  946. # versions than we have now
  947. printf 'INSERT IGNORE INTO `versions` (`epoch`,`version`)'
  948. printf ' SELECT DISTINCT `%s`.`epoch`,`%s`.`version`' \
  949. "${lib_link%:*}" "${lib_link%:*}"
  950. printf ' FROM `%s`;\n' \
  951. "${lib_link%:*}"
  952. printf 'COMMIT;\n'
  953. if [ "${lib_link%:*}" = 'pl' ]; then
  954. printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`,`install_target_is_group`,`version`)'
  955. else
  956. # clear out the old dependencies - we will add the ones from the built package
  957. printf 'DELETE `dependencies`'
  958. printf ' FROM `nl`'
  959. printf ' JOIN `dependencies`'
  960. printf ' ON `dependencies`.`depending_on`=`nl`.`pkgid`;\n'
  961. printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`,`version_relation`,`version`)'
  962. fi
  963. printf ' SELECT `%s`.`pkgid`,`install_targets`.`id`,' \
  964. "${lib_link%:*}"
  965. if [ "${lib_link%:*}" = 'nl' ]; then
  966. printf '`dependency_types`.`id`,'
  967. printf '`%s`.`relation`,' \
  968. "${lib_link%:*}"
  969. else
  970. printf '0,'
  971. fi
  972. printf '`versions`.`id`'
  973. printf ' FROM `install_targets`'
  974. printf ' JOIN `%s`' \
  975. "${lib_link%:*}"
  976. printf ' ON `%s`.`lib`=`install_targets`.`name`' \
  977. "${lib_link%:*}"
  978. if [ "${lib_link%:*}" = 'nl' ]; then
  979. printf ' JOIN `dependency_types`'
  980. printf ' ON `dependency_types`.`name`=`nl`.`dep_type`'
  981. fi
  982. printf ' JOIN `versions`'
  983. printf ' ON `versions`.`epoch`=`%s`.`epoch`' \
  984. "${lib_link%:*}"
  985. printf ' AND `versions`.`version`=`%s`.`version`' \
  986. "${lib_link%:*}"
  987. printf ';\n'
  988. printf 'COMMIT;\n'
  989. done
  990. # remove build_assignment's markers
  991. printf 'UPDATE `build_assignments`'
  992. printf ' SET'
  993. printf ' `build_assignments`.`is_broken`=0,'
  994. printf ' `build_assignments`.`priority`=0,'
  995. printf ' `build_assignments`.`return_date`=NOW()'
  996. printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \
  997. "$(
  998. printf '%s' "${build_assignment_id}" | \
  999. base64 -w0
  1000. )"
  1001. printf 'COMMIT;\n'
  1002. # insert into appropriate repositories and retrieve ids
  1003. while read -r package_id _ repository_id _; do
  1004. printf 'INSERT INTO `binary_packages_in_repositories` (`package`,`repository`,`is_to_be_deleted`) VALUES '
  1005. printf '(%s,%s,0)' \
  1006. "${package_id}" "${repository_id}"
  1007. printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`binary_packages_in_repositories`.`id`);\n'
  1008. printf 'SELECT LAST_INSERT_ID();\n'
  1009. done < \
  1010. "${tmp_dir}/repository-ids"
  1011. printf 'COMMIT;\n'
  1012. # reschedule toolchain packages if they were not fully unblocked
  1013. printf 'INSERT IGNORE INTO `binary_packages` ('
  1014. printf '`build_assignment`,'
  1015. printf '`epoch`,'
  1016. printf '`pkgver`,'
  1017. printf '`pkgrel`,'
  1018. printf '`has_issues`,'
  1019. printf '`is_tested`,'
  1020. printf '`pkgname`,'
  1021. printf '`architecture`,'
  1022. printf '`sub_pkgrel`)'
  1023. printf ' SELECT '
  1024. printf '`binary_packages`.`%s`,' \
  1025. 'build_assignment' \
  1026. 'epoch' \
  1027. 'pkgver' \
  1028. 'pkgrel' \
  1029. 'has_issues' \
  1030. 'is_tested' \
  1031. 'pkgname' \
  1032. 'architecture'
  1033. printf '`binary_packages`.`sub_pkgrel`+1'
  1034. printf ' FROM `binary_packages`'
  1035. printf ' JOIN `pkg_hashes`'
  1036. printf ' ON `pkg_hashes`.`pkgid`=`binary_packages`.`id`'
  1037. mysql_join_binary_packages_build_assignments
  1038. mysql_join_build_assignments_package_sources
  1039. mysql_join_package_sources_toolchain_order '' 'late'
  1040. printf ' JOIN `toolchain_order` AS `early`'
  1041. printf ' ON `early`.`number`<`late`.`number`'
  1042. printf ' AND `early`.`pkgbase`!=`late`.`pkgbase`'
  1043. mysql_join_toolchain_order_package_sources 'early' 'early_ps'
  1044. mysql_join_package_sources_build_assignments 'early_ps' 'early_ba'
  1045. mysql_join_build_assignments_binary_packages 'early_ba' 'early_bp'
  1046. mysql_join_binary_packages_binary_packages_in_repositories 'early_bp' 'early_bpir'
  1047. printf ' AND `early_bpir`.`repository`=%s' \
  1048. "${repository_ids__any_build_list}"
  1049. printf ' WHERE ('
  1050. printf '`early_bp`.`architecture`=`binary_packages`.`architecture`'
  1051. # shellcheck disable=SC2154
  1052. printf ' OR `%s`.`architecture`=%s' \
  1053. 'early_bp' "${architecture_ids__any}" \
  1054. 'binary_packages' "${architecture_ids__any}"
  1055. printf ') AND NOT EXISTS ('
  1056. printf 'SELECT 1'
  1057. printf ' FROM `binary_packages` AS `early_bp`'
  1058. mysql_join_binary_packages_binary_packages_in_repositories 'early_bp' 'early_bpir'
  1059. mysql_join_binary_packages_in_repositories_repositories 'early_bpir' 'early_r'
  1060. printf ' WHERE `early_r`.`is_on_master_mirror`'
  1061. printf ' AND `early_bp`.`build_assignment`=`early_ba`.`id`'
  1062. printf ');\n'
  1063. join_part=$(
  1064. printf ' JOIN `pkg_hashes`'
  1065. printf ' ON `built_bp`.`id`=`pkg_hashes`.`pkgid`'
  1066. printf ' JOIN `binary_packages` AS `new_bp`'
  1067. printf ' ON'
  1068. printf ' `built_bp`.`%s`=`new_bp`.`%s` AND' \
  1069. 'build_assignment' 'build_assignment' \
  1070. 'epoch' 'epoch' \
  1071. 'pkgver' 'pkgver' \
  1072. 'pkgrel' 'pkgrel' \
  1073. 'has_issues' 'has_issues' \
  1074. 'is_tested' 'is_tested' \
  1075. 'pkgname' 'pkgname' \
  1076. 'architecture' 'architecture'
  1077. printf ' `built_bp`.`sub_pkgrel`+1=`new_bp`.`sub_pkgrel`'
  1078. )
  1079. printf 'INSERT IGNORE INTO `binary_packages_in_repositories`'
  1080. printf ' (`package`,`repository`,`is_to_be_deleted`)'
  1081. printf ' SELECT `new_bp`.`id`,%s,0' \
  1082. "${repository_ids__any_build_list}"
  1083. printf ' FROM `binary_packages` AS `built_bp`'
  1084. printf '%s;\n' "${join_part}"
  1085. printf 'INSERT IGNORE INTO `dependencies`'
  1086. printf ' (`dependent`,`depending_on`,`dependency_type`,`version`,`version_relation`)'
  1087. printf ' SELECT `new_bp`.`id`'
  1088. printf ',`dependencies`.`%s`' \
  1089. 'depending_on' \
  1090. 'dependency_type' \
  1091. 'version' \
  1092. 'version_relation'
  1093. printf ' FROM `dependencies`'
  1094. mysql_join_dependencies_binary_packages '' 'built_bp'
  1095. printf '%s;\n' "${join_part}"
  1096. # TODO: set _correct_ version of install_target_providers
  1097. printf 'INSERT IGNORE INTO `install_target_providers`'
  1098. printf ' (`package`,`install_target`,`version`,`install_target_is_group`)'
  1099. printf ' SELECT `new_bp`.`id`'
  1100. printf ',`install_target_providers`.`%s`' \
  1101. 'install_target' \
  1102. 'version' \
  1103. 'install_target_is_group'
  1104. printf ' FROM `install_target_providers`'
  1105. mysql_join_install_target_providers_binary_packages '' 'built_bp'
  1106. printf '%s;\n' "${join_part}"
  1107. # remove from build-list
  1108. printf 'DELETE `binary_packages_in_repositories`'
  1109. printf ' FROM `binary_packages_in_repositories`'
  1110. printf ' JOIN `pkg_hashes`'
  1111. printf ' ON `pkg_hashes`.`pkgid`=`binary_packages_in_repositories`.`package`'
  1112. printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
  1113. "${repository_ids__any_build_list}"
  1114. printf 'COMMIT;\n'
  1115. # update package information
  1116. printf 'UPDATE `binary_packages`'
  1117. printf ' SET'
  1118. printf ' `binary_packages`.`has_issues`=0,'
  1119. printf ' `binary_packages`.`is_tested`=0'
  1120. printf ' WHERE `binary_packages`.`id` IN ('
  1121. cut -d' ' -f1 < \
  1122. "${tmp_dir}/package-ids" | \
  1123. base64_encode_each | \
  1124. sed '
  1125. s/^/from_base64("/
  1126. s/$/"),/
  1127. $ s/,$//
  1128. '
  1129. printf ');\n'
  1130. printf 'COMMIT;\n'
  1131. # remove from build slave's `currently_building`
  1132. printf 'UPDATE `build_slaves`'
  1133. printf ' SET `build_slaves`.`currently_building`=NULL,'
  1134. printf ' `build_slaves`.`last_action`=NULL,'
  1135. printf ' `build_slaves`.`logged_lines`=NULL,'
  1136. printf ' `build_slaves`.`trials`=NULL'
  1137. printf ' WHERE `build_slaves`.`currently_building`=from_base64("%s");\n' \
  1138. "$(
  1139. printf '%s' "${build_assignment_id}" | \
  1140. base64 -w0
  1141. )"
  1142. printf 'COMMIT;\n'
  1143. # remove broken loops
  1144. printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n'
  1145. printf 'INSERT IGNORE INTO `loops_to_delete`'
  1146. printf ' SELECT `build_dependency_loops`.`loop`'
  1147. printf ' FROM `build_dependency_loops`'
  1148. mysql_join_build_dependency_loops_binary_packages
  1149. mysql_join_binary_packages_binary_packages_in_repositories
  1150. printf ' WHERE NOT `binary_packages_in_repositories`.`repository`=%s;\n' \
  1151. "${repository_ids__any_build_list}"
  1152. printf 'COMMIT;\n'
  1153. printf 'DELETE FROM `build_dependency_loops`'
  1154. printf ' WHERE EXISTS ('
  1155. printf 'SELECT 1'
  1156. printf ' FROM `loops_to_delete`'
  1157. printf ' WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`'
  1158. printf ');\n'
  1159. printf 'DROP TEMPORARY TABLE `loops_to_delete`;\n'
  1160. } | \
  1161. mysql_run_query | \
  1162. sort -u | \
  1163. remove_old_package_versions
  1164. trigger_mirror_refreshs