Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

build-packages 37KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098
  1. #!/bin/sh
  2. # build packages one by one, then upload the binary package to the repository server
  3. # shellcheck disable=SC2119,SC2120
  4. # shellcheck source=../lib/load-configuration
  5. . "${0%/*}/../lib/load-configuration"
  6. # TODO: report back memory and hdd stats to the build master on successful build
  7. # TODO: some build failures should skip straws - e.g. if dependencies are unavailable,
  8. # we should not try to rebuild with a cleaned chroot
  9. # TODO: some build environments should be archived when the build fails - but:
  10. # which/how?
  11. # TODO: it might be worth trying to build with makedepends installed from
  12. # [testing] and only depends installed from [staging]
  13. # TODO: the following things need proper locking and/or proper handling of
  14. # external locks:
  15. # - `git` on the package source
  16. # - `archbuild` (e.g. /var/lib/archbuilds/staging-i686-build/root.lock)
  17. # - `find ... -exec umount` on the build chroot
  18. # TODO: releave some locking conditions - meta goal: be able to run multiple
  19. # build slaves on one host from within one archlinux32/builder repository
  20. # shellcheck disable=SC2016
  21. usage() {
  22. >&2 echo ''
  23. >&2 echo 'build-packages: build package(s) on the build-list'
  24. >&2 echo ''
  25. >&2 echo 'possible options:'
  26. >&2 echo ' -d|--diff source-directory:'
  27. >&2 echo ' Apply the diff between $source-directory/PKGBUILD and'
  28. >&2 echo ' $source-directory/../../trunk/PKGBUILD before building.'
  29. >&2 echo ' This option is intended for building from trunk where minor'
  30. >&2 echo ' fixes did not get released yet. Hence, it conflicts intentionally'
  31. >&2 echo ' with -n, -l, -t or -x and it requires -p.'
  32. >&2 echo ' -h|--help: Show this help and exit.'
  33. >&2 echo ' -l|--local pkgname.git-revision.git-mod-revision.repository.architecture:'
  34. >&2 echo ' Build the given package without asking / reporting to the'
  35. >&2 echo ' build master. Cannot be combined with -n, -p, -t or -x.'
  36. >&2 echo ' where:'
  37. >&2 echo ' - git-revision: packages-HEAD or community-HEAD or a valid commit'
  38. >&2 echo ' - git-mod-revision: work-tree or a valid commit'
  39. >&2 echo ' example:'
  40. >&2 echo ' build-package -l which.packages-HEAD.work-tree.core.i686'
  41. >&2 echo ' -n count: Build $count packages (if available), then exit.'
  42. >&2 echo ' $count=0 is interpreted as infinity.'
  43. >&2 echo ' The default is $count=1 or 0 iff -t or -x is given.'
  44. >&2 echo ' Cannot be combined with -l.'
  45. >&2 echo ' -p|--prefer pkgname:'
  46. >&2 echo ' Ask the build master to get an assignment for the given'
  47. >&2 echo ' package but built what is offered anyway. Cannot be combined'
  48. >&2 echo ' with -l.'
  49. >&2 echo ' -s|--straw $straw:'
  50. >&2 echo ' Use this straw instead of the preconfigured ones. -- May be'
  51. >&2 echo ' given multiple times to allow using multiple straws.'
  52. >&2 echo ' -t seconds: Do not request new assignment(s) $seconds seconds after start.'
  53. >&2 echo ' Cannot be combined with -l.'
  54. >&2 echo ' -x: If package build fails, do not request new assignment(s).'
  55. >&2 echo ' Cannot be combined with -l.'
  56. >&2 echo ''
  57. >&2 echo 'known straws (separated by and enclosed in ":", sets of straws separated by " "):'
  58. >&2 echo ' :clean_chroot:'
  59. >&2 echo ' clean the chroot before building'
  60. >&2 echo ' :haskell_without_check:'
  61. >&2 echo ' try with :without_check: iff this is a haskell-*, python-* or python2-* package'
  62. >&2 echo ' :mirrored_source:'
  63. >&2 echo ' compile from source tarbal from sources.archlinux.org if possible'
  64. >&2 echo ' :mirrored_source_by_hash:'
  65. >&2 echo ' download sources from sources.archlinux32.org by hash if possible'
  66. >&2 echo ' :without_check:'
  67. >&2 echo ' run makepkg with "--no-check"'
  68. >&2 echo ' :without_systemd_nspawn:'
  69. >&2 echo ' do not invoke archbuild'"'"'s systemd-nspawn fanciness [RISKY]'
  70. >&2 echo ' :with_build_support:'
  71. >&2 echo ' allow using [build-support]'
  72. >&2 echo ' :with_/dev/fuse:'
  73. >&2 echo ' mount /dev/fuse into the chroot'
  74. [ -z "$1" ] && exit 1 || exit "$1"
  75. }
  76. eval set -- "$(
  77. getopt -o d:hl:n:p:s:t:ux \
  78. --long diff: \
  79. --long help \
  80. --long local: \
  81. --long prefer: \
  82. --long straw: \
  83. --long upload \
  84. -n "$(basename "$0")" -- "$@" || \
  85. echo usage
  86. )"
  87. unset count
  88. unset diff_source_dir
  89. unset forced_package
  90. unset forced_straws
  91. unset prefered_package
  92. exit_after_failure=false
  93. timeout=0
  94. while true
  95. do
  96. case "$1" in
  97. -d|--diff)
  98. shift
  99. if [ -n "${diff_source_dir}" ]; then
  100. >&2 echo 'Option -d, --diff can be given only once.'
  101. usage
  102. fi
  103. diff_source_dir="$1"
  104. ;;
  105. -h|--help)
  106. usage 0
  107. ;;
  108. -l|--local)
  109. shift
  110. if [ -n "${forced_package}" ]; then
  111. >&2 echo 'Option -l, --local can be given only once.'
  112. usage
  113. fi
  114. forced_package="$1"
  115. ;;
  116. -n)
  117. shift
  118. count="$1"
  119. [ "${count}" -eq 0 ] && \
  120. count=-1
  121. ;;
  122. -p|--prefer)
  123. shift
  124. if [ -n "${prefered_package}" ]; then
  125. >&2 echo 'Option -p, --prefer can be given only once.'
  126. usage
  127. fi
  128. prefered_package="$1"
  129. ;;
  130. -s|--straw)
  131. shift
  132. forced_straws="${forced_straws} $1"
  133. ;;
  134. -t)
  135. shift
  136. timeout="$1"
  137. ;;
  138. -x)
  139. exit_after_failure=true
  140. ;;
  141. --)
  142. shift
  143. break
  144. ;;
  145. *)
  146. >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
  147. exit 42
  148. ;;
  149. esac
  150. shift
  151. done
  152. if [ $# -ne 0 ]; then
  153. >&2 echo 'Too many arguments.'
  154. usage
  155. fi
  156. if [ -n "${diff_source_dir}" ]; then
  157. if [ -n "${count}" ] || \
  158. [ -n "${forced_package}" ] || \
  159. [ "${timeout}" -ne 0 ] || \
  160. ${exit_after_failure} || \
  161. [ -z "${prefered_package}" ]; then
  162. >&2 echo 'Conflicting flags.'
  163. usage
  164. fi
  165. fi
  166. if [ -n "${forced_package}" ]; then
  167. if [ -n "${count}" ] || \
  168. [ "${timeout}" -ne 0 ] || \
  169. [ -n "${prefered_package}" ] || \
  170. ${exit_after_failure}; then
  171. >&2 echo 'Conflicting flags.'
  172. usage
  173. fi
  174. upload_to_build_master=false
  175. else
  176. upload_to_build_master=true
  177. fi
  178. if [ -z "${count}" ]; then
  179. if [ "${timeout}" -ne 0 ] || ${exit_after_failure}; then
  180. count=-1
  181. else
  182. count=1
  183. fi
  184. fi
  185. if [ -n "${forced_straws}" ]; then
  186. straws_that_might_repair_failing_builds="${forced_straws# }"
  187. fi
  188. if [ "${timeout}" -ne 0 ]; then
  189. timeout=$((timeout+$(date +%s)))
  190. fi
  191. checksum=$(
  192. calculate_script_checksum
  193. )
  194. # When this script or a script in lib/ got updated, we do not request
  195. # any new assignments. This script should rather exit and be restarted.
  196. while [ "${count}" -ne 0 ] && \
  197. [ "$(calculate_script_checksum)" = "${checksum}" ]; do
  198. if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ];
  199. then
  200. break
  201. fi
  202. err=0
  203. case "$(uname -m)" in
  204. 'i486')
  205. my_arch='i486'
  206. ;;
  207. 'i686'|'x86_64')
  208. if grep '^flags\s*:' '/proc/cpuinfo' | \
  209. grep -qwF 'sse2'; then
  210. my_arch='pentium4'
  211. else
  212. my_arch='i686'
  213. fi
  214. ;;
  215. *)
  216. >&2 printf 'Sorry, architecture %s does not work (yet) as a build slave.\n' \
  217. "$(uname -m)"
  218. exit 2
  219. ;;
  220. esac
  221. if [ -z "${forced_package}" ]; then
  222. package=$(
  223. # shellcheck disable=SC2029
  224. ssh \
  225. -i "${master_build_server_identity}" \
  226. -p "${master_build_server_port}" \
  227. "${master_build_server_user}@${master_build_server}" \
  228. 'get-assignment' "${my_arch}" "${prefered_package}"
  229. ) || err=$?
  230. expected_packages=$(
  231. printf '%s\n' "${package}" | \
  232. sed '1d'
  233. )
  234. package=$(
  235. printf '%s\n' "${package}" | \
  236. sed -n '1p'
  237. )
  238. else
  239. package=$(
  240. echo "${forced_package}" | \
  241. sed '
  242. s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3 0 \4|
  243. '
  244. )
  245. expected_packages=''
  246. fi
  247. case ${err} in
  248. # 0: ok, I gave you an assignment
  249. 0)
  250. [ ${count} -gt 0 ] && \
  251. count=$((count-1))
  252. arch="${package##* }"
  253. package="${package% *}"
  254. sub_pkgrel="${package##* }"
  255. package="${package% *}"
  256. repository="${package##* }"
  257. package="${package% *}"
  258. mod_git_revision="${package##* }"
  259. package="${package% *}"
  260. git_revision="${package##* }"
  261. package="${package% *}"
  262. if [ "${arch}" = 'any' ]; then
  263. arch="${my_arch}"
  264. fi
  265. if [ -n "${diff_source_dir}" ] &&
  266. [ "${prefered_package}" != "${package}" ]; then
  267. >&2 echo 'The prefered package was not handed out.'
  268. >&2 echo 'Because -d was given, I will abort.'
  269. # shellcheck disable=SC2029
  270. ssh \
  271. -i "${master_build_server_identity}" \
  272. -p "${master_build_server_port}" \
  273. "${master_build_server_user}@${master_build_server}" \
  274. 'return-assignment' 'ABORT'
  275. exit 2
  276. fi
  277. if [ "${git_revision##*-}" = 'HEAD' ]; then
  278. git_revision=$(
  279. repo_name="${git_revision%-*}"
  280. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  281. if [ -z "${repo_path}" ]; then
  282. >&2 printf 'Unknown git repository "%s".\n' "${repo_name}"
  283. exit 2
  284. fi
  285. git -C "${repo_path}" rev-parse HEAD
  286. )
  287. fi
  288. if [ "${mod_git_revision}" = 'work-tree' ]; then
  289. mod_git_revision=$(
  290. # we can't just create an empty index-file with mktemp, because git doesn't like it
  291. find . \
  292. -mindepth 1 \
  293. -maxdepth 1 \
  294. -name 'tmp.build-packages.git.*' \
  295. -exec rm -rf --one-file-system {} \;
  296. tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
  297. trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
  298. export GIT_INDEX_FILE="${tmp_subdir}/index.new"
  299. git -C "${repo_paths__archlinux32}" add -A
  300. git -C "${repo_paths__archlinux32}" write-tree
  301. )
  302. fi
  303. # Update git repositories (official packages, community packages and the repository of package customizations).
  304. for repo_name in ${repo_names}; do
  305. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  306. if [ -d "${repo_path}/.git" ]; then
  307. git -C "${repo_path}" remote update
  308. else
  309. git -C "${repo_path}" fetch origin master:master
  310. fi || \
  311. true
  312. done
  313. # trigger update of mirror (if configured)
  314. if [ -n "${mirror_update_command}" ]; then
  315. ${mirror_update_command}
  316. fi
  317. bail_out() {
  318. err=$?
  319. if [ -n "$1" ]; then
  320. err="$1"
  321. fi
  322. cd "${base_dir}"
  323. recursively_umount_and_rm "${tmp_dir}"
  324. if [ -z "${forced_package}" ]; then
  325. flock -u 9 || true
  326. fi
  327. exit "${err}"
  328. }
  329. find "${work_dir}" \
  330. -mindepth 1 \
  331. -maxdepth 1 \
  332. -name 'tmp.build-packages.??????' \
  333. -printf '%p\n' | \
  334. while read -r old_tmp_dir; do
  335. find "${old_tmp_dir}" -xdev -not -type l -exec chmod 777 {} \;
  336. rm -rf --one-file-system "${old_tmp_dir}"
  337. done
  338. tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
  339. trap bail_out EXIT
  340. if ! git_repo=$(find_repository_with_commit "${git_revision}") || \
  341. [ -z "${git_repo}" ] || \
  342. ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" || \
  343. ! extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}" || \
  344. ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then
  345. # report local failure (probably a missing commit) to build-master
  346. # shellcheck disable=SC2029
  347. ssh \
  348. -i "${master_build_server_identity}" \
  349. -p "${master_build_server_port}" \
  350. "${master_build_server_user}@${master_build_server}" \
  351. 'return-assignment' 'ABORT'
  352. recursively_umount_and_rm "${tmp_dir}"
  353. if [ -z "${forced_package}" ]; then
  354. flock -u 9 || true
  355. fi
  356. trap - EXIT
  357. continue
  358. fi
  359. cd "${tmp_dir}"
  360. echo 'nothing' > "${tmp_dir}/.ping-build-master"
  361. if [ -z "${forced_package}" ]; then
  362. # we get a lock on "${work_dir}/ping-build-master.lock",
  363. # if we release that lock, ping-to-master should stop _immediately_
  364. exec 9> "${work_dir}/ping-build-master.lock"
  365. if ! verbose_flock -n 9; then
  366. >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
  367. exit 2
  368. fi
  369. "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
  370. fi
  371. success=false
  372. if printf '%s\n' "${package}" | \
  373. grep -q '^\(haskell\|python2\?\)-'; then
  374. straws_that_might_repair_failing_builds=$(
  375. # shellcheck disable=SC2086
  376. printf '%s\n' ${straws_that_might_repair_failing_builds} | \
  377. sed '
  378. s/^\(.*\):haskell_without_check:\(.*\)$/\1:\2\n\1:without_check:\2/
  379. '
  380. )
  381. fi
  382. for straw in ${straws_that_might_repair_failing_builds}; do
  383. if ${success}; then
  384. break
  385. fi
  386. echo 'preparing' > "${tmp_dir}/.ping-build-master"
  387. outerParameters="-r ${archbuild_chroots}"
  388. if echo "${straw}" | \
  389. grep -qF ':clean_chroot:'; then
  390. outerParameters="-c ${outerParameters}"
  391. fi
  392. innerParameters='--nocolor'
  393. if echo "${straw}" | \
  394. grep -qF ':without_check:'; then
  395. innerParameters="${innerParameters} --nocheck"
  396. fi
  397. if echo "${straw}" | \
  398. grep -qF ':with_/dev/fuse:'; then
  399. middleParameters='-d /dev/fuse'
  400. else
  401. middleParameters=''
  402. fi
  403. if echo "${straw}" | \
  404. grep -qF ':with_build_support:'; then
  405. build_command='staging-with-build-support-'"${arch}"'-build'
  406. elif echo "${straw}" | \
  407. grep -qF ':without_systemd_nspawn:'; then
  408. if [ -z "${prefered_package}" ]; then
  409. >&2 echo 'straw :without_systemd_nspawn: only allowed with -p'
  410. exit 2
  411. fi
  412. if [ "${prefered_package}" != "${package}" ]; then
  413. >&2 echo 'The prefered package was not handed out.'
  414. >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.'
  415. # shellcheck disable=SC2029
  416. ssh \
  417. -i "${master_build_server_identity}" \
  418. -p "${master_build_server_port}" \
  419. "${master_build_server_user}@${master_build_server}" \
  420. 'return-assignment' 'ABORT'
  421. exit 2
  422. fi
  423. # we're not interested in what this cpu /can/ do, but what
  424. # architecture the installed packages actually are
  425. if ! pacman -Qi pacman | \
  426. sed 's/^Architecture\s*:\s*//;t;d' | \
  427. grep -qxF "${arch}"; then
  428. >&2 echo 'straw :without_systemd_nspawn: requires running build-packages on the'
  429. >&2 echo 'architecture for which the package should be built.'
  430. >&2 printf '"%s" != "%s"\n' \
  431. "$(
  432. pacman -Qi pacman | \
  433. sed 's/^Architecture\s*:\s*//;t;d'
  434. )" \
  435. "${arch}"
  436. exit 2
  437. fi
  438. build_command='makepkg'
  439. outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver"
  440. middleParameters=''
  441. innerParameters=''
  442. else
  443. build_command='staging-'"${arch}"'-build'
  444. fi
  445. find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
  446. rm {} \;
  447. echo 'checking-source' > "${tmp_dir}/.ping-build-master"
  448. success=false
  449. verifysource_trial=0
  450. while [ ${verifysource_trial} -lt 5 ]; do
  451. verifysource_trial=$((verifysource_trial + 1))
  452. log_file="${tmp_dir}/$(
  453. date -u --iso-8601=seconds | \
  454. cut -d+ -f1
  455. ).build-log"
  456. if CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then
  457. success=true
  458. rm "${log_file}"
  459. break
  460. fi
  461. # receive specific missing keys
  462. if [ ${verifysource_trial} -eq 1 ]; then
  463. missing_keys=$(
  464. sed -n '
  465. s/^.* FAILED (unknown public key \([0-9A-F]\{16\}\)).*$/0x\1/
  466. T
  467. p
  468. ' "${log_file}"
  469. )
  470. if [ -n "${missing_keys}" ]; then
  471. # TODO: how about putting all the missing keys somewhere on archlinux32.org?
  472. if gpg --recv-keys "${missing_keys}"; then
  473. continue
  474. fi
  475. fi
  476. verifysource_trial=$((verifysource_trial + 1))
  477. fi
  478. # download the repository key from github
  479. if [ ${verifysource_trial} -eq 2 ]; then
  480. if grep -q ' FAILED (unknown public key \([0-9A-F]\{16\}\))' -- "${log_file}"; then
  481. # TODO: get the name of the key file from its finger print or
  482. # some other information inside the repository
  483. if makepkg --printsrcinfo | \
  484. sed -n '
  485. s,^\ssource = git+\(https://github\.com/[^/[:space:]]\+\)/[^/]*$,\1.gpg,
  486. T
  487. p
  488. ' | \
  489. xargs -rn1 curl -s | \
  490. gpg --import; then
  491. continue
  492. fi
  493. fi
  494. verifysource_trial=$((verifysource_trial + 1))
  495. fi
  496. # try to download source from sources.archlinux.org/sources/$repo/$source
  497. if [ ${verifysource_trial} -eq 3 ]; then
  498. if echo "${straw}" | \
  499. grep -qF ':mirrored_source:'; then
  500. source_name=$(
  501. makepkg --printsrcinfo | \
  502. sed -n '
  503. /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
  504. /^pkgname = /q
  505. ' | \
  506. sed '
  507. s|^pkgbase = \(.*\)$|0 \1-|
  508. s|^epoch = \(.*\)$|1 \1:|
  509. s|^pkgver = \(.*\)$|2 \1-|
  510. s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz|
  511. ' | \
  512. sort -k1n,1 | \
  513. sed '
  514. s|^[0-9] ||
  515. :a
  516. N
  517. s|\n[0-9] \(\S\+\)$|\1|
  518. ta
  519. '
  520. )
  521. if wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
  522. # shellcheck disable=SC2046
  523. tar -xz --overwrite \
  524. -f "${source_name}" \
  525. --exclude PKGBUILD \
  526. $(
  527. if [ -n "${PKGBUILD_mod}" ]; then
  528. git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
  529. tar -t | \
  530. sed 's/^/--exclude /'
  531. fi
  532. ) \
  533. --strip-components=1 \
  534. || true
  535. continue
  536. fi
  537. fi
  538. verifysource_trial=$((verifysource_trial + 1))
  539. fi
  540. # try to download source from sources.archlinux32.org by its hash
  541. if [ ${verifysource_trial} -eq 4 ]; then
  542. if echo "${straw}" | \
  543. grep -qF ':mirrored_source_by_hash:'; then
  544. if download_sources_by_hash; then
  545. continue
  546. fi
  547. fi
  548. verifysource_trial=$((verifysource_trial + 1))
  549. fi
  550. done
  551. if ${success}; then
  552. echo 'building' > "${tmp_dir}/.ping-build-master"
  553. >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) for %s ...' \
  554. "$(date +'%Y-%m-%d %T')" \
  555. "${package}" \
  556. "${git_revision}" \
  557. "${mod_git_revision}" \
  558. "${repository}" \
  559. "${straw}" \
  560. "${arch}"
  561. log_file="${tmp_dir}/$(
  562. date -u --iso-8601=seconds | \
  563. cut -d+ -f1
  564. ).build-log"
  565. # by piping the log, we don't see anything in the terminal,
  566. # but all ways to duplicate the logs seem pretty elaborate
  567. # shellcheck disable=SC2024,SC2086
  568. if ! "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
  569. "${log_file}" 2>&1; then
  570. success=false
  571. build_dir="/var/lib/archbuild/${build_command%-build}/$(whoami)"
  572. if [ -d "${build_dir}" ]; then
  573. sed '
  574. s/^Full log written to\s\+\(\S\+\)\s*$/\1/
  575. t
  576. /# An error report file with more information is saved as:/ {
  577. N
  578. s/^.*\n#\s\+//
  579. t
  580. d
  581. }
  582. s/^See also "\(\S\+\)"\.\s*$/\1/
  583. t
  584. d
  585. ' "${log_file}" | \
  586. while read -r extra_log_file; do
  587. grep -HF '' "${build_dir}/${extra_log_file}" || true
  588. done | \
  589. sponge -a "${log_file}"
  590. fi
  591. fi
  592. fi
  593. if ${success}; then
  594. # build successful
  595. >&2 printf ' ok.\n'
  596. if [ "${build_command}" = 'makepkg' ]; then
  597. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  598. -exec sh -c 'namcap "$1" > "$1-namcap.log"' '_' '{}' \;
  599. fi
  600. tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
  601. echo 'post-build' > "${tmp_dir}/.ping-build-master"
  602. # remove unexpected packages
  603. if [ -n "${expected_packages}" ]; then
  604. {
  605. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' -printf '%f\n'
  606. printf '%s\n' "${expected_packages}" | \
  607. sed 'p'
  608. } | \
  609. sort | \
  610. uniq -u | \
  611. while read -r unexpected_package; do
  612. rm "${unexpected_package}"*
  613. done
  614. fi
  615. >&2 printf 'signing package(s)\n'
  616. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  617. -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
  618. -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \
  619. -printf '%f\n' | \
  620. sponge | \
  621. while read -r pkg_file; do
  622. {
  623. pacman -Spdd --print-format '%l' --noconfirm "${pkg_file%-*-*-*}" 2>/dev/null | \
  624. sed '
  625. s|/[^/]\+\.pkg\.tar\.xz$||
  626. '
  627. # shellcheck disable=SC2016
  628. curl -Ss 'https://www.archlinux.org/mirrorlist/?country=all&protocol=https&tier=1&use_mirror_status=on' | \
  629. sed -n '
  630. s/^#Server = //
  631. T
  632. s/\$repo/'"${repository}"'/g
  633. s/\$arch/x86_64/g
  634. p
  635. ' | \
  636. shuf
  637. } | \
  638. sed '
  639. s|$|/'"${pkg_file}"'|
  640. s/\.[0-9]\+\(-[^-]\+\)$/\1/
  641. s/-'"${arch}"'\+\(\.pkg\.tar\.xz\)$/-x86_64\1/
  642. ' | \
  643. while read -r url; do
  644. >&2 printf 'downloading "%s" ...' "${url}"
  645. if wget -q --timeout=15 -nd "${url}"; then
  646. >&2 printf ' ok.\n'
  647. break;
  648. fi
  649. >&2 printf ' failed. Next ...\n'
  650. done
  651. done
  652. >&2 printf 'searching for provided libraries\n'
  653. find "${tar_content_dir}" -maxdepth 1 \
  654. -name '*.pkg.tar.xz' \
  655. -printf '%p\n' | \
  656. while read -r pkgfile; do
  657. pacman -Qqlp "${pkgfile}" | \
  658. sed -n '
  659. s,^.*/,,
  660. /\.so\(\..\+\)\?$/ {
  661. :a
  662. p
  663. s/\(\.so\(\..\+\)\?\)\.[^.]\+$/\1/
  664. t a
  665. }
  666. ' | \
  667. sort -u > \
  668. "${pkgfile}.so.provides"
  669. done
  670. >&2 printf 'searching for required and more provided libraries\n'
  671. package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
  672. find "${tar_content_dir}" -maxdepth 1 \
  673. -name '*.pkg.tar.xz' | \
  674. while read -r pkgfile; do
  675. if printf '%s\n' "${pkgfile}" | \
  676. grep -vq -- '-any\.pkg\.tar\.xz$'; then
  677. # we do not check "any" packages for linked libraries
  678. # (why do they have them in the first place?)
  679. mkdir "${package_content_dir}/${pkgfile##*/}"
  680. tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
  681. # we rely on "${checksum}" not appearing in any objdump output
  682. : "${checksum?umm, checksum is unset - this will break below sed-fu}"
  683. # TODO: symbols may be in object files _inside_archives_
  684. # In that case, we're interested in the object file's name,
  685. # not the archive's file name
  686. find "${package_content_dir}/${pkgfile##*/}" \
  687. -name 'opt' -prune , \
  688. -type f \
  689. -exec objdump -x '{}' \; 2>/dev/null | \
  690. sed '
  691. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  692. \@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\1\.[0-9.]\+/@ {
  693. s@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\(\1\(\.[0-9.]\+\)\?\)\.\([0-9]\+\)/.*$@\2 \4@
  694. w '"${pkgfile}"'.needed-perl-version
  695. d
  696. }
  697. /\sNEEDED\s/ {
  698. s/^\s*\S\+\s\+\(\S\+\)\(\s.*\)\?$/\1/
  699. /\.c32$/d
  700. s,^.*/,,
  701. t
  702. }
  703. /^Version References:$/,/^$/ {
  704. /^\s*required from / {
  705. s/^\s*required from \(\S\+\):\s*$/'"${checksum}"'\1/
  706. T end
  707. h
  708. d
  709. }
  710. s/^\s*\(0x[0-9a-fA-F]\+\)\s\+0x[0-9a-fA-F]\+\s\+[0-9]\+\s\+\(\S\+\)$/\2-\1/
  711. T end
  712. G
  713. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  714. t
  715. }
  716. :end
  717. d
  718. ' | \
  719. sort -u > \
  720. "${pkgfile}.so.needs"
  721. sed -i '
  722. /^\(libav.*\.so\)\.[0-9]\+$/ {
  723. :a
  724. N
  725. s/^\(libav.*\.so\)\(\.[0-9]\+\)\?\n\1\.[0-9]\+$/\1/
  726. t a
  727. P
  728. D
  729. }
  730. ' "${pkgfile}.so.needs"
  731. sed '
  732. s/^installed = \(qt[0-9]\+-\S\+\)-\([0-9.]\+\)\(-[^-]\+\)\{2\}$/\1 \2/
  733. t
  734. d
  735. ' "${package_content_dir}/${pkgfile##*/}/.BUILDINFO" | \
  736. sort -u | \
  737. sort -k1,1 > \
  738. "${tmp_dir}/installed-qt-versions"
  739. sed '
  740. s/^depend = \(qt[0-9]\+-\S\+\)$/\1/
  741. t
  742. d
  743. ' "${package_content_dir}/${pkgfile##*/}/.PKGINFO" | \
  744. sort -u | \
  745. join -1 1 -2 1 -o 2.1,2.2 - "${tmp_dir}/installed-qt-versions" | \
  746. tr ' ' '=' >> \
  747. "${pkgfile}.so.needs"
  748. find "${package_content_dir}/${pkgfile##*/}" \
  749. -name 'opt' -prune , \
  750. \( -type f -o -type l \) \
  751. -printf "${checksum}"'%f\n' \
  752. -exec objdump -x '{}' \; 2>/dev/null | \
  753. sed '
  754. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  755. /^'"${checksum}"'/ {
  756. h
  757. d
  758. }
  759. /\sSONAME\s/ {
  760. s/^\s*\S\+\s\+\(\S\+\)\s.*$/\1/
  761. t
  762. }
  763. /^Version definitions:$/,/^$/ {
  764. s/^[0-9]\+\s\+0x[0-9a-fA-F]\+\s\+\(0x[0-9a-fA-F]\+\)\s\+\(\S\+\)$/\2-\1/
  765. T end
  766. G
  767. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  768. t
  769. }
  770. :end
  771. d
  772. ' | \
  773. sort -u >> \
  774. "${pkgfile}.so.provides"
  775. find "${package_content_dir:?}/${pkgfile##*/}" -xdev -not -type l -exec chmod 777 '{}' \;
  776. rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
  777. fi
  778. {
  779. tar -tJf "${pkgfile}" 2>/dev/null | \
  780. sed -n '
  781. s,^usr/lib/python\(2\?\)\([013-9.][0-9.]*\)/$,python\1 \1\2,
  782. t print
  783. s,^usr/lib/perl[^/]\+/\([0-9.]\+\)/$,perl \1,
  784. t print
  785. s,^usr/lib/ruby/\([0-9.]\+\)/$,ruby \1,
  786. t print
  787. b
  788. :print
  789. s/\.\([^.]\+\)$/ \1/
  790. p
  791. '
  792. if [ -f "${pkgfile}.needed-perl-versions" ]; then
  793. sed '
  794. s/^/perl /
  795. ' "${pkgfile}.needed-perl-versions"
  796. rm "${pkgfile}.needed-perl-versions"
  797. fi
  798. } | \
  799. while read -r name major_version minor_version; do
  800. printf '%s>=%s.%s\n%s<%s.%s\n' \
  801. "${name}" \
  802. "${major_version}" \
  803. "${minor_version}" \
  804. "${name}" \
  805. "${major_version}" \
  806. "$((minor_version+1))"
  807. done >> \
  808. "${pkgfile}.so.needs"
  809. done
  810. >&2 printf 'running namcap ...'
  811. if [ "${repository}" = 'multilib' ]; then
  812. x86_64_build_command='multilib-build'
  813. else
  814. x86_64_build_command='extra-x86_64-build'
  815. fi
  816. # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
  817. # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
  818. sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
  819. sudo "${x86_64_build_command}" -c -- -- --version > /dev/null 2>&1 || \
  820. true
  821. >&2 printf ' ok.\n'
  822. >&2 printf 'smoothen namcap log ...'
  823. # now we generate diffs from the namcap.logs
  824. find . "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%p\n' | \
  825. while read -r log; do
  826. smoothen_namcap_log "${log}"
  827. done
  828. find "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%f\n' | \
  829. sed '
  830. s|\(^.*\)-'"${arch}"'\(\.pkg\.tar\.xz-namcap\.log\)$|\0 \1-x86_64\2|
  831. s|^.*-any\.pkg\.tar\.xz-namcap\.log$|\0 \0|
  832. ' | \
  833. while read -r log x86_64_log; do
  834. if [ -f "${x86_64_log}" ]; then
  835. diff -u "${x86_64_log}" "${tar_content_dir}/${log}" | \
  836. sed '
  837. 1,3d
  838. /^[^+-]/d
  839. ' | \
  840. sponge "${tar_content_dir}/${log}"
  841. else
  842. sed -i 's|^|*|' "${tar_content_dir}/${log}"
  843. fi
  844. done
  845. >&2 printf ' ok.\n'
  846. echo 'uploading' > "${tmp_dir}/.ping-build-master"
  847. if ${upload_to_build_master}; then
  848. find "${tar_content_dir}/" -maxdepth 1 \
  849. \( \
  850. -name '*.pkg.tar.xz-namcap.log' -o \
  851. -name '*.pkg.tar.xz.so.needs' -o \
  852. -name '*.pkg.tar.xz.so.provides' \
  853. \) \
  854. -execdir gzip '{}' \;
  855. else
  856. find "${tar_content_dir}/" -maxdepth 1 \
  857. -name '*.pkg.tar.xz-namcap.log' \
  858. -execdir grep -HF '' '{}' \;
  859. fi
  860. # shellcheck disable=SC2046
  861. tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
  862. find "${tar_content_dir}/" -maxdepth 1 \
  863. \( \
  864. -name '*.pkg.tar.xz' -o \
  865. -name '*.pkg.tar.xz.sig' -o \
  866. -name '*.pkg.tar.xz-namcap.log.gz' -o \
  867. -name '*.pkg.tar.xz.so.needs.gz' -o \
  868. -name '*.pkg.tar.xz.so.provides.gz' \
  869. \) \
  870. -printf '%f\n'
  871. ) || \
  872. tar -cf 'package.tar' -T /dev/null
  873. if ${upload_to_build_master}; then
  874. # shellcheck disable=SC2046
  875. rsync $(
  876. find "${tar_content_dir}/" -maxdepth 1 \
  877. \( \
  878. -name '*.pkg.tar.xz' -o \
  879. -name '*.pkg.tar.xz.sig' \
  880. \) \
  881. -printf '%p\n'
  882. ) "rsync://mirror.archlinux32.org/transfer32/" || true
  883. fi
  884. while ${upload_to_build_master}; do
  885. err=0
  886. # shellcheck disable=SC2029
  887. ssh \
  888. -i "${master_build_server_identity}" \
  889. -p "${master_build_server_port}" \
  890. "${master_build_server_user}@${master_build_server}" \
  891. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" "${sub_pkgrel}" \
  892. < 'package.tar' || \
  893. err=$?
  894. case ${err} in
  895. 0)
  896. # upload successful
  897. break
  898. ;;
  899. 1)
  900. >&2 echo '"return-assignment" was running already.'
  901. wait_some_time 30
  902. ;;
  903. 2)
  904. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  905. break
  906. ;;
  907. 3)
  908. >&2 echo "'return-assignment' reports a signature error."
  909. exit 1
  910. ;;
  911. 4)
  912. >&2 echo "'return-assignment' reports too many or missing packages."
  913. exit 1
  914. ;;
  915. 5)
  916. >&2 echo "'return-assignment' was called with wrong arguments."
  917. exit 1
  918. ;;
  919. *)
  920. >&2 echo "unknown return code ${err} from 'return-assignment'"
  921. exit ${err}
  922. esac
  923. done
  924. break
  925. fi
  926. echo 'failure' > "${tmp_dir}/.ping-build-master"
  927. >&2 printf ' failed.\n'
  928. done
  929. if ! ${success}; then
  930. for log in *'.build-log'; do
  931. if [ -f "${log}" ]; then
  932. if ${upload_to_build_master}; then
  933. printf '%s@%s\n' \
  934. "$(whoami)" \
  935. "$(hostname)" >> \
  936. "${log}"
  937. gzip "${log}"
  938. else
  939. grep -HF '' "${log}"
  940. fi
  941. fi
  942. done
  943. if ${upload_to_build_master} && \
  944. tar -cf 'build-logs.gz.tar' \
  945. -- *'.build-log.gz'; then
  946. while true; do
  947. err=0
  948. # shellcheck disable=SC2029
  949. ssh \
  950. -i "${master_build_server_identity}" \
  951. -p "${master_build_server_port}" \
  952. "${master_build_server_user}@${master_build_server}" \
  953. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" 'ERROR' \
  954. < 'build-logs.gz.tar' || \
  955. err=$?
  956. case ${err} in
  957. 0)
  958. # upload successful
  959. break
  960. ;;
  961. 1)
  962. >&2 echo '"return-assignment" was running already.'
  963. wait_some_time 30
  964. ;;
  965. 2)
  966. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  967. break
  968. ;;
  969. 5)
  970. >&2 echo '"return-assignment" was called with wrong arguments.'
  971. exit 1
  972. ;;
  973. *)
  974. >&2 echo "unknown return code ${err} from 'return-assignment'"
  975. exit ${err}
  976. esac
  977. done
  978. fi
  979. if ${exit_after_failure}; then
  980. >&2 echo 'Build failed, exiting now'
  981. exit 0
  982. fi
  983. fi
  984. # clean up tmp_dir
  985. cd "${base_dir}"
  986. recursively_umount_and_rm "${tmp_dir}"
  987. if [ -z "${forced_package}" ]; then
  988. flock -u 9 || true
  989. fi
  990. trap - EXIT
  991. continue
  992. ;;
  993. 1)
  994. >&2 echo 'get-assignment told me:'
  995. >&2 echo ' come back (shortly) later - I was running already'
  996. wait_some_time 30
  997. continue
  998. ;;
  999. 2)
  1000. >&2 echo 'get-assignment told me:'
  1001. >&2 echo ' 2: come back later - there are still packages to be built,'
  1002. >&2 echo ' but currently none has all its dependencies ready'
  1003. wait_some_time 30
  1004. continue
  1005. ;;
  1006. 3)
  1007. >&2 echo 'get-assignment told me:'
  1008. >&2 echo ' 3: come back after the next run of get-package-updates or when'
  1009. >&2 echo ' some currently building packages are returned - currently'
  1010. >&2 echo ' there are no pending buildable packages'
  1011. exit 0
  1012. ;;
  1013. 4)
  1014. >&2 echo 'get-assignment told me:'
  1015. >&2 echo ' 4: wrong number of arguments'
  1016. exit 5
  1017. ;;
  1018. *)
  1019. >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
  1020. exit ${err}
  1021. ;;
  1022. esac
  1023. done
  1024. >&2 echo 'Done.'