Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

build-packages 38KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110
  1. #!/bin/sh
  2. # build packages one by one, then upload the binary package to the repository server
  3. # shellcheck disable=SC2119,SC2120
  4. # shellcheck source=../lib/load-configuration
  5. . "${0%/*}/../lib/load-configuration"
  6. # TODO: report back memory and hdd stats to the build master on successful build
  7. # TODO: some build failures should skip straws - e.g. if dependencies are unavailable,
  8. # we should not try to rebuild with a cleaned chroot
  9. # TODO: some build environments should be archived when the build fails - but:
  10. # which/how?
  11. # TODO: it might be worth trying to build with makedepends installed from
  12. # [testing] and only depends installed from [staging]
  13. # TODO: the following things need proper locking and/or proper handling of
  14. # external locks:
  15. # - `git` on the package source
  16. # - `archbuild` (e.g. /var/lib/archbuilds/staging-i686-build/root.lock)
  17. # - `find ... -exec umount` on the build chroot
  18. # TODO: releave some locking conditions - meta goal: be able to run multiple
  19. # build slaves on one host from within one archlinux32/builder repository
  20. # shellcheck disable=SC2016
  21. usage() {
  22. >&2 echo ''
  23. >&2 echo 'build-packages: build package(s) on the build-list'
  24. >&2 echo ''
  25. >&2 echo 'possible options:'
  26. >&2 echo ' -d|--diff source-directory:'
  27. >&2 echo ' Apply the diff between $source-directory/PKGBUILD and'
  28. >&2 echo ' $source-directory/../../trunk/PKGBUILD before building.'
  29. >&2 echo ' This option is intended for building from trunk where minor'
  30. >&2 echo ' fixes did not get released yet. Hence, it conflicts intentionally'
  31. >&2 echo ' with -n, -l, -t or -x and it requires -p.'
  32. >&2 echo ' -h|--help: Show this help and exit.'
  33. >&2 echo ' -l|--local pkgname.git-revision.git-mod-revision.repository.architecture:'
  34. >&2 echo ' Build the given package without asking / reporting to the'
  35. >&2 echo ' build master. Cannot be combined with -n, -p, -t or -x.'
  36. >&2 echo ' where:'
  37. >&2 echo ' - git-revision: packages-HEAD or community-HEAD or a valid commit'
  38. >&2 echo ' - git-mod-revision: work-tree or a valid commit'
  39. >&2 echo ' example:'
  40. >&2 echo ' build-package -l which.packages-HEAD.work-tree.core.i686'
  41. >&2 echo ' -n count: Build $count packages (if available), then exit.'
  42. >&2 echo ' $count=0 is interpreted as infinity.'
  43. >&2 echo ' The default is $count=1 or 0 iff -t or -x is given.'
  44. >&2 echo ' Cannot be combined with -l.'
  45. >&2 echo ' -p|--prefer pkgname:'
  46. >&2 echo ' Ask the build master to get an assignment for the given'
  47. >&2 echo ' package but built what is offered anyway. Cannot be combined'
  48. >&2 echo ' with -l.'
  49. >&2 echo ' -s|--straw $straw:'
  50. >&2 echo ' Use this straw instead of the preconfigured ones. -- May be'
  51. >&2 echo ' given multiple times to allow using multiple straws.'
  52. >&2 echo ' -t seconds: Do not request new assignment(s) $seconds seconds after start.'
  53. >&2 echo ' Cannot be combined with -l.'
  54. >&2 echo ' -x: If package build fails, do not request new assignment(s).'
  55. >&2 echo ' Cannot be combined with -l.'
  56. >&2 echo ''
  57. >&2 echo 'known straws (separated by and enclosed in ":", sets of straws separated by " "):'
  58. >&2 echo ' :clean_chroot:'
  59. >&2 echo ' clean the chroot before building'
  60. >&2 echo ' :haskell_without_check:'
  61. >&2 echo ' try with :without_check: iff this is a haskell-*, python-* or python2-* package'
  62. >&2 echo ' :mirrored_source:'
  63. >&2 echo ' compile from source tarbal from sources.archlinux.org if possible'
  64. >&2 echo ' :mirrored_source_by_hash:'
  65. >&2 echo ' download sources from sources.archlinux32.org by hash if possible'
  66. >&2 echo ' :without_check:'
  67. >&2 echo ' run makepkg with "--no-check"'
  68. >&2 echo ' :without_systemd_nspawn:'
  69. >&2 echo ' do not invoke archbuild'"'"'s systemd-nspawn fanciness [RISKY]'
  70. >&2 echo ' :with_build_support:'
  71. >&2 echo ' allow using [build-support]'
  72. >&2 echo ' :with_/dev/fuse:'
  73. >&2 echo ' mount /dev/fuse into the chroot'
  74. [ -z "$1" ] && exit 1 || exit "$1"
  75. }
  76. eval set -- "$(
  77. getopt -o d:hl:n:p:s:t:ux \
  78. --long diff: \
  79. --long help \
  80. --long local: \
  81. --long prefer: \
  82. --long straw: \
  83. --long upload \
  84. -n "$(basename "$0")" -- "$@" || \
  85. echo usage
  86. )"
  87. unset count
  88. unset diff_source_dir
  89. unset forced_package
  90. unset forced_straws
  91. unset prefered_package
  92. exit_after_failure=false
  93. timeout=0
  94. while true
  95. do
  96. case "$1" in
  97. -d|--diff)
  98. shift
  99. if [ -n "${diff_source_dir}" ]; then
  100. >&2 echo 'Option -d, --diff can be given only once.'
  101. usage
  102. fi
  103. diff_source_dir="$1"
  104. ;;
  105. -h|--help)
  106. usage 0
  107. ;;
  108. -l|--local)
  109. shift
  110. if [ -n "${forced_package}" ]; then
  111. >&2 echo 'Option -l, --local can be given only once.'
  112. usage
  113. fi
  114. forced_package="$1"
  115. ;;
  116. -n)
  117. shift
  118. count="$1"
  119. [ "${count}" -eq 0 ] && \
  120. count=-1
  121. ;;
  122. -p|--prefer)
  123. shift
  124. if [ -n "${prefered_package}" ]; then
  125. >&2 echo 'Option -p, --prefer can be given only once.'
  126. usage
  127. fi
  128. prefered_package="$1"
  129. ;;
  130. -s|--straw)
  131. shift
  132. forced_straws="${forced_straws} $1"
  133. ;;
  134. -t)
  135. shift
  136. timeout="$1"
  137. ;;
  138. -x)
  139. exit_after_failure=true
  140. ;;
  141. --)
  142. shift
  143. break
  144. ;;
  145. *)
  146. >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
  147. exit 42
  148. ;;
  149. esac
  150. shift
  151. done
  152. if [ $# -ne 0 ]; then
  153. >&2 echo 'Too many arguments.'
  154. usage
  155. fi
  156. if [ -n "${diff_source_dir}" ]; then
  157. if [ -n "${count}" ] || \
  158. [ -n "${forced_package}" ] || \
  159. [ "${timeout}" -ne 0 ] || \
  160. ${exit_after_failure} || \
  161. [ -z "${prefered_package}" ]; then
  162. >&2 echo 'Conflicting flags.'
  163. usage
  164. fi
  165. fi
  166. if [ -n "${forced_package}" ]; then
  167. if [ -n "${count}" ] || \
  168. [ "${timeout}" -ne 0 ] || \
  169. [ -n "${prefered_package}" ] || \
  170. ${exit_after_failure}; then
  171. >&2 echo 'Conflicting flags.'
  172. usage
  173. fi
  174. upload_to_build_master=false
  175. else
  176. upload_to_build_master=true
  177. fi
  178. if [ -z "${count}" ]; then
  179. if [ "${timeout}" -ne 0 ] || ${exit_after_failure}; then
  180. count=-1
  181. else
  182. count=1
  183. fi
  184. fi
  185. if [ -n "${forced_straws}" ]; then
  186. straws_that_might_repair_failing_builds="${forced_straws# }"
  187. fi
  188. if [ "${timeout}" -ne 0 ]; then
  189. timeout=$((timeout+$(date +%s)))
  190. fi
  191. checksum=$(
  192. calculate_script_checksum
  193. )
  194. # When this script or a script in lib/ got updated, we do not request
  195. # any new assignments. This script should rather exit and be restarted.
  196. while [ "${count}" -ne 0 ] && \
  197. [ "$(calculate_script_checksum)" = "${checksum}" ]; do
  198. if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ];
  199. then
  200. break
  201. fi
  202. err=0
  203. case "$(uname -m)" in
  204. 'i486')
  205. my_arch='i486'
  206. ;;
  207. 'i686'|'x86_64')
  208. if grep '^flags\s*:' '/proc/cpuinfo' | \
  209. grep -qwF 'sse2'; then
  210. my_arch='pentium4'
  211. else
  212. my_arch='i686'
  213. fi
  214. ;;
  215. *)
  216. >&2 printf 'Sorry, architecture %s does not work (yet) as a build slave.\n' \
  217. "$(uname -m)"
  218. exit 2
  219. ;;
  220. esac
  221. if [ -z "${forced_package}" ]; then
  222. package=$(
  223. # shellcheck disable=SC2029
  224. ssh \
  225. -i "${master_build_server_identity}" \
  226. -p "${master_build_server_port}" \
  227. "${master_build_server_user}@${master_build_server}" \
  228. 'get-assignment' "${my_arch}" "${prefered_package}"
  229. ) || err=$?
  230. expected_packages=$(
  231. printf '%s\n' "${package}" | \
  232. sed '1d'
  233. )
  234. package=$(
  235. printf '%s\n' "${package}" | \
  236. sed -n '1p'
  237. )
  238. else
  239. package=$(
  240. echo "${forced_package}" | \
  241. sed '
  242. s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3 0 \4|
  243. '
  244. )
  245. expected_packages=''
  246. fi
  247. case ${err} in
  248. # 0: ok, I gave you an assignment
  249. 0)
  250. [ ${count} -gt 0 ] && \
  251. count=$((count-1))
  252. arch="${package##* }"
  253. package="${package% *}"
  254. sub_pkgrel="${package##* }"
  255. package="${package% *}"
  256. repository="${package##* }"
  257. package="${package% *}"
  258. mod_git_revision="${package##* }"
  259. package="${package% *}"
  260. git_revision="${package##* }"
  261. package="${package% *}"
  262. if [ "${arch}" = 'any' ]; then
  263. arch="${my_arch}"
  264. fi
  265. if [ -n "${diff_source_dir}" ] &&
  266. [ "${prefered_package}" != "${package}" ]; then
  267. >&2 echo 'The prefered package was not handed out.'
  268. >&2 echo 'Because -d was given, I will abort.'
  269. # shellcheck disable=SC2029
  270. ssh \
  271. -i "${master_build_server_identity}" \
  272. -p "${master_build_server_port}" \
  273. "${master_build_server_user}@${master_build_server}" \
  274. 'return-assignment' 'ABORT'
  275. exit 2
  276. fi
  277. if [ "${git_revision##*-}" = 'HEAD' ]; then
  278. git_revision=$(
  279. repo_name="${git_revision%-*}"
  280. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  281. if [ -z "${repo_path}" ]; then
  282. >&2 printf 'Unknown git repository "%s".\n' "${repo_name}"
  283. exit 2
  284. fi
  285. git -C "${repo_path}" rev-parse HEAD
  286. )
  287. fi
  288. if [ "${mod_git_revision}" = 'work-tree' ]; then
  289. mod_git_revision=$(
  290. # we can't just create an empty index-file with mktemp, because git doesn't like it
  291. find . \
  292. -mindepth 1 \
  293. -maxdepth 1 \
  294. -name 'tmp.build-packages.git.*' \
  295. -exec rm -rf --one-file-system {} \;
  296. tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
  297. trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
  298. export GIT_INDEX_FILE="${tmp_subdir}/index.new"
  299. git -C "${repo_paths__archlinux32}" add -A
  300. git -C "${repo_paths__archlinux32}" write-tree
  301. )
  302. fi
  303. # Update git repositories (official packages, community packages and the repository of package customizations).
  304. for repo_name in ${repo_names}; do
  305. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  306. if [ -d "${repo_path}/.git" ]; then
  307. git -C "${repo_path}" remote update
  308. else
  309. git -C "${repo_path}" fetch origin master:master
  310. fi || \
  311. true
  312. done
  313. # trigger update of mirror (if configured)
  314. if [ -n "${mirror_update_command}" ]; then
  315. ${mirror_update_command}
  316. fi
  317. bail_out() {
  318. err=$?
  319. if [ -n "$1" ]; then
  320. err="$1"
  321. fi
  322. cd "${base_dir}"
  323. recursively_umount_and_rm "${tmp_dir}"
  324. if [ -z "${forced_package}" ]; then
  325. flock -u 9 || true
  326. fi
  327. exit "${err}"
  328. }
  329. find "${work_dir}" \
  330. -mindepth 1 \
  331. -maxdepth 1 \
  332. -name 'tmp.build-packages.??????' \
  333. -printf '%p\n' | \
  334. while read -r old_tmp_dir; do
  335. find "${old_tmp_dir}" -xdev -not -type l -exec chmod 777 {} \;
  336. rm -rf --one-file-system "${old_tmp_dir}"
  337. done
  338. tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
  339. trap bail_out EXIT
  340. if ! git_repo=$(find_repository_with_commit "${git_revision}") || \
  341. [ -z "${git_repo}" ] || \
  342. ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" || \
  343. ! extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}" || \
  344. ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then
  345. # report local failure (probably a missing commit) to build-master
  346. # shellcheck disable=SC2029
  347. ssh \
  348. -i "${master_build_server_identity}" \
  349. -p "${master_build_server_port}" \
  350. "${master_build_server_user}@${master_build_server}" \
  351. 'return-assignment' 'ABORT'
  352. recursively_umount_and_rm "${tmp_dir}"
  353. if [ -z "${forced_package}" ]; then
  354. flock -u 9 || true
  355. fi
  356. trap - EXIT
  357. continue
  358. fi
  359. cd "${tmp_dir}"
  360. echo 'nothing' > "${tmp_dir}/.ping-build-master"
  361. if [ -z "${forced_package}" ]; then
  362. # we get a lock on "${work_dir}/ping-build-master.lock",
  363. # if we release that lock, ping-to-master should stop _immediately_
  364. exec 9> "${work_dir}/ping-build-master.lock"
  365. if ! verbose_flock -n 9; then
  366. >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
  367. exit 2
  368. fi
  369. "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
  370. fi
  371. success=false
  372. if printf '%s\n' "${package}" | \
  373. grep -q '^\(haskell\|python2\?\)-'; then
  374. straws_that_might_repair_failing_builds=$(
  375. # shellcheck disable=SC2086
  376. printf '%s\n' ${straws_that_might_repair_failing_builds} | \
  377. sed '
  378. s/^\(.*\):haskell_without_check:\(.*\)$/\1:\2\n\1:without_check:\2/
  379. '
  380. )
  381. fi
  382. for straw in ${straws_that_might_repair_failing_builds}; do
  383. if ${success}; then
  384. break
  385. fi
  386. echo 'preparing' > "${tmp_dir}/.ping-build-master"
  387. outerParameters="-r ${archbuild_chroots}"
  388. if echo "${straw}" | \
  389. grep -qF ':clean_chroot:'; then
  390. outerParameters="-c ${outerParameters}"
  391. fi
  392. innerParameters='--nocolor'
  393. if echo "${straw}" | \
  394. grep -qF ':without_check:'; then
  395. innerParameters="${innerParameters} --nocheck"
  396. fi
  397. if echo "${straw}" | \
  398. grep -qF ':with_/dev/fuse:'; then
  399. middleParameters='-d /dev/fuse'
  400. else
  401. middleParameters=''
  402. fi
  403. if echo "${straw}" | \
  404. grep -qF ':with_build_support:'; then
  405. build_command='staging-with-build-support-'"${arch}"'-build'
  406. elif echo "${straw}" | \
  407. grep -qF ':without_systemd_nspawn:'; then
  408. if [ -z "${prefered_package}" ]; then
  409. >&2 echo 'straw :without_systemd_nspawn: only allowed with -p'
  410. exit 2
  411. fi
  412. if [ "${prefered_package}" != "${package}" ]; then
  413. >&2 echo 'The prefered package was not handed out.'
  414. >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.'
  415. # shellcheck disable=SC2029
  416. ssh \
  417. -i "${master_build_server_identity}" \
  418. -p "${master_build_server_port}" \
  419. "${master_build_server_user}@${master_build_server}" \
  420. 'return-assignment' 'ABORT'
  421. exit 2
  422. fi
  423. # we're not interested in what this cpu /can/ do, but what
  424. # architecture the installed packages actually are
  425. if ! pacman -Qi pacman | \
  426. sed 's/^Architecture\s*:\s*//;t;d' | \
  427. grep -qxF "${arch}"; then
  428. >&2 echo 'straw :without_systemd_nspawn: requires running build-packages on the'
  429. >&2 echo 'architecture for which the package should be built.'
  430. >&2 printf '"%s" != "%s"\n' \
  431. "$(
  432. pacman -Qi pacman | \
  433. sed 's/^Architecture\s*:\s*//;t;d'
  434. )" \
  435. "${arch}"
  436. exit 2
  437. fi
  438. build_command='makepkg'
  439. outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver"
  440. middleParameters=''
  441. innerParameters=''
  442. else
  443. build_command='staging-'"${arch}"'-build'
  444. fi
  445. find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
  446. rm {} \;
  447. echo 'checking-source' > "${tmp_dir}/.ping-build-master"
  448. success=false
  449. verifysource_trial=0
  450. while [ ${verifysource_trial} -lt 5 ]; do
  451. verifysource_trial=$((verifysource_trial + 1))
  452. log_file="${tmp_dir}/$(
  453. date -u --iso-8601=seconds | \
  454. cut -d+ -f1
  455. ).build-log"
  456. if CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then
  457. success=true
  458. rm "${log_file}"
  459. break
  460. fi
  461. # receive specific missing keys
  462. if [ ${verifysource_trial} -eq 1 ]; then
  463. missing_keys=$(
  464. sed -n '
  465. s/^.* FAILED (unknown public key \([0-9A-F]\{16\}\)).*$/0x\1/
  466. T
  467. p
  468. ' "${log_file}"
  469. )
  470. if [ -n "${missing_keys}" ]; then
  471. if printf '%s\n' "${missing_keys}" \
  472. | sed '
  473. s@^@https://archlinux32.org/keys.php?k=@
  474. ' \
  475. | xargs curl -Ss \
  476. | gpg --import; then
  477. continue
  478. fi
  479. if gpg --recv-keys "${missing_keys}"; then
  480. continue
  481. fi
  482. fi
  483. verifysource_trial=$((verifysource_trial + 1))
  484. fi
  485. # download the repository key from github
  486. if [ ${verifysource_trial} -eq 2 ]; then
  487. if grep -q ' FAILED (unknown public key \([0-9A-F]\{16\}\))' -- "${log_file}"; then
  488. # TODO: get the name of the key file from its finger print or
  489. # some other information inside the repository
  490. if makepkg --printsrcinfo | \
  491. sed -n '
  492. s,^\ssource = git+\(https://github\.com/[^/[:space:]]\+\)/[^/]*$,\1.gpg,
  493. T
  494. p
  495. ' | \
  496. xargs -rn1 curl -s | \
  497. gpg --import; then
  498. continue
  499. fi
  500. fi
  501. verifysource_trial=$((verifysource_trial + 1))
  502. fi
  503. # try to download source from sources.archlinux.org/sources/$repo/$source
  504. if [ ${verifysource_trial} -eq 3 ]; then
  505. if echo "${straw}" | \
  506. grep -qF ':mirrored_source:'; then
  507. source_name=$(
  508. makepkg --printsrcinfo | \
  509. sed -n '
  510. /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
  511. /^pkgname = /q
  512. ' | \
  513. sed '
  514. s|^pkgbase = \(.*\)$|0 \1-|
  515. s|^epoch = \(.*\)$|1 \1:|
  516. s|^pkgver = \(.*\)$|2 \1-|
  517. s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz|
  518. ' | \
  519. sort -k1n,1 | \
  520. sed '
  521. s|^[0-9] ||
  522. :a
  523. N
  524. s|\n[0-9] \(\S\+\)$|\1|
  525. ta
  526. '
  527. )
  528. if wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
  529. # shellcheck disable=SC2046
  530. tar -xz --overwrite \
  531. -f "${source_name}" \
  532. --exclude PKGBUILD \
  533. $(
  534. if [ -n "${PKGBUILD_mod}" ]; then
  535. git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
  536. tar -t | \
  537. sed 's/^/--exclude /'
  538. fi
  539. ) \
  540. --strip-components=1 \
  541. || true
  542. continue
  543. fi
  544. fi
  545. verifysource_trial=$((verifysource_trial + 1))
  546. fi
  547. # try to download source from sources.archlinux32.org by its hash
  548. if [ ${verifysource_trial} -eq 4 ]; then
  549. if echo "${straw}" | \
  550. grep -qF ':mirrored_source_by_hash:'; then
  551. if download_sources_by_hash; then
  552. continue
  553. fi
  554. fi
  555. verifysource_trial=$((verifysource_trial + 1))
  556. fi
  557. done
  558. if ${success}; then
  559. echo 'building' > "${tmp_dir}/.ping-build-master"
  560. >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) for %s ...' \
  561. "$(date +'%Y-%m-%d %T')" \
  562. "${package}" \
  563. "${git_revision}" \
  564. "${mod_git_revision}" \
  565. "${repository}" \
  566. "${straw}" \
  567. "${arch}"
  568. log_file="${tmp_dir}/$(
  569. date -u --iso-8601=seconds | \
  570. cut -d+ -f1
  571. ).build-log"
  572. # by piping the log, we don't see anything in the terminal,
  573. # but all ways to duplicate the logs seem pretty elaborate
  574. # shellcheck disable=SC2024,SC2086
  575. if ! "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
  576. "${log_file}" 2>&1; then
  577. success=false
  578. build_dir="/var/lib/archbuild/${build_command%-build}/$(whoami)"
  579. if [ -d "${build_dir}" ]; then
  580. sed '
  581. s/^Full log written to\s\+\(\S\+\)\s*$/\1/
  582. t
  583. /# An error report file with more information is saved as:/ {
  584. N
  585. s/^.*\n#\s\+//
  586. t
  587. d
  588. }
  589. s/^See also "\(\S\+\)"\.\s*$/\1/
  590. t
  591. s/^See \(\S\+\)\s*$/\1/
  592. t
  593. d
  594. ' "${log_file}" | \
  595. while read -r extra_log_file; do
  596. grep -HF '' "${build_dir}/${extra_log_file}" || true
  597. done | \
  598. sponge -a "${log_file}"
  599. fi
  600. printf 'used straw: %s\n' "${straw}" >> "${log_file}"
  601. fi
  602. fi
  603. if ${success}; then
  604. # build successful
  605. >&2 printf ' ok.\n'
  606. if [ "${build_command}" = 'makepkg' ]; then
  607. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  608. -exec sh -c 'namcap "$1" > "$1-namcap.log"' '_' '{}' \;
  609. fi
  610. tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
  611. echo 'post-build' > "${tmp_dir}/.ping-build-master"
  612. # remove unexpected packages
  613. if [ -n "${expected_packages}" ]; then
  614. {
  615. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' -printf '%f\n'
  616. printf '%s\n' "${expected_packages}" | \
  617. sed 'p'
  618. } | \
  619. sort | \
  620. uniq -u | \
  621. while read -r unexpected_package; do
  622. rm "${unexpected_package}"*
  623. done
  624. fi
  625. >&2 printf 'signing package(s)\n'
  626. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  627. -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
  628. -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \
  629. -printf '%f\n' | \
  630. sponge | \
  631. while read -r pkg_file; do
  632. {
  633. pacman -Spdd --print-format '%l' --noconfirm "${pkg_file%-*-*-*}" 2>/dev/null | \
  634. sed '
  635. s|/[^/]\+\.pkg\.tar\.xz$||
  636. '
  637. # shellcheck disable=SC2016
  638. curl -Ss 'https://www.archlinux.org/mirrorlist/?country=all&protocol=https&tier=1&use_mirror_status=on' | \
  639. sed -n '
  640. s/^#Server = //
  641. T
  642. s/\$repo/'"${repository}"'/g
  643. s/\$arch/x86_64/g
  644. p
  645. ' | \
  646. shuf
  647. } | \
  648. sed '
  649. s|$|/'"${pkg_file}"'|
  650. s/\.[0-9]\+\(-[^-]\+\)$/\1/
  651. s/-'"${arch}"'\+\(\.pkg\.tar\.xz\)$/-x86_64\1/
  652. ' | \
  653. while read -r url; do
  654. >&2 printf 'downloading "%s" ...' "${url}"
  655. if wget -q --timeout=15 -nd "${url}"; then
  656. >&2 printf ' ok.\n'
  657. break;
  658. fi
  659. >&2 printf ' failed. Next ...\n'
  660. done
  661. done
  662. >&2 printf 'searching for provided libraries\n'
  663. find "${tar_content_dir}" -maxdepth 1 \
  664. -name '*.pkg.tar.xz' \
  665. -printf '%p\n' | \
  666. while read -r pkgfile; do
  667. pacman -Qqlp "${pkgfile}" | \
  668. sed -n '
  669. s,^.*/,,
  670. /\.so\(\..\+\)\?$/ {
  671. :a
  672. p
  673. s/\(\.so\(\..\+\)\?\)\.[^.]\+$/\1/
  674. t a
  675. }
  676. ' | \
  677. sort -u > \
  678. "${pkgfile}.so.provides"
  679. done
  680. >&2 printf 'searching for required and more provided libraries\n'
  681. package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
  682. find "${tar_content_dir}" -maxdepth 1 \
  683. -name '*.pkg.tar.xz' | \
  684. while read -r pkgfile; do
  685. if printf '%s\n' "${pkgfile}" | \
  686. grep -vq -- '-any\.pkg\.tar\.xz$'; then
  687. # we do not check "any" packages for linked libraries
  688. # (why do they have them in the first place?)
  689. mkdir "${package_content_dir}/${pkgfile##*/}"
  690. tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
  691. # we rely on "${checksum}" not appearing in any objdump output
  692. : "${checksum?umm, checksum is unset - this will break below sed-fu}"
  693. # TODO: symbols may be in object files _inside_archives_
  694. # In that case, we're interested in the object file's name,
  695. # not the archive's file name
  696. find "${package_content_dir}/${pkgfile##*/}" \
  697. -name 'opt' -prune , \
  698. -type f \
  699. -exec objdump -x '{}' \; 2>/dev/null | \
  700. sed '
  701. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  702. \@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\1\.[0-9.]\+/@ {
  703. s@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\(\1\(\.[0-9.]\+\)\?\)\.\([0-9]\+\)/.*$@\2 \4@
  704. w '"${pkgfile}"'.needed-perl-version
  705. d
  706. }
  707. /\sNEEDED\s/ {
  708. s/^\s*\S\+\s\+\(\S\+\)\(\s.*\)\?$/\1/
  709. /\.c32$/d
  710. s,^.*/,,
  711. t
  712. }
  713. /^Version References:$/,/^$/ {
  714. /^\s*required from / {
  715. s/^\s*required from \(\S\+\):\s*$/'"${checksum}"'\1/
  716. T end
  717. h
  718. d
  719. }
  720. s/^\s*\(0x[0-9a-fA-F]\+\)\s\+0x[0-9a-fA-F]\+\s\+[0-9]\+\s\+\(\S\+\)$/\2-\1/
  721. T end
  722. G
  723. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  724. t
  725. }
  726. :end
  727. d
  728. ' | \
  729. sort -u > \
  730. "${pkgfile}.so.needs"
  731. sed -i '
  732. /^\(libav.*\.so\)\.[0-9]\+$/ {
  733. :a
  734. N
  735. s/^\(libav.*\.so\)\(\.[0-9]\+\)\?\n\1\.[0-9]\+$/\1/
  736. t a
  737. P
  738. D
  739. }
  740. ' "${pkgfile}.so.needs"
  741. sed '
  742. s/^installed = \(qt[0-9]\+-\S\+\)-\([0-9.]\+\)\(-[^-]\+\)\{2\}$/\1 \2/
  743. t
  744. d
  745. ' "${package_content_dir}/${pkgfile##*/}/.BUILDINFO" | \
  746. sort -u | \
  747. sort -k1,1 > \
  748. "${tmp_dir}/installed-qt-versions"
  749. sed '
  750. s/^depend = \(qt[0-9]\+-\S\+\)$/\1/
  751. t
  752. d
  753. ' "${package_content_dir}/${pkgfile##*/}/.PKGINFO" | \
  754. sort -u | \
  755. join -1 1 -2 1 -o 2.1,2.2 - "${tmp_dir}/installed-qt-versions" | \
  756. tr ' ' '=' >> \
  757. "${pkgfile}.so.needs"
  758. find "${package_content_dir}/${pkgfile##*/}" \
  759. -name 'opt' -prune , \
  760. \( -type f -o -type l \) \
  761. -printf "${checksum}"'%f\n' \
  762. -exec objdump -x '{}' \; 2>/dev/null | \
  763. sed '
  764. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  765. /^'"${checksum}"'/ {
  766. h
  767. d
  768. }
  769. /\sSONAME\s/ {
  770. s/^\s*\S\+\s\+\(\S\+\)\s.*$/\1/
  771. t
  772. }
  773. /^Version definitions:$/,/^$/ {
  774. s/^[0-9]\+\s\+0x[0-9a-fA-F]\+\s\+\(0x[0-9a-fA-F]\+\)\s\+\(\S\+\)$/\2-\1/
  775. T end
  776. G
  777. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  778. t
  779. }
  780. :end
  781. d
  782. ' | \
  783. sort -u >> \
  784. "${pkgfile}.so.provides"
  785. find "${package_content_dir:?}/${pkgfile##*/}" -xdev -not -type l -exec chmod 777 '{}' \;
  786. rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
  787. fi
  788. {
  789. tar -tJf "${pkgfile}" 2>/dev/null | \
  790. sed -n '
  791. s,^usr/lib/python\(2\?\)\([013-9.][0-9.]*\)/$,python\1 \1\2,
  792. t print
  793. s,^usr/lib/perl[^/]\+/\([0-9.]\+\)/$,perl \1,
  794. t print
  795. s,^usr/lib/ruby/\([0-9.]\+\)/$,ruby \1,
  796. t print
  797. b
  798. :print
  799. s/\.\([^.]\+\)$/ \1/
  800. p
  801. '
  802. if [ -f "${pkgfile}.needed-perl-versions" ]; then
  803. sed '
  804. s/^/perl /
  805. ' "${pkgfile}.needed-perl-versions"
  806. rm "${pkgfile}.needed-perl-versions"
  807. fi
  808. } | \
  809. while read -r name major_version minor_version; do
  810. printf '%s>=%s.%s\n%s<%s.%s\n' \
  811. "${name}" \
  812. "${major_version}" \
  813. "${minor_version}" \
  814. "${name}" \
  815. "${major_version}" \
  816. "$((minor_version+1))"
  817. done >> \
  818. "${pkgfile}.so.needs"
  819. # TODO: dependency-adding from make_source_info() should
  820. # be replicated here, somehow ...
  821. done
  822. >&2 printf 'running namcap ...'
  823. if [ "${repository}" = 'multilib' ]; then
  824. x86_64_build_command='multilib-build'
  825. else
  826. x86_64_build_command='extra-x86_64-build'
  827. fi
  828. # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
  829. # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
  830. sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
  831. sudo "${x86_64_build_command}" -c -- -- --version > /dev/null 2>&1 || \
  832. true
  833. >&2 printf ' ok.\n'
  834. >&2 printf 'smoothen namcap log ...'
  835. # now we generate diffs from the namcap.logs
  836. find . "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%p\n' | \
  837. while read -r log; do
  838. smoothen_namcap_log "${log}"
  839. done
  840. find "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%f\n' | \
  841. sed '
  842. s|\(^.*\)-'"${arch}"'\(\.pkg\.tar\.xz-namcap\.log\)$|\0 \1-x86_64\2|
  843. s|^.*-any\.pkg\.tar\.xz-namcap\.log$|\0 \0|
  844. ' | \
  845. while read -r log x86_64_log; do
  846. if [ -f "${x86_64_log}" ]; then
  847. diff -u "${x86_64_log}" "${tar_content_dir}/${log}" | \
  848. sed '
  849. 1,3d
  850. /^[^+-]/d
  851. ' | \
  852. sponge "${tar_content_dir}/${log}"
  853. else
  854. sed -i 's|^|*|' "${tar_content_dir}/${log}"
  855. fi
  856. done
  857. >&2 printf ' ok.\n'
  858. echo 'uploading' > "${tmp_dir}/.ping-build-master"
  859. if ${upload_to_build_master}; then
  860. find "${tar_content_dir}/" -maxdepth 1 \
  861. \( \
  862. -name '*.pkg.tar.xz-namcap.log' -o \
  863. -name '*.pkg.tar.xz.so.needs' -o \
  864. -name '*.pkg.tar.xz.so.provides' \
  865. \) \
  866. -execdir gzip '{}' \;
  867. else
  868. find "${tar_content_dir}/" -maxdepth 1 \
  869. -name '*.pkg.tar.xz-namcap.log' \
  870. -execdir grep -HF '' '{}' \;
  871. fi
  872. # shellcheck disable=SC2046
  873. tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
  874. find "${tar_content_dir}/" -maxdepth 1 \
  875. \( \
  876. -name '*.pkg.tar.xz' -o \
  877. -name '*.pkg.tar.xz.sig' -o \
  878. -name '*.pkg.tar.xz-namcap.log.gz' -o \
  879. -name '*.pkg.tar.xz.so.needs.gz' -o \
  880. -name '*.pkg.tar.xz.so.provides.gz' \
  881. \) \
  882. -printf '%f\n'
  883. ) || \
  884. tar -cf 'package.tar' -T /dev/null
  885. if ${upload_to_build_master}; then
  886. # shellcheck disable=SC2046
  887. rsync $(
  888. find "${tar_content_dir}/" -maxdepth 1 \
  889. \( \
  890. -name '*.pkg.tar.xz' -o \
  891. -name '*.pkg.tar.xz.sig' \
  892. \) \
  893. -printf '%p\n'
  894. ) "rsync://mirror.archlinux32.org/transfer32/" || true
  895. fi
  896. while ${upload_to_build_master}; do
  897. err=0
  898. # shellcheck disable=SC2029
  899. ssh \
  900. -i "${master_build_server_identity}" \
  901. -p "${master_build_server_port}" \
  902. "${master_build_server_user}@${master_build_server}" \
  903. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" "${sub_pkgrel}" \
  904. < 'package.tar' || \
  905. err=$?
  906. case ${err} in
  907. 0)
  908. # upload successful
  909. break
  910. ;;
  911. 1)
  912. >&2 echo '"return-assignment" was running already.'
  913. wait_some_time 30
  914. ;;
  915. 2)
  916. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  917. break
  918. ;;
  919. 3)
  920. >&2 echo "'return-assignment' reports a signature error."
  921. exit 1
  922. ;;
  923. 4)
  924. >&2 echo "'return-assignment' reports too many or missing packages."
  925. exit 1
  926. ;;
  927. 5)
  928. >&2 echo "'return-assignment' was called with wrong arguments."
  929. exit 1
  930. ;;
  931. *)
  932. >&2 echo "unknown return code ${err} from 'return-assignment'"
  933. exit ${err}
  934. esac
  935. done
  936. break
  937. fi
  938. echo 'failure' > "${tmp_dir}/.ping-build-master"
  939. >&2 printf ' failed.\n'
  940. done
  941. if ! ${success}; then
  942. for log in *'.build-log'; do
  943. if [ -f "${log}" ]; then
  944. if ${upload_to_build_master}; then
  945. printf '%s@%s\n' \
  946. "$(whoami)" \
  947. "$(hostname)" >> \
  948. "${log}"
  949. gzip "${log}"
  950. else
  951. grep -HF '' "${log}"
  952. fi
  953. fi
  954. done
  955. if ${upload_to_build_master} && \
  956. tar -cf 'build-logs.gz.tar' \
  957. -- *'.build-log.gz'; then
  958. while true; do
  959. err=0
  960. # shellcheck disable=SC2029
  961. ssh \
  962. -i "${master_build_server_identity}" \
  963. -p "${master_build_server_port}" \
  964. "${master_build_server_user}@${master_build_server}" \
  965. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" 'ERROR' \
  966. < 'build-logs.gz.tar' || \
  967. err=$?
  968. case ${err} in
  969. 0)
  970. # upload successful
  971. break
  972. ;;
  973. 1)
  974. >&2 echo '"return-assignment" was running already.'
  975. wait_some_time 30
  976. ;;
  977. 2)
  978. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  979. break
  980. ;;
  981. 5)
  982. >&2 echo '"return-assignment" was called with wrong arguments.'
  983. exit 1
  984. ;;
  985. *)
  986. >&2 echo "unknown return code ${err} from 'return-assignment'"
  987. exit ${err}
  988. esac
  989. done
  990. fi
  991. if ${exit_after_failure}; then
  992. >&2 echo 'Build failed, exiting now'
  993. exit 0
  994. fi
  995. fi
  996. # clean up tmp_dir
  997. cd "${base_dir}"
  998. recursively_umount_and_rm "${tmp_dir}"
  999. if [ -z "${forced_package}" ]; then
  1000. flock -u 9 || true
  1001. fi
  1002. trap - EXIT
  1003. continue
  1004. ;;
  1005. 1)
  1006. >&2 echo 'get-assignment told me:'
  1007. >&2 echo ' come back (shortly) later - I was running already'
  1008. wait_some_time 30
  1009. continue
  1010. ;;
  1011. 2)
  1012. >&2 echo 'get-assignment told me:'
  1013. >&2 echo ' 2: come back later - there are still packages to be built,'
  1014. >&2 echo ' but currently none has all its dependencies ready'
  1015. wait_some_time 30
  1016. continue
  1017. ;;
  1018. 3)
  1019. >&2 echo 'get-assignment told me:'
  1020. >&2 echo ' 3: come back after the next run of get-package-updates or when'
  1021. >&2 echo ' some currently building packages are returned - currently'
  1022. >&2 echo ' there are no pending buildable packages'
  1023. exit 0
  1024. ;;
  1025. 4)
  1026. >&2 echo 'get-assignment told me:'
  1027. >&2 echo ' 4: wrong number of arguments'
  1028. exit 5
  1029. ;;
  1030. *)
  1031. >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
  1032. exit ${err}
  1033. ;;
  1034. esac
  1035. done
  1036. >&2 echo 'Done.'