Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

build-packages 38KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108
  1. #!/bin/sh
  2. # build packages one by one, then upload the binary package to the repository server
  3. # shellcheck disable=SC2119,SC2120
  4. # shellcheck source=../lib/load-configuration
  5. . "${0%/*}/../lib/load-configuration"
  6. # TODO: report back memory and hdd stats to the build master on successful build
  7. # TODO: some build failures should skip straws - e.g. if dependencies are unavailable,
  8. # we should not try to rebuild with a cleaned chroot
  9. # TODO: some build environments should be archived when the build fails - but:
  10. # which/how?
  11. # TODO: it might be worth trying to build with makedepends installed from
  12. # [testing] and only depends installed from [staging]
  13. # TODO: the following things need proper locking and/or proper handling of
  14. # external locks:
  15. # - `git` on the package source
  16. # - `archbuild` (e.g. /var/lib/archbuilds/staging-i686-build/root.lock)
  17. # - `find ... -exec umount` on the build chroot
  18. # TODO: releave some locking conditions - meta goal: be able to run multiple
  19. # build slaves on one host from within one archlinux32/builder repository
  20. # shellcheck disable=SC2016
  21. usage() {
  22. >&2 echo ''
  23. >&2 echo 'build-packages: build package(s) on the build-list'
  24. >&2 echo ''
  25. >&2 echo 'possible options:'
  26. >&2 echo ' -d|--diff source-directory:'
  27. >&2 echo ' Apply the diff between $source-directory/PKGBUILD and'
  28. >&2 echo ' $source-directory/../../trunk/PKGBUILD before building.'
  29. >&2 echo ' This option is intended for building from trunk where minor'
  30. >&2 echo ' fixes did not get released yet. Hence, it conflicts intentionally'
  31. >&2 echo ' with -n, -l, -t or -x and it requires -p.'
  32. >&2 echo ' -h|--help: Show this help and exit.'
  33. >&2 echo ' -l|--local pkgname.git-revision.git-mod-revision.repository.architecture:'
  34. >&2 echo ' Build the given package without asking / reporting to the'
  35. >&2 echo ' build master. Cannot be combined with -n, -p, -t or -x.'
  36. >&2 echo ' where:'
  37. >&2 echo ' - git-revision: packages-HEAD or community-HEAD or a valid commit'
  38. >&2 echo ' - git-mod-revision: work-tree or a valid commit'
  39. >&2 echo ' example:'
  40. >&2 echo ' build-package -l which.packages-HEAD.work-tree.core.i686'
  41. >&2 echo ' -n count: Build $count packages (if available), then exit.'
  42. >&2 echo ' $count=0 is interpreted as infinity.'
  43. >&2 echo ' The default is $count=1 or 0 iff -t or -x is given.'
  44. >&2 echo ' Cannot be combined with -l.'
  45. >&2 echo ' -p|--prefer pkgname:'
  46. >&2 echo ' Ask the build master to get an assignment for the given'
  47. >&2 echo ' package but built what is offered anyway. Cannot be combined'
  48. >&2 echo ' with -l.'
  49. >&2 echo ' -s|--straw $straw:'
  50. >&2 echo ' Use this straw instead of the preconfigured ones. -- May be'
  51. >&2 echo ' given multiple times to allow using multiple straws.'
  52. >&2 echo ' -t seconds: Do not request new assignment(s) $seconds seconds after start.'
  53. >&2 echo ' Cannot be combined with -l.'
  54. >&2 echo ' -x: If package build fails, do not request new assignment(s).'
  55. >&2 echo ' Cannot be combined with -l.'
  56. >&2 echo ''
  57. >&2 echo 'known straws (separated by and enclosed in ":", sets of straws separated by " "):'
  58. >&2 echo ' :clean_chroot:'
  59. >&2 echo ' clean the chroot before building'
  60. >&2 echo ' :haskell_without_check:'
  61. >&2 echo ' try with :without_check: iff this is a haskell-*, python-* or python2-* package'
  62. >&2 echo ' :mirrored_source:'
  63. >&2 echo ' compile from source tarbal from sources.archlinux.org if possible'
  64. >&2 echo ' :mirrored_source_by_hash:'
  65. >&2 echo ' download sources from sources.archlinux32.org by hash if possible'
  66. >&2 echo ' :without_check:'
  67. >&2 echo ' run makepkg with "--no-check"'
  68. >&2 echo ' :without_systemd_nspawn:'
  69. >&2 echo ' do not invoke archbuild'"'"'s systemd-nspawn fanciness [RISKY]'
  70. >&2 echo ' :with_build_support:'
  71. >&2 echo ' allow using [build-support]'
  72. >&2 echo ' :with_/dev/fuse:'
  73. >&2 echo ' mount /dev/fuse into the chroot'
  74. [ -z "$1" ] && exit 1 || exit "$1"
  75. }
  76. eval set -- "$(
  77. getopt -o d:hl:n:p:s:t:ux \
  78. --long diff: \
  79. --long help \
  80. --long local: \
  81. --long prefer: \
  82. --long straw: \
  83. --long upload \
  84. -n "$(basename "$0")" -- "$@" || \
  85. echo usage
  86. )"
  87. unset count
  88. unset diff_source_dir
  89. unset forced_package
  90. unset forced_straws
  91. unset prefered_package
  92. exit_after_failure=false
  93. timeout=0
  94. while true
  95. do
  96. case "$1" in
  97. -d|--diff)
  98. shift
  99. if [ -n "${diff_source_dir}" ]; then
  100. >&2 echo 'Option -d, --diff can be given only once.'
  101. usage
  102. fi
  103. diff_source_dir="$1"
  104. ;;
  105. -h|--help)
  106. usage 0
  107. ;;
  108. -l|--local)
  109. shift
  110. if [ -n "${forced_package}" ]; then
  111. >&2 echo 'Option -l, --local can be given only once.'
  112. usage
  113. fi
  114. forced_package="$1"
  115. ;;
  116. -n)
  117. shift
  118. count="$1"
  119. [ "${count}" -eq 0 ] && \
  120. count=-1
  121. ;;
  122. -p|--prefer)
  123. shift
  124. if [ -n "${prefered_package}" ]; then
  125. >&2 echo 'Option -p, --prefer can be given only once.'
  126. usage
  127. fi
  128. prefered_package="$1"
  129. ;;
  130. -s|--straw)
  131. shift
  132. forced_straws="${forced_straws} $1"
  133. ;;
  134. -t)
  135. shift
  136. timeout="$1"
  137. ;;
  138. -x)
  139. exit_after_failure=true
  140. ;;
  141. --)
  142. shift
  143. break
  144. ;;
  145. *)
  146. >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
  147. exit 42
  148. ;;
  149. esac
  150. shift
  151. done
  152. if [ $# -ne 0 ]; then
  153. >&2 echo 'Too many arguments.'
  154. usage
  155. fi
  156. if [ -n "${diff_source_dir}" ]; then
  157. if [ -n "${count}" ] || \
  158. [ -n "${forced_package}" ] || \
  159. [ "${timeout}" -ne 0 ] || \
  160. ${exit_after_failure} || \
  161. [ -z "${prefered_package}" ]; then
  162. >&2 echo 'Conflicting flags.'
  163. usage
  164. fi
  165. fi
  166. if [ -n "${forced_package}" ]; then
  167. if [ -n "${count}" ] || \
  168. [ "${timeout}" -ne 0 ] || \
  169. [ -n "${prefered_package}" ] || \
  170. ${exit_after_failure}; then
  171. >&2 echo 'Conflicting flags.'
  172. usage
  173. fi
  174. upload_to_build_master=false
  175. else
  176. upload_to_build_master=true
  177. fi
  178. if [ -z "${count}" ]; then
  179. if [ "${timeout}" -ne 0 ] || ${exit_after_failure}; then
  180. count=-1
  181. else
  182. count=1
  183. fi
  184. fi
  185. if [ -n "${forced_straws}" ]; then
  186. straws_that_might_repair_failing_builds="${forced_straws# }"
  187. fi
  188. if [ "${timeout}" -ne 0 ]; then
  189. timeout=$((timeout+$(date +%s)))
  190. fi
  191. checksum=$(
  192. calculate_script_checksum
  193. )
  194. # When this script or a script in lib/ got updated, we do not request
  195. # any new assignments. This script should rather exit and be restarted.
  196. while [ "${count}" -ne 0 ] && \
  197. [ "$(calculate_script_checksum)" = "${checksum}" ]; do
  198. if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ];
  199. then
  200. break
  201. fi
  202. err=0
  203. case "$(uname -m)" in
  204. 'i486')
  205. my_arch='i486'
  206. ;;
  207. 'i686'|'x86_64')
  208. if grep '^flags\s*:' '/proc/cpuinfo' | \
  209. grep -qwF 'sse2'; then
  210. my_arch='pentium4'
  211. else
  212. my_arch='i686'
  213. fi
  214. ;;
  215. *)
  216. >&2 printf 'Sorry, architecture %s does not work (yet) as a build slave.\n' \
  217. "$(uname -m)"
  218. exit 2
  219. ;;
  220. esac
  221. if [ -z "${forced_package}" ]; then
  222. package=$(
  223. # shellcheck disable=SC2029
  224. ssh \
  225. -i "${master_build_server_identity}" \
  226. -p "${master_build_server_port}" \
  227. "${master_build_server_user}@${master_build_server}" \
  228. 'get-assignment' "${my_arch}" "${prefered_package}"
  229. ) || err=$?
  230. expected_packages=$(
  231. printf '%s\n' "${package}" | \
  232. sed '1d'
  233. )
  234. package=$(
  235. printf '%s\n' "${package}" | \
  236. sed -n '1p'
  237. )
  238. else
  239. package=$(
  240. echo "${forced_package}" | \
  241. sed '
  242. s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3 0 \4|
  243. '
  244. )
  245. expected_packages=''
  246. fi
  247. case ${err} in
  248. # 0: ok, I gave you an assignment
  249. 0)
  250. [ ${count} -gt 0 ] && \
  251. count=$((count-1))
  252. arch="${package##* }"
  253. package="${package% *}"
  254. sub_pkgrel="${package##* }"
  255. package="${package% *}"
  256. repository="${package##* }"
  257. package="${package% *}"
  258. mod_git_revision="${package##* }"
  259. package="${package% *}"
  260. git_revision="${package##* }"
  261. package="${package% *}"
  262. if [ "${arch}" = 'any' ]; then
  263. arch="${my_arch}"
  264. fi
  265. if [ -n "${diff_source_dir}" ] &&
  266. [ "${prefered_package}" != "${package}" ]; then
  267. >&2 echo 'The prefered package was not handed out.'
  268. >&2 echo 'Because -d was given, I will abort.'
  269. # shellcheck disable=SC2029
  270. ssh \
  271. -i "${master_build_server_identity}" \
  272. -p "${master_build_server_port}" \
  273. "${master_build_server_user}@${master_build_server}" \
  274. 'return-assignment' 'ABORT'
  275. exit 2
  276. fi
  277. if [ "${git_revision##*-}" = 'HEAD' ]; then
  278. git_revision=$(
  279. repo_name="${git_revision%-*}"
  280. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  281. if [ -z "${repo_path}" ]; then
  282. >&2 printf 'Unknown git repository "%s".\n' "${repo_name}"
  283. exit 2
  284. fi
  285. git -C "${repo_path}" rev-parse HEAD
  286. )
  287. fi
  288. if [ "${mod_git_revision}" = 'work-tree' ]; then
  289. mod_git_revision=$(
  290. # we can't just create an empty index-file with mktemp, because git doesn't like it
  291. find . \
  292. -mindepth 1 \
  293. -maxdepth 1 \
  294. -name 'tmp.build-packages.git.*' \
  295. -exec rm -rf --one-file-system {} \;
  296. tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
  297. trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
  298. export GIT_INDEX_FILE="${tmp_subdir}/index.new"
  299. git -C "${repo_paths__archlinux32}" add -A
  300. git -C "${repo_paths__archlinux32}" write-tree
  301. )
  302. fi
  303. # Update git repositories (official packages, community packages and the repository of package customizations).
  304. for repo_name in ${repo_names}; do
  305. eval repo_path='"${repo_paths__'"${repo_name}"'}"'
  306. if [ -d "${repo_path}/.git" ]; then
  307. git -C "${repo_path}" remote update
  308. else
  309. git -C "${repo_path}" fetch origin master:master
  310. fi || \
  311. true
  312. done
  313. # trigger update of mirror (if configured)
  314. if [ -n "${mirror_update_command}" ]; then
  315. ${mirror_update_command}
  316. fi
  317. bail_out() {
  318. err=$?
  319. if [ -n "$1" ]; then
  320. err="$1"
  321. fi
  322. cd "${base_dir}"
  323. recursively_umount_and_rm "${tmp_dir}"
  324. if [ -z "${forced_package}" ]; then
  325. flock -u 9 || true
  326. fi
  327. exit "${err}"
  328. }
  329. find "${work_dir}" \
  330. -mindepth 1 \
  331. -maxdepth 1 \
  332. -name 'tmp.build-packages.??????' \
  333. -printf '%p\n' | \
  334. while read -r old_tmp_dir; do
  335. find "${old_tmp_dir}" -xdev -not -type l -exec chmod 777 {} \;
  336. rm -rf --one-file-system "${old_tmp_dir}"
  337. done
  338. tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
  339. trap bail_out EXIT
  340. if ! git_repo=$(find_repository_with_commit "${git_revision}") || \
  341. [ -z "${git_repo}" ] || \
  342. ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" || \
  343. ! extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}" || \
  344. ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then
  345. # report local failure (probably a missing commit) to build-master
  346. # shellcheck disable=SC2029
  347. ssh \
  348. -i "${master_build_server_identity}" \
  349. -p "${master_build_server_port}" \
  350. "${master_build_server_user}@${master_build_server}" \
  351. 'return-assignment' 'ABORT'
  352. recursively_umount_and_rm "${tmp_dir}"
  353. if [ -z "${forced_package}" ]; then
  354. flock -u 9 || true
  355. fi
  356. trap - EXIT
  357. continue
  358. fi
  359. cd "${tmp_dir}"
  360. echo 'nothing' > "${tmp_dir}/.ping-build-master"
  361. if [ -z "${forced_package}" ]; then
  362. # we get a lock on "${work_dir}/ping-build-master.lock",
  363. # if we release that lock, ping-to-master should stop _immediately_
  364. exec 9> "${work_dir}/ping-build-master.lock"
  365. if ! verbose_flock -n 9; then
  366. >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
  367. exit 2
  368. fi
  369. "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
  370. fi
  371. success=false
  372. if printf '%s\n' "${package}" | \
  373. grep -q '^\(haskell\|python2\?\)-'; then
  374. straws_that_might_repair_failing_builds=$(
  375. # shellcheck disable=SC2086
  376. printf '%s\n' ${straws_that_might_repair_failing_builds} | \
  377. sed '
  378. s/^\(.*\):haskell_without_check:\(.*\)$/\1:\2\n\1:without_check:\2/
  379. '
  380. )
  381. fi
  382. for straw in ${straws_that_might_repair_failing_builds}; do
  383. if ${success}; then
  384. break
  385. fi
  386. echo 'preparing' > "${tmp_dir}/.ping-build-master"
  387. outerParameters="-r ${archbuild_chroots}"
  388. if echo "${straw}" | \
  389. grep -qF ':clean_chroot:'; then
  390. outerParameters="-c ${outerParameters}"
  391. fi
  392. innerParameters='--nocolor'
  393. if echo "${straw}" | \
  394. grep -qF ':without_check:'; then
  395. innerParameters="${innerParameters} --nocheck"
  396. fi
  397. if echo "${straw}" | \
  398. grep -qF ':with_/dev/fuse:'; then
  399. middleParameters='-d /dev/fuse'
  400. else
  401. middleParameters=''
  402. fi
  403. if echo "${straw}" | \
  404. grep -qF ':with_build_support:'; then
  405. build_command='staging-with-build-support-'"${arch}"'-build'
  406. elif echo "${straw}" | \
  407. grep -qF ':without_systemd_nspawn:'; then
  408. if [ -z "${prefered_package}" ]; then
  409. >&2 echo 'straw :without_systemd_nspawn: only allowed with -p'
  410. exit 2
  411. fi
  412. if [ "${prefered_package}" != "${package}" ]; then
  413. >&2 echo 'The prefered package was not handed out.'
  414. >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.'
  415. # shellcheck disable=SC2029
  416. ssh \
  417. -i "${master_build_server_identity}" \
  418. -p "${master_build_server_port}" \
  419. "${master_build_server_user}@${master_build_server}" \
  420. 'return-assignment' 'ABORT'
  421. exit 2
  422. fi
  423. # we're not interested in what this cpu /can/ do, but what
  424. # architecture the installed packages actually are
  425. if ! pacman -Qi pacman | \
  426. sed 's/^Architecture\s*:\s*//;t;d' | \
  427. grep -qxF "${arch}"; then
  428. >&2 echo 'straw :without_systemd_nspawn: requires running build-packages on the'
  429. >&2 echo 'architecture for which the package should be built.'
  430. >&2 printf '"%s" != "%s"\n' \
  431. "$(
  432. pacman -Qi pacman | \
  433. sed 's/^Architecture\s*:\s*//;t;d'
  434. )" \
  435. "${arch}"
  436. exit 2
  437. fi
  438. build_command='makepkg'
  439. outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver"
  440. middleParameters=''
  441. innerParameters=''
  442. else
  443. build_command='staging-'"${arch}"'-build'
  444. fi
  445. find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \
  446. rm {} \;
  447. echo 'checking-source' > "${tmp_dir}/.ping-build-master"
  448. success=false
  449. verifysource_trial=0
  450. while [ ${verifysource_trial} -lt 5 ]; do
  451. verifysource_trial=$((verifysource_trial + 1))
  452. log_file="${tmp_dir}/$(
  453. date -u --iso-8601=seconds | \
  454. cut -d+ -f1
  455. ).build-log"
  456. if CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then
  457. success=true
  458. rm "${log_file}"
  459. break
  460. fi
  461. # receive specific missing keys
  462. if [ ${verifysource_trial} -eq 1 ]; then
  463. missing_keys=$(
  464. sed -n '
  465. s/^.* FAILED (unknown public key \([0-9A-F]\{16\}\)).*$/0x\1/
  466. T
  467. p
  468. ' "${log_file}"
  469. )
  470. if [ -n "${missing_keys}" ]; then
  471. if printf '%s\n' "${missing_keys}" \
  472. | sed '
  473. s@^@https://archlinux32.org/keys.php?k=@
  474. ' \
  475. | xargs curl -Ss \
  476. | gpg --import; then
  477. continue
  478. fi
  479. if gpg --recv-keys "${missing_keys}"; then
  480. continue
  481. fi
  482. fi
  483. verifysource_trial=$((verifysource_trial + 1))
  484. fi
  485. # download the repository key from github
  486. if [ ${verifysource_trial} -eq 2 ]; then
  487. if grep -q ' FAILED (unknown public key \([0-9A-F]\{16\}\))' -- "${log_file}"; then
  488. # TODO: get the name of the key file from its finger print or
  489. # some other information inside the repository
  490. if makepkg --printsrcinfo | \
  491. sed -n '
  492. s,^\ssource = git+\(https://github\.com/[^/[:space:]]\+\)/[^/]*$,\1.gpg,
  493. T
  494. p
  495. ' | \
  496. xargs -rn1 curl -s | \
  497. gpg --import; then
  498. continue
  499. fi
  500. fi
  501. verifysource_trial=$((verifysource_trial + 1))
  502. fi
  503. # try to download source from sources.archlinux.org/sources/$repo/$source
  504. if [ ${verifysource_trial} -eq 3 ]; then
  505. if echo "${straw}" | \
  506. grep -qF ':mirrored_source:'; then
  507. source_name=$(
  508. makepkg --printsrcinfo | \
  509. sed -n '
  510. /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
  511. /^pkgname = /q
  512. ' | \
  513. sed '
  514. s|^pkgbase = \(.*\)$|0 \1-|
  515. s|^epoch = \(.*\)$|1 \1:|
  516. s|^pkgver = \(.*\)$|2 \1-|
  517. s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz|
  518. ' | \
  519. sort -k1n,1 | \
  520. sed '
  521. s|^[0-9] ||
  522. :a
  523. N
  524. s|\n[0-9] \(\S\+\)$|\1|
  525. ta
  526. '
  527. )
  528. if wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
  529. # shellcheck disable=SC2046
  530. tar -xz --overwrite \
  531. -f "${source_name}" \
  532. --exclude PKGBUILD \
  533. $(
  534. if [ -n "${PKGBUILD_mod}" ]; then
  535. git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
  536. tar -t | \
  537. sed 's/^/--exclude /'
  538. fi
  539. ) \
  540. --strip-components=1 \
  541. || true
  542. continue
  543. fi
  544. fi
  545. verifysource_trial=$((verifysource_trial + 1))
  546. fi
  547. # try to download source from sources.archlinux32.org by its hash
  548. if [ ${verifysource_trial} -eq 4 ]; then
  549. if echo "${straw}" | \
  550. grep -qF ':mirrored_source_by_hash:'; then
  551. if download_sources_by_hash; then
  552. continue
  553. fi
  554. fi
  555. verifysource_trial=$((verifysource_trial + 1))
  556. fi
  557. done
  558. if ${success}; then
  559. echo 'building' > "${tmp_dir}/.ping-build-master"
  560. >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) for %s ...' \
  561. "$(date +'%Y-%m-%d %T')" \
  562. "${package}" \
  563. "${git_revision}" \
  564. "${mod_git_revision}" \
  565. "${repository}" \
  566. "${straw}" \
  567. "${arch}"
  568. log_file="${tmp_dir}/$(
  569. date -u --iso-8601=seconds | \
  570. cut -d+ -f1
  571. ).build-log"
  572. # by piping the log, we don't see anything in the terminal,
  573. # but all ways to duplicate the logs seem pretty elaborate
  574. # shellcheck disable=SC2024,SC2086
  575. if ! "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
  576. "${log_file}" 2>&1; then
  577. success=false
  578. build_dir="/var/lib/archbuild/${build_command%-build}/$(whoami)"
  579. if [ -d "${build_dir}" ]; then
  580. sed '
  581. s/^Full log written to\s\+\(\S\+\)\s*$/\1/
  582. t
  583. /# An error report file with more information is saved as:/ {
  584. N
  585. s/^.*\n#\s\+//
  586. t
  587. d
  588. }
  589. s/^See also "\(\S\+\)"\.\s*$/\1/
  590. t
  591. d
  592. ' "${log_file}" | \
  593. while read -r extra_log_file; do
  594. grep -HF '' "${build_dir}/${extra_log_file}" || true
  595. done | \
  596. sponge -a "${log_file}"
  597. fi
  598. printf 'used straw: %s\n' "${straw}" >> "${log_file}"
  599. fi
  600. fi
  601. if ${success}; then
  602. # build successful
  603. >&2 printf ' ok.\n'
  604. if [ "${build_command}" = 'makepkg' ]; then
  605. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  606. -exec sh -c 'namcap "$1" > "$1-namcap.log"' '_' '{}' \;
  607. fi
  608. tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
  609. echo 'post-build' > "${tmp_dir}/.ping-build-master"
  610. # remove unexpected packages
  611. if [ -n "${expected_packages}" ]; then
  612. {
  613. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' -printf '%f\n'
  614. printf '%s\n' "${expected_packages}" | \
  615. sed 'p'
  616. } | \
  617. sort | \
  618. uniq -u | \
  619. while read -r unexpected_package; do
  620. rm "${unexpected_package}"*
  621. done
  622. fi
  623. >&2 printf 'signing package(s)\n'
  624. find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \
  625. -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
  626. -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \
  627. -printf '%f\n' | \
  628. sponge | \
  629. while read -r pkg_file; do
  630. {
  631. pacman -Spdd --print-format '%l' --noconfirm "${pkg_file%-*-*-*}" 2>/dev/null | \
  632. sed '
  633. s|/[^/]\+\.pkg\.tar\.xz$||
  634. '
  635. # shellcheck disable=SC2016
  636. curl -Ss 'https://www.archlinux.org/mirrorlist/?country=all&protocol=https&tier=1&use_mirror_status=on' | \
  637. sed -n '
  638. s/^#Server = //
  639. T
  640. s/\$repo/'"${repository}"'/g
  641. s/\$arch/x86_64/g
  642. p
  643. ' | \
  644. shuf
  645. } | \
  646. sed '
  647. s|$|/'"${pkg_file}"'|
  648. s/\.[0-9]\+\(-[^-]\+\)$/\1/
  649. s/-'"${arch}"'\+\(\.pkg\.tar\.xz\)$/-x86_64\1/
  650. ' | \
  651. while read -r url; do
  652. >&2 printf 'downloading "%s" ...' "${url}"
  653. if wget -q --timeout=15 -nd "${url}"; then
  654. >&2 printf ' ok.\n'
  655. break;
  656. fi
  657. >&2 printf ' failed. Next ...\n'
  658. done
  659. done
  660. >&2 printf 'searching for provided libraries\n'
  661. find "${tar_content_dir}" -maxdepth 1 \
  662. -name '*.pkg.tar.xz' \
  663. -printf '%p\n' | \
  664. while read -r pkgfile; do
  665. pacman -Qqlp "${pkgfile}" | \
  666. sed -n '
  667. s,^.*/,,
  668. /\.so\(\..\+\)\?$/ {
  669. :a
  670. p
  671. s/\(\.so\(\..\+\)\?\)\.[^.]\+$/\1/
  672. t a
  673. }
  674. ' | \
  675. sort -u > \
  676. "${pkgfile}.so.provides"
  677. done
  678. >&2 printf 'searching for required and more provided libraries\n'
  679. package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
  680. find "${tar_content_dir}" -maxdepth 1 \
  681. -name '*.pkg.tar.xz' | \
  682. while read -r pkgfile; do
  683. if printf '%s\n' "${pkgfile}" | \
  684. grep -vq -- '-any\.pkg\.tar\.xz$'; then
  685. # we do not check "any" packages for linked libraries
  686. # (why do they have them in the first place?)
  687. mkdir "${package_content_dir}/${pkgfile##*/}"
  688. tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null
  689. # we rely on "${checksum}" not appearing in any objdump output
  690. : "${checksum?umm, checksum is unset - this will break below sed-fu}"
  691. # TODO: symbols may be in object files _inside_archives_
  692. # In that case, we're interested in the object file's name,
  693. # not the archive's file name
  694. find "${package_content_dir}/${pkgfile##*/}" \
  695. -name 'opt' -prune , \
  696. -type f \
  697. -exec objdump -x '{}' \; 2>/dev/null | \
  698. sed '
  699. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  700. \@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\1\.[0-9.]\+/@ {
  701. s@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\(\1\(\.[0-9.]\+\)\?\)\.\([0-9]\+\)/.*$@\2 \4@
  702. w '"${pkgfile}"'.needed-perl-version
  703. d
  704. }
  705. /\sNEEDED\s/ {
  706. s/^\s*\S\+\s\+\(\S\+\)\(\s.*\)\?$/\1/
  707. /\.c32$/d
  708. s,^.*/,,
  709. t
  710. }
  711. /^Version References:$/,/^$/ {
  712. /^\s*required from / {
  713. s/^\s*required from \(\S\+\):\s*$/'"${checksum}"'\1/
  714. T end
  715. h
  716. d
  717. }
  718. s/^\s*\(0x[0-9a-fA-F]\+\)\s\+0x[0-9a-fA-F]\+\s\+[0-9]\+\s\+\(\S\+\)$/\2-\1/
  719. T end
  720. G
  721. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  722. t
  723. }
  724. :end
  725. d
  726. ' | \
  727. sort -u > \
  728. "${pkgfile}.so.needs"
  729. sed -i '
  730. /^\(libav.*\.so\)\.[0-9]\+$/ {
  731. :a
  732. N
  733. s/^\(libav.*\.so\)\(\.[0-9]\+\)\?\n\1\.[0-9]\+$/\1/
  734. t a
  735. P
  736. D
  737. }
  738. ' "${pkgfile}.so.needs"
  739. sed '
  740. s/^installed = \(qt[0-9]\+-\S\+\)-\([0-9.]\+\)\(-[^-]\+\)\{2\}$/\1 \2/
  741. t
  742. d
  743. ' "${package_content_dir}/${pkgfile##*/}/.BUILDINFO" | \
  744. sort -u | \
  745. sort -k1,1 > \
  746. "${tmp_dir}/installed-qt-versions"
  747. sed '
  748. s/^depend = \(qt[0-9]\+-\S\+\)$/\1/
  749. t
  750. d
  751. ' "${package_content_dir}/${pkgfile##*/}/.PKGINFO" | \
  752. sort -u | \
  753. join -1 1 -2 1 -o 2.1,2.2 - "${tmp_dir}/installed-qt-versions" | \
  754. tr ' ' '=' >> \
  755. "${pkgfile}.so.needs"
  756. find "${package_content_dir}/${pkgfile##*/}" \
  757. -name 'opt' -prune , \
  758. \( -type f -o -type l \) \
  759. -printf "${checksum}"'%f\n' \
  760. -exec objdump -x '{}' \; 2>/dev/null | \
  761. sed '
  762. /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
  763. /^'"${checksum}"'/ {
  764. h
  765. d
  766. }
  767. /\sSONAME\s/ {
  768. s/^\s*\S\+\s\+\(\S\+\)\s.*$/\1/
  769. t
  770. }
  771. /^Version definitions:$/,/^$/ {
  772. s/^[0-9]\+\s\+0x[0-9a-fA-F]\+\s\+\(0x[0-9a-fA-F]\+\)\s\+\(\S\+\)$/\2-\1/
  773. T end
  774. G
  775. s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
  776. t
  777. }
  778. :end
  779. d
  780. ' | \
  781. sort -u >> \
  782. "${pkgfile}.so.provides"
  783. find "${package_content_dir:?}/${pkgfile##*/}" -xdev -not -type l -exec chmod 777 '{}' \;
  784. rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
  785. fi
  786. {
  787. tar -tJf "${pkgfile}" 2>/dev/null | \
  788. sed -n '
  789. s,^usr/lib/python\(2\?\)\([013-9.][0-9.]*\)/$,python\1 \1\2,
  790. t print
  791. s,^usr/lib/perl[^/]\+/\([0-9.]\+\)/$,perl \1,
  792. t print
  793. s,^usr/lib/ruby/\([0-9.]\+\)/$,ruby \1,
  794. t print
  795. b
  796. :print
  797. s/\.\([^.]\+\)$/ \1/
  798. p
  799. '
  800. if [ -f "${pkgfile}.needed-perl-versions" ]; then
  801. sed '
  802. s/^/perl /
  803. ' "${pkgfile}.needed-perl-versions"
  804. rm "${pkgfile}.needed-perl-versions"
  805. fi
  806. } | \
  807. while read -r name major_version minor_version; do
  808. printf '%s>=%s.%s\n%s<%s.%s\n' \
  809. "${name}" \
  810. "${major_version}" \
  811. "${minor_version}" \
  812. "${name}" \
  813. "${major_version}" \
  814. "$((minor_version+1))"
  815. done >> \
  816. "${pkgfile}.so.needs"
  817. # TODO: dependency-adding from make_source_info() should
  818. # be replicated here, somehow ...
  819. done
  820. >&2 printf 'running namcap ...'
  821. if [ "${repository}" = 'multilib' ]; then
  822. x86_64_build_command='multilib-build'
  823. else
  824. x86_64_build_command='extra-x86_64-build'
  825. fi
  826. # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
  827. # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too)
  828. sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
  829. sudo "${x86_64_build_command}" -c -- -- --version > /dev/null 2>&1 || \
  830. true
  831. >&2 printf ' ok.\n'
  832. >&2 printf 'smoothen namcap log ...'
  833. # now we generate diffs from the namcap.logs
  834. find . "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%p\n' | \
  835. while read -r log; do
  836. smoothen_namcap_log "${log}"
  837. done
  838. find "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.xz-namcap.log' -printf '%f\n' | \
  839. sed '
  840. s|\(^.*\)-'"${arch}"'\(\.pkg\.tar\.xz-namcap\.log\)$|\0 \1-x86_64\2|
  841. s|^.*-any\.pkg\.tar\.xz-namcap\.log$|\0 \0|
  842. ' | \
  843. while read -r log x86_64_log; do
  844. if [ -f "${x86_64_log}" ]; then
  845. diff -u "${x86_64_log}" "${tar_content_dir}/${log}" | \
  846. sed '
  847. 1,3d
  848. /^[^+-]/d
  849. ' | \
  850. sponge "${tar_content_dir}/${log}"
  851. else
  852. sed -i 's|^|*|' "${tar_content_dir}/${log}"
  853. fi
  854. done
  855. >&2 printf ' ok.\n'
  856. echo 'uploading' > "${tmp_dir}/.ping-build-master"
  857. if ${upload_to_build_master}; then
  858. find "${tar_content_dir}/" -maxdepth 1 \
  859. \( \
  860. -name '*.pkg.tar.xz-namcap.log' -o \
  861. -name '*.pkg.tar.xz.so.needs' -o \
  862. -name '*.pkg.tar.xz.so.provides' \
  863. \) \
  864. -execdir gzip '{}' \;
  865. else
  866. find "${tar_content_dir}/" -maxdepth 1 \
  867. -name '*.pkg.tar.xz-namcap.log' \
  868. -execdir grep -HF '' '{}' \;
  869. fi
  870. # shellcheck disable=SC2046
  871. tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
  872. find "${tar_content_dir}/" -maxdepth 1 \
  873. \( \
  874. -name '*.pkg.tar.xz' -o \
  875. -name '*.pkg.tar.xz.sig' -o \
  876. -name '*.pkg.tar.xz-namcap.log.gz' -o \
  877. -name '*.pkg.tar.xz.so.needs.gz' -o \
  878. -name '*.pkg.tar.xz.so.provides.gz' \
  879. \) \
  880. -printf '%f\n'
  881. ) || \
  882. tar -cf 'package.tar' -T /dev/null
  883. if ${upload_to_build_master}; then
  884. # shellcheck disable=SC2046
  885. rsync $(
  886. find "${tar_content_dir}/" -maxdepth 1 \
  887. \( \
  888. -name '*.pkg.tar.xz' -o \
  889. -name '*.pkg.tar.xz.sig' \
  890. \) \
  891. -printf '%p\n'
  892. ) "rsync://mirror.archlinux32.org/transfer32/" || true
  893. fi
  894. while ${upload_to_build_master}; do
  895. err=0
  896. # shellcheck disable=SC2029
  897. ssh \
  898. -i "${master_build_server_identity}" \
  899. -p "${master_build_server_port}" \
  900. "${master_build_server_user}@${master_build_server}" \
  901. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" "${sub_pkgrel}" \
  902. < 'package.tar' || \
  903. err=$?
  904. case ${err} in
  905. 0)
  906. # upload successful
  907. break
  908. ;;
  909. 1)
  910. >&2 echo '"return-assignment" was running already.'
  911. wait_some_time 30
  912. ;;
  913. 2)
  914. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  915. break
  916. ;;
  917. 3)
  918. >&2 echo "'return-assignment' reports a signature error."
  919. exit 1
  920. ;;
  921. 4)
  922. >&2 echo "'return-assignment' reports too many or missing packages."
  923. exit 1
  924. ;;
  925. 5)
  926. >&2 echo "'return-assignment' was called with wrong arguments."
  927. exit 1
  928. ;;
  929. *)
  930. >&2 echo "unknown return code ${err} from 'return-assignment'"
  931. exit ${err}
  932. esac
  933. done
  934. break
  935. fi
  936. echo 'failure' > "${tmp_dir}/.ping-build-master"
  937. >&2 printf ' failed.\n'
  938. done
  939. if ! ${success}; then
  940. for log in *'.build-log'; do
  941. if [ -f "${log}" ]; then
  942. if ${upload_to_build_master}; then
  943. printf '%s@%s\n' \
  944. "$(whoami)" \
  945. "$(hostname)" >> \
  946. "${log}"
  947. gzip "${log}"
  948. else
  949. grep -HF '' "${log}"
  950. fi
  951. fi
  952. done
  953. if ${upload_to_build_master} && \
  954. tar -cf 'build-logs.gz.tar' \
  955. -- *'.build-log.gz'; then
  956. while true; do
  957. err=0
  958. # shellcheck disable=SC2029
  959. ssh \
  960. -i "${master_build_server_identity}" \
  961. -p "${master_build_server_port}" \
  962. "${master_build_server_user}@${master_build_server}" \
  963. 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" 'ERROR' \
  964. < 'build-logs.gz.tar' || \
  965. err=$?
  966. case ${err} in
  967. 0)
  968. # upload successful
  969. break
  970. ;;
  971. 1)
  972. >&2 echo '"return-assignment" was running already.'
  973. wait_some_time 30
  974. ;;
  975. 2)
  976. >&2 echo 'I was too slow, the package is outdated. I will continue ...'
  977. break
  978. ;;
  979. 5)
  980. >&2 echo '"return-assignment" was called with wrong arguments.'
  981. exit 1
  982. ;;
  983. *)
  984. >&2 echo "unknown return code ${err} from 'return-assignment'"
  985. exit ${err}
  986. esac
  987. done
  988. fi
  989. if ${exit_after_failure}; then
  990. >&2 echo 'Build failed, exiting now'
  991. exit 0
  992. fi
  993. fi
  994. # clean up tmp_dir
  995. cd "${base_dir}"
  996. recursively_umount_and_rm "${tmp_dir}"
  997. if [ -z "${forced_package}" ]; then
  998. flock -u 9 || true
  999. fi
  1000. trap - EXIT
  1001. continue
  1002. ;;
  1003. 1)
  1004. >&2 echo 'get-assignment told me:'
  1005. >&2 echo ' come back (shortly) later - I was running already'
  1006. wait_some_time 30
  1007. continue
  1008. ;;
  1009. 2)
  1010. >&2 echo 'get-assignment told me:'
  1011. >&2 echo ' 2: come back later - there are still packages to be built,'
  1012. >&2 echo ' but currently none has all its dependencies ready'
  1013. wait_some_time 30
  1014. continue
  1015. ;;
  1016. 3)
  1017. >&2 echo 'get-assignment told me:'
  1018. >&2 echo ' 3: come back after the next run of get-package-updates or when'
  1019. >&2 echo ' some currently building packages are returned - currently'
  1020. >&2 echo ' there are no pending buildable packages'
  1021. exit 0
  1022. ;;
  1023. 4)
  1024. >&2 echo 'get-assignment told me:'
  1025. >&2 echo ' 4: wrong number of arguments'
  1026. exit 5
  1027. ;;
  1028. *)
  1029. >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
  1030. exit ${err}
  1031. ;;
  1032. esac
  1033. done
  1034. >&2 echo 'Done.'