| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/build-packages | 188 |
diff --git a/bin/build-packages b/bin/build-packages index 9c5ab03..b39546d 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -21,7 +21,7 @@ # TODO: the following things need proper locking and/or proper handling of # external locks: # - `git` on the package source -# - `archbuild` (e.g. /var/lib/archbuilds/staging-i686-build/root.lock) +# - `archbuild` (e.g. /var/lib/archbuilds/extra-staging-i686-build/root.lock) # - `find ... -exec umount` on the build chroot # TODO: releave some locking conditions - meta goal: be able to run multiple @@ -38,12 +38,6 @@ usage() { >&2 echo 'build-packages: build package(s) on the build-list' >&2 echo '' >&2 echo 'possible options:' - >&2 echo ' -d|--diff source-directory:' - >&2 echo ' Apply the diff between $source-directory/PKGBUILD and' - >&2 echo ' $source-directory/../../trunk/PKGBUILD before building.' - >&2 echo ' This option is intended for building from trunk where minor' - >&2 echo ' fixes did not get released yet. Hence, it conflicts intentionally' - >&2 echo ' with -n, -l, -t or -x and it requires -p.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -l|--local pkgname.git-revision.git-mod-revision.repository.architecture:' >&2 echo ' Build the given package without asking / reporting to the' @@ -74,10 +68,10 @@ usage() { >&2 echo ' clean the chroot before building' >&2 echo ' :haskell_without_check:' >&2 echo ' try with :without_check: iff this is a haskell-*, python-* or python2-* package' - >&2 echo ' :mirrored_source:' - >&2 echo ' compile from source tarbal from sources.archlinux.org if possible' >&2 echo ' :mirrored_source_by_hash:' >&2 echo ' download sources from sources.archlinux32.org by hash if possible' + >&2 echo ' :on_x86_64:' + >&2 echo ' build any package with x86_64 arch' >&2 echo ' :without_check:' >&2 echo ' run makepkg with "--no-check"' >&2 echo ' :without_systemd_nspawn:' @@ -90,8 +84,7 @@ usage() { } eval set -- "$( - getopt -o d:hl:n:p:s:t:ux \ - --long diff: \ + getopt -o hl:n:p:s:t:ux \ --long help \ --long local: \ --long prefer: \ @@ -102,7 +95,6 @@ eval set -- "$( )" unset count -unset diff_source_dir unset forced_package unset forced_straws unset prefered_package @@ -112,14 +104,6 @@ timeout=0 while true do case "$1" in - -d|--diff) - shift - if [ -n "${diff_source_dir}" ]; then - >&2 echo 'Option -d, --diff can be given only once.' - usage - fi - diff_source_dir="$1" - ;; -h|--help) usage 0 ;; @@ -173,17 +157,6 @@ if [ $# -ne 0 ]; then usage fi -if [ -n "${diff_source_dir}" ]; then - if [ -n "${count}" ] || \ - [ -n "${forced_package}" ] || \ - [ "${timeout}" -ne 0 ] || \ - ${exit_after_failure} || \ - [ -z "${prefered_package}" ]; then - >&2 echo 'Conflicting flags.' - usage - fi -fi - if [ -n "${forced_package}" ]; then if [ -n "${count}" ] || \ [ "${timeout}" -ne 0 ] || \ @@ -222,6 +195,14 @@ checksum=$( while [ "${count}" -ne 0 ] && \ [ "$(calculate_script_checksum)" = "${checksum}" ]; do + if [ -f '/tmp/do-not-run-build-slave' ]; then + >&2 echo 'build slave is on halt.' + >&2 echo 'I will terminate the loop and stop now.' + # non-zero exit code to signal systemctl that the process should not + # be restarted automatically + exit 1 + fi + if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ]; then break @@ -249,7 +230,7 @@ while [ "${count}" -ne 0 ] && \ if [ -z "${forced_package}" ]; then package=$( # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -277,7 +258,7 @@ while [ "${count}" -ne 0 ] && \ # 0: ok, I gave you an assignment 0) - [ ${count} -gt 0 ] && \ + [ "${count}" -gt 0 ] && \ count=$((count-1)) arch="${package##* }" package="${package% *}" @@ -291,19 +272,9 @@ while [ "${count}" -ne 0 ] && \ package="${package% *}" if [ "${arch}" = 'any' ]; then arch="${my_arch}" - fi - - if [ -n "${diff_source_dir}" ] && - [ "${prefered_package}" != "${package}" ]; then - >&2 echo 'The prefered package was not handed out.' - >&2 echo 'Because -d was given, I will abort.' - # shellcheck disable=SC2029 - ssh \ - -i "${master_build_server_identity}" \ - -p "${master_build_server_port}" \ - "${master_build_server_user}@${master_build_server}" \ - 'return-assignment' 'ABORT' - exit 2 + assignment_was_any=true + else + assignment_was_any=false fi if [ "${git_revision##*-}" = 'HEAD' ]; then @@ -376,14 +347,10 @@ while [ "${count}" -ne 0 ] && \ tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX") trap bail_out EXIT - if ! git_repo=$(find_repository_with_commit "${git_revision}") || \ - [ -z "${git_repo}" ] || \ - ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" || \ - ! extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}" || \ - ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then + if ! extract_source_directory "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}"; then # report local failure (probably a missing commit) to build-master # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -394,17 +361,21 @@ while [ "${count}" -ne 0 ] && \ continue fi + if [ -d "${tmp_dir}/keys/pgp" ]; then + find "${tmp_dir}/keys/pgp" -type f -exec gpg --import {} + + fi + cd "${tmp_dir}" echo 'nothing' > "${tmp_dir}/.ping-build-master" if [ -z "${forced_package}" ]; then - # we get a lock on "${work_dir}/ping-build-master.lock", + # we get a lock on "${tmp_dir}/ping-build-master.lock", # if we release that lock, ping-to-master should stop _immediately_ - exec 9> "${work_dir}/ping-build-master.lock" + exec 9> "${tmp_dir}/ping-build-master.lock" if ! verbose_flock -n 9; then >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.' exit 2 fi - "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" & + "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" "${tmp_dir}/ping-build-master.lock" & fi success=false @@ -448,7 +419,7 @@ while [ "${count}" -ne 0 ] && \ if echo "${straw}" | \ grep -qF ':with_build_support:'; then - build_command='staging-with-build-support-'"${arch}"'-build' + build_command="${repository}-staging-with-build-support-${arch}-build" elif echo "${straw}" | \ grep -qF ':without_systemd_nspawn:'; then if [ -z "${prefered_package}" ]; then @@ -459,7 +430,7 @@ while [ "${count}" -ne 0 ] && \ >&2 echo 'The prefered package was not handed out.' >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.' # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -485,8 +456,30 @@ while [ "${count}" -ne 0 ] && \ outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver" middleParameters='' innerParameters='' + elif echo "${straw}" | \ + grep -qF ':on_x86_64:'; then + if [ -z "${prefered_package}" ]; then + >&2 echo 'straw :on_x86_64: only allowed with -p' + exit 2 + fi + if [ "${prefered_package}" != "${package}" ]; then + >&2 echo 'The prefered package was not handed out.' + >&2 echo 'Because straw :on_x86_64: is active, I will abort.' + # shellcheck disable=SC2029 + ssh -o PasswordAuthentication=No \ + -i "${master_build_server_identity}" \ + -p "${master_build_server_port}" \ + "${master_build_server_user}@${master_build_server}" \ + 'return-assignment' 'ABORT' + exit 2 + fi + if ! ${assignment_was_any}; then + >&2 printf 'Can only build "any" packages with :on_x86_64:, but got a "%s" package.\n' "${arch}" + exit 2 + fi + build_command="${repository}-staging-x86_64-build" else - build_command='staging-'"${arch}"'-build' + build_command="${repository}-staging-${arch}-build" fi find . -maxdepth 1 -type f \( \ @@ -501,13 +494,13 @@ while [ "${count}" -ne 0 ] && \ echo 'checking-source' > "${tmp_dir}/.ping-build-master" success=false verifysource_trial=0 - while [ ${verifysource_trial} -lt 5 ]; do + while [ ${verifysource_trial} -lt 4 ]; do verifysource_trial=$((verifysource_trial + 1)) log_file="${tmp_dir}/$( date -u --iso-8601=seconds | \ cut -d+ -f1 ).build-log" - if CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then + if GIT_TERMINAL_PROMPT=0 CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then success=true rm "${log_file}" break @@ -557,53 +550,8 @@ while [ "${count}" -ne 0 ] && \ verifysource_trial=$((verifysource_trial + 1)) fi - # try to download source from sources.archlinux.org/sources/$repo/$source - if [ ${verifysource_trial} -eq 3 ]; then - if echo "${straw}" | \ - grep -qF ':mirrored_source:'; then - source_name=$( - makepkg --printsrcinfo | \ - sed -n ' - /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p} - /^pkgname = /q - ' | \ - sed ' - s|^pkgbase = \(.*\)$|0 \1-| - s|^epoch = \(.*\)$|1 \1:| - s|^pkgver = \(.*\)$|2 \1-| - s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz| - ' | \ - sort -k1n,1 | \ - sed ' - s|^[0-9] || - :a - N - s|\n[0-9] \(\S\+\)$|\1| - ta - ' - ) - if wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then - # shellcheck disable=SC2046 - tar -zst --overwrite \ - -f "${source_name}" \ - --exclude PKGBUILD \ - $( - if [ -n "${PKGBUILD_mod}" ]; then - git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \ - tar -t | \ - sed 's/^/--exclude /' - fi - ) \ - --strip-components=1 \ - || true - continue - fi - fi - verifysource_trial=$((verifysource_trial + 1)) - fi - # try to download source from sources.archlinux32.org by its hash - if [ ${verifysource_trial} -eq 4 ]; then + if [ ${verifysource_trial} -eq 3 ]; then if echo "${straw}" | \ grep -qF ':mirrored_source_by_hash:'; then if download_sources_by_hash; then @@ -730,13 +678,22 @@ while [ "${count}" -ne 0 ] && \ } | \ sort | \ uniq -u | \ - while read -r unexpected_package; do - >&2 printf 'removing unexpected build artifact "%s"\n' \ - "${unexpected_package}" - rm "${unexpected_package}"* - done + { + removed_something_unexpected=false + while read -r unexpected_package; do + >&2 printf 'removing unexpected build artifact "%s"\n' \ + "${unexpected_package}" + rm "${unexpected_package}"* + removed_something_unexpected=true + done + if "${removed_something_unexpected}"; then + >&2 printf 'I was only expecting:\n%s\n' \ + "${expected_packages}" + fi + } fi >&2 printf 'signing package(s)\n' + mv "${log_file}" "${tar_content_dir}/" find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \ -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \ -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \ @@ -977,7 +934,8 @@ while [ "${count}" -ne 0 ] && \ \( \ -name '*.pkg.tar.zst-namcap.log' -o \ -name '*.pkg.tar.zst.so.needs' -o \ - -name '*.pkg.tar.zst.so.provides' \ + -name '*.pkg.tar.zst.so.provides' -o \ + -name '*.build-log' \ \) \ -execdir gzip '{}' \; else @@ -1015,7 +973,7 @@ while [ "${count}" -ne 0 ] && \ while ${upload_to_build_master}; do err=0 # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -1078,7 +1036,7 @@ while [ "${count}" -ne 0 ] && \ while true; do err=0 # shellcheck disable=SC2029 - ssh \ + ssh -o PasswordAuthentication=No \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ @@ -1174,7 +1132,7 @@ while [ "${count}" -ne 0 ] && \ done -# remove the slave switch file if it exists (and if we are runnig on a slave) +# remove the slave switch file if it exists (and if we are running on a slave) if ! ${i_am_the_master}; then SLAVE=$(whoami) [ -f "/tmp/do-not-run-build-slave.$SLAVE" ] && rm "/tmp/do-not-run-build-slave.$SLAVE" |