#!/bin/bash # build packages one by one, then upload the binary package to the repository server # Details: # https://github.com/archlinux32/builder/wiki/Build-system#build-packages # TODOs: # use different build commands for different repositories - do we need this actually? # send logs of failed builds # handle if build fails due to "local issues" (e.g. unclean build environment, wrong mirror, ...) # force different cache for builds (since we don't want to build against an empty i686 cache on a x86_64 host) . "${0%/*}/../conf/default.conf" usage() { >&2 echo '' >&2 echo 'build-packages: build package(s) on the build-list' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -n count: Build $count packages (if available), then exit.' >&2 echo ' $count=0 is interpreted as infinity.' >&2 echo ' The default is $count=1 or 0 iff -t or -x is given.' >&2 echo ' -t seconds: Do not request new assignment(s) $seconds seconds after start.' >&2 echo ' -x: If package build fails, do not request new assignment(s).' [ -z "$1" ] && exit 1 || exit $1 } eval set -- "$( getopt -o hn:t:x \ --long help \ -n "$(basename "$0")" -- "$@" || \ echo usage )" unset count exit_after_failure=false timeout=0 while true do case "$1" in -h|--help) usage 0 ;; -n) shift count="$1" [ ${count} -eq 0 ] && \ count=-1 ;; -t) shift timeout="$1" ;; -x) exit_after_failure=true ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit -1 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi if [ -z "${count}" ]; then if [ ${timeout} -ne 0 ] || ${exit_after_failure}; then count=-1 else count=1 fi fi if [ ${timeout} -ne 0 ]; then timeout=$[${timeout}+$(date +%s)] fi while [ ${count} -ne 0 ]; do if [ ${timeout} -ne 0 ] && [ ${timeout} -lt $(date +%s) ]; then break fi err=0 package="$( ssh \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ 'get-assignment' )" || err=$? case ${err} in # 0: ok, I gave you an assignment 0) [ ${count} -gt 0 ] && \ count=$[${count}-1] repository="${package##* }" package="${package% *}" mod_git_revision="${package##* }" package="${package% *}" git_revision="${package##* }" package="${package% *}" # Update git repositories (official packages, community packages and the repository of package customizations). for repo in "${repo_paths[@]}"; do git -C "${repo}" clean -df git -C "${repo}" reset --hard git -C "${repo}" checkout master git -C "${repo}" pull done git -C "${repo_paths["$(find_repository_with_commit "${git_revision}")"]}" checkout "${git_revision}" &> /dev/null git -C "${repo_paths["archlinux32"]}" checkout "${mod_git_revision}" &> /dev/null PKGBUILD="$(find_pkgbuild "${package}" "${repository}")" if [ ! -r "${PKGBUILD}" ]; then echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'" exit 1 fi cd "${PKGBUILD%/*}" apply_package_customizations success=false for mirrored_source in false true; do if ${mirror_source}; then # maybe a missing source is/was the problem? # try to download them from sources.archlinux.org/sources/$repo/$source source_name="$( makepkg --printsrcinfo | \ sed -n ' /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p} /^pkgname = /q ' | \ sed ' s|^pkgbase = \(.*\)$|0 \1-| s|^epoch = \(.*\)$|1 \1:| s|^pkgver = \(.*\)$|2 \1-| s|^pkgrel = \(.*\)$|3 \1.src.tar.gz| ' | \ sort -k1n,1 | \ sed ' s|^[0-9] || :a N s|\n[0-9] || ta ' )" git_repo="$(find_git_repository_to_package_repository "${repository}")" if wget -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then tar -xzkf "${source_name}" --exclude PKGBUILD --strip-components=1 || true fi fi for parameters in '' '-c'; do rm -f *.pkg.tar.xz{,.sig} if staging-i686-build ${parameters}; then # build successful ls -1 *.pkg.tar.xz | \ xargs -rn1 gpg --local-user="${package_key}" --detach-sign tar -cf 'package.tar' *.pkg.tar.xz{,.sig} while true; do err=0 ssh \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" \ < 'package.tar' || \ err=$? case ${err} in 0) # upload successful break ;; 1) # 'return-assignment' was running already sleep $[15+$RANDOM%30] ;; 2) >&2 echo 'I was too slow, the package is outdated. I will continue ...' break ;; 3) >& echo "'return-assignment' reports a signature error." exit 1 ;; *) >&2 echo "unknown return code ${err} from 'return-assignment'" exit 1 esac done success=true break fi done if ${success}; then break fi done if ! ${success}; then ssh \ -i "${master_build_server_identity}" \ -p "${master_build_server_port}" \ "${master_build_server_user}@${master_build_server}" \ 'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" 'ERROR' if ${exit_after_failure}; then >&2 echo 'Build failed, exiting now' exit fi fi continue ;; # 1: come back (shortly) later - I was running already 1) sleep $[15+$RANDOM%30] continue ;; # 2: come back later - there are still packages to be built, # but currently none has all its dependencies ready 2) sleep $[60+$RANDOM%30] continue ;; # 3: come back after the next run of get-package-updates - currently # there are no pending packages 3) >&2 echo 'Done. No more packages left to build.' exit 0 ;; # 4: come back, when you've done your work - you hit the limit on # maximum allowed parallel jobs per ip 4) >&2 echo 'ERROR: There are too many parallel builds running on this machine.' exit 1 ;; *) >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'." exit 1 ;; esac done >&2 echo 'Done.'