path: root/bin/build-packages

#!/bin/sh

# build packages one by one, then upload the binary package to the repository server
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#build-packages

# TODOs:

#  repair build for signed git repositories (e.g. community/bitcoin)
#   - this should be ok now (updated devtools32 accordingly)

#  use different build commands for different repositories - do we need
#   this actually?

#  handle if build fails due to "local issues" (e.g. unclean
#   build environment, wrong mirror, ...) - how do we detect this (and
#   not fix the actual issue then)?

#  force different cache for builds (since we don't want to build
#   against an empty i686 cache on a x86_64 host)

#  avoid any-packages from x86_64 mirrors in /var/cache/pacman/pkg of
#   build slave

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'build-packages: build package(s) on the build-list'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:   Show this help and exit.'
  >&2 echo '  -n count:    Build $count packages (if available), then exit.'
  >&2 echo '               $count=0 is interpreted as infinity.'
  >&2 echo '               The default is $count=1 or 0 iff -t or -x is given.'
  >&2 echo '  -t seconds:  Do not request new assignment(s) $seconds seconds after start.'
  >&2 echo '  -x:          If package build fails, do not request new assignment(s).'
  [ -z "$1" ] && exit 1 || exit $1
}

eval set -- "$(
  getopt -o hn:t:x \
    --long help \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

unset count
exit_after_failure=false
timeout=0

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -n)
      shift
      count="$1"
      [ ${count} -eq 0 ] && \
        count=-1
    ;;
    -t)
      shift
      timeout="$1"
    ;;
    -x)
      exit_after_failure=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -z "${count}" ]; then
  if [ ${timeout} -ne 0 ] || ${exit_after_failure}; then
    count=-1
  else
    count=1
  fi
fi

if [ ${timeout} -ne 0 ]; then
  timeout=$((${timeout}+$(date +%s)))
fi

while [ ${count} -ne 0 ]; do

  if [ ${timeout} -ne 0 ] && [ ${timeout} -lt $(date +%s) ];
  then
    break
  fi

  err=0
  package=$(
    ssh \
      -i "${master_build_server_identity}" \
      -p "${master_build_server_port}" \
      "${master_build_server_user}@${master_build_server}" \
      'get-assignment'
  ) || err=$?

  case ${err} in

    #  0: ok, I gave you an assignment
    0)
      [ ${count} -gt 0 ] && \
        count=$((${count}-1))
      repository="${package##* }"
      package="${package% *}"
      mod_git_revision="${package##* }"
      package="${package% *}"
      git_revision="${package##* }"
      package="${package% *}"

      # Update git repositories (official packages, community packages and the repository of package customizations).

      for repo_name in ${repo_names}; do
        eval repo_path='"${repo_paths__'"${repo_name}"'}"'
        git -C "${repo_path}" fetch
      done

      git_repo=$(find_repository_with_commit "${git_revision}")

      find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"

      bail_out() {
        err=$?
        if [ -n $1 ]; then
          err=$1
        fi
        cd "${base_dir}"
        recursively_umount_and_rm "${tmp_dir}"
        exit ${err}
      }
      tmp_dir=$(mktemp -d "${work_dir}/tmp.XXXXXX")
      trap bail_out EXIT

      extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}"

      cd "${tmp_dir}"
      success=false
      for straw in ${straws_that_might_repair_failing_builds}; do

        if echo "${straw}" | \
          grep -qF ':mirrored_source:'; then
          # maybe a missing source is/was the problem?
          # try to download them from sources.archlinux.org/sources/$repo/$source
          source_name=$(
            makepkg --printsrcinfo | \
              sed -n '
                /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
                /^pkgname = /q
              ' | \
              sed '
                s|^pkgbase = \(.*\)$|0 \1-|
                s|^epoch = \(.*\)$|1 \1:|
                s|^pkgver = \(.*\)$|2 \1-|
                s|^pkgrel = \(.*\)$|3 \1.src.tar.gz|
              ' | \
              sort -k1n,1 | \
              sed '
                s|^[0-9] ||
                :a
                  N
                  s|\n[0-9] ||
                ta
              '
          )
          if ! wget -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
            # we can't improve anything
            continue
          fi
          tar -xz --overwrite -f "${source_name}" --exclude PKGBUILD --strip-components=1 || true
        fi

        if echo "${straw}" | \
          grep -qF ':with_build_support:'; then
          build_command='staging-with-build-support-i686-build'
        else
          build_command='staging-i686-build'
        fi
        if echo "${straw}" | \
          grep -qF ':clean_chroot:'; then
          parameters='-c'
        else
          parameters=''
        fi

        rm -f *".pkg.tar.xz" *".pkg.tar.xz.sig"

        >&2 printf '%s: building package "%s" (straw: %s) ...' \
          "$(date +'%Y-%m-%d %T')" \
          "${package}" \
          "${straw}"
        # by piping the log, we don't see anything in the terminal,
        # but all ways to duplicate the logs seem pretty elaborate
        if "${build_command}" ${parameters} > \
          "$(
            date -u --iso-8601=seconds | \
              cut -d+ -f1
          ).build-log" 2>&1; then
          # build successful
          >&2 printf ' ok.\n'
          ls -1 *.pkg.tar.xz | \
            xargs -rn1 gpg --local-user="${package_key}" --detach-sign
          tar -cf 'package.tar' $(
            ls -1 | \
              grep '\.pkg\.tar\.xz\(\.sig\)\?$' | \
              grep -v -- '-debug-'
            )
          while true; do
            err=0
            ssh \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" \
              < 'package.tar' || \
              err=$?
            case ${err} in
              0)
                # upload successful
                break
              ;;
              1)
                >&2 echo '"return-assignment" was running already.'
                wait_some_time 15
              ;;
              2)
                >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                break
              ;;
              3)
                >&2 echo "'return-assignment' reports a signature error."
                bail_out 1
              ;;
              4)
                >&2 echo "'return-assignment' reports too many or missing packages."
                bail_out 1
              ;;
              *)
                >&2 echo "unknown return code ${err} from 'return-assignment'"
                bail_out 1
            esac
          done
          success=true
          break
        fi
        >&2 printf ' failed.\n'
      done

      if ! ${success}; then
        for log in *'.build-log'; do
          if [ -f "${log}" ]; then
            gzip "${log}"
          fi
        done
        if tar -cf 'build-logs.gz.tar' \
          *'.build-log.gz'; then
          while ! ssh \
            -i "${master_build_server_identity}" \
            -p "${master_build_server_port}" \
            "${master_build_server_user}@${master_build_server}" \
            'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" 'ERROR' < \
            'build-logs.gz.tar'; do
            wait_some_time 15
          done
        fi
        if ${exit_after_failure}; then
          >&2 echo 'Build failed, exiting now'
          bail_out 0
        fi
      fi

      # clean up tmp_dir
      cd "${base_dir}"
      recursively_umount_and_rm "${tmp_dir}"
      trap - EXIT

      continue

    ;;

    1)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' come back (shortly) later - I was running already'
      wait_some_time 15
      continue

    ;;

    2)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 2: come back later - there are still packages to be built,'
      >&2 echo ' but currently none has all its dependencies ready'
      wait_some_time 60
      continue

    ;;

    3)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 3: come back after the next run of get-package-updates - currently'
      >&2 echo ' there are no pending packages'
      exit 0

    ;;

    4)

      >&2 echo 'get-assignment told me:'
      >&2 echo " come back, when you've done your work - you hit the limit on"
      >&2 echo ' maximum allowed parallel jobs per ip'
      exit 1

    ;;

    *)

      >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
      exit 1

    ;;

  esac

done

>&2 echo 'Done.'