path: root/bin/build-packages

#!/bin/sh

# build packages one by one, then upload the binary package to the repository server
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#build-packages

# TODOs:

#  repair build for signed git repositories (e.g. community/bitcoin)
#   - this should be ok now (updated devtools32 accordingly)

#  use different build commands for different repositories - do we need
#   this actually?

#  handle if build fails due to "local issues" (e.g. unclean
#   build environment, wrong mirror, ...) - how do we detect this (and
#   not fix the actual issue then)?

#  force different cache for builds (since we don't want to build
#   against an empty i686 cache on a x86_64 host)

#  avoid any-packages from x86_64 mirrors in /var/cache/pacman/pkg of
#   build slave

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'build-packages: build package(s) on the build-list'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:   Show this help and exit.'
  >&2 echo '  -n count:    Build $count packages (if available), then exit.'
  >&2 echo '               $count=0 is interpreted as infinity.'
  >&2 echo '               The default is $count=1 or 0 iff -t or -x is given.'
  >&2 echo '  -t seconds:  Do not request new assignment(s) $seconds seconds after start.'
  >&2 echo '  -x:          If package build fails, do not request new assignment(s).'
  [ -z "$1" ] && exit 1 || exit $1
}

eval set -- "$(
  getopt -o hn:t:x \
    --long help \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

unset count
exit_after_failure=false
timeout=0

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -n)
      shift
      count="$1"
      [ ${count} -eq 0 ] && \
        count=-1
    ;;
    -t)
      shift
      timeout="$1"
    ;;
    -x)
      exit_after_failure=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -z "${count}" ]; then
  if [ ${timeout} -ne 0 ] || ${exit_after_failure}; then
    count=-1
  else
    count=1
  fi
fi

if [ ${timeout} -ne 0 ]; then
  timeout=$((${timeout}+$(date +%s)))
fi

while [ ${count} -ne 0 ]; do

  if [ ${timeout} -ne 0 ] && [ ${timeout} -lt $(date +%s) ];
  then
    break
  fi

  err=0
  package="$(
    ssh \
      -i "${master_build_server_identity}" \
      -p "${master_build_server_port}" \
      "${master_build_server_user}@${master_build_server}" \
      'get-assignment'
  )" || err=$?

  case ${err} in

    #  0: ok, I gave you an assignment
    0)
      [ ${count} -gt 0 ] && \
        count=$((${count}-1))
      repository="${package##* }"
      package="${package% *}"
      mod_git_revision="${package##* }"
      package="${package% *}"
      git_revision="${package##* }"
      package="${package% *}"

      # Update git repositories (official packages, community packages and the repository of package customizations).

      for repo_name in ${repo_names}; do
        eval repo_path='"${repo_paths__'"${repo_name}"'}"'
        git -C "${repo_path}" fetch
      done

      git_repo="$(find_repository_with_commit "${git_revision}")"

      find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"

      bail_out() {
        err=$?
        if [ -n $1 ]; then
          err=$1
        fi
        cd "${base_dir}"
        rm -rf --one-file-system "${tmp_dir}"
        exit ${err}
      }
      tmp_dir="$(mktemp -d "${work_dir}/tmp.XXXXXX")"

      if [ -n "${PKGBUILD}" ]; then
        eval 'git -C "'"$(printf '${repo_paths__%s}' "${git_repo}")"'"'" archive '${git_revision}' -- '${PKGBUILD%/*}'" | \
          tar -x --strip-components=3 -C "${tmp_dir}"
      fi
      if [ -n "${PKGBUILD_mod}" ]; then
        git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" -- "${PKGBUILD_mod%/*}" | \
          tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${tmp_dir}"
        git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" -- "${PKGBUILD_mod}" | \
          tar -Ox 'PKGBUILD' >> \
          "${tmp_dir}/PKGBUILD"
      fi

      cd "${tmp_dir}"
      success=false
      for mirrored_source in false true; do

        if ${mirrored_source}; then
          # maybe a missing source is/was the problem?
          # try to download them from sources.archlinux.org/sources/$repo/$source
          source_name="$(
            makepkg --printsrcinfo | \
              sed -n '
                /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
                /^pkgname = /q
              ' | \
              sed '
                s|^pkgbase = \(.*\)$|0 \1-|
                s|^epoch = \(.*\)$|1 \1:|
                s|^pkgver = \(.*\)$|2 \1-|
                s|^pkgrel = \(.*\)$|3 \1.src.tar.gz|
              ' | \
              sort -k1n,1 | \
              sed '
                s|^[0-9] ||
                :a
                  N
                  s|\n[0-9] ||
                ta
              '
          )"
          if ! wget -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
            # we can't improve anything
            break
          fi
          tar -xzkf "${source_name}" --exclude PKGBUILD --strip-components=1 || true
        fi

        for parameters in '' '-c'; do
          rm -f *".pkg.tar.xz" *".pkg.tar.xz.sig"

          >&2 printf 'building package "%s" ...' "${package}"
          # by piping the log, we don't see anything in the terminal,
          # but all ways to duplicate the logs seem pretty elaborate
          if staging-i686-build ${parameters} > \
            "$(
              date -u --iso-8601=seconds | \
                cut -d+ -f1
            ).build-log" 2>&1; then
            # build successful
            >&2 printf ' ok.\n'
            ls -1 *.pkg.tar.xz | \
              xargs -rn1 gpg --local-user="${package_key}" --detach-sign || \
              bail_out
            tar -cf 'package.tar' $(
                ls -1 | \
                  grep '\.pkg\.tar\.xz\(\.sig\)\?$' | \
                  grep -v -- '-debug-'
              ) || \
              bail_out
            while true; do
              err=0
              ssh \
                -i "${master_build_server_identity}" \
                -p "${master_build_server_port}" \
                "${master_build_server_user}@${master_build_server}" \
                'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" \
                < 'package.tar' || \
                err=$?
              case ${err} in
                0)
                  # upload successful
                  break
                ;;
                1)
                  >&2 echo '"return-assignment" was running already.'
                  wait_some_time 15
                ;;
                2)
                  >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                  break
                ;;
                3)
                  >&2 echo "'return-assignment' reports a signature error."
                  bail_out 1
                ;;
                4)
                  >&2 echo "'return-assignment' reports too many or missing packages."
                  bail_out 1
                ;;
                *)
                  >&2 echo "unknown return code ${err} from 'return-assignment'"
                  bail_out 1
              esac
            done
            success=true
            break
          fi
          >&2 printf ' failed.\n'
        done
        if ${success}; then
          break
        fi
      done

      if ! ${success}; then
        for log in *'.build-log'; do
          if [ -f "${log}" ]; then
            gzip "${log}" || \
              bail_out
          fi
        done
        tar -cf 'build-logs.gz.tar' \
          *'.build-log.gz' || \
          continue # this might happen if we didn't create any logs
        while ! ssh \
          -i "${master_build_server_identity}" \
          -p "${master_build_server_port}" \
          "${master_build_server_user}@${master_build_server}" \
          'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" 'ERROR' < \
          'build-logs.gz.tar'; do
          wait_some_time 15
        done
        if ${exit_after_failure}; then
          >&2 echo 'Build failed, exiting now'
          bail_out 0
        fi
      fi

      # clean up tmp_dir
      cd "${base_dir}"
      rm -rf --one-file-system "${tmp_dir}"

      continue

    ;;

    1)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' come back (shortly) later - I was running already'
      wait_some_time 15
      continue

    ;;

    2)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 2: come back later - there are still packages to be built,'
      >&2 echo ' but currently none has all its dependencies ready'
      wait_some_time 60
      continue

    ;;

    3)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 3: come back after the next run of get-package-updates - currently'
      >&2 echo ' there are no pending packages'
      exit 0

    ;;

    4)

      >&2 echo 'get-assignment told me:'
      >&2 echo " come back, when you've done your work - you hit the limit on"
      >&2 echo ' maximum allowed parallel jobs per ip'
      exit 1

    ;;

    *)

      >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
      exit 1

    ;;

  esac

done

>&2 echo 'Done.'