path: root/bin/get-package-updates

#!/bin/sh

# check for packages that need to be built, and build a list in the proper build order
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates

# TODOs:

#  be more secure in case of update while build(s) is/are still in progress
#   -> (stale) lock files, moving (or changing content of) loop lock files

#  (might be connected to above): find out who deletes loop-files which
#   should not be deleted

#  dependencies declared inside a PKGBUILD's package function do not
#   correctly take into account CARCH (see package "gens" from community)

#  remove extra case for "python-pysocks" and "gens"

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'get-package-updates: check for packages that need to be built,'
  >&2 echo ' and build a list in the proper build order'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--block:   If necessary, wait for lock blocking.'
  >&2 echo '  -h|--help:    Show this help and exit.'
  >&2 echo '  -n|--no-pull: Do not pull git repos, merely reorder build list.'
  [ -z "$1" ] && exit 1 || exit $1
}

eval set -- "$(
  getopt -o bhn \
    --long block \
    --long help \
    --long no-pull \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
pull=true

while true
do
  case "$1" in
    -b|--block)
      block_flag=''
    ;;
    -h|--help)
      usage 0
    ;;
    -n|--no-pull)
      pull=false
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if ! "${base_dir}/bin/sanity-check" -r; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

# delete_package package
# mark $package for deletion
delete_package() {
  echo "$1" >> \
    "${work_dir}/deletion-list.new"
  sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
}

# Update git repositories (official packages, community packages and the repository of package customizations).

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  # TODO:
  #  this is somewhat redundant and slow -- improve it!
  git -C "${repo_path}" checkout -f master
  git -C "${repo_path}" clean -xdf
  git -C "${repo_path}" fetch
  if ${pull}; then
    git -C "${repo_path}" reset --hard origin/master
  fi
done

# Read previous git revision numbers from files.

something_new=false

for repo in ${repo_names}; do
  eval "old_repo_revisions__${repo}='$(
    cat "${work_dir}/${repo}.revision" 2> /dev/null || \
      echo NONE
  )'"
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  eval "new_repo_revisions__${repo}='$(
    git -C "${repo_path}" rev-parse HEAD | \
      tee "${work_dir}/${repo}.revision.new"
  )'"
  eval 'if ! [ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]; then
    something_new=true
  fi'
done

if ${pull} && \
  ! ${something_new}; then
  >&2 echo 'Nothing changed.'
  exit
fi

# Create a lock file for build list.

exec 9> "${build_list_lock_file}"
if ! flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit
fi

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

grep -vxF 'break_loops' \
  "${work_dir}/build-list" > \
  "${work_dir}/build-list.new" || \
  true
cp \
  "${work_dir}/deletion-list" \
  "${work_dir}/deletion-list.new"

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
  eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"'
  (
    # if old revision unknown, mimic "git diff"-output
    if [ "${old_repo_revision}" = "NONE" ]; then
      git -C "${repo_path}" archive --format=tar HEAD | \
        tar -t | \
        sed 's|^|A\t|'
    else
      git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
    fi
  ) | \
    # only track changes in PKGBUILDs
    grep '/PKGBUILD$' | \
    if [ "${repo}" = "archlinux32" ]; then
    # modify the directory structure from the modifiaction-repository
    # to the one of an original source repository
      sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
        while read -r pkg_repo rest; do
          eval "printf '%s %s\n'" \
            "$(printf '"${new_repo_revisions__%s}"' "$(find_git_repository_to_package_repository "${pkg_repo}")")" \
            "'${rest}'"
        done
    else
      sed "s|^|${new_repo_revision} |"
    fi | \
    grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
    # ignore i686
    grep -v -- '-i686/PKGBUILD$' | \
    # ignore staging and testing
    grep -v -- '[-/]\(staging\|testing\)-[^/]\+/PKGBUILD$' | \
    sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|'
done | \
  sort -u | \
  while read -r mode package git_revision repository; do
    case "${mode}" in

      # new or modified PKGBUILD
      "A"|"M")
        sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
        echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
          "${work_dir}/build-list.new"
        sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
      ;;

      # deleted PKGBUILD
      "D")
        delete_package "${package}"
      ;;

      *)
        >&2 echo "unknown git diff mode '${mode}'"
        exit 1
      ;;

    esac
  done

sort -u "${work_dir}/deletion-list.new" > \
  "${work_dir}/deletion-list.new.new"

mv \
  "${work_dir}/deletion-list.new.new" \
  "${work_dir}/deletion-list.new"

echo 'Extract dependencies of packages.'

# First, we extract the dependencies of each package.

mkdir -p "${work_dir}/package-infos"

while read -r package git_revision mod_git_revision repository; do

  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"

done < "${work_dir}/build-list.new"

echo 'apply blacklisting'
# ignore blacklisted packages and dependent packages
# this is the first time when all the information is available and up to date

black_listed=''
black_listed_new="$(
  cat "${repo_paths__archlinux32}/blacklist"
  ls "${work_dir}/package-infos" | \
    grep '\.depends$' | \
    sed "s|^|${work_dir}/package-infos/|" | \
    xargs -r grep -l '^lib32-' | \
    sed '
      s|^.*/||
      s|\(\.[^.]\+\)\{3\}$||
    '
)"

while [ -n "${black_listed_new}" ]; do
  black_listed="$(
    printf '%s\n%s' "${black_listed}" "${black_listed_new}"
  )"
  black_listed_new="$(
    echo "${black_listed_new}" | \
      while read -r bl_package; do
        ls "${work_dir}/package-infos" | \
          grep '\.depends$' | \
          sed "s|^|${work_dir}/package-infos/|" | \
          xargs -r grep -lxF "${bl_package}" | \
          sed '
            s|^.*/||
            s|\(\.[^.]\+\)\{3\}$||
          '
      done | \
      while read -r pkg; do
        if [ "${pkg}" = 'python-pysocks' ] || \
          [ "${pkg}" = 'gens' ]; then
          >&2 printf 'blacklisting: %s would be black listed, but is ignored.' "${pkg}"
        else
          echo "${pkg}"
        fi
      done | \
      sort -u
  )"
  black_listed_new="$(
    printf '%s\n' "${black_listed}" "${black_listed}" "${black_listed_new}" | \
      sort | \
      uniq -u
  )"
done
echo "${black_listed}" | \
  while read -r package; do
    [ -n "${package}" ] && \
      delete_package "${package}"
  done

sort -u "${work_dir}/deletion-list.new" > \
  "${work_dir}/deletion-list.new.new"

mv \
  "${work_dir}/deletion-list.new.new" \
  "${work_dir}/deletion-list.new"

# Now we create the partial order.

while read -r package git_revision mod_git_revision repository; do
  # add "$pkgname -> $build-target" to build-order list
  sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.builds"
  # add "$dependency -> $pkgname" to build-order list
  sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
done \
  < "${work_dir}/build-list.new" \
  > "${work_dir}/build-order"

if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then
  >&2 echo 'Created partial order file "build-order" is invalid.'
  exit 2
fi

echo 'Now actually sort it.'

(
  # this part will have the correct build order, but all the infos are missing
  tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
    nl -ba | \
    awk '{print $1 " not-git also-not-git whatever " $2}'
  # this part has all the infos, but possibly the wrong order
  awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new"
) | \
  sort -k5,5 -k1nr | \
  # now, we have the correct order and the infos, but in adjacent lines
  uniq -f4 -D | \
  sed '/^0 /d;N;s|\n| |' | \
  # now in one line, each
  sort -k1n,1 | \
  awk '{print $5 " " $7 " " $8 " " $9}' > \
  "${work_dir}/build-list.new.new"

rm --one-file-system -rf "${work_dir}/build-list.loops.new"
mkdir "${work_dir}/build-list.loops.new"

if [ -s "${work_dir}/tsort.error" ]; then
  >&2 echo 'WARNING: There is a dependency cycle!'
  >&2 cat "${work_dir}/tsort.error"
  >&2 echo
  >&2 echo 'I will continue anyway.'
  # save loops in separate files each, so breaking them is easier
  awk '
    /^tsort: \S+: input contains a loop:$/{
      n++;
      getline
    }
    {
      print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n
    }
    ' "${work_dir}/tsort.error"

  # remove lines from loop files which are no packages
  ls "${work_dir}/build-list.loops.new" | \
    grep '^loop_[0-9]\+$' | \
    while read -r loop; do
      (
        sort -u "${work_dir}/build-list.loops.new/${loop}"
        cut -d' ' -f1 "${work_dir}/build-list.new.new" | \
          sort -u
      ) | \
        sort | \
        uniq -d > \
        "${work_dir}/build-list.loops.new/${loop}.new"
      mv \
        "${work_dir}/build-list.loops.new/${loop}.new" \
        "${work_dir}/build-list.loops.new/${loop}"
    done

  # add orders to break loops to build list
  insert_break_loops_orders "${work_dir}/build-list.new.new"

else
  rm "${work_dir}/tsort.error"
fi

# Move the .new-files to the actual files

rm -rf --one-file-system "${work_dir}/build-list.loops"
(
  printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list"
  printf '%s.revision\n' ${repo_names}
) | \
  while read -r file; do
    mv "${work_dir}/${file}.new" "${work_dir}/${file}"
  done

# Delete old package meta data
delete_old_metadata

# Remove the lock file

rm -f "${build_list_lock_file}"