path: root/bin/get-package-updates

#!/bin/sh

# check for packages that need to be built

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: keep database clean in case of abort

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'get-package-updates: check for packages that need to be built,'
  >&2 echo ' and build a list in the proper build order'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -d|--date $datetime:'
  >&2 echo '                Pull latest commit before $datetime'
  >&2 echo '                (yyyy-mm-ddThh:mm:ss). Conflicts -n.'
  >&2 echo '  -h|--help:    Show this help and exit.'
  >&2 echo '  -i|--ignore-insanity:'
  >&2 echo '    Do not abort when insane.'
  >&2 echo '  -n|--no-pull: Do not pull git repos, merely reorder build list.'
  >&2 echo '                Conflicts -d.'
  >&2 echo '  -r|--recent-modifications:'
  >&2 echo '                Use the latest commit for the modifications'
  >&2 echo '                git repository (e.g. ignore -d for it).'
  >&2 echo '                Requires -d.'
  >&2 echo '  -w|--wait:    If necessary, wait for lock blocking.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o d:hinrw \
    --long date: \
    --long help \
    --long ignore-insanity \
    --long no-pull \
    --long recent-modifications \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
date_time=''
ignore_insanity=false
pull=true
recent_modifications=false

while true
do
  case "$1" in
    -d|--date)
      shift
      date_time="$1"
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore-insanity)
      ignore_insanity=true
    ;;
    -n|--no-pull)
      pull=false
    ;;
    -r|--recent-modifications)
      recent_modifications=true
    ;;
    -w|--wait)
      block_flag=''
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -n "${date_time}" ] && ! ${pull}; then
  >&2 printf -- '-d and -n are mutually exclusive.\n'
  usage
fi

if ${recent_modifications} && [ -z "${date_time}" ]; then
  >&2 printf -- '-r requires -d.\n'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  if ! ${ignore_insanity}; then
    exit
  fi
fi

# delete_package arch package
# mark $arch/$package for deletion
delete_package() {
  # shellcheck disable=SC2016
  query_delete_packages=$(
    printf '`architectures` AS `d_a`'
    printf ' JOIN `architecture_compatibilities` AS `a_c`'
    printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
    printf ' AND `d_a`.`name`=from_base64("%s")' \
      "$(printf '%s' "$1" | base64 -w0)"
    printf ' JOIN `build_assignments`'
    printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
    # "any" references all architectures, but this is not represented
    # in `architecture_compatibilities`: If a package is not buildable
    # for "any", this means literally, that it is not buildable for
    # the _least_ architecture (e.g. it is not generic)
    printf ' OR `d_a`.`name`="any")'
    mysql_join_build_assignments_package_sources
    printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
      "$(printf '%s' "$2" | base64 -w0)"
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
  )
  # shellcheck disable=SC2016
  {
    # packages from the build-list/to-be-decided go straight to the deletion-list
    # this happens in two steps, because we need to create one item per
    # target architecture
    printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
      printf '`package`,'
      printf '`repository`,'
      printf '`last_moved`,'
      printf '`is_to_be_deleted`'
    printf ') SELECT'
    printf ' `binary_packages`.`id`,'
    printf '`d_r`.`id`,'
    printf 'NOW(),'
    printf '1'
    printf ' FROM'
    printf ' %s' "${query_delete_packages}"
    mysql_join_binary_packages_in_repositories_repositories
    # shellcheck disable=SC2154
    printf ' AND `repositories`.`stability` in (%s,%s)' \
      "${repository_stability_ids__unbuilt}" \
      "${repository_stability_ids__virtual}"
    mysql_join_build_assignments_architectures '' 'ba_a'
    printf ' JOIN `repositories` AS `d_r`'
    printf ' ON ('
      # arch-specific build_assignments must match exactly
      printf '`d_r`.`architecture`=`ba_a`.`id`'
      # "any" build_assignments build for all architectures
      printf ' OR `ba_a`.`name`="any"'
    printf ')'
    # shellcheck disable=SC2154
    printf ' AND `d_r`.`stability`=%s;\n' \
      "${repository_stability_ids__forbidden}"

    printf 'COMMIT;\n'

    printf 'DELETE `binary_packages_in_repositories`'
    printf ' FROM %s' "${query_delete_packages}"
    mysql_join_binary_packages_in_repositories_repositories
    # shellcheck disable=SC2154
    printf ' AND `repositories`.`stability` in (%s,%s);\n' \
      "${repository_stability_ids__unbuilt}" \
      "${repository_stability_ids__virtual}"

    printf 'COMMIT;\n'

    # other packages are marked as `is_to_be_deleted`
    printf 'UPDATE %s' "${query_delete_packages}"
    printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
  } | \
    mysql_run_query
}

something_new=false

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  # Update git repositories (official packages, community packages and the repository of package customizations).
  if [ -d "${repo_path}/.git" ]; then
    git -C "${repo_path}" pull --ff-only
  else
    git -C "${repo_path}" fetch origin master:master
  fi || \
    true
  # read previous git revision numbers from database.
  # shellcheck disable=SC2016
  eval "old_repo_revisions__${repo}='$(
    {
      printf 'SELECT `git_repositories`.`head`'
      printf ' FROM `git_repositories`'
      printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
        "$(printf '%s' "${repo}" | base64 -w0)"
    } | \
      mysql_run_query
  )'"
  # determine new git revision
  if ${pull}; then
    if ${recent_modifications} && \
      [ "${repo}" = 'archlinux32' ] || \
      [ -z "${date_time}" ]; then
      eval "new_repo_revisions__${repo}='$(
        git -C "${repo_path}" rev-parse HEAD
      )'"
    else
      new_rev=$(
        git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD
      )
      eval 'old_rev="${old_repo_revisions__'"${repo}"'}"'
      # do not go backwards in time
      # shellcheck disable=SC2154
      if ! git -C "${repo_path}" merge-base --is-ancestor "${old_rev}" "${new_rev}"; then
        new_rev="${old_rev}"
      fi
      eval "new_repo_revisions__${repo}='${new_rev}'"
    fi
  else
    eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"'
  fi
  if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
    something_new=true
  fi
done

if ${pull} && \
  ! ${something_new}; then
  >&2 echo 'Nothing changed.'
  exit
fi

# Create a lock file for build list.

exec 9> "${build_list_lock_file}"
if ! verbose_flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit
fi

exec 8> "${sanity_check_lock_file}"
if ! verbose_flock -s ${block_flag} 8; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit
fi

cleanup() {
  mysql_cleanup
  rm -rf --one-file-system "${tmp_dir:?}"
}
tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)

trap cleanup EXIT

# shellcheck disable=SC2119
mysql_cleanup

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

# shellcheck disable=SC2016
{
  printf 'SELECT DISTINCT'
  printf ' `package_sources`.`pkgbase`,'
  printf '`upstream_repositories`.`name`,'
  printf 'IF(`build_assignments`.`architecture`=%s,"any","x86_64")' \
    "${architecture_ids__any}"
  printf ' FROM `binary_packages`'
  mysql_join_binary_packages_binary_packages_in_repositories
  mysql_join_binary_packages_build_assignments
  mysql_join_build_assignments_package_sources
  mysql_join_package_sources_upstream_repositories
  printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
    "${repository_ids__any_build_list}"
} | \
  mysql_run_query | \
  sed '
    s@^\(\S*\)\t\(\S*\)\t\(\S*\)$@s,^\\(.\\t\1/repos/\2-\3/\\)[^/]\\+$,\\1PKGBUILD,\ns,^\\(.\\t\2/\1/\\)[^/]\\+$,\\1PKGBUILD,@
  ' > \
  "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression"

{
  # trigger rebuild of packages removed from blacklist
  # shellcheck disable=SC2016
  {
    printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`,`git_repositories`.`name`'
    printf ' FROM `package_sources`'
    mysql_join_package_sources_upstream_repositories
    mysql_join_upstream_repositories_git_repositories
    printf ' WHERE `package_sources`.`pkgbase` IN ('
    # shellcheck disable=SC2154
    git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
      sed '
        s@^D\tblacklist/[^/]\+/[^/]\+/@@
        t
        d
      ' | \
      base64_encode_each | \
      sed '
        s/^/from_base64("/
        s/$/")/
      ' | \
      tr '\n' ','
    printf '"");\n'
  } | \
    mysql_run_query | \
    while read -r pkgbase repository git_repository; do
      printf 'A %s ' \
        "${pkgbase}"
      eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${git_repository}"'}"'
      printf ' %s\n' \
        "${repository}"
    done
  # actual updates/removes
  for repo in ${repo_names}; do
    eval repo_path='"${repo_paths__'"${repo}"'}"'
    eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
    eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"'
    # if old revision unknown, mimic "git diff"-output
    # shellcheck disable=SC2154
    if [ "${old_repo_revision}" = "NONE" ]; then
      git -C "${repo_path}" archive --format=tar HEAD | \
        tar -t | \
        sed 's|^|A\t|'
    else
      git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
    fi | \
      # rename any file to "PKGBUILD" for packages on the build-list
      sed -f "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" | \
      # only track changes in PKGBUILDs
      grep '/PKGBUILD$' | \
      if [ "${repo}" = "archlinux32" ]; then
      # modify the directory structure from the modifiaction-repository
      # to the one of an original source repository
        # shellcheck disable=SC2016
        sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
          while read -r pkg_repo rest; do
            repo=$(find_git_repository_to_package_repository "${pkg_repo}")
            eval 'printf '"'"'%s %s\n'"'" \
              "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \
              "'${rest}'"
          done
      else
        sed "s|^|${new_repo_revision} |"
      fi | \
      grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
      # ignore i686
      grep -- '-\(any\|x86_64\)/PKGBUILD$' | \
      # ignore staging and testing
      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
      sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|'
  done | \
    sort -u | \
    sed '
      s|^D\s|0 \0|
      t
      s|^[AM]\s|1 \0|
      t
      s|^|2 |
    ' | \
    sort -k1,1 | \
    sed 's|^[012] ||'
} | \
  while read -r mode package git_revision repository; do
    if [ "${mode}" = 'D' ]; then
      # deleted PKGBUILD
      # shellcheck disable=SC2154
      git_revision="${new_repo_revisions__archlinux32}"
      found_package=false
      # we need to test archlinux32 last, because otherwise
      # find_package_repository_to_package might look in the wrong git
      # repository of a package w/o upstream
      for repository in ${repo_names} archlinux32; do
        eval 'repo_path="${repo_paths__'"${repository}"'}"'
        if [ "${repository}" = "archlinux32" ]; then
          if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \
            tar -t 2> /dev/null | \
            grep -q '^[^/]\+/'"$(str_to_regex "${package}")"'/PKGBUILD$'; then
            found_package=true
          fi
        else
          # shellcheck disable=SC2154
          if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos"' 2> /dev/null | \
            tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \
            cut -d/ -f3 | \
            grep -v 'staging\|testing\|-unstable' | \
            grep -vq -- '-i686$'; then
            eval 'git_revision="${new_repo_revisions__'"${repository}"'}"'
            found_package=true
            break
          fi
        fi
      done
      if ${found_package}; then
        mode='M'
        repository=$(
          find_package_repository_to_package "${package}" "${repository}" "${git_revision}"
        )
      else
        delete_package 'any' "${package}"
        continue
      fi
    fi
    if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
      # shellcheck disable=SC2016
      {
        # delete old binary packages which are not yet built or on the
        # deletion list and their build_assignment, if it only built
        # deleted packages (first try deleting the build assignment,
        # otherwise we loose the handle on it!)
        for with_ba in true false; do
          printf 'DELETE '
          if "${with_ba}"; then
            # try to delete build-assignment, too
            printf '`build_assignments`,'
          fi
          printf '`binary_packages`,`binary_packages_in_repositories`'
          printf ' FROM `binary_packages`'
          mysql_join_binary_packages_build_assignments
          mysql_join_build_assignments_package_sources
          if "${with_ba}"; then
            # some package not unbuilt or not on the deletion-list?
            mysql_join_binary_packages_binary_packages_in_repositories '' 'keep_bpir'
            printf ' LEFT'
            mysql_join_binary_packages_in_repositories_repositories 'keep_bpir' 'keep_r'
            # shellcheck disable=SC2154
            printf ' AND `all_r`.`stability` NOT IN (%s,%s);\n' \
              "${repository_stability_ids__unbuilt}" \
              "${repository_stability_ids__forbidden}"
          fi
          mysql_join_binary_packages_binary_packages_in_repositories
          mysql_join_binary_packages_in_repositories_repositories
          printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
            "$(
              printf '%s' "${package}" | \
                base64 -w0
            )"
          if "${with_ba}"; then
            # only if all packages unbuilt or on the deletion-list
            printf ' AND `keep_r`.`id` IS NULL'
          fi
          # shellcheck disable=SC2154
          printf ' AND `repositories`.`stability` IN (%s,%s);\n' \
            "${repository_stability_ids__unbuilt}" \
            "${repository_stability_ids__forbidden}"
        done
        # remove is-to-be-deleted marker from old binary packages
        printf 'UPDATE `binary_packages_in_repositories`'
        mysql_join_binary_packages_in_repositories_binary_packages
        mysql_join_binary_packages_build_assignments
        mysql_join_build_assignments_package_sources
        printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0'
        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
          "$(
            printf '%s' "${package}" | \
              base64 -w0
          )"
      } | \
        mysql_run_query
      printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages"
      # shellcheck disable=SC2154
      printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2
# TODO: sometimes we schedule the identical package twice (once "normal"
# and once, because we found the deleted package elsewhere) - this could
# be avoided
      mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
      printf '\n' >&2
      continue
    fi

    >&2 echo "unknown git diff mode '${mode}'"
    exit 1
  done

if [ -s "${tmp_dir}/modified-packages" ] || ! ${pull}; then
  echo 'Delete black-listed packages.'
  if [ -s "${tmp_dir}/modified-packages" ]; then
    sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages"
  fi
  # extract black-listed packages
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
    tar -t 'blacklist' | \
    sed '
      s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
      t
      d
    ' | \
    expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" | \
    if [ -s "${tmp_dir}/modified-packages" ]; then
      sort -k2,2 | \
      join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" -
      sed '
        /^lib32-/ s/^/any /
        t
        d
      ' "${tmp_dir}/modified-packages"
    else
      cat
    fi | \
    sort -u | \
    while read -r arch pkgbase; do
      delete_package "${arch}" "${pkgbase}"
    done
  rm "${tmp_dir}/architecture-compatibilities"
fi

echo 'Done - mark decisions as final.'

# shellcheck disable=SC2016
{
  # save blacklist into database
  printf 'CREATE TEMPORARY TABLE `blacklist` (`arch` VARCHAR(16), `pkgbase` VARCHAR(64), `reason` TEXT);\n'
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
    tar -x --to-command 'sed "s@^@$TAR_FILENAME @"' 'blacklist' | \
    sed '
      s@^blacklist/\([^/[:space:]]\+\)/\S\+/\([^/[:space:]]\+\) @\1 \2 @
      t
      d
    ' | \
    while read -r arch pkgbase reason; do
      printf '(from_base64("%s"),from_base64("%s"),from_base64("%s")),\n' \
        "$(printf '%s' "${arch}" | base64 -w0)" \
        "$(printf '%s' "${pkgbase}" | base64 -w0)" \
        "$(printf '%s' "${reason}" | base64 -w0)"
    done | \
    sed '
      1 i INSERT IGNORE INTO `blacklist` (`arch`,`pkgbase`,`reason`) VALUES
      $ s/,$/;/
    '
  printf 'UPDATE `build_assignments`'
  printf ' SET `build_assignments`.`is_black_listed`=NULL;\n'
  printf 'UPDATE `blacklist`'
  printf ' JOIN `architectures`'
  printf ' ON `architectures`.`name`=`blacklist`.`arch`'
  printf ' JOIN `package_sources`'
  printf ' ON `blacklist`.`pkgbase`=`package_sources`.`pkgbase`'
  mysql_join_package_sources_build_assignments
  printf ' JOIN `architecture_compatibilities`'
  printf ' ON `build_assignments`.`architecture`=`architecture_compatibilities`.`built_for`'
  printf ' AND ('
    printf '`architectures`.`id`=`architecture_compatibilities`.`runs_on`'
    # shellcheck disable=SC2154
    printf ' OR `architectures`.`id`=%s' \
      "${architecture_ids__any}"
  printf ')'
  printf ' SET `build_assignments`.`is_black_listed`=`blacklist`.`reason`;\n'
  printf 'DROP TEMPORARY TABLE `blacklist`;\n'
  printf 'COMMIT;\n'

  # update hashes of repositories in mysql database
  for repo in ${repo_names}; do
    printf 'UPDATE `git_repositories`'
    printf ' SET `git_repositories`.`head`=from_base64("%s")' \
      "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}"' | base64 -w0)"
    printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
      "$(printf '%s' "${repo}" | base64 -w0)"
  done
  # move binary_packages from "to-be-decided" to "build-list"
  printf 'UPDATE `binary_packages_in_repositories`'
  mysql_join_binary_packages_in_repositories_binary_packages
  printf ' SET `binary_packages_in_repositories`.`repository`=%s' \
    "${repository_ids__any_build_list}"
  printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
    "${repository_ids__any_to_be_decided}"
} | \
  mysql_run_query

echo 'Aftermath - sort versions.'

mysql_sort_versions

echo 'Aftermath - find assignment loops.'

# update loop list in database (beware, the packages are expected to be in "build-list",
# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
mysql_find_build_assignment_loops

echo 'Aftermath - remove duplicate binary_packages.'

# remove duplicate binary_packages from "build-list"
mysql_query_remove_old_binary_packages_from_build_list | \
  mysql_run_query 'unimportant'