path: root/bin/get-package-updates

#!/bin/sh

# check for packages that need to be built

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: keep database clean in case of abort

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'get-package-updates: check for packages that need to be built,'
  >&2 echo ' and build a list in the proper build order'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -d|--date $datetime:'
  >&2 echo '                Pull latest commit before $datetime'
  >&2 echo '                (yyyy-mm-ddThh:mm:ss). Conflicts -n.'
  >&2 echo '  -h|--help:    Show this help and exit.'
  >&2 echo '  -i|--ignore-insanity:'
  >&2 echo '                Do not abort when insane.'
  >&2 echo '  -m|--max-upstream-packages $number:'
  >&2 echo '                Do not update more than $number upstream packages.'
  >&2 echo '                Do not update git head of state repository.'
  >&2 echo '                USE WITH CAUTION: This may remove packages, if they'
  >&2 echo '                were moved. Always do a final run without -m.'
  >&2 echo '  -n|--no-pull: Do not pull git repos, merely reorder build list.'
  >&2 echo '                Conflicts -d.'
  >&2 echo '  -r|--recent-modifications:'
  >&2 echo '                Use the latest commit for the modifications'
  >&2 echo '                git repository (e.g. ignore -d for it).'
  >&2 echo '                Requires -d.'
  >&2 echo '  -w|--wait:    If necessary, wait for lock blocking.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o d:him:nrw \
    --long date: \
    --long help \
    --long ignore-insanity \
    --long max-upstream-packages: \
    --long no-pull \
    --long recent-modifications \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
date_time=''
ignore_insanity=false
max_upstream_packages=''
pull=true
recent_modifications=false

while true
do
  case "$1" in
    -d|--date)
      shift
      date_time="$1"
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore-insanity)
      ignore_insanity=true
    ;;
    -m|--max-upstream-packages)
      shift
      max_upstream_packages="$1"
    ;;
    -n|--no-pull)
      pull=false
    ;;
    -r|--recent-modifications)
      recent_modifications=true
    ;;
    -w|--wait)
      block_flag=''
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -n "${date_time}" ] && ! ${pull}; then
  >&2 printf -- '-d and -n are mutually exclusive.\n'
  usage
fi

if ${recent_modifications} && [ -z "${date_time}" ]; then
  >&2 printf -- '-r requires -d.\n'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  if ! ${ignore_insanity}; then
    exit
  fi
fi

# delete_package arch package repository
# mark $arch/$package for deletion

# shellcheck disable=SC3043
delete_package() {
  local architecture="$1"
  local pkgbase="$2"
  local repository="$3"
  >&2 printf 'delete_package %s %s %s\n' "${architecture}" "${pkgbase}" "${repository}"
  # shellcheck disable=SC2016
  query_delete_packages=$(
    printf '`architectures` AS `d_a`'
    printf ' JOIN `architecture_compatibilities` AS `a_c`'
    printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
    printf ' AND `d_a`.`name`=from_base64("%s")' \
      "$(printf '%s' "${architecture}" | base64 -w0)"
    printf ' JOIN `build_assignments`'
    printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
    # "any" references all architectures, but this is not represented
    # in `architecture_compatibilities`: If a package is not buildable
    # for "any", this means literally, that it is not buildable for
    # the _least_ architecture (e.g. it is not generic)
    printf ' OR `d_a`.`name`="any")'
    mysql_join_build_assignments_package_sources
    printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
      "$(printf '%s' "${pkgbase}" | base64 -w0)"
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    mysql_join_package_sources_upstream_repositories
    printf ' AND `upstream_repositories`.`name` LIKE from_base64("%s")' \
      "$(printf '%s' "${repository}" | base64 -w0)"
  )
  # shellcheck disable=SC2016
  {
    # packages from the build-list/to-be-decided go straight to the deletion-list
    # this happens in two steps, because we need to create one item per
    # target architecture
    printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
      printf '`package`,'
      printf '`repository`,'
      printf '`last_moved`,'
      printf '`is_to_be_deleted`'
    printf ') SELECT'
    printf ' `binary_packages`.`id`,'
    printf '`d_r`.`id`,'
    printf 'NOW(),'
    printf '1'
    printf ' FROM'
    printf ' %s' "${query_delete_packages}"
    mysql_join_binary_packages_in_repositories_repositories
    # shellcheck disable=SC2154
    printf ' AND `repositories`.`stability` in (%s,%s)' \
      "${repository_stability_ids__unbuilt}" \
      "${repository_stability_ids__virtual}"
    mysql_join_build_assignments_architectures '' 'ba_a'
    printf ' JOIN `repositories` AS `d_r`'
    printf ' ON ('
      # arch-specific build_assignments must match exactly
      printf '`d_r`.`architecture`=`ba_a`.`id`'
      # "any" build_assignments build for all architectures
      printf ' OR `ba_a`.`name`="any"'
    printf ')'
    # shellcheck disable=SC2154
    printf ' AND `d_r`.`stability`=%s;\n' \
      "${repository_stability_ids__forbidden}"

    printf 'COMMIT;\n'

    # no need to remove the binary_package, because it is now on the
    # deletion-lists
    printf 'DELETE `binary_packages_in_repositories`'
    printf ' FROM %s' "${query_delete_packages}"
    mysql_join_binary_packages_in_repositories_repositories
    # shellcheck disable=SC2154
    printf ' AND `repositories`.`stability` in (%s,%s);\n' \
      "${repository_stability_ids__unbuilt}" \
      "${repository_stability_ids__virtual}"

    printf 'COMMIT;\n'

    # other packages are marked as `is_to_be_deleted`
    printf 'UPDATE %s' "${query_delete_packages}"
    printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
  } | \
    mysql_run_query
}

something_new=false

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  if [ "${repo}" = 'archlinux32' ]; then
    branch='master'
  else
    branch='main'
  fi
  # Update git repositories (upstream state and our package customizations).
  if [ -d "${repo_path}/.git" ]; then
    git -C "${repo_path}" pull --ff-only
  else
    git -C "${repo_path}" fetch origin "${branch}:${branch}"
  fi || \
    true
  # read previous git revision numbers from database.
  # shellcheck disable=SC2016
  eval "old_repo_revisions__${repo}='$(
    {
      printf 'SELECT `git_repositories`.`head`'
      printf ' FROM `git_repositories`'
      printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
        "$(printf '%s' "${repo}" | base64 -w0)"
    } | \
      mysql_run_query
  )'"
  # determine new git revision
  if ${pull}; then
    if ${recent_modifications} && \
      [ "${repo}" = 'archlinux32' ] || \
      [ -z "${date_time}" ]; then
      eval "new_repo_revisions__${repo}='$(
        git -C "${repo_path}" rev-parse HEAD
      )'"
    else
      new_rev=$(
        git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD
      )
      eval 'old_rev="${old_repo_revisions__'"${repo}"'}"'
      # do not go backwards in time
      # shellcheck disable=SC2154
      if ! git -C "${repo_path}" merge-base --is-ancestor "${old_rev}" "${new_rev}"; then
        new_rev="${old_rev}"
      fi
      eval "new_repo_revisions__${repo}='${new_rev}'"
    fi
  else
    eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"'
  fi
  if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
    something_new=true
  fi
done

if ${pull} && \
  ! ${something_new}; then
  >&2 echo 'Nothing changed.'
  exit
fi

exec 9> "${sanity_check_lock_file}"
# shellcheck disable=SC2086
if ! verbose_flock -s ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit
fi

cleanup() {
  mysql_cleanup
  rm -rf --one-file-system "${tmp_dir:?}"
}
tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)

trap cleanup EXIT

# shellcheck disable=SC2119
mysql_cleanup

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

{
  directories=$(
    # shellcheck disable=SC2154
    git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \
    | tar -t \
    | cut -d/ -f1 \
    | grep -vF -- '-testing-' \
    | grep -vF -- '-staging-' \
    | sort -u
  )
  # shellcheck disable=SC2016
  {
    printf 'SELECT DISTINCT `package_sources`.`pkgbase`,'
    printf '`package_sources`.`git_revision`,'
    printf '`upstream_repositories`.`name`'
    printf ' FROM `package_sources`'
    mysql_join_package_sources_upstream_repositories
    mysql_join_package_sources_build_assignments
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
  } \
  | mysql_run_query \
  | tr '\t' ' ' \
  | sort -u \
  > "${tmp_dir}/mysql-packages"
  for directory in ${directories}; do
    # shellcheck disable=SC2154
    git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- "${directory}" \
    | tar -Ox \
    | grep -v '^\S\+\( \S\+\)\1\1$' \
    | grep ' [0-9a-f]\{40\}$' \
    | cut -d' ' -f1,4 \
    | sed '
      s@$@ '"${directory%-*}"'@
    '
  done \
  | grep -v '^lib32-' \
  | grep -vF '03fd4a0a1cee6144848a5e5de6e03f0319e9d118' \
  | grep -vF 'e383f4debfe90631a2eefabff9c6704f26bc3249' \
  | sort -u \
  > "${tmp_dir}/upstream-packages"
  diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \
  | grep '^>' \
  | awk '{print $3 " " $2 " " $4}' \
  | sort -k2,2 -k3,3 \
  | uniq -uf1 \
  | while read -r git_revision pkgbase repository; do
    # shellcheck disable=SC2154
    if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then
      mod_git_revision="${new_repo_revisions__archlinux32}"
    else
      mod_git_revision='0000000000000000000000000000000000000000'
    fi
    printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${mod_git_revision}"
  done \
  | if [ -n "${max_upstream_packages}" ]; then
    head -n"${max_upstream_packages}"
  else
    cat
  fi
  # shellcheck disable=SC2154
  git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" --name-status \
  | tr '\t/' ' ' \
  | grep '^[AM]' \
  | cut -d' ' -f2,3 \
  | grep -v '^blacklist ' \
  | while read -r repository pkgbase; do
    git_revision=$(
      # shellcheck disable=SC2046
      git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- $(
        printf '%s\n' "${directories}" \
        | grep '^'"${repository}"'-'
      ) \
      | tar -Ox \
      | sort -k1,1 \
      > "${tmp_dir}/git-revisions"
      echo "${pkgbase}" \
      | join -1 1 -2 1 -o 1.4 "${tmp_dir}/git-revisions" -
    )
    if [ -z "${git_revision}" ]; then
      git_revision='0000000000000000000000000000000000000000'
    fi
    if [ ${#git_revision} != 40 ] \
    || printf '%s\n' "${git_revision}" \
    | grep -vq '[0-9]'; then
      >&2 printf 'invalid git revision "%s"\n' "${git_revision}"
      exit 1
    fi
    printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${new_repo_revisions__archlinux32}"
  done
} \
| sort -u \
| sort -k1,1 -k2,2 \
> "${tmp_dir}/modified-packages"

errors=$(
  awk '{print $3 " " $4 " " $1 " " $2}' \
  < "${tmp_dir}/modified-packages" \
  | uniq -Df2
)
if [ -n "${errors}" ]; then
  >&2 printf 'Some packages are scheduled with different versions - this should not happen:\n'
  >&2 printf '%s\n' "${errors}"
  exit 1
fi

echo 'Mark to be deleted packages for removal'

cut -d' ' -f1,3 "${tmp_dir}/mysql-packages" \
| sort -u \
| sponge "${tmp_dir}/mysql-packages"

{
  cut -d' ' -f1,3 "${tmp_dir}/upstream-packages"
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" \
  | tar -t \
  | grep -v '^blacklist/' \
  | tr '/' ' ' \
  | awk '{print $2 " " $1}'
} \
| sort -u \
| sponge "${tmp_dir}/upstream-packages"

diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \
| grep '^<' \
| cut -d' ' -f2,3 \
| while read -r pkgbase repository; do
  delete_package 'any' "${pkgbase}" "${repository}"
done

echo 'Insert updated packages'

while read -r pkgbase repository git_revision mod_git_revision; do
  if test "$repository" = "kde-unstable"; then
    continue
  fi
  if test "$repository" = "gnome-unstable"; then
    continue
  fi
  # shellcheck disable=SC2016
  {
    # delete old binary packages which are not yet built or on the
    # deletion list
    mysql_query_delete_packages \
      '`package_sources`.`pkgbase`=from_base64("'"$(
        printf '%s' "${pkgbase}" \
        | base64 -w0
      )"'")' \
      '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')'
    # remove is-to-be-deleted marker from old binary packages
    printf 'UPDATE `binary_packages_in_repositories`'
    mysql_join_binary_packages_in_repositories_binary_packages
    mysql_join_binary_packages_build_assignments
    mysql_join_build_assignments_package_sources
    mysql_join_package_sources_upstream_repositories
    printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0'
    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
      "$(
        printf '%s' "${pkgbase}" \
        | base64 -w0
      )"
    printf ' AND `upstream_repositories`.`name`=from_base64("%s");\n' \
      "$(
        printf '%s' "${repository}" \
        | base64 -w0
      )"
  } \
  | mysql_run_query
  # shellcheck disable=SC2154
  >&2 printf '%s ' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
  mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
  >&2 printf '\n'
done \
< "${tmp_dir}/modified-packages"

echo 'remove blacklisted packages'

{
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' \
  | tar -t 'blacklist' \
  | sed '
    s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@
    t
    d
  ' \
  | sort -k2,2 \
  | join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" -
  git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' \
  | sed '
    s@^[AM]\sblacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@
    t
    d
  '
} \
| sort -u \
| while read -r arch pkgbase; do
  delete_package "${arch}" "${pkgbase}" '%'
done

echo 'Done - mark decisions as final.'

# shellcheck disable=SC2016
{
  # save blacklist into database
  printf 'CREATE TEMPORARY TABLE `blacklist` (`arch` VARCHAR(16), `pkgbase` VARCHAR(64), `reason` TEXT);\n'
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
    tar -x --to-command 'sed "s@^@$TAR_FILENAME @"' 'blacklist' | \
    sed '
      s@^blacklist/\([^/[:space:]]\+\)/\S\+/\([^/[:space:]]\+\) @\1 \2 @
      t
      d
    ' | \
    while read -r arch pkgbase reason; do
      printf '(from_base64("%s"),from_base64("%s"),from_base64("%s")),\n' \
        "$(printf '%s' "${arch}" | base64 -w0)" \
        "$(printf '%s' "${pkgbase}" | base64 -w0)" \
        "$(printf '%s' "${reason}" | base64 -w0)"
    done | \
    sed '
      1 i INSERT IGNORE INTO `blacklist` (`arch`,`pkgbase`,`reason`) VALUES
      $ s/,$/;/
    '
  printf 'UPDATE `build_assignments`'
  printf ' SET `build_assignments`.`is_black_listed`=NULL;\n'
  printf 'UPDATE `blacklist`'
  printf ' JOIN `architectures`'
  printf ' ON `architectures`.`name`=`blacklist`.`arch`'
  printf ' JOIN `package_sources`'
  printf ' ON `blacklist`.`pkgbase`=`package_sources`.`pkgbase`'
  mysql_join_package_sources_build_assignments
  printf ' JOIN `architecture_compatibilities`'
  printf ' ON `build_assignments`.`architecture`=`architecture_compatibilities`.`built_for`'
  printf ' AND ('
    printf '`architectures`.`id`=`architecture_compatibilities`.`runs_on`'
    # shellcheck disable=SC2154
    printf ' OR `architectures`.`id`=%s' \
      "${architecture_ids__any}"
  printf ')'
  printf ' SET `build_assignments`.`is_black_listed`=`blacklist`.`reason`;\n'
  printf 'DROP TEMPORARY TABLE `blacklist`;\n'
  printf 'COMMIT;\n'

  # update hashes of repositories in mysql database
  for repo in ${repo_names}; do
    if [ -n "${max_upstream_packages}" ] \
    && [ "${repo}" = 'state' ]; then
      continue
    fi
    printf 'UPDATE `git_repositories`'
    printf ' SET `git_repositories`.`head`=from_base64("%s")' \
      "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}"' | base64 -w0)"
    printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
      "$(printf '%s' "${repo}" | base64 -w0)"
  done
  # move binary_packages from "to-be-decided" to "build-list"
  printf 'UPDATE `binary_packages_in_repositories`'
  mysql_join_binary_packages_in_repositories_binary_packages
  printf ' SET `binary_packages_in_repositories`.`repository`=%s' \
    "${repository_ids__any_build_list}"
  printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
    "${repository_ids__any_to_be_decided}"
} | \
  mysql_run_query

echo 'Aftermath - sort versions.'

mysql_sort_versions

echo 'Aftermath - find assignment loops.'

# update loop list in database (beware, the packages are expected to be in "build-list",
# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
mysql_find_build_assignment_loops

echo 'Aftermath - remove duplicate binary_packages.'

# remove duplicate binary_packages from "build-list"
mysql_query_remove_old_binary_packages_from_build_list | \
  mysql_run_query 'unimportant'