path: root/bin/delete-packages

#!/bin/sh

# delete obsolete binary packages

# 1] Condition for deleting a package A is that:
#   a) nothing on the build-list (make|check|)depends on A and
#   b) no built package B which is not being deleted depends on A

# "Package x depends on package y" means, that something needed by x
# is provided by y and no other package which will not be deleted.

# shellcheck disable=SC2039,SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'delete-packages [options]:'
  >&2 echo ' delete obsolete binary packages.'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--build-support $package-list:'
  >&2 echo '    Only delete the given packages from [build-support].'
  >&2 echo '    $package-list contains one package per line of'
  >&2 echo '    the form "$repo_arch $pkgfile".'
  >&2 echo '    conflicts -f; only accepted once'
  >&2 echo '  -f|--force $package:'
  >&2 echo '    Only delete the selected packages (given by'
  >&2 echo '    $repo_arch/$repo/$pkgname) - but delete'
  >&2 echo '    in any case. conflicts -b'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  >&2 echo '  -i|--ignore-insanity:'
  >&2 echo '    Do not abort when insane.'
  >&2 echo '  -n|--no-action:'
  >&2 echo '    Only print what would be deleted.'
  >&2 echo '  -w|--wait:'
  >&2 echo '    If necessary, wait for lock blocking.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o b:f:hinw \
    --long build-support: \
    --long force: \
    --long help \
    --long ignore-insanity \
    --long no-action \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
build_support=''
force_packages=''
ignore_insanity=false
no_action_option=''

while true
do
  case "$1" in
    -b|--build-support)
      if [ -n "${build_support}" ]; then
        >&2 printf 'Option given multiple times: %s\n' "$1"
        usage
      fi
      shift
      build_support="$1"
    ;;
    -f|--force)
      shift
      force_packages="${force_packages} $1"
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore-insanity)
      ignore_insanity=true
    ;;
    -n|--no-action)
      no_action_option='-n'
    ;;
    -w|--wait)
      block_flag=''
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ -n "${build_support}" ] && \
  [ -n "${force_packages}" ]; then
  >&2 echo 'Conflicting flags: -b & -f.'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  if ! ${ignore_insanity}; then
    exit
  fi
fi

# Create a lock file and a trap.

if [ -z "${no_action_option}" ]; then

#  exec 9> "${build_list_lock_file}"
#  if ! verbose_flock ${block_flag} 9; then
#    >&2 echo 'come back (shortly) later - I cannot lock build list.'
#    exit 0
#  fi

  exec 8> "${package_database_lock_file}"
  # shellcheck disable=SC2086
  if ! verbose_flock ${block_flag} 8; then
    >&2 echo 'come back (shortly) later - I cannot lock package database.'
    exit 0
  fi

  # shellcheck disable=SC2086
  if intentions_left ${block_flag}; then
    >&2 echo 'come back (shortly) later - There are still intentions in the queue.'
    exit 0
  fi

  exec 7> "${sanity_check_lock_file}"
  # shellcheck disable=SC2086
  if ! verbose_flock -s ${block_flag} 7; then
    >&2 echo 'come back (shortly) later - sanity-check running.'
    exit 0
  fi

fi

tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT

export TMPDIR="${tmp_dir}"

# shellcheck disable=SC2016
{
  printf 'CREATE TEMPORARY TABLE `bpir_to_delete` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  printf 'INSERT IGNORE INTO `bpir_to_delete`'
  # shellcheck disable=SC2030
  if [ -n "${force_packages}" ]; then
    printf ' SELECT'
    printf ' `binary_packages_in_repositories`.`id`'
    printf ' FROM `binary_packages_in_repositories`'
    mysql_join_binary_packages_in_repositories_binary_packages
    mysql_join_binary_packages_in_repositories_repositories
    mysql_join_repositories_architectures
    printf ' WHERE `repositories`.`is_on_master_mirror`'
    printf ' AND CONCAT('
      printf '`architectures`.`name`,"/",'
      printf '`repositories`.`name`,"/",'
      printf '`binary_packages`.`pkgname`'
    printf ') IN ('
    # shellcheck disable=SC2086
    printf '%s\n' ${force_packages} | \
      base64_encode_each | \
      sed '
        s/^/(from_base64("/
        s/$/")),/
        $ s/,$/);/
      '
  elif [ -n "${build_support}" ]; then
    printf ' SELECT DISTINCT `binary_packages_in_repositories`.`id`'
    printf ' FROM `binary_packages_in_repositories`'
    mysql_join_binary_packages_in_repositories_binary_packages
    mysql_join_binary_packages_in_repositories_repositories
    mysql_join_binary_packages_architectures
    printf ' LEFT'
    mysql_join_binary_packages_compressions
    mysql_join_repositories_architectures '' 'r_a'
    printf ' WHERE `repositories`.`name`="build-support"'
    printf ' AND CONCAT(`r_a`.`name`," ",'
    mysql_package_name_query
    printf ') IN ('
    base64_encode_each < "${build_support}" | \
      sed '
        s/^/from_base64("/
        s/$/"),/
        $ s/,$/);/
      '
  else
    printf ' SELECT DISTINCT `binary_packages_in_repositories`.`id`'
    printf ' FROM `binary_packages_in_repositories`'
    mysql_join_binary_packages_in_repositories_binary_packages
    mysql_join_binary_packages_in_repositories_repositories
    printf ' LEFT' # should not be necessary, but is formally more correct
    mysql_join_binary_packages_install_target_providers
    printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`'
    printf ' AND `repositories`.`is_on_master_mirror`'
    printf ' AND NOT EXISTS ('
      # no packages depending on that one exist
      printf 'SELECT 1'
      printf ' FROM `dependencies`'
      mysql_join_dependencies_dependency_types
      printf ' AND `dependency_types`.`relevant_for_binary_packages`'
      mysql_join_dependencies_binary_packages '' 'd_bp'
      mysql_join_binary_packages_binary_packages_in_repositories 'd_bp' 'd_bpir'
      # deliberately break dependencies of deletion-list packages
      printf ' AND NOT `d_bpir`.`is_to_be_deleted`'
      mysql_join_binary_packages_in_repositories_repositories 'd_bpir' 'd_r'
      # this is deliberately less restrict than `d_r`.`is_on_master_mirror`
      # shellcheck disable=SC2154
      printf ' AND `d_r`.`stability` NOT IN (%s,%s)' \
        "${repository_stability_ids__forbidden}" \
        "${repository_stability_ids__virtual}"
      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
      printf ' AND `d_r`.`architecture`=`repositories`.`architecture`'
      printf ' AND NOT EXISTS ('
        printf 'SELECT 1'
        printf ' FROM `binary_packages` AS `s_bp`'
        mysql_join_binary_packages_binary_packages_in_repositories 's_bp' 's_bpir'
        printf ' AND NOT `s_bpir`.`is_to_be_deleted`'
        mysql_join_binary_packages_install_target_providers 's_bp' 's_itp'
        mysql_join_binary_packages_in_repositories_repositories 's_bpir' 's_r'
        printf ' AND `s_r`.`is_on_master_mirror`'
        printf ' JOIN `repository_stability_relations`'
          printf ' ON `repository_stability_relations`.`more_stable`=`s_r`.`stability`'
        printf ' WHERE `s_itp`.`install_target`=`install_target_providers`.`install_target`'
        printf ' AND `s_r`.`architecture`=`repositories`.`architecture`'
        printf ' AND `repository_stability_relations`.`less_stable`=`d_r`.`stability`'
      printf ')'
    printf ');\n'
  fi

  # first enter all potentially to-be-deleted binary_packages into the table
  printf 'CREATE TEMPORARY TABLE `bp_to_delete` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  printf 'INSERT IGNORE INTO `bp_to_delete`'
  printf ' SELECT `binary_packages_in_repositories`.`package`'
  printf ' FROM `bpir_to_delete`'
  printf ' JOIN `binary_packages_in_repositories`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`;\n'

  # then remove all bp_to_delete which are actually still needed
  printf 'DELETE `bp_to_delete`'
  printf ' FROM `bp_to_delete`'
  printf ' JOIN `binary_packages`'
  printf ' ON `binary_packages`.`id`=`bp_to_delete`.`id`'
  mysql_join_binary_packages_binary_packages_in_repositories
  mysql_join_binary_packages_in_repositories_repositories
  printf ' WHERE `repositories`.`is_on_master_mirror`'
  printf ' AND NOT EXISTS ('
    printf 'SELECT 1'
    printf ' FROM `bpir_to_delete`'
    printf ' WHERE `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  printf ');\n'

  printf 'SELECT DISTINCT'
  printf ' "repo",'
  printf '`architectures`.`name`,'
  printf '`repositories`.`name`'
  printf ' FROM `bpir_to_delete`'
  printf ' JOIN `binary_packages_in_repositories`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  mysql_join_binary_packages_in_repositories_binary_packages
  mysql_join_binary_packages_in_repositories_repositories
  mysql_join_repositories_architectures
  printf ';\n'

  printf 'SELECT'
  printf ' "package",'
  printf '`architectures`.`name`,'
  printf '`repositories`.`name`,'
  printf '`binary_packages`.`pkgname`'
  printf ' FROM `bpir_to_delete`'
  printf ' JOIN `binary_packages_in_repositories`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  mysql_join_binary_packages_in_repositories_binary_packages
  mysql_join_binary_packages_in_repositories_repositories
  mysql_join_repositories_architectures
  printf ';\n'

  printf 'SELECT "package-file",CONCAT('
  printf '`r_a`.`name`,"/",'
  printf '`repositories`.`name`,"/",'
  mysql_package_name_query
  printf ') FROM `bpir_to_delete`'
  printf ' JOIN `binary_packages_in_repositories`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  mysql_join_binary_packages_in_repositories_binary_packages
  mysql_join_binary_packages_in_repositories_repositories
  mysql_join_binary_packages_architectures
  printf ' LEFT'
  mysql_join_binary_packages_compressions
  mysql_join_repositories_architectures '' 'r_a'
  printf ';\n'

  printf 'SELECT "package-in-repository-id",`bpir_to_delete`.`id`'
  printf ' FROM `bpir_to_delete`'
  printf ';\n'

  printf 'SELECT "package-file",CONCAT('
  printf '"pool/",'
  mysql_package_name_query
  printf ') FROM `bp_to_delete`'
  printf ' JOIN `binary_packages`'
  printf ' ON `bp_to_delete`.`id`=`binary_packages`.`id`'
  mysql_join_binary_packages_architectures
  printf ' LEFT'
  mysql_join_binary_packages_compressions
  printf ';\n'

  # we do not need the ids of the to-be-deleted binary_packages, because
  # they stay in the database and are merely moved to the deletion-list
} | \
  mysql_run_query | \
  sed '
    y/\t/ /
    /^repo /{
      s/^\S\+ //
      w '"${tmp_dir}"'/repositories
      d
    }
    /^package /{
      s/^\S\+ //
      w '"${tmp_dir}"'/packages
      d
    }
    /^package-file /{
      s/^\S\+ //
      w '"${tmp_dir}"'/package-files
      s/$/.sig/
      w '"${tmp_dir}"'/package-files
      d
    }
    /^package-in-repository-id /{
      s/^\S\+ //
      w '"${tmp_dir}"'/package-in-repository-ids
      d
    }
  '

# shellcheck disable=SC2031
if [ -n "${build_support}" ] && \
  [ -w "${build_support}" ] && \
  [ ! -p "${build_support}" ]; then
  sed -n '
    \,^pool/, d
    s/\.sig$//
    T
    p
  ' "${tmp_dir}/package-files" > \
    "${build_support}"
fi

if [ ! -s "${tmp_dir}/packages" ]; then
  printf 'Nothing to delete.\n'
  exit
fi

mkdir "${tmp_dir}/repos"

while read -r arch repo; do
  {
    printf 'mkdir -p "%s/repos/%s"\n' "${tmp_dir}" "${arch}"
    printf 'failsafe_rsync'
    for suffix in 'db' 'files'; do
      printf ' "%s/%s/%s/%s.%s.tar.gz"' \
        "${master_mirror_rsync_directory}" \
        "${arch}" \
        "${repo}" \
        "${repo}" \
        "${suffix}"
    done
    printf ' "%s/repos/%s"\n' "${tmp_dir}" "${arch}"
  } \
  | intent_something "${no_action_option}"
  {
    printf 'repo-remove "%s/repos/%s/%s.db.tar.gz" ' \
      "${tmp_dir}" \
      "${arch}" \
      "${repo}"
    grep "^$(str_to_regex "${arch} ${repo}") " "${tmp_dir}/packages" \
    | cut -d' ' -f3 \
    | tr '\n' ' '
    printf '\n'
    printf 'recompress_gz "%s"' \
      "${tmp_dir}"
    for suffix in '' '.old'; do
      printf ' "%s/repos/%s/%s."*".tar.gz%s"' \
        "${tmp_dir}" "${arch}" "${repo}" "${suffix}"
    done
    printf '\n'
  } \
  | intent_something "${no_action_option}"
  {
    printf 'failsafe_rsync'
    for suffix in 'db' 'files'; do
      printf ' "%s/repos/%s/%s.%s."*' \
        "${tmp_dir}" \
        "${arch}" \
        "${repo}" \
        "${suffix}"
    done
    printf ' "%s/%s/%s/"\n' \
      "${master_mirror_rsync_directory}" \
      "${arch}" \
      "${repo}"
  } \
  | intent_something "${no_action_option}"
done \
<"${tmp_dir}/repositories"

# shellcheck disable=SC2016
{
  printf 'mysql_run_query <<END_OF_MYSQL_QUERY\n'
  printf 'CREATE TEMPORARY TABLE `bpir_to_delete` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `bpir_to_delete`;\n' \
    "${tmp_dir}/package-in-repository-ids"

  printf 'UPDATE `binary_packages_in_repositories`'
  printf ' JOIN `bpir_to_delete`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  mysql_join_binary_packages_in_repositories_repositories
  printf ' JOIN `repositories` AS `d_r`'
  printf ' ON `d_r`.`stability`=%s' \
    "${repository_stability_ids__forbidden}"
  printf ' AND `d_r`.`architecture`=`repositories`.`architecture`'
  printf ' SET `binary_packages_in_repositories`.`repository`=`d_r`.`id`'
  # shellcheck disable=SC2154
  printf ' WHERE `repositories`.`stability`!=%s;\n' \
    "${repository_stability_ids__standalone}"

  printf 'DELETE `binary_packages_in_repositories`'
  printf ' FROM `binary_packages_in_repositories`'
  printf ' JOIN `bpir_to_delete`'
  printf ' ON `bpir_to_delete`.`id`=`binary_packages_in_repositories`.`id`'
  mysql_join_binary_packages_in_repositories_repositories
  printf ' WHERE `repositories`.`stability`!=%s;\n' \
    "${repository_stability_ids__forbidden}"

  printf 'DELETE `binary_packages`'
  printf ' FROM `binary_packages`'
  printf ' WHERE NOT EXISTS ('
    printf 'SELECT 1'
    printf ' FROM `binary_packages_in_repositories`'
    printf ' WHERE `binary_packages_in_repositories`.`package`=`binary_packages`.`id`'
  printf ');\n'
  printf 'END_OF_MYSQL_QUERY\n'
} \
| sed 's/`/\\`/g' \
| intent_something "${no_action_option}"

{
  printf 'failsafe_sftp <<END_OF_SFTP_COMMANDS\n'
  sed '
    s/^/rm "/
    s/$/"/
  ' "${tmp_dir}/package-files"
  printf 'END_OF_SFTP_COMMANDS\n'
} \
| intent_something "${no_action_option}"

# cleanup is now handled by the intentions-queue
printf 'rm -rf --one-file-system "%s"\n' "${tmp_dir}" \
| intent_something "${no_action_option}"
# iff not `-n`
if [ -z "${no_action_option}" ]; then
  trap - EXIT

  execute_all_intentions
fi