From ff8d40d2771eb8e98d03b2d0cf734bc7d2d5faab Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 12:25:40 +0100
Subject: bin/delete-packages: some start

---
 bin/delete-packages | 368 ++++++++++++++++------------------------------------
 1 file changed, 113 insertions(+), 255 deletions(-)

(limited to 'bin/delete-packages')

diff --git a/bin/delete-packages b/bin/delete-packages
index 26d323a..4675c79 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -69,18 +69,15 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
-trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
 # Create a lock file and a trap.
 
 if ! ${no_action}; then
 
-  exec 9> "${build_list_lock_file}"
-  if ! flock ${block_flag} 9; then
-    >&2 echo 'come back (shortly) later - I cannot lock build list.'
-    exit 0
-  fi
+#  exec 9> "${build_list_lock_file}"
+#  if ! flock ${block_flag} 9; then
+#    >&2 echo 'come back (shortly) later - I cannot lock build list.'
+#    exit 0
+#  fi
 
   exec 8> "${package_database_lock_file}"
   if ! flock ${block_flag} 8; then
@@ -96,260 +93,121 @@ if ! ${no_action}; then
 
 fi
 
-clean_up_lock_file() {
-  if ! ${no_action}; then
-    rm -f "${package_database_lock_file}" "${build_list_lock_file}"
-  fi
-  rm -rf --one-file-system "${tmp_dir}"
-}
-
-trap clean_up_lock_file EXIT
-
-cp \
-  "${work_dir}/deletion-list" \
-  "${work_dir}/build-list" \
-  "${tmp_dir}/"
-
-all_repos="${standalone_package_repositories} ${stable_package_repositories} ${testing_package_repositories} ${staging_package_repositories}"
-all_repos="core"
-
-for repo in ${all_repos}; do
-  mkdir "${tmp_dir}/${repo}"
-  ${master_mirror_rsync_command} \
-    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
-    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
-    "${tmp_dir}/${repo}/"
-done
-
-for repo in ${all_repos}; do
-  tar -C "${tmp_dir}/${repo}" -xzf "${tmp_dir}/${repo}/${repo}.db.tar.gz"
-done
+tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX")
+trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
-find "${tmp_dir}" -mindepth 3 -maxdepth 3 -name 'desc' -exec \
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+  printf 'INSERT IGNORE INTO `to_deletes`'
+  printf ' SELECT `binary_packages`.`id`'
+  printf ' FROM `binary_packages`'
+  mysql_join_binary_packages_dependencies
+  mysql_join_dependencies_dependency_types
+  printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+  mysql_join_binary_packages_repositories
+  printf ' AND `repositories`.`is_on_master_mirror`'
+  printf ' WHERE `binary_packages`.`is_to_be_deleted`'
+  printf ' AND NOT EXISTS ('
+    printf 'SELECT * FROM `install_target_providers`'
+    printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+  printf ');\n'
+
+  printf ' SELECT DISTINCT "repo",`repositories`.`name`'
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  printf ';\n'
+
+  printf ' SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`'
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  printf ';\n'
+
+  printf ' SELECT "package-id",`to_deletes`.`id`'
+  printf ' FROM `to_deletes`'
+  printf ';\n'
+} | \
+  mysql_run_query | \
   sed '
-    /^%DEPENDS%$/{
-      s/.*//
-      :dep_loop
-      N
-      /\n$/{
-        s@\(^\|\n\)\(.\)@\1depends \2@g
-        bend
-      }
-      bdep_loop
+    y/\t/ /
+    /^repo /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/repositories
+      d
     }
-    /^%PROVIDES%$/{
-      s/.*//
-      :pro_loop
-      N
-      /\n$/{
-        s@\(^\|\n\)\(.\)@\1provides \2@g
-        bend
-      }
-      bpro_loop
+    /^package /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/packages
+      d
     }
-    d
-    :end
-    s/[<>=]\S*\($\|\n\)/\1/g
-    s#\(^\|\n\)\(.\)#\1{} \2#g
-  ' {} \; \
-  -printf '%p provides %p\n' | \
-  sed '
-    /^$/d
-    s|^\S\+/\([^/ ]\+\)/\([^/ ]\+\)\(-[^-/ ]\+\)\{2\}/desc |\1 \2 |
-    s| \S\+/\([^/]\+\)\(-[^/-]\+\)\{2\}/desc$| \1|
-  ' | \
-  awk '{print $3 " " $1 " " $2 " " $4}' | \
-  sed -n '
-    /^provides /{
-      s/^provides //
-      w '"${tmp_dir}"'/db.provides
-      b
-    }
-    /^depends /{
-      s/^depends //
-      w '"${tmp_dir}"'/db.depends
-      b
+    /^package-id /{
+      s/^\S\+ //
+      w '"${tmp_dir}"'/package-ids
+      d
     }
   '
 
+if [ ! -s "${tmp_dir}/packages" ]; then
+  printf 'Nothing to delete.\n'
+  exit
+fi
 
+if ${no_action}; then
+  printf 'I would delete the following packages:\n'
+  sed '
+    y, ,/,
+    s/^/  /
+  ' "${tmp_dir}/packages"
+  printf 'From these repositories:\n'
+  sed '
+    s/^/  /
+  ' "${tmp_dir}/repositories"
+  exit
+fi
 
-echo 'OK' >&2
-exit 42
+mkdir "${tmp_dir}/repos"
 
-# # sanity check
-#
-# for ending in 'done' 'testing'; do
-#   if [ "${ending}" = 'testing' ] && \
-#     [ -z "${packages_to_stabilize}" ]; then
-#     # if nothing is to be untested, we don't care about duplicate
-#     # testing packages (and maybe an unstaging fixes this anyway)
-#     continue
-#   fi
-#   if [ -n "$(
-#     find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-#       sed 's|\(\.[^.]\+\)\{4\}$||' | \
-#       sort | \
-#       uniq -d
-#     )" ]; then
-#     >&2 echo 'Removing duplicates not yet implemented:'
-#     find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
-#       sed 's|\(\.[^.]\+\)\{4\}$||' | \
-#       sort | \
-#       uniq -d
-#     exit 42
-#   fi
-# done
-#
-# # packages which are done
-#
-# find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
-#   sed '
-#     s|\.done$||
-#   ' | \
-#   sort -u > \
-#   "${tmp_dir}/done-packages"
-#
-# # packages still on the build-list
-#
-# tr ' ' '.' < \
-#   "${work_dir}/build-list" | \
-#   sort -u > \
-#   "${tmp_dir}/keep-packages"
-#
-# find "${work_dir}/package-infos" -name '*.groups' \
-#   -exec grep -qx 'base\(-devel\)\?' {} \; \
-#   -printf '%f\n' | \
-#   sed '
-#     s|\.groups$||
-#   ' | \
-#   sort -u > \
-#   "${tmp_dir}/base-packages"
-#
-# # no base / base-devel packages on the build list?
-# if [ -z "$(
-#     join -j 1 \
-#       "${tmp_dir}/base-packages" \
-#       "${tmp_dir}/keep-packages"
-#   )" ]; then
-#   # unstage all base / base-devel packages from staging
-#   cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages" "${tmp_dir}/keep-packages" | \
-#     sort | \
-#     uniq -u | \
-#     sponge "${tmp_dir}/keep-packages"
-# fi
-#
-# # find all dependencies of the unstageable packages
-# mv \
-#   "${tmp_dir}/keep-packages" \
-#   "${tmp_dir}/new-keep-packages"
-# touch "${tmp_dir}/keep-packages"
-#
-# while [ -s "${tmp_dir}/new-keep-packages" ]; do
-#
-#   cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \
-#     sort -u | \
-#     sponge "${tmp_dir}/keep-packages"
-#
-#   sed '
-#     s|^|'"${work_dir}"'/package-infos/|
-#     s|$|.depends|
-#   ' "${tmp_dir}/keep-packages" | \
-#     xargs -r grep -HF '' | \
-#     sed '
-#       s|^.*/||
-#       s|\.depends:| |
-#     ' | \
-#     sort -u | \
-#     sort -k2,2 > \
-#     "${tmp_dir}/keep-packages.depends"
-#
-#   sed '
-#     s|^|'"${work_dir}"'/package-infos/|
-#     s|$|.builds|
-#   ' "${tmp_dir}/done-packages" | \
-#     xargs -r grep -HF '' | \
-#     sed '
-#       s|^.*/||
-#       s|\.builds:| |
-#     ' | \
-#     sort -u | \
-#     sort -k2,2 > \
-#     "${tmp_dir}/done-packages.builds"
-#
-#   join -j 2 -o 1.1 \
-#     "${tmp_dir}/done-packages.builds" \
-#     "${tmp_dir}/keep-packages.depends" | \
-#     sort -u > \
-#     "${tmp_dir}/new-keep-packages"
-#
-#   # "new" is only what has not been there before
-#   cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \
-#     sort | \
-#     uniq -u | \
-#     sponge "${tmp_dir}/new-keep-packages"
-#
-# done
-#
-# # unlock build list
-#
-# rm -f "${build_list_lock_file}"
-# flock -u 9
-#
-# clean_up_lock_file() {
-#   rm -rf --one-file-system "${tmp_dir}"
-#   rm -f "${package_database_lock_file}"
-# }
-#
-# # calculate unstageable packages from keep_packages and done_packages
-#
-# done_packages=$(
-#   cat \
-#     "${tmp_dir}/keep-packages" \
-#     "${tmp_dir}/keep-packages" \
-#     "${tmp_dir}/done-packages" | \
-#     sort | \
-#     uniq -u
-# )
-#
-# # move packages in packages_to_stabilize from *testing/ to the stable repos
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-#   for package in ${packages_to_stabilize}; do
-#
-#     if [ -z "${package}" ]; then
-#       continue
-#     fi
-#
-#     printf '%s/%s/%s\n' \
-#       "${package}" \
-#       "$(official_or_community "${package}" 'testing')" \
-#       "$(repository_of_package "${package}")"
-#
-#   done
-#
-# )
-#
-# # move packages from *staging to *testing
-#
-# # shellcheck disable=SC2046
-# move_packages $(
-#
-#   for package in ${done_packages}; do
-#
-#     if [ -z "${package}" ]; then
-#       continue
-#     fi
-#
-#     printf '%s/%s/%s\n' \
-#       "${package}" \
-#       "$(official_or_community "${package}" 'staging')" \
-#       "$(official_or_community "${package}" 'testing')"
-#
-#   done
-#
-# )
-#
-# clean_up_lock_file
+while read -r repo; do
+  ${master_mirror_rsync_command} \
+    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \
+    "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \
+    "${tmp_dir}/repos/"
+  # shellcheck disable=SC2046
+  repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \
+    $(
+      grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \
+        cut -d' ' -f2
+    )
+  ${master_mirror_rsync_command} \
+    "${tmp_dir}/repos/${repo}.db.tar.gz" \
+    "${tmp_dir}/repos/${repo}.files.tar.gz" \
+    "${master_mirror_rsync_directory}/i686/${repo}/"
+done < \
+  "${tmp_dir}/repositories"
+
+# shellcheck disable=SC2016
+{
+  printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \
+    "${tmp_dir}/package-ids"
+  printf 'DELETE `binary_packages` FROM `binary_packages` '
+  printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  printf ';\n'
+} | \
+  mysql_run_query
+
+tr ' ' '/' < \
+  "${tmp_dir}/packages" | \
+  sed '
+    p
+    s/$/.sig/
+  ' | \
+  sed '
+    s,^,rm "i686/,
+    s,$,",
+  ' | \
+  ${master_mirror_sftp_command}
-- 
cgit v1.2.3-54-g00ecf


From dbf60cd6d389c2c3c2d42b28ccb7403dcdeebe4f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:13:53 +0100
Subject: bin/delete-packages: clarify TODO

---
 bin/delete-packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'bin/delete-packages')

diff --git a/bin/delete-packages b/bin/delete-packages
index 4675c79..7e843c5 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -13,7 +13,7 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
-# TODO: finish this
+# TODO: delete other to-be-deleted packages if asked to do so
 
 # shellcheck disable=SC2016
 usage() {
-- 
cgit v1.2.3-54-g00ecf


From 080cfc23228f76825805643079f1a14e2d82fa2b Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:48:56 +0100
Subject: bin/delete-packages: ignore deletion-list and other abstract packages
 as install_target_providers

---
 bin/delete-packages | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'bin/delete-packages')

diff --git a/bin/delete-packages b/bin/delete-packages
index 7e843c5..258e441 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -110,7 +110,11 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
   printf ' WHERE `binary_packages`.`is_to_be_deleted`'
   printf ' AND NOT EXISTS ('
     printf 'SELECT * FROM `install_target_providers`'
+    mysql_join_install_target_providers_binary_packages '' 'prov_bp'
+    mysql_join_binary_packages_repositories 'prov_bp' 'prov_r'
+    mysql_join_repositories_repository_stabilities 'prov_r'
     printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+    printf ' AND `repository_stabilities`.`name` NOT IN ("forbidden","virtual")'
   printf ');\n'
 
   printf ' SELECT DISTINCT "repo",`repositories`.`name`'
-- 
cgit v1.2.3-54-g00ecf


From 2a4617fd25cf56e2d81d90c0c2aff81bfd9e9823 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Mon, 12 Mar 2018 13:50:24 +0100
Subject: bin/delete-packages: repair deletion of packages on the mirror

---
 bin/delete-packages | 40 ++++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

(limited to 'bin/delete-packages')

diff --git a/bin/delete-packages b/bin/delete-packages
index 258e441..8a4ea4a 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -129,6 +129,14 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
   mysql_join_binary_packages_repositories
   printf ';\n'
 
+  printf ' SELECT "package-file",`repositories`.`name`,'
+  mysql_package_name_query
+  printf ' FROM `to_deletes`'
+  printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`'
+  mysql_join_binary_packages_repositories
+  mysql_join_binary_packages_architectures
+  printf ';\n'
+
   printf ' SELECT "package-id",`to_deletes`.`id`'
   printf ' FROM `to_deletes`'
   printf ';\n'
@@ -146,6 +154,15 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
       w '"${tmp_dir}"'/packages
       d
     }
+    /^package-file /{
+      s/^\S\+ //
+      s, ,/,
+      s/$/.pkg.tar.xz/
+      w '"${tmp_dir}"'/package-files
+      s/$/.sig/
+      w '"${tmp_dir}"'/package-files
+      d
+    }
     /^package-id /{
       s/^\S\+ //
       w '"${tmp_dir}"'/package-ids
@@ -196,22 +213,21 @@ done < \
   printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
   printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \
     "${tmp_dir}/package-ids"
-  printf 'DELETE `binary_packages` FROM `binary_packages` '
+
+  printf 'UPDATE `binary_packages` '
   printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`'
   mysql_join_binary_packages_repositories
   mysql_join_binary_packages_architectures
-  printf ';\n'
+  printf ' SET `repository`=('
+    printf 'SELECT `repositories`.`id`'
+    printf ' FROM `repositories`'
+    printf ' WHERE `repositories`.`name`="deletion-list"'
+  printf ');\n'
 } | \
   mysql_run_query
 
-tr ' ' '/' < \
-  "${tmp_dir}/packages" | \
-  sed '
-    p
-    s/$/.sig/
-  ' | \
-  sed '
-    s,^,rm "i686/,
-    s,$,",
-  ' | \
+sed '
+  s,^,rm "i686/,
+  s,$,",
+' "${tmp_dir}/package-files" | \
   ${master_mirror_sftp_command}
-- 
cgit v1.2.3-54-g00ecf


From 8abcf920222da35737f5739b32b0bfd76e9bff4f Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Wed, 14 Mar 2018 16:43:04 +0100
Subject: bin/delete-packages: several bug fixes and improvements

---
 bin/delete-packages | 53 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 18 deletions(-)

(limited to 'bin/delete-packages')

diff --git a/bin/delete-packages b/bin/delete-packages
index 8a4ea4a..4343bc7 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -157,7 +157,6 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
     /^package-file /{
       s/^\S\+ //
       s, ,/,
-      s/$/.pkg.tar.xz/
       w '"${tmp_dir}"'/package-files
       s/$/.sig/
       w '"${tmp_dir}"'/package-files
@@ -175,19 +174,6 @@ if [ ! -s "${tmp_dir}/packages" ]; then
   exit
 fi
 
-if ${no_action}; then
-  printf 'I would delete the following packages:\n'
-  sed '
-    y, ,/,
-    s/^/  /
-  ' "${tmp_dir}/packages"
-  printf 'From these repositories:\n'
-  sed '
-    s/^/  /
-  ' "${tmp_dir}/repositories"
-  exit
-fi
-
 mkdir "${tmp_dir}/repos"
 
 while read -r repo; do
@@ -195,19 +181,50 @@ while read -r repo; do
     "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \
     "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \
     "${tmp_dir}/repos/"
+
+  tar -Oxzf "${tmp_dir}/repos/${repo}.db.tar.gz" --wildcards '*/desc' | \
+    sed -n '
+      /^%FILENAME%$/{
+        N
+        s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/
+        T
+        p
+      }
+    ' | \
+    while read -r old new; do
+      printf 's,/%s\\(\\.sig\\)\\?$,/%s\\1,\n' \
+        "$(str_to_regex "${old}")" \
+        "$(str_to_regex "${new}")"
+    done >> \
+    "${tmp_dir}/sub_pkgrel-removal.sed"
+
   # shellcheck disable=SC2046
   repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \
     $(
       grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \
         cut -d' ' -f2
     )
-  ${master_mirror_rsync_command} \
-    "${tmp_dir}/repos/${repo}.db.tar.gz" \
-    "${tmp_dir}/repos/${repo}.files.tar.gz" \
-    "${master_mirror_rsync_directory}/i686/${repo}/"
+  if ! ${no_action}; then
+    ${master_mirror_rsync_command} \
+      "${tmp_dir}/repos/${repo}.db.tar.gz" \
+      "${tmp_dir}/repos/${repo}.files.tar.gz" \
+      "${master_mirror_rsync_directory}/i686/${repo}/"
+  fi
 done < \
   "${tmp_dir}/repositories"
 
+if [ -s "${tmp_dir}/sub_pkgrel-removal.sed" ]; then
+  sed -i -f "${tmp_dir}/sub_pkgrel-removal.sed" "${tmp_dir}/package-files"
+fi
+
+if ${no_action}; then
+  printf 'Now, I would remove the packages from the database and delete the following files from the master mirror:\n'
+  sed '
+    s,^,  ,
+  ' "${tmp_dir}/package-files"
+  exit
+fi
+
 # shellcheck disable=SC2016
 {
   printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n'
-- 
cgit v1.2.3-54-g00ecf