bin/seed-build-list: general overhaul: blacklisting, un-deleting, ignore-list, architecture recognition, ...

1 files changed, 113 insertions, 196 deletions

diff --git a/bin/seed-build-list b/bin/seed-build-list
index 61b187f..832641a 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -2,6 +2,17 @@
 
 # shellcheck disable=SC2119,SC2120
 
+# seed the build list from various sources
+#
+# This uses the following schedule:
+# - find scheduleworthy packages by pkgname/pkgbase (diff to mirror,
+#   broken dependencies, explicite regex) and their architecture
+# - find pkgbases of those packages (ask upstream, if necessary)
+# - remove all ignored packages (seed-igonre, explicitely ignored,
+#   blacklist)
+# - run mysql_generate_package_metadata on each of them once (ignoring
+#   the architecture for now)
+
 # shellcheck source=../lib/load-configuration
 . "${0%/*}/../lib/load-configuration"
 
@@ -9,10 +20,6 @@
 
 # TODO: -a|--auto schedules too much - or is the scheduling really /that/ broken?
 
-# TODO: -u and -m should be merged
-
-# TODO: the blacklist is currently being ignored - why?
-
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
@@ -29,23 +36,20 @@ usage() {
   >&2 echo '    Do not exit if mysql_generate_package_metadata() failed.'
   >&2 echo '  -h|--help:'
   >&2 echo '    Show this help and exit.'
-  >&2 echo '  -i|--ignore $pkgbase:'
-  >&2 echo '    Do not update the given package.'
+  >&2 echo '  -i|--ignore $pkgbase|$arch/$pkgbase:'
+  >&2 echo '    Do not update the given package (for the given $arch).'
   >&2 echo '  -j|--jostle'
   >&2 echo '    Give new build assignments the highest priority.'
   >&2 echo '  -m|--mirror $url:'
-  >&2 echo '    Schedule all packages, that are newer on the given'
-  >&2 echo '    x86_64 mirror - except packages in'
-  >&2 echo '    conf/seed-ignore-packages.'
+  >&2 echo '    Schedule all packages, that are newer (this includes'
+  >&2 echo '    packages currently unavailable on archlinux32) on the'
+  >&2 echo '    given x86_64 mirror - except packages on the blacklist'
+  >&2 echo '    or in conf/seed-ignore-packages.'
   >&2 echo '  -n|--no-action:'
   >&2 echo '    Do not actually update build-list, just print it.'
   >&2 echo '  -p|--package $pkg_regex:'
   >&2 echo '    Reschedule packages with matching pkgname or pkgbase.'
   >&2 echo '    Note, that these packages must be known to the database.'
-  >&2 echo '  -u|--undelete $url: '
-  >&2 echo '    Schedule all former deletion-list packages which do not'
-  >&2 echo '    belong on the deletion-list anymore (e.g. can be built)'
-  >&2 echo '    and which are available on the given x86_64 mirror.'
   >&2 echo '  -w|--wait:'
   >&2 echo '    Wait for lock if necessary.'
   [ -z "$1" ] && exit 1 || exit "$1"
@@ -56,7 +60,7 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
 trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT
 
 eval set -- "$(
-  getopt -o afhi:jm:np:u:w \
+  getopt -o afhi:jm:np:w \
     --long auto \
     --long force \
     --long help \
@@ -65,7 +69,6 @@ eval set -- "$(
     --long mirror: \
     --long no-action \
     --long package: \
-    --long undelete: \
     --long wait \
     -n "$(basename "$0")" -- "$@" || \
     echo usage
@@ -77,11 +80,6 @@ jostle=false
 update=true
 wait_for_lock='-n'
 
-if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
-  sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \
-    "${tmp_dir}/ignore-packages"
-fi
-
 while true
 do
   case "$1" in
@@ -96,8 +94,13 @@ do
     ;;
     -i|--ignore)
       shift
-      printf '3\t%s\n' "$1" >> \
-        "${tmp_dir}/ignore-packages"
+      if [ -z "${1%%*/*}" ]; then
+        printf '%s\n' "$1" \
+        | tr '/' '\t'
+      else
+        printf 'any\t%s\n' "$1"
+      fi \
+      >> "${tmp_dir}/ignore-packages"
     ;;
     -j|--jostle)
       jostle=true
@@ -115,11 +118,6 @@ do
       printf '%s\n' "$1" >> \
         "${tmp_dir}/package-regexes"
     ;;
-    -u|--undelete)
-      shift
-      printf '%s\n' "$1" >> \
-        "${tmp_dir}/undelete-mirrors"
-    ;;
     -w|--wait)
       wait_for_lock=''
     ;;
@@ -142,7 +140,6 @@ fi
 
 if [ ! -s "${tmp_dir}/mirrors" ] && \
   [ ! -s "${tmp_dir}/package-regexes" ] && \
-  [ ! -s "${tmp_dir}/undelete-mirrors" ] && \
   ! ${auto}; then
   # nothing to do
   >&2 echo 'No options given to do anything.'
@@ -178,7 +175,30 @@ repos=$(
   printf '%s\n' 'multilib'
 )
 
-# TODO: translate lib32-* packages instead of ignoring them for -u and -m
+if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
+  sed '
+    s@/@\t@
+    t
+    s/^/any\t/
+  ' "${base_dir}/conf/seed-ignore-packages" >> \
+    "${tmp_dir}/ignore-packages"
+fi
+
+# shellcheck disable=SC2154
+git -C "${repo_paths__archlinux32}" archive "${repo_heads__archlinux32}" -- blacklist \
+| tar -t \
+| sed '
+  s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
+  t
+  d
+' \
+>> "${tmp_dir}/ignore-packages"
+
+expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \
+< "${tmp_dir}/ignore-packages" \
+| sponge "${tmp_dir}/ignore-packages"
+
+# TODO: translate lib32-* packages instead of ignoring them for -m
 
 # harvest pkgnames from mirror delta
 if [ -s "${tmp_dir}/mirrors" ]; then
@@ -196,7 +216,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then
             N
             s/^.*\n//
             /^lib32-/d
-            s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
+            s/^\(.*-\)x86_64\.pkg\.tar\.xz$/\1i486\n\1i686\n\1pentium4/
+            s/^\(.*-any\)\.pkg\.tar\.xz$/\1/
           ' | \
           sed '
             s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
@@ -212,12 +233,14 @@ if [ -s "${tmp_dir}/mirrors" ]; then
       printf ' FROM `binary_packages`'
       mysql_join_binary_packages_architectures
       mysql_join_binary_packages_binary_packages_in_repositories
-      printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
+      mysql_join_binary_packages_in_repositories_repositories
+      printf ' WHERE `repositories`.`is_on_master_mirror`'
+      printf ' OR `repositories`.`name`="build-list"'
     } | \
       mysql_run_query | \
       sed '
-        s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)$/ours \2 \4 \1/
-        s/^\(.* \)any\(\.\S\+ \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
+        s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)\.pkg\.tar\.xz$/ours \2 \4 \1/
+        s/^\(.* \)any\( \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/
       '
   } | \
     expand_version 2 | \
@@ -227,48 +250,13 @@ if [ -s "${tmp_dir}/mirrors" ]; then
     sed -n '
       s/^theirs \(\S\+ \)//
       T
-      s/^\(\S\+ \)//
-      T
       p
     ' | \
-    if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
-      grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
+    if [ -r "${tmp_dir}/ignore-packages" ]; then
+      grep -vxFf "${tmp_dir}/ignore-packages"
     else
       cat
     fi | \
-    sed 's/^/1\t/' | \
-    sort -u >> \
-    "${tmp_dir}/pkgnames"
-fi
-
-# harvest pkgnames from undelete-mirrors
-if [ -s "${tmp_dir}/undelete-mirrors" ]; then
-  while read -r mirror; do
-    if [ -z "${mirror}" ]; then
-      continue
-    fi
-    for repo in ${repos}; do
-      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
-        tar -Oxz --wildcards '*/desc' | \
-        sed '
-          /^%FILENAME%$/!d
-          N
-          s/^.*\n//
-          /^lib32-/d
-          s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/
-        ' | \
-        sed '
-          s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/
-        '
-    done
-  done < \
-    "${tmp_dir}/undelete-mirrors" | \
-    if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
-      grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
-    else
-      cat
-    fi | \
-    sed 's/^/2\t/' | \
     sort -u >> \
     "${tmp_dir}/pkgnames"
 fi
@@ -294,14 +282,15 @@ if [ -s "${tmp_dir}/package-regexes" ]; then
     printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
   } | \
     mysql_run_query | \
+    tr '\t' ' ' | \
     sed -n '
-      /^pkgbase\s/ {
-        s/^\S\+\s/3\t/
+      /^pkgbase / {
+        s/^\S\+ /any /
         w /dev/stdout
         d
       }
-      /^pkgname\s/ {
-        s/^\S\+\s/3\t/
+      /^pkgname / {
+        s/^\S\+ /any /
         w /dev/stderr
         d
       }
@@ -315,14 +304,18 @@ fi
   # create pkgbases to given pkgnames
   if [ -s "${tmp_dir}/pkgnames" ]; then
     printf 'CREATE TEMPORARY TABLE `pkgnames` ('
-      printf '`priority` SMALLINT,'
+      printf '`architecture` VARCHAR(16),'
       printf '`pkgname` VARCHAR(64),'
-      printf 'UNIQUE KEY `pkgname`(`pkgname`)'
+      printf 'UNIQUE KEY `content`(`architecture`,`pkgname`)'
     printf ');\n'
-    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \
+    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` COLUMNS TERMINATED BY " " (`architecture`,`pkgname`);\n' \
       "${tmp_dir}/pkgnames"
 
-    printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+    printf 'SELECT DISTINCT'
+    printf ' "pkgbase",'
+    printf '`pkgnames`.`architecture`,'
+    printf '`package_sources`.`pkgbase`,'
+    printf '`upstream_repositories`.`name`'
     printf ' FROM `pkgnames`'
     printf ' JOIN `binary_packages`'
     printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
@@ -331,7 +324,10 @@ fi
     mysql_join_package_sources_upstream_repositories
     printf ';\n'
 
-    printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`'
+    printf 'SELECT DISTINCT'
+    printf ' "pkgname",'
+    printf '`pkgnames`.`architecture`,'
+    printf '`pkgnames`.`pkgname`'
     printf ' FROM `package_sources`'
     mysql_join_package_sources_upstream_repositories
     mysql_join_package_sources_build_assignments
@@ -344,6 +340,8 @@ fi
   fi
   # auto-detect pkgbases
   if ${auto}; then
+    # TODO: check, if this still works or if it relied on the `priority` magic
+    >&2 echo 'autoscheduling needs testing / inspection - be prepared to schedule too many packages this way!'
     # schedule any package, that:
     #  1) is not on the build-list currently and
     #  2) has some dependency which is not provided by any package which
@@ -413,7 +411,7 @@ fi
     # 1)
     printf ' WHERE `least_stable_bp`.`is_on_build_list`;'
 
-    printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+    printf 'SELECT DISTINCT "pkgbase",`ba_a`.`name`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
     printf ' FROM `least_stable_bp`'
     printf ' JOIN `binary_packages`'
     printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`'
@@ -422,6 +420,7 @@ fi
     printf ' AND `dependency_types`.`relevant_for_binary_packages`'
     mysql_join_dependencies_versions '' 'd_v'
     mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_architectures '' 'ba_a'
     mysql_join_build_assignments_package_sources
     mysql_join_package_sources_upstream_repositories
     # some dependencies are not provided by the least stable packages
@@ -456,14 +455,15 @@ fi
   fi
 } | \
   mysql_run_query | \
+  tr '\t' ' ' | \
   sed -n '
-    /^pkgbase\s/ {
-      s/^\S\+\s//
+    /^pkgbase / {
+      s/^\S\+ //
       w /dev/stderr
       d
     }
-    /^pkgname\s/ {
-      s/^\S\+\s//
+    /^pkgname / {
+      s/^\S\+ //
       w /dev/stdout
       d
     }
@@ -486,7 +486,7 @@ done
 
 # pkgnames -> pkgbases (with help from upstream)
 if [ -s "${tmp_dir}/pkgnames" ]; then
-  while read -r priority pkgname; do
+  while read -r architecture pkgname; do
     content=$(
       curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
         tr ',' '\n'
@@ -512,11 +512,11 @@ if [ -s "${tmp_dir}/pkgnames" ]; then
         '
     )
     if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
-      printf '%s\t%s\n' "${priority}" "${pkgname}"
+      printf '%s %s\n' "${architecture}" "${pkgname}"
       continue
     fi
-    printf '%s\t%s\t%s\n' \
-      "${priority}" \
+    printf '%s %s %s\n' \
+      "${architecture}" \
       "${pkgbase}" \
       "${repo}" >> \
       "${tmp_dir}/pkgbases"
@@ -531,112 +531,30 @@ if [ -s "${tmp_dir}/pkgnames" ]; then
   exit 2
 fi
 
-# now we (re)schedule all pkgbases which are:
-#  - not explicitely ignored (priority 3)             <- explicitely scheduled
-#  - not explicitely/implicitely ignored, but not currently available
-#    (priority 2) <- from undelete-mirror-delta
-#  - neither ignored nor to-be-deleted (priority 1)   <- from mirror-delta or auto-rebuild
+sort -u "${tmp_dir}/pkgbases" \
+| sort -k3,3 \
+| sponge "${tmp_dir}/pkgbases"
+
+# get the current HEADs
 
 # shellcheck disable=SC2016
 {
-  printf 'CREATE TEMPORARY TABLE `ignore_packages` ('
-    printf '`priority` SMALLINT,'
-    printf '`pkgbase` VARCHAR(64),'
-    printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)'
-  printf ');\n'
-  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \
-    "${tmp_dir}/ignore-packages"
-
-  printf 'CREATE TEMPORARY TABLE `pkgbases` ('
-    printf '`priority` SMALLINT,'
-    printf '`pkgbase` VARCHAR(64),'
-    printf '`repo` VARCHAR(64),'
-    printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
-  printf ');\n'
-  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \
-    "${tmp_dir}/pkgbases"
-
-  printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
-  printf ' SELECT 2,`sub_q`.`pkgbase`'
-  printf ' FROM ('
-    printf 'SELECT'
-    printf ' `package_sources`.`pkgbase`,'
-    printf 'GROUP_CONCAT('
-      printf 'DISTINCT'
-      printf ' `build_assignments`.`architecture`'
-      printf ' ORDER BY `build_assignments`.`architecture`'
-    printf ') AS `architectures`'
-    printf ' FROM `package_sources`'
-    mysql_join_package_sources_build_assignments
-    mysql_join_build_assignments_binary_packages
-    mysql_join_binary_packages_binary_packages_in_repositories
-    mysql_join_binary_packages_in_repositories_repositories
-    printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL'
-    printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"'
-    printf ' OR `repositories`.`is_on_master_mirror`'
-    printf ' OR `binary_packages_in_repositories`.`repository`=%s' \
-      "${repository_ids__any_build_list}"
-    printf ' GROUP BY `package_sources`.`pkgbase`'
-  printf ') AS `sub_q`'
-  # shellcheck disable=SC2154
-  printf ' WHERE `sub_q`.`architectures` IN ("%s","%s","%s,%s")' \
-    "${architecture_ids__any}" \
-    "${non_any_architecture_ids}" \
-    "${architecture_ids__any}" \
-    "${non_any_architecture_ids}"
-  printf ';\n'
-
-  printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
-  printf ' SELECT 1,`package_sources`.`pkgbase`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_build_assignments
-  mysql_join_build_assignments_binary_packages
-  mysql_join_binary_packages_binary_packages_in_repositories
-  printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`'
-  printf ';\n'
-
-  printf 'SELECT '
-  printf '`pkgbases`.`pkgbase`,'
-  printf '`git_repositories`.`head`,'
-  printf '('
-    printf 'SELECT `al32`.`head`'
-    printf ' FROM `git_repositories` AS `al32`'
-    printf ' WHERE `al32`.`name`="archlinux32"'
-  printf '),'
-  printf '`pkgbases`.`repo`'
-  printf ' FROM `pkgbases`'
-  printf ' JOIN `upstream_repositories`'
-  printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`'
+  printf 'SELECT'
+  printf ' `upstream_repositories`.`name`,'
+  printf '`git_repositories`.`head`'
+  printf ' FROM `upstream_repositories`'
   mysql_join_upstream_repositories_git_repositories
-  printf ' WHERE NOT EXISTS ('
-    printf 'SELECT 1'
-    printf ' FROM `ignore_packages`'
-    printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`'
-    printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`'
-  printf ') AND ('
-    printf '`pkgbases`.`priority`!=2'
-    printf ' OR NOT EXISTS ('
-      printf 'SELECT 1'
-      printf ' FROM `package_sources`'
-      mysql_join_package_sources_build_assignments
-      mysql_join_build_assignments_binary_packages
-      mysql_join_binary_packages_binary_packages_in_repositories
-      printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
-      mysql_join_binary_packages_in_repositories_repositories
-      printf ' AND ('
-        printf '`repositories`.`is_on_master_mirror`'
-        printf ' OR `repositories`.`name`="build-list"'
-      printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`'
-    printf ')'
-  printf ');\n'
-} | \
-  mysql_run_query | \
-  tr '\t' ' ' | \
-  sort -u > \
-  "${tmp_dir}/update-list"
+} \
+| mysql_run_query \
+| tr '\t' ' ' \
+| sort -k1,1 \
+| join -1 3 -2 1 -o 1.1,1.2,1.3,2.2 "${tmp_dir}/pkgbases" - \
+| sponge "${tmp_dir}/pkgbases"
 
 if ${update}; then
-  while read -r pkgbase git_rev mod_git_rev repo; do
+  cut -d' ' -f 2,3,4 "${tmp_dir}/pkgbases" \
+  | sort -u \
+  | while read -r pkgbase repo git_rev; do
     success=false
     # shellcheck disable=SC2154
     for gr_r in \
@@ -644,9 +562,9 @@ if ${update}; then
       "${repo_heads__packages}:core" \
       "${repo_heads__packages}:extra" \
       "${repo_heads__community}:community"; do
-      printf '%s ' "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}" | \
+      printf '%s ' "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}" | \
         sed 's/ $/\n/'
-      if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}"; then
+      if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}"; then
         success=true
         break
       fi
@@ -655,13 +573,12 @@ if ${update}; then
       ! ${ignore_mysql_generate_package_metadata_errors}; then
       exit 2
     fi
-  done < \
-    "${tmp_dir}/update-list"
+  done
 
   cut -d' ' -f1 < \
-    "${tmp_dir}/update-list" | \
+    "${tmp_dir}/pkgbases" | \
     sort -u | \
-    sponge "${tmp_dir}/update-list"
+    sponge "${tmp_dir}/pkgbases"
 
   # shellcheck disable=SC2016
   {
@@ -670,7 +587,7 @@ if ${update}; then
       printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
     printf ');\n'
     printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \
-      "${tmp_dir}/update-list"
+      "${tmp_dir}/pkgbases"
 
     printf 'DELETE `d_bpir`'
     printf ' FROM `pkgbases`'
@@ -723,5 +640,5 @@ if ${update}; then
   mysql_query_remove_old_binary_packages_from_build_list | \
     mysql_run_query
 else
-  cat "${tmp_dir}/update-list"
+  cat "${tmp_dir}/pkgbases"
 fi