bin/return-assignment: improve haskell-rescheduling-logic: we now also look at the version of installed packages (sometimes a package has an older-than-expected version - this should not trigger a rebuild!)

1 files changed, 112 insertions, 25 deletions

diff --git a/bin/return-assignment b/bin/return-assignment
index e73bec4..8bcc1b9 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -232,11 +232,13 @@ if [ "$6" = 'ERROR' ]; then
 
   if ! ${was_broken_before}; then
 
-    # this will hold a list of "$time_stamp $haskell_package" meaning
-    # that the given $haskell_package did not work at the given
-    # $time_stamp
-    broken_haskell_packages=$(mktemp 'tmp.return-assignment.broken-haskell-packages.XXXXXXXXXX' --tmpdir)
-    trap 'rm "${broken_haskell_packages}"' EXIT
+    mysql_load_min_and_max_versions
+
+    # this will hold a list of "$build_time $haskell_package $version"
+    # meaning that the given $haskell_package with version $version did
+    # not work at $build_time
+    # note, that $build_time is only considered when $architecture or
+    # $version are unavailable
     find "${build_log_directory}/error" -type f \
       -name "$1.$2.$3.$4.$5.*.build-log.gz" \
       -exec zgrep -qF "$(
@@ -253,35 +255,102 @@ if [ "$6" = 'ERROR' ]; then
           date +%s -d"${build_time}"
         )
         zcat "${build_log}" | \
-        sed '
-          s/^installed package \(.*\) is broken due to missing package .*$/\1/
-          t
-          s@^.*Bad interface file: /usr/lib/ghc-[0-9.]\+/site-local/\([^/]\+\)-[0-9.]\+/.*$@\1@
-          t
-          d
-        ' | \
-        tr ' ' '\n' | \
-        tr '[:upper:]' '[:lower:]' | \
-        sed '
-          s/^/'"${build_time}"'\t/
-          s/-[0-9.]\+$//
-        '
+          sed -n '
+            /^Packages ([0-9]\+) /,/^\s*$/ p
+          ' | \
+          tr ' ' '\n' | \
+          sed '
+            1,2 d
+            /^\s*$/d
+            p
+            s/^haskell-//
+            t
+            d
+          ' | \
+          sort -u | \
+          sed '
+            s/-\(\([0-9]\+\):\)\?\([^-:]\+\)-\([^-.]\+\)\(\.\([^-.]\+\)\)\?$/ \2 \3 \4 \6/
+            s/ $/ 0/
+            s/  / 0 /g
+          ' > \
+          "${tmp_dir}/installed-versions"
+        zcat "${build_log}" | \
+          sed '
+            s/^installed package \(.*\) is broken due to missing package .*$/\1/
+            t
+            s@^.*Bad interface file: /usr/lib/ghc-[0-9.]\+/site-local/\([^/]\+\)-[0-9.]\+/.*$@\1@
+            t
+            d
+          ' | \
+          tr ' ' '\n' | \
+          tr '[:upper:]' '[:lower:]' | \
+          sed '
+            s/^/'"${build_time}"' /
+            s/-[0-9.]\+$//
+          ' | \
+          sort -u | \
+          sort -k2,2 > \
+          "${tmp_dir}/broken-packages"
+        {
+          cut -d' ' -f1 < \
+            "${tmp_dir}/installed-versions" | \
+            sed 'p'
+          cut -d' ' -f2 < \
+            "${tmp_dir}/broken-packages"
+        } | \
+          sort | \
+          uniq -u | \
+          sed '
+            s/$/ '"${max_version%%:*}"' '"${max_version#*:}"' 0 0/
+          ' | \
+          sponge -a "${tmp_dir}/installed-versions"
+        sort -k1,1 < \
+          "${tmp_dir}/installed-versions" | \
+          sponge "${tmp_dir}/installed-versions"
+        join -1 2 -2 1 \
+          "${tmp_dir}/broken-packages" \
+          "${tmp_dir}/installed-versions"
       done | \
-      sort -k2,2 -k1nr,1 | \
-      uniq -f1 > \
-      "${tmp_dir}/broken-haskell-packages"
+      sort -k2 -k1nr,1 | \
+      uniq -f1 | \
+      tr ' ' '\t' > \
+      "${tmp_dir}/broken-packages-with-version"
 
     # now we look if the broken packages have been rebuilt in the meantime
     haskell_rebuild_packages=$(
       # shellcheck disable=SC2016
       {
-        printf 'CREATE TEMPORARY TABLE `broken`(`time_stamp` BIGINT,`pkgname` VARCHAR(64),UNIQUE KEY `pkgname`(`pkgname`),KEY `time_stamp`(`time_stamp`));\n'
-        printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `broken`(`time_stamp`,`pkgname`);\n' \
-          "${tmp_dir}/broken-haskell-packages"
+        printf 'CREATE TEMPORARY TABLE `broken`('
+          printf '`time_stamp` BIGINT,'
+          printf '`pkgname` VARCHAR(64),'
+          printf '`epoch` MEDIUMINT,'
+          printf '`pkgver` VARCHAR(64),'
+          printf '`pkgrel` MEDIUMINT,'
+          printf '`sub_pkgrel` MEDIUMINT,'
+          printf 'KEY `time_stamp`(`time_stamp`),'
+          printf 'KEY `pkgname`(`pkgname`),'
+          printf 'KEY `epoch`(`epoch`),'
+          printf 'KEY `pkgver`(`pkgver`),'
+          printf 'KEY `pkgrel`(`pkgrel`),'
+          printf 'KEY `sub_pkgrel`(`sub_pkgrel`),'
+          printf 'UNIQUE KEY `pkgfile`(`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`)'
+        printf ');\n'
+        printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `broken`(' \
+          "${tmp_dir}/broken-packages-with-version"
+          printf '`%s`,' \
+            'pkgname' \
+            'time_stamp' \
+            'epoch' \
+            'pkgver' \
+            'pkgrel' \
+            'sub_pkgrel' | \
+            sed 's/,$//'
+        printf ');\n'
+
         printf 'SELECT DISTINCT `broken`.`pkgname`'
         printf ' FROM `broken`'
         printf ' WHERE NOT EXISTS ('
-          printf 'SELECT 1'
+          printf 'SELECT *'
           printf ' FROM `binary_packages`'
           mysql_join_binary_packages_binary_packages_in_repositories
           mysql_join_binary_packages_in_repositories_repositories
@@ -306,6 +375,24 @@ if [ "$6" = 'ERROR' ]; then
             printf ' OR `binary_packages`.`pkgname`=CONCAT("haskell-",`broken`.`pkgname`)'
           printf ') AND ('
             printf '`build_assignments`.`return_date`>FROM_UNIXTIME(`broken`.`time_stamp`)'
+            printf ' OR ('
+              printf '`binary_packages`.`epoch`>`broken`.`epoch`'
+              printf ' OR ('
+                printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+                printf ' AND `binary_packages`.`pkgver`>`broken`.`pkgver`'
+              printf ')'
+              printf ' OR ('
+                printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+                printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
+                printf ' AND `binary_packages`.`pkgrel`>`broken`.`pkgrel`'
+              printf ')'
+              printf ' OR ('
+                printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+                printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
+                printf ' AND `binary_packages`.`pkgrel`=`broken`.`pkgrel`'
+                printf ' AND `binary_packages`.`sub_pkgrel`>`broken`.`sub_pkgrel`'
+              printf ')'
+            printf ')'
             printf ' OR `repositories`.`id`=%s' \
               "${repository_ids__any_build_list}"
           printf ')'