bin/get-assignment: honor order of toolchain when handing out assignments

2 files changed, 102 insertions, 3 deletions

diff --git a/bin/get-assignment b/bin/get-assignment
index f8d5468..f3028e2 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -17,9 +17,6 @@
 # shellcheck source=../lib/load-configuration
 . "${0%/*}/../lib/load-configuration"
 
-# TODO: honor manual build order of tool-chain:
-# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
-
 # hand_out_assignment $build_assignments.id
 hand_out_assignment() {
 
@@ -142,6 +139,29 @@ fi
 next_building=$(
   # shellcheck disable=SC2016
   {
+    # We force the following build order of the tool-chain:
+    # linux-api-headers->glibc->binutils->gcc->binutils->glibc
+    for suffix in '' '_copy'; do
+      printf 'CREATE TEMPORARY TABLE `toolchain_order%s` (' \
+        "${suffix}"
+        printf '`number` SMALLINT,'
+        printf '`pkgbase` VARCHAR(64),'
+        printf 'PRIMARY KEY `number`(`number`)'
+      printf ');\n'
+    done
+    printf 'INSERT INTO `toolchain_order` (`number`,`pkgbase`) VALUES '
+    printf '(%s,"%s"),' \
+      0 'linux-api-headers' \
+      1 'glibc' \
+      2 'binutils' \
+      3 'gcc' \
+      4 'binutils' \
+      5 'glibc' | \
+      sed 's/,$/;\n/'
+    printf 'INSERT INTO `toolchain_order_copy` (`number`,`pkgbase`)'
+    printf ' SELECT `toolchain_order`.`number`,`toolchain_order`.`pkgbase`'
+    printf ' FROM `toolchain_order`;\n'
+
     printf 'SELECT `q`.`ba_id` FROM'
     printf '('
       printf 'SELECT '
@@ -150,6 +170,12 @@ next_building=$(
           printf '%s' "$1" | \
             base64 -w0
         )"
+      printf 'IFNULL(`toolchain_query`.`priority`,3) AS `toolchain_priority`,'
+      # 0: in toolchain, never blocked
+      # 1: in toolchain, unblocked exists, not-yet-built
+      # 2: in toolchain, unblocked exists, already built
+      # (should normally become "0", before being handed out)
+      # 3: not in toolchain or only blocked in toolchain
       printf '`build_assignments`.`priority`,'
       printf 'COALESCE('
       printf 'MAX(`failed_builds`.`date`),0'
@@ -168,6 +194,77 @@ next_building=$(
       printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
       printf ' LEFT'
       mysql_join_build_assignments_failed_builds
+      printf ' LEFT JOIN ('
+        # Does this job wait for something else in `toolchain_order`?
+        printf 'SELECT '
+        printf '`tc_q`.`pkgbase`,'
+        printf '`tc_q`.`architecture`,'
+        printf 'IF('
+          printf '`tc_q`.`is_blocked`,'
+          printf 'IF('
+            printf '`tc_q`.`is_built`,'
+            printf '2,'
+            printf '1'
+          printf '),'
+          printf '0'
+        printf ') AS `priority`'
+        printf ' FROM ('
+          printf 'SELECT '
+          printf '`late`.`pkgbase`,'
+          printf '`binary_packages`.`architecture`,'
+          printf 'MAX('
+            printf 'IF('
+              printf 'EXISTS ('
+                printf 'SELECT 1'
+                printf ' FROM `binary_packages` AS `built_bp`'
+                mysql_join_binary_packages_binary_packages_in_repositories 'built_bp' 'built_bpir'
+                mysql_join_binary_packages_in_repositories_repositories 'built_bpir' 'built_r'
+                printf ' AND `built_r`.`is_on_master_mirror`'
+                printf ' WHERE `built_bp`.`build_assignment`=`build_assignments`.`id`'
+              printf '),'
+              printf '1,'
+              printf '0'
+            printf ')'
+          printf ') AS `is_built`,'
+          printf 'MAX('
+            printf 'IF('
+              printf 'EXISTS ('
+                printf 'SELECT 1'
+                printf ' FROM `package_sources` AS `block_ps`'
+                mysql_join_package_sources_build_assignments 'block_ps' 'block_ba'
+                mysql_join_build_assignments_binary_packages 'block_ba' 'block_bp'
+                mysql_join_binary_packages_binary_packages_in_repositories 'block_bp' 'block_bpir'
+                printf ' AND `block_bpir`.`repository`=%s' \
+                  "${repository_ids__any_build_list}"
+                printf ' JOIN `toolchain_order_copy` AS `early`'
+                printf ' ON `early`.`pkgbase`=`block_ps`.`pkgbase`'
+                printf ' WHERE `early`.`number`<`late`.`number`'
+                printf ' AND `early`.`pkgbase`!=`late`.`pkgbase`'
+                printf ' AND ('
+                  printf '`block_bp`.`architecture`=`binary_packages`.`architecture`'
+                  # shellcheck disable=SC2154
+                  printf ' OR `%s`.`architecture`=%s' \
+                    'block_bp' "${architecture_ids__any}" \
+                    'binary_packages' "${architecture_ids__any}"
+                printf ')'
+              printf '),'
+              printf '1,'
+              printf '0'
+            printf ')'
+          printf ') AS `is_blocked`'
+          printf ' FROM `toolchain_order` AS `late`'
+          printf ' JOIN `package_sources`'
+          printf ' ON `package_sources`.`pkgbase`=`late`.`pkgbase`'
+          mysql_join_package_sources_build_assignments
+          mysql_join_build_assignments_binary_packages
+          mysql_join_binary_packages_binary_packages_in_repositories
+          printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
+            "${repository_ids__any_build_list}"
+          printf ' GROUP BY CONCAT(`late`.`pkgbase`,"-",`binary_packages`.`architecture`)'
+        printf ') AS `tc_q`'
+      printf ') AS `toolchain_query`'
+      printf ' ON `toolchain_query`.`pkgbase`=`package_sources`.`pkgbase`'
+      printf ' AND `toolchain_query`.`architecture`=`binary_packages`.`architecture`'
       printf ' WHERE NOT EXISTS ('
         printf ' SELECT 1'
         printf ' FROM `build_slaves`'
@@ -192,6 +289,7 @@ next_building=$(
       printf ' GROUP BY `build_assignments`.`id`'
       printf ' ORDER BY'
       printf ' `requested` DESC,'
+      printf ' `toolchain_priority`,'
       printf ' `priority` DESC,'
       printf ' `last_trial`,'
       printf ' `architectures`.`id`=`build_assignments`.`architecture` DESC,'
diff --git a/bin/return-assignment b/bin/return-assignment
index d9d87be..b837cac 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -679,6 +679,7 @@ trigger_mirror_refreshs
     "${tmp_dir}/repository-ids"
   printf 'COMMIT;\n'
 
+# TODO: do not remove toolchain build assignments if they need to be compiled a second time
   # remove from build-list
   printf 'DELETE FROM `binary_packages_in_repositories`'
   printf ' WHERE `binary_packages_in_repositories`.`package` IN ('