| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/bootstrap-mysql | 72 | ||||
| -rwxr-xr-x | bin/build-master-status | 25 | ||||
| -rwxr-xr-x | bin/build-packages | 3 | ||||
| -rwxr-xr-x | bin/cleanup | 4 | ||||
| -rwxr-xr-x | bin/copy-to-build-support | 2 | ||||
| -rwxr-xr-x | bin/db-update | 6 | ||||
| -rwxr-xr-x | bin/delete-packages | 10 | ||||
| -rwxr-xr-x | bin/find-obsolete-packages | 127 | ||||
| -rwxr-xr-x | bin/get-assignment | 138 | ||||
| -rwxr-xr-x | bin/get-package-updates | 181 | ||||
| -rwxr-xr-x | bin/ii-connect | 18 | ||||
| -rwxr-xr-x | bin/ping-from-slave | 3 | ||||
| -rwxr-xr-x | bin/return-assignment | 10 | ||||
| -rwxr-xr-x | bin/sanity-check | 216 | ||||
| -rwxr-xr-x | bin/slave-build-connect | 29 |
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 822d995..35f5398 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -367,6 +367,25 @@ if [ ! "$1" = 'slim' ]; then printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \ "${copy}" done + printf 'DROP TEMPORARY TABLE IF EXISTS `package_blobs`;\n' + printf 'CREATE TEMPORARY TABLE `package_blobs` (`ps_a` BIGINT, `ps_b` BIGINT, UNIQUE KEY `content` (`ps_a`,`ps_b`));\n' + # all packages being vaguely relevant enter `package_blobs`, because + # they might block any other package from moving + printf 'INSERT IGNORE INTO `package_blobs` (`ps_a`,`ps_b`)' + printf ' SELECT `a_ps`.`id`,`b_ps`.`id`' + printf ' FROM `package_sources` AS `a_ps`' + printf ' JOIN `package_sources` AS `b_ps`' + printf ' ON UNIX_TIMESTAMP(`a_ps`.`commit_time`) - UNIX_TIMESTAMP(`b_ps`.`commit_time`) BETWEEN -10 AND 10' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' + mysql_join_binary_packages_repositories 'a_bp' 'a_r' + mysql_join_binary_packages_repositories 'b_bp' 'b_r' + mysql_join_repositories_repository_stabilities 'a_r' 'a_rs' + mysql_join_repositories_repository_stabilities 'b_r' 'b_rs' + printf ' WHERE `a_rs`.`name` = `from_stability`' + printf ' AND `b_rs`.`name` = `from_stability`;\n' # these packages are considered for moving: printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' @@ -408,6 +427,8 @@ if [ ! "$1" = 'slim' ]; then # now we delete all unreplaceable and unmoveable packages from the respective # list until no further changes are required printf 'REPEAT\n' + printf 'SET row_count_saved = 0;\n' + # create copies of our temporary tables *yuck* for table in 'replaced' 'moveable'; do for copy in '' '2'; do @@ -433,23 +454,23 @@ if [ ! "$1" = 'slim' ]; then printf ' AND `dependency_types`.`relevant_for_binary_packages`' # ... by a not-deleted, "more stable" package already in place or ... printf ' WHERE NOT EXISTS (' - printf 'SELECT * FROM `install_target_providers`' + printf 'SELECT 1 FROM `install_target_providers`' mysql_join_install_target_providers_binary_packages '' 'prov_bp' mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`' printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf 'SELECT 1 FROM `replaced_binary_packages_copy`' printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' printf ')' # ... by a moved package printf ') AND NOT EXISTS (' - printf 'SELECT * FROM `install_target_providers`' + printf 'SELECT 1 FROM `install_target_providers`' printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' - printf 'SET @row_count_saved = ROW_COUNT();\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' # refresh copies of our temporary tables *yuck* for table in 'replaced' 'moveable'; do @@ -485,7 +506,7 @@ if [ ! "$1" = 'slim' ]; then mysql_join_install_target_providers_dependencies printf ' AND NOT EXISTS (' # dependencies of replaced packages don't matter - printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf 'SELECT 1 FROM `replaced_binary_packages_copy`' printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`' printf ')' mysql_join_dependencies_dependency_types @@ -502,7 +523,7 @@ if [ ! "$1" = 'slim' ]; then # a) ^ printf ' WHERE NOT EXISTS (' # no moved package ... - printf 'SELECT *' + printf 'SELECT 1' printf ' FROM `moveable_binary_packages_copy`' mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp' # ... provides the same @@ -510,7 +531,7 @@ if [ ! "$1" = 'slim' ]; then # b) 1) ^ printf ') AND NOT EXISTS (' # no current package ... - printf 'SELECT *' + printf 'SELECT 1' printf ' FROM `binary_packages` AS `subst_bp`' mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp' # ... in a repository ... @@ -522,14 +543,47 @@ if [ ! "$1" = 'slim' ]; then printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`' printf ' AND NOT EXISTS (' # ... and which is not replaced ... - printf 'SELECT * FROM `replaced_binary_packages_copy2`' + printf 'SELECT 1 FROM `replaced_binary_packages_copy2`' printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`' printf ')' # ... and provides the same printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' # b) 2) ^ printf ');\n' - printf 'UNTIL row_count_saved=0 AND ROW_COUNT()=0\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' + + # refresh copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" + done + done + + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' + printf ' RIGHT JOIN `moveable_binary_packages`' + printf ' ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' + printf ' JOIN `binary_packages`' + printf ' ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_build_assignments + printf ' JOIN `package_blobs`' + printf ' ON `build_assignments`.`package_source`=`package_blobs`.`ps_a`' + printf ' JOIN `build_assignments` AS `bl_ba`' + printf ' ON `bl_ba`.`package_source`=`package_blobs`.`ps_b`' + mysql_join_build_assignments_binary_packages 'bl_ba' 'bl_bp' + printf ' WHERE NOT EXISTS (' + printf 'SELECT 1 FROM `moveable_binary_packages_copy`' + printf ' WHERE `moveable_binary_packages_copy`.`id`=`bl_bp`.`id`' + printf ');\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' + + printf 'UNTIL row_count_saved=0\n' printf 'END REPEAT;\n' for table in 'moveable' 'replaced'; do for copy in '' '2'; do diff --git a/bin/build-master-status b/bin/build-master-status index 9503d71..7ab38d4 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -19,7 +19,7 @@ fi mysql_cleanup # update todos -find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ +find "${base_dir}/bin/" "${base_dir}/conf/" "${base_dir}/lib/" -type f \ -exec grep -nHF '' '{}' \; | \ sed 's,^'"$(str_to_regex "${base_dir}")"'/,,' | \ sed -n ' @@ -206,15 +206,20 @@ if [ ! -s "${work_dir}/build-master-sanity" ]; then mysql_join_build_assignments_binary_packages mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `dependencies`' - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_binary_packages`' - mysql_join_dependencies_install_target_providers - mysql_join_install_target_providers_binary_packages '' 'prov_bp' - mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' - printf ' WHERE `prov_r`.`name`="build-list"' - printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`' + printf ' AND (' + printf 'NOT EXISTS (' + printf 'SELECT 1 FROM `dependencies`' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' WHERE `prov_r`.`name`="build-list"' + printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`' + printf ') OR EXISTS (' + printf 'SELECT 1 FROM `build_dependency_loops`' + printf ' WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`' + printf ')' printf '))' printf ');\n' } | \ diff --git a/bin/build-packages b/bin/build-packages index e9ac0c4..fc80ab1 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -466,6 +466,9 @@ while [ "${count}" -ne 0 ]; do find "${package_content_dir}/${pkgfile##*/}" \ -name 'opt' -prune , \ -exec objdump -x '{}' \; 2>/dev/null | \ + sed ' + /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d + ' | \ grep -w 'NEEDED' | \ awk '{print $2}' | \ sed ' diff --git a/bin/cleanup b/bin/cleanup index 83bb209..830cb94 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -60,10 +60,6 @@ find "${build_log_directory}/error" -maxdepth 1 -type f -printf '%f\n' | \ done } -# only keep last 50 lines of ssh-log -tail -n50 "${work_dir}/ssh-log" | \ - sponge "${work_dir}/ssh-log" - # only keep namcap logs of last 2 weeks for succeeded packages find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \ -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \ diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 5382235..1164c1d 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -2,6 +2,8 @@ # copy the given package(s) into build-support +# TODO: enable separate repositories for each architecture + # shellcheck source=../conf/default.conf . "${0%/*}/../conf/default.conf" diff --git a/bin/db-update b/bin/db-update index 54d2f87..28e9861 100755 --- a/bin/db-update +++ b/bin/db-update @@ -12,6 +12,8 @@ # TODO: we should delete more packages than just the ones in repositories # where we move to (think of [extra] -> [community]) +# TODO: enable separate repositories for each architecture + # shellcheck disable=SC2039 # shellcheck source=../conf/default.conf . "${0%/*}/../conf/default.conf" @@ -392,7 +394,9 @@ for source_stability in 'testing' 'staging'; do printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n' printf 'UPDATE `binary_packages`' printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`' - printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' + printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`,' + printf ' `binary_packages`.`last_moved`=NOW()' + printf ' WHERE `binary_packages`.`repository`!=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query done diff --git a/bin/delete-packages b/bin/delete-packages index 7d2a3bd..5f60b40 100755 --- a/bin/delete-packages +++ b/bin/delete-packages @@ -15,6 +15,8 @@ # TODO: delete other to-be-deleted packages if asked to do so +# TODO: enable separate repositories for each architecture + # shellcheck disable=SC2016 usage() { >&2 echo '' @@ -135,19 +137,19 @@ export TMPDIR="${tmp_dir}" printf ')' printf ');\n' - printf ' SELECT DISTINCT "repo",`repositories`.`name`' + printf 'SELECT DISTINCT "repo",`repositories`.`name`' printf ' FROM `to_deletes`' printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' mysql_join_binary_packages_repositories printf ';\n' - printf ' SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`' + printf 'SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`' printf ' FROM `to_deletes`' printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' mysql_join_binary_packages_repositories printf ';\n' - printf ' SELECT "package-file",`repositories`.`name`,' + printf 'SELECT "package-file",`repositories`.`name`,' mysql_package_name_query printf ' FROM `to_deletes`' printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' @@ -155,7 +157,7 @@ export TMPDIR="${tmp_dir}" mysql_join_binary_packages_architectures printf ';\n' - printf ' SELECT "package-id",`to_deletes`.`id`' + printf 'SELECT "package-id",`to_deletes`.`id`' printf ' FROM `to_deletes`' printf ';\n' } | \ diff --git a/bin/find-obsolete-packages b/bin/find-obsolete-packages new file mode 100755 index 0000000..3a4bebc --- /dev/null +++ b/bin/find-obsolete-packages @@ -0,0 +1,127 @@ +#!/bin/sh + +# shellcheck source=../conf/default.conf +. "${0%/*}/../conf/default.conf" + +usage() { + >&2 echo '' + >&2 echo 'find-obsolete-packages: find packages which are no longer available upstream.' + >&2 echo '' + >&2 echo 'possible options:' + >&2 echo ' -h|--help:' + >&2 echo ' Show this help and exit.' + >&2 echo ' -m|--mirror https://mirror.example.com/archlinux' + >&2 echo ' Mirror url to take upstream'"'"'s packages from.' + >&2 echo ' -n|--no-action:' + >&2 echo ' Do not mark packages as to_be_deleted.' + >&2 echo ' -w|--wait:' + >&2 echo ' Wait for lock if necessary.' + [ -z "$1" ] && exit 1 || exit "$1" +} + +eval set -- "$( + getopt -o hm:nw \ + --long help \ + --long mirror: \ + --long no-action \ + --long wait \ + -n "$(basename "$0")" -- "$@" || \ + echo usage +)" + +mirror='' +no_action=false +wait_for_lock='-n' + +while true +do + case "$1" in + -h|--help) + usage 0 + ;; + -m|--mirror) + if [ -n "${mirror}" ]; then + >&2 echo 'Already have one -m flag.' + usage + fi + shift + mirror="$1" + ;; + -n|--no-action) + no_action=true + ;; + -w|--wait) + wait_for_lock='' + ;; + --) + shift + break + ;; + *) + >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' + exit 42 + ;; + esac + shift +done + +if [ $# -ne 0 ]; then + >&2 echo 'Too many arguments.' + usage +fi + +if [ -z "${mirror}" ]; then + >&2 echo 'Flag -m missing' + usage +fi + +if ! ${no_action}; then + exec 9> "${sanity_check_lock_file}" + if ! flock -s ${wait_for_lock} 9; then + >&2 echo 'Cannot get sanity-check lock.' + exit 1 + fi + + exec 8> "${build_list_lock_file}" + if ! flock ${wait_for_lock} 8; then + >&2 echo 'Cannot get build-list lock.' + exit 1 + fi +fi + +{ + for repo in core extra community; do + curl -Ss "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ + tar -tz + done | \ + sed -n ' + s,\(-[^-]*\)\{2\}/desc$,, + T + p + p + ' + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT `binary_packages`.`pkgname`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`is_on_master_mirror`' + printf ' AND NOT `binary_packages`.`is_to_be_deleted`' + } | \ + mysql_run_query +} | \ + sort | \ + uniq -u | \ + if ${no_action}; then + cat + else + # shellcheck disable=SC2016 + base64_encode_each | \ + sed ' + s/^/from_base64("/ + s/$/"),/ + 1 s/^/UPDATE `binary_packages` SET `is_to_be_deleted`=1 WHERE `binary_packages`.`pkgname` IN (/ + $ s/,$/);/ + ' | \ + mysql_run_query + fi diff --git a/bin/get-assignment b/bin/get-assignment index 7f08d42..34a990e 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -18,6 +18,7 @@ # TODO: honor manual build order of tool-chain: # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc +# hand_out_assignment $build_assignments.id hand_out_assignment() { # shellcheck disable=SC2016 @@ -32,46 +33,28 @@ hand_out_assignment() { mysql_join_build_assignments_binary_packages mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND `package_sources`.`%s`=from_base64("%s")' \ - 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" - printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ - "$(printf '%s' "$4" | base64 -w0)" + printf ' AND `build_assignments`.`id`=from_base64("%s")' \ + "$(printf '%s' "$1" | base64 -w0)" printf ' LIMIT 1;\n' + } | \ + mysql_run_query | \ + tr '\t' ' ' + # shellcheck disable=SC2016 + { printf 'UPDATE `build_slaves`' - printf ' SET `currently_building` = (' - printf ' SELECT `build_assignments`.`id`' - printf ' FROM `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE' - printf ' `package_sources`.`%s` = from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" - printf ' `upstream_repositories`.`name` = from_base64("%s")' \ - "$(printf '%s' "$4" | base64 -w0)" - printf ')' + printf ' SET `currently_building` = from_base64("%s")' \ + "$(printf '%s' "$1" | base64 -w0)" # shellcheck disable=SC2154 - printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${slave}" | base64 -w0)" + printf ' WHERE `build_slaves`.`id`=from_base64("%s");\n' \ + "$(printf '%s' "${slave_id}" | base64 -w0)" printf 'UPDATE `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`priority`=0' - printf ' WHERE' - printf ' `package_sources`.`%s` = from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" - printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ - "$(printf '%s' "$4" | base64 -w0)" + printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \ + "$(printf '%s' "${slave_id}" | base64 -w0)" } | \ - mysql_run_query | \ - tr '\t' ' ' + mysql_run_query exit 0 @@ -108,13 +91,13 @@ trap clean_up EXIT currently_building=$( # shellcheck disable=SC2016 { - printf 'SELECT ' - mysql_query_select_pkgbase_and_revision + printf 'SELECT `build_assignments`.`id`' + printf ' FROM `build_assignments`' mysql_join_build_assignments_build_slaves mysql_join_build_assignments_binary_packages mysql_join_binary_packages_repositories - printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ - "$(printf '%s' "${slave}" | base64 -w0)" + printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \ + "$(printf '%s' "${slave_id}" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ @@ -122,8 +105,7 @@ currently_building=$( ) if [ -n "${currently_building}" ]; then - # shellcheck disable=SC2086 - hand_out_assignment ${currently_building} + hand_out_assignment "${currently_building}" fi # a package with [all dependencies met or which is part of a loop] @@ -135,54 +117,54 @@ fi next_building=$( # shellcheck disable=SC2016 { - printf 'SELECT ' - printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \ - "$( - printf '%s' "$1" | \ - base64 -w0 - )" - printf '`build_assignments`.`priority`,' - printf 'COALESCE(' - printf 'MAX(`failed_builds`.`date`),0' - printf ') AS `last_trial`,' - mysql_query_is_part_of_loop '`build_assignments`.`id`' - printf ' AS `part_of_loop`,' - mysql_query_select_pkgbase_and_revision - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' LEFT' - mysql_join_build_assignments_failed_builds - printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND NOT EXISTS (' - printf ' SELECT *' - printf ' FROM `build_slaves`' - printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`' - printf ') AND (' - printf '`build_assignments`.`is_blocked` IS NULL' - printf ' OR' - printf ' `package_sources`.`pkgbase`=from_base64("%s")' \ + printf 'SELECT `q`.`ba_id` FROM' + printf '(' + printf 'SELECT ' + printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \ "$( printf '%s' "$1" | \ base64 -w0 )" - printf ') AND (' + printf '`build_assignments`.`priority`,' + printf 'COALESCE(' + printf 'MAX(`failed_builds`.`date`),0' + printf ') AS `last_trial`,' mysql_query_is_part_of_loop '`build_assignments`.`id`' - printf ' OR NOT ' - mysql_query_has_pending_dependencies '`build_assignments`.`id`' - printf ')' - printf ' GROUP BY `build_assignments`.`id`' - printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' - printf ' LIMIT 1;\n' + printf ' AS `part_of_loop`,' + printf '`build_assignments`.`id` AS `ba_id`' + printf ' FROM `build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' LEFT' + mysql_join_build_assignments_failed_builds + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND NOT EXISTS (' + printf ' SELECT 1' + printf ' FROM `build_slaves`' + printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`' + printf ') AND (' + printf '`build_assignments`.`is_blocked` IS NULL' + printf ' OR' + printf ' `package_sources`.`pkgbase`=from_base64("%s")' \ + "$( + printf '%s' "$1" | \ + base64 -w0 + )" + printf ') AND (' + mysql_query_is_part_of_loop '`build_assignments`.`id`' + printf ' OR NOT ' + mysql_query_has_pending_dependencies '`build_assignments`.`id`' + printf ')' + printf ' GROUP BY `build_assignments`.`id`' + printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' + printf ' LIMIT 1' + printf ') AS `q`;\n' } | \ - mysql_run_query | \ - sed ' - y/\t/ / - s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ - ' + mysql_run_query ) if [ -n "${next_building}" ]; then - # shellcheck disable=SC2086 - hand_out_assignment ${next_building} + hand_out_assignment "${next_building}" fi # Check if there are any pending packages at all diff --git a/bin/get-package-updates b/bin/get-package-updates index f581e13..73766c8 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -14,9 +14,6 @@ # TODO: keep database clean in case of abort -# TODO: removing something from the blacklist should trigger a build -# (if it is found in upstreams repositories) - # shellcheck disable=SC2016 usage() { >&2 echo '' @@ -25,8 +22,12 @@ usage() { >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -b|--block: If necessary, wait for lock blocking.' + >&2 echo ' -d|--date $datetime:' + >&2 echo ' Pull latest commit before $datetime' + >&2 echo ' (yyyy-mm-ddThh:mm:ss). Conflicts -n.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -n|--no-pull: Do not pull git repos, merely reorder build list.' + >&2 echo ' Conflicts -d.' >&2 echo ' -x|--test-exclusion $package:' >&2 echo ' Print additionally deleted/excluded packages if' >&2 echo ' "$package" would be black listed.' @@ -34,8 +35,9 @@ usage() { } eval set -- "$( - getopt -o bhnx: \ + getopt -o bd:hnx: \ --long block \ + --long date: \ --long help \ --long no-pull \ --long test-exclusion: \ @@ -44,6 +46,7 @@ eval set -- "$( )" block_flag='-n' +date_time='' test_exclusion='' pull=true @@ -53,6 +56,10 @@ do -b|--block) block_flag='' ;; + -d|--date) + shift + date_time="$1" + ;; -h|--help) usage 0 ;; @@ -85,6 +92,11 @@ if [ $# -ne 0 ]; then usage fi +if [ -n "${date_time}" ] && ! ${pull}; then + >&2 printf -- '-d and -n are mutually exclusive.\n' + usage +fi + if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit @@ -146,9 +158,15 @@ for repo in ${repo_names}; do )'" # determine new git revision if ${pull}; then - eval "new_repo_revisions__${repo}='$( - git -C "${repo_path}" rev-parse HEAD - )'" + if [ -z "${date_time}" ]; then + eval "new_repo_revisions__${repo}='$( + git -C "${repo_path}" rev-parse HEAD + )'" + else + eval "new_repo_revisions__${repo}='$( + git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD + )'" + fi else eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"' fi @@ -189,11 +207,66 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -for repo in ${repo_names}; do - eval repo_path='"${repo_paths__'"${repo}"'}"' - eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' - eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"' +trigger_rebuild_on_any_file_sed_expression=$( + # note: This only works for upstream sources! + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT CONCAT(' + printf '`package_sources`.`pkgbase`,"/repos/",' + printf '`upstream_repositories`.`name`,"-",' + printf 'IF(`architectures`.`name`="any","any","x86_64"))' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_architectures + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE `repositories`.`name`="build-list";\n' + } | \ + mysql_run_query | \ + sed ' + s@.*@s,^\\(.\\t\0/\\)[^/]\\+,\\1PKGBUILD,@ + ' +) + +{ + # trigger rebuild of packages removed from blacklist + # shellcheck disable=SC2016 { + printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`,`git_repositories`.`name`' + printf ' FROM `package_sources`' + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_git_repositories + printf ' WHERE `package_sources`.`pkgbase` IN (' + # shellcheck disable=SC2154 + git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \ + sed -n ' + s/^-\([^-]\)/\1/ + T + s/\s*#.*$// + p + ' | \ + base64_encode_each | \ + sed ' + s/^/from_base64("/ + s/$/")/ + ' | \ + tr '\n' ',' + printf '"");\n' + } | \ + mysql_run_query | \ + while read -r pkgbase repository git_repository; do + printf 'A %s ' \ + "${pkgbase}" + eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${git_repository}"'}"' + printf ' empty %s\n' \ + "${repository}" + done + # actual updates/removes + for repo in ${repo_names}; do + eval repo_path='"${repo_paths__'"${repo}"'}"' + eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' + eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"' # if old revision unknown, mimic "git diff"-output # shellcheck disable=SC2154 if [ "${old_repo_revision}" = "NONE" ]; then @@ -202,45 +275,48 @@ for repo in ${repo_names}; do sed 's|^|A\t|' else git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" - fi - } | \ - # only track changes in PKGBUILDs - # TODO: However, packages which are already on the build list should - # receive a git_revision bump if _any_ file changed. - grep '/PKGBUILD$' | \ - if [ "${repo}" = "archlinux32" ]; then - # modify the directory structure from the modifiaction-repository - # to the one of an original source repository - # shellcheck disable=SC2016 - sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ - while read -r pkg_repo rest; do - repo=$(find_git_repository_to_package_repository "${pkg_repo}") - eval 'printf '"'"'%s %s %s\n'"'" \ - "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \ - "$(printf '"${old_repo_revisions__%s}"' "${repo}")" \ - "'${rest}'" - done - else - sed "s|^|${new_repo_revision} ${old_repo_revision} |" fi | \ - grep '^\S\+ \S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - # ignore i686 - grep -v -- '-i686/PKGBUILD$' | \ - # ignore staging and testing - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sed 's|^\(\S\+ \S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' -done | \ - sort -u | \ - sed ' - s|^D\s|0 \0| - t - s|^[AM]\s|1 \0| - t - s|^|2 | - ' | \ - sort -k1,1 | \ - sed 's|^[012] ||' | \ + # rename any file to "PKGBUILD" for packages on the build-list + sed "${trigger_rebuild_on_any_file_sed_expression}" | \ + # only track changes in PKGBUILDs + grep '/PKGBUILD$' | \ + if [ "${repo}" = "archlinux32" ]; then + # modify the directory structure from the modifiaction-repository + # to the one of an original source repository + # shellcheck disable=SC2016 + sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ + while read -r pkg_repo rest; do + repo=$(find_git_repository_to_package_repository "${pkg_repo}") + eval 'printf '"'"'%s %s %s\n'"'" \ + "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \ + "$(printf '"${old_repo_revisions__%s}"' "${repo}")" \ + "'${rest}'" + done + else + sed "s|^|${new_repo_revision} ${old_repo_revision} |" + fi | \ + grep '^\S\+ \S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ + # ignore i686 + grep -v -- '-i686/PKGBUILD$' | \ + # ignore staging and testing + grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ + sed 's|^\(\S\+ \S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' + done | \ + sort -u | \ + sed ' + s|^D\s|0 \0| + t + s|^[AM]\s|1 \0| + t + s|^|2 | + ' | \ + sort -k1,1 | \ + sed 's|^[012] ||' +} | \ while read -r mode package git_revision old_git_revision repository; do + if [ "${old_git_revision}" = 'empty' ]; then + old_git_revision='' + fi if [ "${mode}" = 'D' ]; then # deleted PKGBUILD # shellcheck disable=SC2154 @@ -376,11 +452,14 @@ while [ -n "${black_listed_new}" ]; do mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' mysql_join_binary_packages_dependencies 'a_bp' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + printf ' AND `dependency_types`.`relevant_for_binary_packages`' printf ' WHERE NOT EXISTS (' - printf ' SELECT *' + printf ' SELECT 1' printf ' FROM `install_target_providers`' printf ' WHERE NOT EXISTS (' - printf ' SELECT *' + printf ' SELECT 1' printf ' FROM `bl`' printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' @@ -391,7 +470,7 @@ while [ -n "${black_listed_new}" ]; do printf ') AND EXISTS (' # TODO: This should be corrected at the root: automatic install targets, which are bogus should # not be added in the first place - but how do we detect that? - printf ' SELECT *' + printf ' SELECT 1' printf ' FROM `install_target_providers`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' diff --git a/bin/ii-connect b/bin/ii-connect index ca8ea99..5fc5f37 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -79,8 +79,10 @@ if [ "$1" = 'watch' ]; then channel="${channel##*/}" if [ -z "${channel%%#*}" ]; then prefix='buildmaster: ' + sloppy_salutation='buildmaster' else prefix='' + sloppy_salutation='' fi regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' if grep -q "${regex}" "${out_file}"; then @@ -130,6 +132,22 @@ if [ "$1" = 'watch' ]; then done sed -i "/${regex}/d" "${out_file}" fi + regex='^\(\S\+ \)\?\S\+ <\S\+> .*[Ww]hat'"'"'\?s *[Uu]p' + if grep "${regex}" "${out_file}" | \ + grep -q "${sloppy_salutation}"; then + done_something=true + sed -i "/${regex}/d" "${out_file}" + printf 'up? I'"'"'m up for %s, %s\n' "$(uptime -p | sed 's/^up //')" "$(uptime | sed 's/.*, //')" | \ + sponge "${out_file%/out}/in" + fi + regex='^\(\S\+ \)\?\S\+ <\S\+> \(.* \)\?[Tt]h\(anks\|x\)\([ ,.!]\|$\)' + if grep "${regex}" "${out_file}" | \ + grep -q "${sloppy_salutation}"; then + done_something=true + sed -i "/${regex}/d" "${out_file}" + printf 'np\n' | \ + sponge "${out_file%/out}/in" + fi done sleep 1 done diff --git a/bin/ping-from-slave b/bin/ping-from-slave index 3c03b45..7611f44 100755 --- a/bin/ping-from-slave +++ b/bin/ping-from-slave @@ -18,7 +18,8 @@ if [ "$( printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" } | \ - mysql_run_query 'unimportant' + mysql_run_query 'unimportant' || \ + echo '1' )" -ne 1 ]; then >&2 echo 'You do not build anything currently - abort whatever you are doing.' exit 2 diff --git a/bin/return-assignment b/bin/return-assignment index 9e64ff0..bcdb717 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -99,7 +99,6 @@ if [ "$5" = 'ERROR' ]; then 'log_file' 'VARCHAR(512)' | \ sed 's/,$//' printf ');\n' - printf 'INSERT INTO `failures` (`date`,`reason`,`log_file`) VALUES' fail_reason_identifiers=$( { printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' @@ -131,8 +130,10 @@ if [ "$5" = 'ERROR' ]; then fi done done | \ - sed 's/,$//' - printf ';\n' + sed ' + 1 s/^/INSERT INTO `failures` (`date`,`reason`,`log_file`) VALUES / + s/,$/;\n/ + ' printf 'INSERT INTO `failed_builds` (`build_slave`,`build_assignment`,`date`,`reason`,`log_file`)' printf ' SELECT ' printf '`build_slaves`.`%s`,' \ @@ -536,7 +537,8 @@ trigger_mirror_refreshs base64 -w0 )" printf ' `binary_packages`.`has_issues`=0,' - printf ' `binary_packages`.`is_tested`=0' + printf ' `binary_packages`.`is_tested`=0,' + printf ' `binary_packages`.`last_moved`=NOW()' printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ diff --git a/bin/sanity-check b/bin/sanity-check index 06da288..96f028f 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -41,6 +41,7 @@ eval set -- "$( silence=0 repos="${standalone_package_repositories} ${stable_package_repositories} ${testing_package_repositories} ${staging_package_repositories}" +archs='i686' web=false while true @@ -218,11 +219,13 @@ while [ $# -gt 0 ]; do errors=$( { # shellcheck disable=SC2086 - printf 'expected %s\n' ${repos} - ls_master_mirror 'i686' | \ - sed 's|^|found |' + for arch in ${archs}; do + printf 'expected '"${arch}"' %s\n' ${repos} + ls_master_mirror "${arch}" | \ + sed 's|^|found '"${arch}"' |' + done } | \ - sort -k2 | \ + sort -k2,3 | \ uniq -uf1 ) if [ -n "${errors}" ]; then @@ -242,109 +245,111 @@ while [ $# -gt 0 ]; do package-database) - for repo in ${repos}; do - - [ ${silence} -gt 0 ] || \ - printf 'checking consistency of repository "%s" on the master mirror ...' "${repo}" >> \ - "${tmp_dir}/messages" + for arch in ${archs}; do + for repo in ${repos}; do - packages=$( - ls_master_mirror "i686/${repo}" | \ - grep '\.pkg\.tar\.xz\(\.sig\)\?$' - ) || true - - errors=$( - echo "${packages}" | \ - grep '\S' | \ - sed ' - s|^\(.*\.pkg\.tar\.xz\)$|package \1| - s|^\(.*\.pkg\.tar\.xz\)\.sig$|signature \1| - ' | \ - sort -k2 | \ - uniq -cf1 | \ - grep -v '^\s*2\s' | \ - awk '{print $2 " " $3}' - ) || true - if [ -n "${errors}" ]; then - if [ ${silence} -le 1 ]; then - printf '\nThe following packages in %s are missing a signature or vice versa:\n%s\n' \ - "${repo}" \ - "${errors}" >> \ - "${tmp_dir}/messages" - fi - i_am_insane - fi + [ ${silence} -gt 0 ] || \ + printf 'checking consistency of repository "%s/%s" on the master mirror ...' "${arch}" "${repo}" >> \ + "${tmp_dir}/messages" - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \ - "${tmp_dir}/" + packages=$( + ls_master_mirror "${arch}/${repo}" | \ + grep '\.pkg\.tar\.xz\(\.sig\)\?$' + ) || true - errors=$( - { - tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \ - sed -n ' - /^%FILENAME%$/ { - N - s/^.*\n/in_database / - p - } - ' + errors=$( echo "${packages}" | \ + grep '\S' | \ sed ' - /\.pkg\.tar\.xz$/ !d - s/^/in_repository / + s|^\(.*\.pkg\.tar\.xz\)$|package \1| + s|^\(.*\.pkg\.tar\.xz\)\.sig$|signature \1| ' | \ - sort -u - } | \ - sort -k2 | \ - uniq -uf1 - ) - if [ -n "${errors}" ]; then - if [ ${silence} -le 1 ]; then - printf '\nThe following packages in %s are missing from the database or vice versa:\n%s\n' \ - "${repo}" \ - "${errors}" >> \ - "${tmp_dir}/messages" + sort -k2 | \ + uniq -cf1 | \ + grep -v '^\s*2\s' | \ + awk '{print $2 " " $3}' + ) || true + if [ -n "${errors}" ]; then + if [ ${silence} -le 1 ]; then + printf '\nThe following packages in %s are missing a signature or vice versa:\n%s\n' \ + "${repo}" \ + "${errors}" >> \ + "${tmp_dir}/messages" + fi + i_am_insane fi - i_am_insane - fi - errors=$( - { - tar -tzf "${tmp_dir}/${repo}.files.tar.gz" | \ - grep '/$' | \ - sed ' - s|/$|| - s|^|in_database | - ' - echo "${packages}" | \ - grep '\S' | \ - sed ' - s|-[^-]\+$|| - s|^|in_repository | - ' | \ - sort -u - } | \ - sort -k2 | \ - uniq -uf1 - ) - if [ -n "${errors}" ]; then - if [ ${silence} -le 1 ]; then - printf '\nThe following packages in %s are missing from the file-database or vice versa:\n%s\n' \ - "${repo}" \ - "${errors}" >> \ - "${tmp_dir}/messages" + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db.tar.gz" \ + "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files.tar.gz" \ + "${tmp_dir}/" + + errors=$( + { + tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \ + sed -n ' + /^%FILENAME%$/ { + N + s/^.*\n/in_database / + p + } + ' + echo "${packages}" | \ + sed ' + /\.pkg\.tar\.xz$/ !d + s/^/in_repository / + ' | \ + sort -u + } | \ + sort -k2 | \ + uniq -uf1 + ) + if [ -n "${errors}" ]; then + if [ ${silence} -le 1 ]; then + printf '\nThe following packages in %s are missing from the database or vice versa:\n%s\n' \ + "${repo}" \ + "${errors}" >> \ + "${tmp_dir}/messages" + fi + i_am_insane + fi + + errors=$( + { + tar -tzf "${tmp_dir}/${repo}.files.tar.gz" | \ + grep '/$' | \ + sed ' + s|/$|| + s|^|in_database | + ' + echo "${packages}" | \ + grep '\S' | \ + sed ' + s|-[^-]\+$|| + s|^|in_repository | + ' | \ + sort -u + } | \ + sort -k2 | \ + uniq -uf1 + ) + if [ -n "${errors}" ]; then + if [ ${silence} -le 1 ]; then + printf '\nThe following packages in %s are missing from the file-database or vice versa:\n%s\n' \ + "${repo}" \ + "${errors}" >> \ + "${tmp_dir}/messages" + fi + i_am_insane fi - i_am_insane - fi - find "${tmp_dir:?}" -mindepth 1 \( -not -name 'messages' \) -delete + find "${tmp_dir:?}" -mindepth 1 \( -not -name 'messages' \) -delete - [ ${silence} -gt 0 ] || \ - echo ' passed.' >> \ - "${tmp_dir}/messages" + [ ${silence} -gt 0 ] || \ + echo ' passed.' >> \ + "${tmp_dir}/messages" + done done ;; @@ -359,23 +364,26 @@ while [ $# -gt 0 ]; do { # shellcheck disable=SC2016 { - printf 'SELECT "mysql",CONCAT(`repositories`.`name`,"/",' + printf 'SELECT "mysql",CONCAT(`r_a`.`name`,"/",`repositories`.`name`,"/",' mysql_package_name_query printf ') FROM `binary_packages`' mysql_join_binary_packages_repositories printf ' AND `repositories`.`is_on_master_mirror`' mysql_join_binary_packages_architectures + mysql_join_repositories_architectures '' 'r_a' } | \ mysql_run_query | \ tr '\t' ' ' - ls_master_mirror 'i686' | \ - while read -r repo; do - ls_master_mirror "i686/${repo}" | \ - sed ' - /\.pkg\.tar\.xz$/!d - s,^,package-file '"${repo}"'/, - ' - done + for arch in ${archs}; do + ls_master_mirror "${arch}" | \ + while read -r repo; do + ls_master_mirror "${arch}/${repo}" | \ + sed ' + /\.pkg\.tar\.xz$/!d + s,^,package-file '"${arch}"'/'"${repo}"'/, + ' + done + done } | \ sed 's/\(-[0-9]\+\)\.0\(-[^- ]\+$\)/\1\2/' | \ sort -k2 | \ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index b176c8f..4fcb38a 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -17,9 +17,27 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ fi # shellcheck disable=SC2016 + slave_id=$( + { + printf 'SELECT `build_slaves`.`id` FROM `build_slaves`' + printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ + "$( + printf '%s' "$1" | \ + base64 -w0 + )" + } | \ + mysql_run_query + ) + if [ -z "${slave_id}" ]; then + >&2 printf 'Build slave "%s" is unnknown to the database.\n' "$1" + exit 42 + fi + + # shellcheck disable=SC2016 { printf 'INSERT INTO `ssh_log` (`build_slave`,`action`,`parameters`)' - printf ' SELECT `build_slaves`.`id`' + printf ' VALUES (%s' \ + "${slave_id}" printf ',from_base64("%s")' \ "$( printf '%s' "${SSH_ORIGINAL_COMMAND%% *}" | \ @@ -29,16 +47,11 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ printf '%s' "${SSH_ORIGINAL_COMMAND#* }" | \ base64 -w0 )" - printf ' FROM `build_slaves`' - printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ - "$( - printf '%s' "$1" | \ - base64 -w0 - )" + printf ');\n' } | \ mysql_run_query 'unimportant' - slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" + slave="$1" slave_id="${slave_id}" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" else |