From 0553c6d12a5e310899e8a5679913faafa72b17fb Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 09:57:59 +0100 Subject: bin/build-master-status: write todos into database --- bin/build-master-status | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 4d47976..2477e3b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -502,6 +502,7 @@ if ${web}; then s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p ' | \ + tee "${tmp_dir}/todos" | \ sed ' :a N @@ -517,6 +518,62 @@ if ${web}; then } > \ "${tmp_dir}/todos.html" + if [ -s "${tmp_dir}/todos" ]; then + sed ' + :a + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | + s/\n$// + s/\n/\\n/g + ' -i "${tmp_dir}/todos" + while read -r file line desc; do + printf '%s %s %s\n' \ + "$(printf '%s' "${file}" | base64 -w0)" \ + "$(printf '%s' "${line}" | base64 -w0)" \ + "$(printf '%s' "${desc}" | base64 -w0)" + done < \ + "${tmp_dir}/todos" | \ + sponge "${tmp_dir}/todos" + # update todos + # shellcheck disable=SC2016 + while read -r file line desc; do + printf 'UPDATE TABLE `todos`' + printf ' SET `todos`.`line`=from_base64("%s")' \ + "${line}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`desc`=from_base64("%s");\n' \ + "${desc}" + + printf 'UPDATE TABLE `todos`' + printf ' SET `todos`.`desc`=from_base64("%s")' \ + "${desc}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`line`=from_base64("%s");\n' \ + "${line}" + done | \ + ${mysql_command} + # insert unfound todos + # shellcheck disable=SC2016 + while read -r file line desc; do + printf ' (' + printf 'from_base64("%s"),' \ + "${file}" \ + "${line}" \ + "${desc}" | \ + sed 's/,$/),/' + done < \ + "${tmp_dir}/todos" | \ + sed ' + 1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES + s/,$/;\n/ + ' | \ + ${mysql_command} + rm -f "${tmp_dir}/todos" + fi + { printf '%s\n' \ '' \ -- cgit v1.2.3-70-g09d2 From b8a0ed85a347f35a6c34723f80b4a7355f577785 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 10:09:11 +0100 Subject: bin/build-master-status: bugfix --- bin/build-master-status | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 2477e3b..1672789 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -538,22 +538,23 @@ if ${web}; then # update todos # shellcheck disable=SC2016 while read -r file line desc; do - printf 'UPDATE TABLE `todos`' + printf 'UPDATE `todos`' printf ' SET `todos`.`line`=from_base64("%s")' \ "${line}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ "${file}" - printf ' AND `todos`.`desc`=from_base64("%s");\n' \ + printf ' AND `todos`.`description`=from_base64("%s");\n' \ "${desc}" - printf 'UPDATE TABLE `todos`' - printf ' SET `todos`.`desc`=from_base64("%s")' \ + printf 'UPDATE `todos`' + printf ' SET `todos`.`description`=from_base64("%s")' \ "${desc}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ "${file}" printf ' AND `todos`.`line`=from_base64("%s");\n' \ "${line}" - done | \ + done < \ + "${tmp_dir}/todos" | \ ${mysql_command} # insert unfound todos # shellcheck disable=SC2016 -- cgit v1.2.3-70-g09d2 From 8d3d7cabdad3b873722064632968f3ce916b3e99 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 11:51:58 +0100 Subject: add many mysql-TODOs --- bin/build-master-status | 2 ++ bin/build-master-status-from-mysql | 2 ++ bin/build-packages | 2 ++ bin/calculate-dependent-packages | 4 ++++ bin/check-bugtracker | 2 ++ bin/cleanup | 2 ++ bin/db-update | 6 ++---- bin/get-package-updates | 2 ++ bin/interpret-mail | 2 ++ bin/prioritize-build-list | 2 ++ bin/return-assignment | 2 ++ bin/sanity-check | 2 ++ bin/seed-build-list | 2 ++ bin/show-dependencies | 2 ++ bin/why-dont-you | 2 ++ 15 files changed, 32 insertions(+), 4 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 1672789..b117e4e 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: replace by build-master-status-from-mysql + usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 44e07c1..16353dd 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: add all (necessary) features from build-master-status + tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT diff --git a/bin/build-packages b/bin/build-packages index 81afa7f..94292c7 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -10,6 +10,8 @@ # TODO: report back in regular intervals, so the build master can # abort if the package is outdated and/or someone else finished it. +# TODO: build other "architectures", too (pentium4, i486) + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages index bfb87a1..3208e61 100755 --- a/bin/calculate-dependent-packages +++ b/bin/calculate-dependent-packages @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: save information in database + +# TODO: read information from database + if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 46e78db..6b304e3 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -8,6 +8,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT diff --git a/bin/cleanup b/bin/cleanup index 78daece..15b6fc9 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: clean database, too + # we only clean if run interactive or if no one is logged in if ! tty -s && \ [ -n "$(users)" ]; then diff --git a/bin/db-update b/bin/db-update index fe45f41..35fd223 100755 --- a/bin/db-update +++ b/bin/db-update @@ -20,11 +20,9 @@ # track packages in stable, too # know _exactly_ in which repository the packages are -# TODO: fully integrate database (in other scripts too) - e.g. rely on -# info in db instead of state-files +# TODO: save information in database -# TODO: add possibility to manually clone a (binary) package (e.g. -# for build-support) - this should also be possible via email +# TODO: read information from database # shellcheck disable=SC2039 # shellcheck source=conf/default.conf diff --git a/bin/get-package-updates b/bin/get-package-updates index 55efee4..91fb6be 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -11,6 +11,8 @@ # TODO: test_exclusion does not yet cooperate with the database +# TODO: read information from database + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/interpret-mail b/bin/interpret-mail index d16c07f..03336c2 100755 --- a/bin/interpret-mail +++ b/bin/interpret-mail @@ -5,6 +5,8 @@ # TODO: enable email interface to delete packages +# TODO: read information from database (?) + if [ $# -ne 0 ]; then >&2 echo '' >&2 echo 'usage: interpret-mail' diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index 06d4c06..5655223 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -3,6 +3,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: save information in database + if [ $# -ne 1 ]; then echo 'Usage: prioritize-build-list pkg-regex' echo ' moves packages matching pkg-regex to front of build list' diff --git a/bin/return-assignment b/bin/return-assignment index 9de8add..42393ac 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -16,6 +16,8 @@ # TODO: sign database +# TODO: read information from database + # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" diff --git a/bin/sanity-check b/bin/sanity-check index 8736d47..e31a5ba 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + usage() { >&2 echo '' >&2 echo 'sanity-check [options] [checks]: check sanity of build master' diff --git a/bin/seed-build-list b/bin/seed-build-list index cd3152d..08f6a42 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/show-dependencies b/bin/show-dependencies index efdf745..ee7db19 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -3,6 +3,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + # Create a lock file. if [ $# -eq 0 ]; then diff --git a/bin/why-dont-you b/bin/why-dont-you index 2e3ea86..830048b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -7,6 +7,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + action="$1" shift -- cgit v1.2.3-70-g09d2 From a154303d59fc2d6e8928df6351dcf76e8aefe64b Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 14:02:05 +0100 Subject: bin/build-master-status: update todos correctly --- bin/build-master-status | 59 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 15 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index b117e4e..ce49332 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -540,7 +540,7 @@ if ${web}; then # update todos # shellcheck disable=SC2016 while read -r file line desc; do - printf 'UPDATE `todos`' + printf 'UPDATE IGNORE `todos`' printf ' SET `todos`.`line`=from_base64("%s")' \ "${line}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ @@ -548,7 +548,7 @@ if ${web}; then printf ' AND `todos`.`description`=from_base64("%s");\n' \ "${desc}" - printf 'UPDATE `todos`' + printf 'UPDATE IGNORE `todos`' printf ' SET `todos`.`description`=from_base64("%s")' \ "${desc}" printf ' WHERE `todos`.`file`=from_base64("%s")' \ @@ -560,19 +560,48 @@ if ${web}; then ${mysql_command} # insert unfound todos # shellcheck disable=SC2016 - while read -r file line desc; do - printf ' (' - printf 'from_base64("%s"),' \ - "${file}" \ - "${line}" \ - "${desc}" | \ - sed 's/,$/),/' - done < \ - "${tmp_dir}/todos" | \ - sed ' - 1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES - s/,$/;\n/ - ' | \ + { + printf 'CREATE TEMPORARY TABLE `td` (' + printf '`file` VARCHAR(64), ' + printf '`line` MEDIUMINT, ' + printf '`description` VARCHAR(128), ' + printf 'UNIQUE KEY `position` (`file`,`line`), ' + printf 'UNIQUE KEY `content` (`file`,`description`)' + printf ');' + printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' + while read -r file line desc; do + printf '(' + printf 'from_base64("%s"),' \ + "${file}" \ + "${line}" \ + "${desc}" | \ + sed 's/,$/),/' + done < \ + "${tmp_dir}/todos" | \ + sed ' + s/,$// + ' + printf ';\n' + printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' + printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' + printf 'FROM `td`;\n' + printf 'DELETE FROM `todos` WHERE NOT EXISTS (' + printf 'SELECT * FROM `td`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');' + printf 'DROP TABLE `td`;\n' + printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' + printf ') OR NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' + printf ');\n' + } | \ ${mysql_command} rm -f "${tmp_dir}/todos" fi -- cgit v1.2.3-70-g09d2 From 27106d9e0c70524a42cad4b2d662177c8490c174 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 4 Feb 2018 12:07:03 +0100 Subject: bin/build-master-status: only add not-yet-existent todos newly to database --- bin/build-master-status | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index ce49332..0e80bd6 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -584,7 +584,16 @@ if ${web}; then printf ';\n' printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' - printf 'FROM `td`;\n' + printf 'FROM `td` ' + printf 'WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');\n' + printf 'DELETE FROM `todos` WHERE NOT EXISTS (' printf 'SELECT * FROM `td`' printf ' AND `td`.`%s`=`todos`.`%s`' \ -- cgit v1.2.3-70-g09d2 From 9ef68dd176d10bebf92cd937f8b6115a9d40b0bf Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 4 Feb 2018 13:40:19 +0100 Subject: bin/build-master-status: be more flexible when creating the temporary todos-table --- bin/build-master-status | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 0e80bd6..d74ff8a 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -561,13 +561,13 @@ if ${web}; then # insert unfound todos # shellcheck disable=SC2016 { - printf 'CREATE TEMPORARY TABLE `td` (' - printf '`file` VARCHAR(64), ' - printf '`line` MEDIUMINT, ' - printf '`description` VARCHAR(128), ' - printf 'UNIQUE KEY `position` (`file`,`line`), ' - printf 'UNIQUE KEY `content` (`file`,`description`)' - printf ');' + printf 'SHOW CREATE TABLE `todos`' | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + ' + printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' while read -r file line desc; do printf '(' -- cgit v1.2.3-70-g09d2 From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-70-g09d2 From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:05:51 +0100 Subject: use "mysql -N" instead of cumbersome sedding --- bin/build-master-status | 5 ++--- bin/build-master-status-from-mysql | 46 ++++++++++---------------------------- bin/check-bugtracker | 5 ++--- bin/copy-to-build-support | 7 ++---- bin/db-update | 14 ++++-------- bin/get-assignment | 16 +++++-------- bin/get-package-updates | 6 ++--- bin/modify-package-state | 3 +-- bin/return-assignment | 20 +++++------------ bin/seed-build-list | 13 ++++------- bin/show-dependencies | 3 +-- bin/why-dont-you | 7 ++---- lib/common-functions | 7 ++---- lib/mysql-functions | 19 +++++----------- 14 files changed, 51 insertions(+), 120 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 9be7823..deb58da 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,10 +562,9 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d - 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index fa54177..d1985e7 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,11 +27,8 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" sort "${work_dir}/build-list" > \ @@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \ @@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" { @@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then # TODO: consider versions dependencies, too diff --git a/bin/check-bugtracker b/bin/check-bugtracker index f15757b..156f438 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,8 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' @@ -61,5 +60,5 @@ bug_list=$( tr '\n' ',' printf '"");\n' done - done | tee /dev/stderr | \ + done | \ mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 0b87107..e447596 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,11 +44,8 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${id}" ]; then continue diff --git a/bin/db-update b/bin/db-update index 534c245..78ff622 100755 --- a/bin/db-update +++ b/bin/db-update @@ -457,11 +457,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ sort -u @@ -496,11 +493,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ sort -u diff --git a/bin/get-assignment b/bin/get-assignment index bc67fab..964c6b7 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,11 +58,8 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' { # shellcheck disable=SC2154 @@ -157,8 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ -n "${currently_building}" ]; then @@ -214,9 +210,8 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ + mysql_run_query -N --batch --raw | \ sed ' - 1d y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ ' @@ -238,8 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index 11f384c..e855b19 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,8 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch } | \ sort -u ) @@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 8c5fac7..68b2384 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,8 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index ed29f34..a7e6623 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' @@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -283,9 +279,8 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ - grep -vxF '0'; then + mysql_run_query -N --raw --batch | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi @@ -441,10 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - ' + mysql_run_query -N --raw --batch } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index b9ae193..def9d56 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ @@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sort -u | \ - sed ' - y/\t/ / - / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d - ' | \ + tr '\t' ' ' | \ if ${update}; then # always block if locked exec 9> "${build_list_lock_file}" diff --git a/bin/show-dependencies b/bin/show-dependencies index 9e94fe3..ed7ca87 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,9 +10,8 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/\s.*$// ' | \ sort -u diff --git a/bin/why-dont-you b/bin/why-dont-you index b5d73c5..35b9419 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,11 +85,8 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' / NULL \S\+$/ b multi-dep diff --git a/lib/common-functions b/lib/common-functions index 740c266..ba434ad 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,11 +501,8 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ shrink_version 4 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 6cbc502..3e7e19f 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -668,9 +668,8 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - /^CONCAT("/d s,^,, s,$,, ' @@ -699,9 +698,8 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s,\t,/, s,\t,-, s,\t,:, @@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ { @@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-70-g09d2 From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 16:30:10 +0100 Subject: make -N --raw --batch the default arguments to mysql, explicitely override them if required --- bin/build-master-status | 2 +- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 2 +- bin/copy-to-build-support | 2 +- bin/db-update | 4 ++-- bin/get-assignment | 8 ++++---- bin/get-package-updates | 4 ++-- bin/modify-package-state | 2 +- bin/return-assignment | 8 ++++---- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/why-dont-you | 2 +- lib/common-functions | 2 +- lib/mysql-functions | 18 +++++++++--------- 14 files changed, 40 insertions(+), 40 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index deb58da..006750b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,7 +562,7 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7cf1b95..a72d396 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" @@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" @@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" @@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 156f438..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index e447596..b3decd4 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${id}" ]; then diff --git a/bin/db-update b/bin/db-update index 5efa220..5756494 100755 --- a/bin/db-update +++ b/bin/db-update @@ -467,7 +467,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ @@ -503,7 +503,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ diff --git a/bin/get-assignment b/bin/get-assignment index 964c6b7..2df3059 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' { @@ -154,7 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ -n "${currently_building}" ]; then @@ -210,7 +210,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw | \ + mysql_run_query | \ sed ' y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ @@ -233,7 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index e855b19..a95bd94 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -u ) @@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 68b2384..9b0571b 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index a7e6623..717a30e 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then @@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query -N --raw --batch + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -279,7 +279,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 @@ -436,7 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index def9d56..d49a8f3 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' @@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u | \ tr '\t' ' ' | \ if ${update}; then diff --git a/bin/show-dependencies b/bin/show-dependencies index ed7ca87..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/\s.*$// ' | \ @@ -177,7 +177,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/why-dont-you b/bin/why-dont-you index 35b9419..64dfa1b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' diff --git a/lib/common-functions b/lib/common-functions index 0f75cb8..c7a64fe 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 9d705d2..26d3db8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,17 +26,17 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then - ${mysql_command} "$@" + ${mysql_command} -N --raw --batch "$@" else { - printf '%s %s: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ | tee -a "${work_dir}/mysql.stdin" \ | tee -a "${work_dir}/mysql.stdout" \ >> "${work_dir}/mysql.stderr" tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ + | ${mysql_command} -N --raw --batch "$@" \ 2>> "${work_dir}/mysql.stderr" \ | tee -a "${work_dir}/mysql.stdout" for s in \ @@ -193,7 +193,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - mysql_run_query --html + mysql_run_query --html --column-names } # mysql_add_install_target $install_target @@ -693,7 +693,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,^,, s,$,, @@ -723,7 +723,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,\t,/, s,\t,-, @@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ @@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-70-g09d2 From 2747cf135820de7334144fd4e4a81c458093a08f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Mar 2018 15:47:09 +0100 Subject: bin/build-master-status: rely less on state files --- bin/build-master-status | 174 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 58 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 006750b..aa15fd4 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -64,63 +64,93 @@ tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT stable=$( - ls_master_mirror 'i686' | \ - grep -v 'testing$\|staging$\|-unstable$' | \ - while read -r dir; do - ls_master_mirror "i686/${dir}" - done | \ - grep -c '\.pkg\.tar\.xz$' + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="stable";\n' + } | \ + mysql_run_query ) tasks=$( - grep -c '^\S\+ \S\+ \S\+ \S\+$' \ - "${work_dir}/build-list" -) || true + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list";\n' + } | \ + mysql_run_query +) pending_packages=$( - grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ - tr ' ' '.' | \ - while read -r package; do - generate_package_metadata "${package}" 2>&1 > /dev/null - cat "${work_dir}/package-infos/${package}.packages" - done | - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list";\n' + } | \ + mysql_run_query ) next_tasks=$( + # shellcheck disable=SC2016 { - cat "${work_dir}/build-list" - find "${work_dir}/package-states" -maxdepth 1 \ - \( -name '*.broken' -o -name '*.blocked' \) \ - -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - p - ' + printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `dependencies`' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' WHERE `prov_r`.`name`="build-list"' + printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`' + printf ')' + printf ';\n' } | \ - sort | \ - uniq -u | \ - while read -r package git_revision mod_git_revision repository; do - if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then - echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - fi - done | \ - wc -l + mysql_run_query ) staging=$( - find "${work_dir}/package-states" -name '*.done' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="staging";\n' + } | \ + mysql_run_query ) testing=$( - find "${work_dir}/package-states" -name '*.testing' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="testing"' + printf ' AND NOT `binary_packages`.`is_tested`;\n' + } | \ + mysql_run_query ) tested=$( - find "${work_dir}/package-states" -name '*.tested' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="testing"' + printf ' AND `binary_packages`.`is_tested`;\n' + } | \ + mysql_run_query ) { find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ @@ -145,28 +175,56 @@ tested=$( sort -u > \ "${tmp_dir}/broken-packages-names" broken=$( - wc -l < \ - "${tmp_dir}/broken-packages-names" + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_assignments`.`is_broken`;\n' + } | \ + mysql_run_query ) blocked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_assignments`.`is_blocked` IS NOT NULL;\n' + } | \ + mysql_run_query ) locked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM' + printf ' `build_slaves`' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list";\n' + } | \ + mysql_run_query ) loops=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM' + printf ' `build_dependency_loops`' + } | \ + mysql_run_query ) looped_packages=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l + # shellcheck disable=SC2016 + { + printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM' + printf ' `build_dependency_loops`' + } | \ + mysql_run_query ) { -- cgit v1.2.3-70-g09d2 From 31b97c033f957bbd0cd3f63eef1dc38e7e4eb5d6 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 12:19:57 +0100 Subject: bin/build-master-status: cleanup --- bin/build-master-status | 870 +++++++++++------------------------------------- 1 file changed, 193 insertions(+), 677 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index aa15fd4..9e3f377 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -12,32 +12,24 @@ usage() { >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' - >&2 echo ' -w|--web:' - >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o hw \ + getopt -o h \ --long help \ - --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" -web=false - while true do case "$1" in -h|--help) usage 0 ;; - -w|--web) - web=true - ;; --) shift break @@ -63,691 +55,215 @@ fi tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -stable=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' - printf ' `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_repositories_repository_stabilities - printf ' WHERE `repository_stabilities`.`name`="stable";\n' - } | \ - mysql_run_query -) -tasks=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' - printf ' `build_assignments`' - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list";\n' - } | \ - mysql_run_query -) -pending_packages=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' - printf ' `binary_packages`' - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list";\n' - } | \ - mysql_run_query -) -next_tasks=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' - printf ' `build_assignments`' - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `dependencies`' - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_binary_packages`' - mysql_join_dependencies_install_target_providers - mysql_join_install_target_providers_binary_packages '' 'prov_bp' - mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' - printf ' WHERE `prov_r`.`name`="build-list"' - printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`' - printf ')' - printf ';\n' - } | \ - mysql_run_query -) -staging=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' - printf ' `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_repositories_repository_stabilities - printf ' WHERE `repository_stabilities`.`name`="staging";\n' - } | \ - mysql_run_query -) -testing=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' - printf ' `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_repositories_repository_stabilities - printf ' WHERE `repository_stabilities`.`name`="testing"' - printf ' AND NOT `binary_packages`.`is_tested`;\n' - } | \ - mysql_run_query -) -tested=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' - printf ' `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_repositories_repository_stabilities - printf ' WHERE `repository_stabilities`.`name`="testing"' - printf ' AND `binary_packages`.`is_tested`;\n' - } | \ - mysql_run_query -) -{ - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ - while read -r pkg rev mod_rev repo; do - if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then - echo "${pkg}" - fi - done - { - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}||' | \ - sort -u - } | \ - sort | \ - uniq -d -} | \ - sort -u > \ - "${tmp_dir}/broken-packages-names" -broken=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' - printf ' `build_assignments`' - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND `build_assignments`.`is_broken`;\n' - } | \ - mysql_run_query -) -blocked=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' - printf ' `build_assignments`' - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND `build_assignments`.`is_blocked` IS NOT NULL;\n' - } | \ - mysql_run_query -) -locked=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM' - printf ' `build_slaves`' - mysql_join_build_slaves_build_assignments - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="build-list";\n' - } | \ - mysql_run_query -) -loops=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM' - printf ' `build_dependency_loops`' - } | \ - mysql_run_query -) -looped_packages=$( - # shellcheck disable=SC2016 - { - printf 'SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM' - printf ' `build_dependency_loops`' - } | \ - mysql_run_query -) +"${base_dir}/bin/calculate-dependent-packages" { - printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ - "${stable}" \ - "$((staging+testing+tested+pending_packages))" - printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ - "$((tasks-broken))" \ - "${tasks}" \ - "${pending_packages}" \ - "${next_tasks}" - printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \ - "$((testing+tested))" \ - "${tested}" \ - "${staging}" - printf 'There are %d broken package builds.\n' \ - "${broken}" - if [ "${loops}" -ne 0 ]; then - printf 'There are %d loops containing %d package builds.\n' \ - "${loops}" \ - "${looped_packages}" - fi - if [ $((broken+testing+tested+staging)) -ne 0 ]; then - printf '%.1f%% of all packages are broken.\n' \ - "$( - echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \ - bc - )" - fi - if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then - printf '%.1f%% of the planned work has been done.\n' \ - "$( - echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \ - bc - )" - fi -} > \ - "${tmp_dir}/build-master-status.html" - -if ${web}; then - "${base_dir}/bin/calculate-dependent-packages" - { - printf '%s\n' \ - '' \ - '' \ - 'Status of archlinux32 build master' \ - '' \ - '' \ - '' - sed 's|$|
|' "${tmp_dir}/build-master-status.html" - printf '%s\n' \ - '
' \ - 'currently building packages:
' \ - '' - printf '' - printf '' \ - 'since (UTC)' \ - 'pkgname' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'build slave' - printf '' - find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \ - -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \ - -execdir sed ' - :a - $!{ - N - s/\n/, / - ba - } - ' '{}' \; | \ - sort -k1n,1 | \ - sed ' - s|^\S\+ || - s|\.locked | | - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - ' | \ - while read -r date time pkg rev mod_rev repo slaves; do - printf '' - printf '' \ - "${date} ${time}" \ - "${pkg}" \ - "

${rev}

" \ - "

$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")

" \ - "${repo}" \ - "${slaves}" - printf '\n' - done - printf '%s\n' \ - '
%s
%s
' \ - '' \ - '' - } | \ - sponge "${tmp_dir}/build-master-status.html" - end=$(($(date +%s)-7*24*60*60)) - { - [ -f "${webserver_directory}/statistics" ] && \ - cat "${webserver_directory}/statistics" - printf '%s ' \ - "$(date +%s)" \ - "${stable}" \ - "${tasks}" \ - "${pending_packages}" \ - "${staging}" \ - "${testing}" \ - "${broken}" \ - "${loops}" \ - "${looped_packages}" \ - "${locked}" \ - "${blocked}" \ - "${next_tasks}" \ - "${tested}" | \ - sed 's| $|\n|' - echo "${end}" - } | \ - sort -k1nr,1 | \ - sed -n " - /^${end}\$/q + printf '%s\n' \ + '' \ + '' \ + 'Todos in the build scripts' \ + '' \ + '' + find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ + -exec grep -nHF '' '{}' \; | \ + awk ' + { print $0 } + /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} + ' | \ + sed -n ' + s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ + T + N + s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ + :a + N + s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ + ta + s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p - " | \ - tac > \ - "${tmp_dir}/statistics" - - find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \ - -printf '%f build()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \ - -printf '%f check()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \ - -printf '%f prepare()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \ - -printf '%f package()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \ - -printf '%f source\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \ - -printf '%f dependencies\n' \ - \) -o \ - \( \ - -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \ - -printf '%f package-cache\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \ - -printf '%f run-as-root\n' \ - \) -o \ - -printf '%f unknown\n' \ - \) | \ - sed ' - s|\(\.[^.]\+\)\{3\} | | ' | \ - sort -u | \ + tee "${tmp_dir}/todos" | \ sed ' :a - $!N - s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/ - ta - P - D + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|TODO #\2 - \1 (line \3):\n| ' | \ - sort -k1,1 > \ - "${tmp_dir}/broken-packages.reason" - - { - printf '%s\n' \ - '' \ - '' \ - 'List of broken package builds' \ - '' \ - '' \ - '' \ - 'build logs
' \ - '' \ - '' - printf '' \ - 'package' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'compilations' \ - 'dependent' \ - 'build error' \ - 'blocked' - printf '\n' - find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.broken$||' | \ - sort -k1,1 | \ - join -j 1 - "${tmp_dir}/broken-packages.reason" | \ - sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \ - while read -r sf pkg rev mod_rev repo build_error; do - if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then - printf '1 ' - else - printf '0 ' - fi - printf '%s ' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" \ - "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ - "$( - # shellcheck disable=SC2010 - ls -t "${webserver_directory}/build-logs/error" | \ - grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$' - )" \ - "$( - { - grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ - echo 'x  ' - } | \ - cut -d' ' -f2 - )" \ - "${build_error}" - if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then - sed ' - s|\s\(wait for \)|\n\1|g - ' "${work_dir}/package-states/${sf}.blocked" | \ - while read -r blocked_reason; do - if echo "${blocked_reason}" | \ - grep -q '^wait for '; then - printf 'wait for ' - echo "${blocked_reason}" | \ - sed ' - s|^wait for || - s@\( and \| or \)@\n\1\n@ - ' | \ - while read -r reason; do - if [ "FS#${reason#FS#}" = "${reason}" ]; then - printf '%s' \ - "${reason#FS#}" \ - "${reason}" - elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then - printf '%s' \ - "${reason#FS32#}" \ - "${reason}" - elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then - printf '%s' \ - "${reason}" \ - "${reason}" - elif [ "${reason% *}" != "${reason}" ]; then - printf '%s' \ - "${reason}" - else - printf '%s' \ - "${reason}" - fi - if read -r operator; then - printf ' %s ' "${operator}" - fi - done - else - echo "${blocked_reason}" - fi - done | \ - tr '\n' ' ' - else - printf ' ' - fi - printf '\n' - done | \ - sort -k6n,6 | \ - while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do - if [ "${buildable}" -eq 0 ]; then - left='(' - right=')' - else - unset left - unset right - fi - printf '' - mod_rev=$( - modification_revision_link "${mod_rev}" "${repo}" "${pkg}" - ) - build_error=$( - echo "${build_error}" | \ - sed 's|,|, |g' - ) - printf '' \ - ''"${left}${pkg}${right}"'' \ - "

${rev}

" \ - "

${mod_rev}

" \ - "${repo}" \ - ''"${count}"'' \ - "${dependent}" \ - "${build_error}" \ - "${reason}" - printf '\n' - done - printf '%s\n' \ - '
%s
%s
' \ - '' \ - '' - } > \ - "${tmp_dir}/broken-packages.html" + sed ' + s|$|
| + ' + printf '%s\n' \ + '' \ + '' +} > \ + "${tmp_dir}/todos.html" - rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason" +if [ -s "${tmp_dir}/todos" ]; then + sed ' + :a + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | + s/\n$// + s/\n/\\n/g + ' -i "${tmp_dir}/todos" + while read -r file line desc; do + printf '%s %s %s\n' \ + "$(printf '%s' "${file}" | base64 -w0)" \ + "$(printf '%s' "${line}" | base64 -w0)" \ + "$(printf '%s' "${desc}" | base64 -w0)" + done < \ + "${tmp_dir}/todos" | \ + sponge "${tmp_dir}/todos" + # update todos + # shellcheck disable=SC2016 + while read -r file line desc; do + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`line`=from_base64("%s")' \ + "${line}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`description`=from_base64("%s");\n' \ + "${desc}" + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`description`=from_base64("%s")' \ + "${desc}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`line`=from_base64("%s");\n' \ + "${line}" + done < \ + "${tmp_dir}/todos" | \ + mysql_run_query + # insert unfound todos + # shellcheck disable=SC2016 { - printf '%s\n' \ - '' \ - '' \ - 'Todos in the build scripts' \ - '' \ - '' - find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ - -exec grep -nHF '' '{}' \; | \ - awk ' - { print $0 } - /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} - ' | \ - sed -n ' - s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ - T - N - s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ - :a - N - s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ - ta - s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ - p - ' | \ - tee "${tmp_dir}/todos" | \ + printf 'SHOW CREATE TABLE `todos`' | \ + mysql_run_query | \ sed ' - :a - N - /\n$/!ba - s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|TODO #\2 - \1 (line \3):\n| - ' | \ - sed ' - s|$|
| + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' - printf '%s\n' \ - '' \ - '' - } > \ - "${tmp_dir}/todos.html" - - if [ -s "${tmp_dir}/todos" ]; then - sed ' - :a - N - /\n$/!ba - s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | - s/\n$// - s/\n/\\n/g - ' -i "${tmp_dir}/todos" - while read -r file line desc; do - printf '%s %s %s\n' \ - "$(printf '%s' "${file}" | base64 -w0)" \ - "$(printf '%s' "${line}" | base64 -w0)" \ - "$(printf '%s' "${desc}" | base64 -w0)" - done < \ - "${tmp_dir}/todos" | \ - sponge "${tmp_dir}/todos" - # update todos - # shellcheck disable=SC2016 + printf ';\n' + printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' while read -r file line desc; do - printf 'UPDATE IGNORE `todos`' - printf ' SET `todos`.`line`=from_base64("%s")' \ - "${line}" - printf ' WHERE `todos`.`file`=from_base64("%s")' \ - "${file}" - printf ' AND `todos`.`description`=from_base64("%s");\n' \ - "${desc}" - - printf 'UPDATE IGNORE `todos`' - printf ' SET `todos`.`description`=from_base64("%s")' \ - "${desc}" - printf ' WHERE `todos`.`file`=from_base64("%s")' \ - "${file}" - printf ' AND `todos`.`line`=from_base64("%s");\n' \ - "${line}" + printf '(' + printf 'from_base64("%s"),' \ + "${file}" \ + "${line}" \ + "${desc}" | \ + sed 's/,$/),/' done < \ "${tmp_dir}/todos" | \ - mysql_run_query - # insert unfound todos - # shellcheck disable=SC2016 - { - printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query | \ - sed ' - 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / - ' - printf ';\n' - printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' - while read -r file line desc; do - printf '(' - printf 'from_base64("%s"),' \ - "${file}" \ - "${line}" \ - "${desc}" | \ - sed 's/,$/),/' - done < \ - "${tmp_dir}/todos" | \ - sed ' - s/,$// - ' - printf ';\n' - printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' - printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' - printf 'FROM `td` ' - printf 'WHERE NOT EXISTS (' - printf 'SELECT * FROM `todos`' - printf ' AND `td`.`%s`=`todos`.`%s`' \ - 'file' 'file' \ - 'line' 'line' \ - 'description' 'description' | \ - sed 's/^ AND / WHERE /' - printf ');\n' - - printf 'DELETE FROM `todos` WHERE NOT EXISTS (' - printf 'SELECT * FROM `td`' - printf ' AND `td`.`%s`=`todos`.`%s`' \ - 'file' 'file' \ - 'line' 'line' \ - 'description' 'description' | \ - sed 's/^ AND / WHERE /' - printf ');' - printf 'DROP TABLE `td`;\n' - printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' - printf 'SELECT * FROM `todos` ' - printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' - printf ') OR NOT EXISTS (' - printf 'SELECT * FROM `todos` ' - printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' - printf ');\n' - } | \ - mysql_run_query - rm -f "${tmp_dir}/todos" - fi - - { - printf '%s\n' \ - '' \ - '' \ - 'Blacklisted packages' \ - '' \ - '' \ - '' \ - '' - printf '' - printf '' \ - 'package' \ - 'reason' - printf '\n' - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ - tar -Ox | \ sed ' - s@FS#\([0-9]\+\)@\0@ - s@FS32#\([0-9]\+\)@\0@ - /.#/!s/$/#/ - s|\(.\)#|\1| + s/,$// ' - printf '%s\n' \ - '
%s
| - /^\s*#/{ - s/^\s*#\s*// - s|\s*\(\)|\1| - s/^// - } - s|^|
| - s|$|
' \ - '' \ - '' - } > \ - "${tmp_dir}/blacklist.html" + printf ';\n' + printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' + printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' + printf 'FROM `td` ' + printf 'WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');\n' - { - printf '%s\n' \ - '' \ - '' \ - 'log of ssh connections from build slaves' \ - '' \ - '' \ - '' - printf '' - printf '' \ - 'time' \ - 'build slave' \ - 'command' \ - 'arguments' - printf '\n' - if [ -r "${work_dir}/ssh-log" ]; then - tac "${work_dir}/ssh-log" | \ - while read -r date time slave command arguments; do - printf '' - printf '' \ - "${date} ${time}" \ - "${slave}" \ - "${command}" \ - "${arguments}" - printf '\n' - done - fi - printf '%s\n' \ - '
%s
%s
' \ - '' \ - '' - } > \ - "${tmp_dir}/ssh-log.html" + printf 'DELETE FROM `todos` WHERE NOT EXISTS (' + printf 'SELECT * FROM `td`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');' + printf 'DROP TABLE `td`;\n' + printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' + printf ') OR NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' + printf ');\n' + } | \ + mysql_run_query + rm -f "${tmp_dir}/todos" +fi - find "${tmp_dir}" -maxdepth 1 -type f | \ - while read -r file; do - cat "${file}" > \ - "${webserver_directory}/${file##*/}" - done +{ + printf '%s\n' \ + '' \ + '' \ + 'Blacklisted packages' \ + '' \ + '' \ + '' \ + '' + printf '' + printf '' \ + 'package' \ + 'reason' + printf '\n' + git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ + tar -Ox | \ + sed ' + s@FS#\([0-9]\+\)@\0@ + s@FS32#\([0-9]\+\)@\0@ + /.#/!s/$/#/ + s|\(.\)#|\1| + ' + printf '%s\n' \ + '
%s
| + /^\s*#/{ + s/^\s*#\s*// + s|\s*\(\)|\1| + s/^// + } + s|^|
| + s|$|
' \ + '' \ + '' +} > \ + "${tmp_dir}/blacklist.html" -else - cat "${tmp_dir}/build-master-status.html" -fi +{ + printf '%s\n' \ + '' \ + '' \ + 'log of ssh connections from build slaves' \ + '' \ + '' \ + '' + printf '' + printf '' \ + 'time' \ + 'build slave' \ + 'command' \ + 'arguments' + printf '\n' + if [ -r "${work_dir}/ssh-log" ]; then + tac "${work_dir}/ssh-log" | \ + while read -r date time slave command arguments; do + printf '' + printf '' \ + "${date} ${time}" \ + "${slave}" \ + "${command}" \ + "${arguments}" + printf '\n' + done + fi + printf '%s\n' \ + '
%s
%s
' \ + '' \ + '' +} > \ + "${tmp_dir}/ssh-log.html" + +find "${tmp_dir}" -maxdepth 1 -type f | \ + while read -r file; do + cat "${file}" > \ + "${webserver_directory}/${file##*/}" + done -- cgit v1.2.3-70-g09d2 From 112da5617dba1c83e5c3cfc10092c24ebc652a5f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 12:23:44 +0100 Subject: bin/build-master-status: do not call calculate-dependent-packages - noone uses the results currently --- bin/build-master-status | 2 -- 1 file changed, 2 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 9e3f377..18a5d5b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -55,8 +55,6 @@ fi tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -"${base_dir}/bin/calculate-dependent-packages" - { printf '%s\n' \ '' \ -- cgit v1.2.3-70-g09d2 From 3211e6e8de91e6cb462a1935032ffa209470f550 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 10:52:44 +0100 Subject: bin/build-master-status: blacklist.html is obsolete --- bin/build-master-status | 36 ------------------------------------ 1 file changed, 36 deletions(-) (limited to 'bin/build-master-status') diff --git a/bin/build-master-status b/bin/build-master-status index 18a5d5b..c93a9aa 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -190,42 +190,6 @@ if [ -s "${tmp_dir}/todos" ]; then rm -f "${tmp_dir}/todos" fi -{ - printf '%s\n' \ - '' \ - '' \ - 'Blacklisted packages' \ - '' \ - '' \ - '' \ - '' - printf '' - printf '' \ - 'package' \ - 'reason' - printf '\n' - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ - tar -Ox | \ - sed ' - s@FS#\([0-9]\+\)@\0@ - s@FS32#\([0-9]\+\)@\0@ - /.#/!s/$/#/ - s|\(.\)#|\1| - ' - printf '%s\n' \ - '
%s
| - /^\s*#/{ - s/^\s*#\s*// - s|\s*\(\)|\1| - s/^// - } - s|^|
| - s|$|
' \ - '' \ - '' -} > \ - "${tmp_dir}/blacklist.html" - { printf '%s\n' \ '' \ -- cgit v1.2.3-70-g09d2