From baba38d4dd908df6a1b24eeec935f3dad3d89a93 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 23 Apr 2018 15:16:58 +0200 Subject: lib/common-functions: decomission command-log --- lib/common-functions | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'lib') diff --git a/lib/common-functions b/lib/common-functions index 59af46f..d959094 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -9,17 +9,6 @@ if [ -z "${base_dir}" ]; then . '../conf/default.conf' fi -if [ ! -s "${work_dir}/build-master-sanity" ]; then - { - date - printf 'sourcing common-functions for %s\n' "$0" - printf '%s parameters:' "$#" - printf ' "%s"' "$@" - printf '\n' - } >> \ - "${work_dir}/command-log" -fi - # find_pkgbuilds package repository git_repository git_revision mod_git_revision # find the PKGBUILD and modification of $package from $repository # sets $PKGBUILD and $PKGBUILD_mod -- cgit v1.2.3-54-g00ecf From 42235c298c9666b06cf1d6ced7346882ce0f1b33 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 25 Apr 2018 14:14:39 +0200 Subject: lib/mysql-functions: mysql_cleanup: a build_assignment might be handed out a couple of times, with one set of built packages not on the build-list anymore - do not remove build_assignments in that case --- lib/mysql-functions | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index 8101e0c..a15a8bb 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -882,16 +882,19 @@ mysql_cleanup() { printf ');\n' # remove jobs from build slaves that are not on the build-list if [ "${operator}" = 'DELETE' ]; then - printf 'UPDATE `build_slaves`' + printf 'UPDATE `repositories`' else - printf 'SELECT COUNT(DISTINCT `build_slaves`.`id`) FROM `build_slaves`' + printf 'SELECT COUNT(DISTINCT `build_slaves`.`id`) FROM `repositories`' fi - mysql_join_build_slaves_binary_packages - mysql_join_binary_packages_repositories + mysql_join_repositories_binary_packages + printf ' AND `repositories`.`name`="build-list"' + printf ' RIGHT' + mysql_join_binary_packages_build_slaves if [ "${operator}" = 'DELETE' ]; then printf ' SET `build_slaves`.`currently_building`=NULL' fi - printf ' WHERE `repositories`.`name`!="build-list";\n' + printf ' WHERE `build_slaves`.`currently_building` IS NOT NULL' + printf ' AND `repositories`.`id` IS NULL;\n' # remove build orders from build slaves which have not connected within 1h if [ "${operator}" = 'DELETE' ]; then printf 'UPDATE `build_slaves`' -- cgit v1.2.3-54-g00ecf From d65b3fe76adfad2c1db5285bf6fe35aee0f777f2 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 25 Apr 2018 14:40:16 +0200 Subject: lib/mysql-functions: add TODO --- lib/mysql-functions | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index a15a8bb..359caf3 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -9,6 +9,9 @@ if [ -z "${base_dir}" ]; then . '../conf/default.conf' fi +# TODO: replace most/all insert-select queries by separate select and insert +# queries + # base64_encode_each encode each line of stdin with base64 base64_encode_each() { -- cgit v1.2.3-54-g00ecf From debfc92ee0263d726c4bb9587351e7f9f2f8aef3 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 26 Apr 2018 11:13:12 +0200 Subject: lib/mysql-functions: mysql_add_package_source: insert-select -> select + insert --- lib/mysql-functions | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index 359caf3..6739485 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -109,6 +109,7 @@ mysql_add_package_source() { local repo_path local commit_time local pkgbuild + local upstream_package_repository_id if grep -qx 'PKGBUILD_mod = \S\+' "${srcinfo_file}"; then uses_modification=1 @@ -148,11 +149,31 @@ mysql_add_package_source() { fi printf 'I' >&2 + upstream_package_repository_id=$( + { + printf 'SELECT `upstream_repositories`.`id`' + printf ' FROM `upstream_repositories`' + printf ' WHERE `upstream_repositories`.`name` = from_base64("%s")' \ + "$( + printf '%s' "${upstream_package_repository}" | \ + base64 -w0 + )" + printf ' LIMIT 1;\n' + } | \ + mysql_run_query + ) + + if [ -z "${upstream_package_repository_id}" ]; then + >&2 printf 'Cannot find upstream package repository "%s" in the database.\n' \ + "${upstream_package_repository}" + exit 2 + fi + { printf 'INSERT IGNORE INTO package_sources' printf ' (`pkgbase`,`git_revision`,`mod_git_revision`,`upstream_package_repository`,`uses_upstream`,`uses_modification`,`commit_time`)' - printf ' SELECT' - printf ' from_base64("%s"),' \ + printf ' VALUES (' + printf 'from_base64("%s"),' \ "$( printf '%s' "${pkgbase}" | \ base64 -w0 @@ -165,14 +186,12 @@ mysql_add_package_source() { printf '%s' "${mod_git_revision}" | \ base64 -w0 )" - printf ' `upstream_repositories`.`id`,%s,%s,from_unixtime(%s)' \ - "${uses_upstream}" "${uses_modification}" "${commit_time}" - printf ' FROM `upstream_repositories`' - printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ - "$( - printf '%s' "${upstream_package_repository}" | \ - base64 -w0 - )" + printf '%s,' \ + "${upstream_package_repository_id}" \ + "${uses_upstream}" \ + "${uses_modification}" + printf 'from_unixtime(%s));\n' \ + "${commit_time}" } | \ mysql_run_query printf 'J' >&2 -- cgit v1.2.3-54-g00ecf From fb4e4757f7ca9ea8c12ed97c1887bb67ebd5505f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 2 May 2018 11:25:16 +0200 Subject: lib/mysql-functions: mysql_cleanup is "unimportant" --- lib/mysql-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index 6739485..dad6bc6 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -941,7 +941,7 @@ mysql_cleanup() { printf ' AND (`build_slaves`.`last_action` IS NOT NULL' printf ' OR `build_slaves`.`logged_lines` IS NOT NULL);\n' } | \ - mysql_run_query + mysql_run_query 'unimportant' } # mysql_query_has_pending_dependencies `build_assignment`.`id` -- cgit v1.2.3-54-g00ecf From 71e15c002d5596d84b52c05625478a2ed3123695 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 2 May 2018 11:32:17 +0200 Subject: lib/mysql-functions: remove residues of unimportant mysql queries in case of error --- lib/mysql-functions | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index dad6bc6..5ce7e0c 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -62,21 +62,27 @@ mysql_run_query() { # a present query_file means there was an error if [ -f "${query_stdin}" ]; then >&2 printf 'I could not complete a mysql query!\n' - if [ ! -s "${work_dir}/build-master-sanity" ]; then + if [ ! -s "${work_dir}/build-master-sanity" ] && \ + [ -z "${file_name_extra}" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_stdin##*/}" \ | irc_say - for file in \ - "${query_stdin}" \ - "${query_stdout}" \ - "${query_stderr}"; do - cp "${file}" "${webserver_directory}/mysql-queries/${file##*/}.txt" - chmod go+r "${webserver_directory}/mysql-queries/${file##*/}.txt" - done fi + for file in \ + "${query_stdin}" \ + "${query_stdout}" \ + "${query_stderr}"; do + cp "${file}" "${webserver_directory}/mysql-queries/${file##*/}.txt" + chmod go+r "${webserver_directory}/mysql-queries/${file##*/}.txt" + done if [ -z "${file_name_extra}" ]; then echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" + else + rm -f \ + "${query_stdin}" \ + "${query_stdout}" \ + "${query_stderr}" fi return 2 fi -- cgit v1.2.3-54-g00ecf From 62e0e1d35072dcd91ad19a863b27f069c6ed0f38 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 3 May 2018 14:13:40 +0200 Subject: lib/mysql-functions: new columns => new joins --- lib/mysql-functions | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index 5ce7e0c..df9cf77 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -1186,6 +1186,7 @@ for link in \ 'package_sources:upstream_package_repository:upstream_repositories' \ \ 'repositories:stability:repository_stabilities' \ + 'repositories:architecture:architectures' \ \ 'repository_moves:upstream_package_repository:upstream_repositories' \ 'repository_moves:upstream_package_repository upstream_package_repository:package_sources' \ -- cgit v1.2.3-54-g00ecf From 05f9e7441da092c9353c8bd4a050aab9868b2e22 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 3 May 2018 14:34:19 +0200 Subject: lib/mysql-functions: check more architectures in mysql_sanity_check() --- lib/mysql-functions | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index df9cf77..b925b2e 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -746,29 +746,34 @@ mysql_sanity_check() { temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - for dir in $(ls_master_mirror 'i686'); do - ls_master_mirror "i686/${dir}" | \ - sed -n ' - s/\.pkg\.tar\.xz$// - T - s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ - s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ - s|^|'"${dir}"'/| - p - ' + # shellcheck disable=SC2041 + for arch in 'i686'; do + for dir in $(ls_master_mirror "${arch}"); do + ls_master_mirror "${arch}/${dir}" | \ + sed -n ' + s/\.pkg\.tar\.xz$// + T + s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ + s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ + s|^|'"${arch}"'/'"${dir}"'/| + p + ' + done done | \ sort > \ "${temp_dir}/master-mirror-listing" { - printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' + printf 'SELECT `r_a`.`name`,`repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures mysql_join_binary_packages_repositories + mysql_join_repositories_architectures '' 'r_a' printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ mysql_run_query | \ sed ' + s,\t,/, s,\t,/, s,\t,-, s,\t,:, -- cgit v1.2.3-54-g00ecf From 0dc5104fae587127b9219ca9e79f434d64c5bb90 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 3 May 2018 14:36:14 +0200 Subject: add several todos --- bin/copy-to-build-support | 2 ++ bin/db-update | 2 ++ bin/delete-packages | 2 ++ lib/common-functions | 2 ++ lib/mysql-functions | 1 + 5 files changed, 9 insertions(+) (limited to 'lib') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 5382235..1164c1d 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -2,6 +2,8 @@ # copy the given package(s) into build-support +# TODO: enable separate repositories for each architecture + # shellcheck source=../conf/default.conf . "${0%/*}/../conf/default.conf" diff --git a/bin/db-update b/bin/db-update index 98bc884..28e9861 100755 --- a/bin/db-update +++ b/bin/db-update @@ -12,6 +12,8 @@ # TODO: we should delete more packages than just the ones in repositories # where we move to (think of [extra] -> [community]) +# TODO: enable separate repositories for each architecture + # shellcheck disable=SC2039 # shellcheck source=../conf/default.conf . "${0%/*}/../conf/default.conf" diff --git a/bin/delete-packages b/bin/delete-packages index 1d4b230..5f60b40 100755 --- a/bin/delete-packages +++ b/bin/delete-packages @@ -15,6 +15,8 @@ # TODO: delete other to-be-deleted packages if asked to do so +# TODO: enable separate repositories for each architecture + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/lib/common-functions b/lib/common-functions index d959094..038af38 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -550,6 +550,8 @@ recursively_umount_and_rm() { # mangle $arch in PKBUILDs to contain i486, i586, i686 # append $sub_pkgrel to the pkgrel +# TODO: actually include the other architectures + mangle_pkgbuild() { local PKGBUILD="$1" local sub_pkgrel="$2" diff --git a/lib/mysql-functions b/lib/mysql-functions index b925b2e..83b9296 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -292,6 +292,7 @@ mysql_generate_package_metadata() { exit 2 fi # remove empty lines and unsupported architectures + # TODO: support more architectures sed -i ' /^[^=]*=\s*$/d /^\s*arch = /{ -- cgit v1.2.3-54-g00ecf