From 2e6d97b8ddcd16f029673bc6f101fc3baf141e51 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sat, 24 Feb 2018 19:41:56 +0100 Subject: bin/common-functions -> lib/common-functions, bin/mysql-functions -> lib/mysql-functions --- lib/mysql-functions | 1098 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1098 insertions(+) create mode 100755 lib/mysql-functions (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions new file mode 100755 index 0000000..185d9a9 --- /dev/null +++ b/lib/mysql-functions @@ -0,0 +1,1098 @@ +#!/bin/sh + +# contains functions used to access mysql db + +# shellcheck disable=SC2016,SC2039 + +if [ -z "${base_dir}" ]; then + # just to make shellcheck happy + . 'conf/default.conf' +fi + +# base64_encode_each encode each line of stdin with base64 + +base64_encode_each() { + local line + + while read -r line; do + printf '%s' \ + "${line}" | \ + base64 -w0 + printf '\n' + done +} + +# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository + +# shellcheck disable=SC2086 +mysql_add_package_source() { + local names='pkgbase git_revision mod_git_revision upstream_package_repository' + local values + local uses_upstream + local uses_modification + local repo + + if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then + uses_modification=1 + else + uses_modification=0 + fi + uses_upstream=0 + for repo in ${repo_names}; do + if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \ + tar -t 2>/dev/null | \ + sed 's,-x86_64/,-any/,' | \ + grep -qFx "$1/repos/$4-any/PKGBUILD"; then + uses_upstream=1 + fi + done + + for _ in ${names}; do + values="${values}$( + printf '%s' "$1" | \ + base64 -w0 + ) " + shift + done + values="${values% }" + + { + printf 'INSERT IGNORE INTO package_sources' + printf ' (' + printf '`%s`, ' ${names} + printf '`uses_upstream`,`uses_modification`' + printf ') SELECT' + printf ' from_base64("%s"), ' ${values% *} + printf ' `upstream_repositories`.`id`,%s,%s' \ + ${uses_upstream} ${uses_modification} + printf ' FROM `upstream_repositories`' + printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ + "${values##* }" + } | \ + ${mysql_command} +} + +# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository + +# shellcheck disable=SC2031,SC2086,SC2154 +mysql_add_binary_package() { + local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository' + local name + for name in ${names}; do + eval 'local '"${name}" + eval "${name}"'=$( + printf "%s" "$1" | + base64 -w0 + )' + shift + done + + { + printf 'INSERT IGNORE INTO binary_packages' + printf ' (' + printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted' + printf ') SELECT' + printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" + printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' + printf ' 0, 0, 0' + printf ' FROM' + printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' + printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' + printf ' WHERE' + printf ' `%s`.`name` = from_base64("%s") AND' \ + 'repositories' "${repository}" \ + 'architectures' "${architecture}" + printf ' `package_sources`.`%s` = from_base64("%s") AND' \ + 'pkgbase' "${pkgbase}" \ + 'git_revision' "${git_revision}" \ + 'mod_git_revision' "${mod_git_revision}" + printf ' `upstream_repositories`.`name` = from_base64("%s")' \ + "${upstream_package_repository}" + } | \ + sed ' + s|, )|)|g + s|, FROM| FROM|g + ' | \ + ${mysql_command} +} + +# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel + +# shellcheck disable=SC2031,SC2086,SC2154 +mysql_show_binary_package() { + local names='pkgname pkgver pkgrel sub_pkgrel' + local name + for name in ${names}; do + eval 'local '"${name}" + eval "${name}"'=$( + printf "%s" "$1" | + base64 -w0 + )' + shift + done + + { + printf 'SELECT' + printf ' `%s`.`%s`,' \ + 'repositories' 'name' \ + 'binary_packages' 'pkgname' \ + 'package_sources' 'pkgver' \ + 'package_sources' 'pkgrel' \ + 'binary_packages' 'sub_pkgrel' \ + 'architectures' 'name' \ + 'package_sources' 'pkgbase' \ + 'package_sources' 'git_revision' \ + 'package_sources' 'mod_git_revision' \ + 'upstream_repositories' 'name' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE' + printf ' `%s`.`%s` = from_base64("%s") AND' \ + 'binary_packages' 'pkgname' "${pkgname}" \ + 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ + 'package_sources' 'pkgver' "${pkgver}" \ + 'package_sources' 'pkgrel' "${pkgrel}" + printf ';' + } | \ + sed ' + s|, FROM| FROM|g + s|AND;|;|g + ' | \ + ${mysql_command} --html +} + +# mysql_add_install_target $install_target + +# shellcheck disable=2086 +mysql_add_install_target() { + local install_target + install_target=$( + printf "%s" "$1" | \ + base64 -w0 + ) + + { + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' VALUES (from_base64("%s"))' \ + "${install_target}" + } | \ + ${mysql_command} +} + +# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository +# or +# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository +# if sub_pkgrel should be determined automatically +# and +# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository +# or +# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository +# if $sub_pkgrel should be forced + +# generate the meta data of a package (dependencies, built packages, ...) in the database + +mysql_generate_package_metadata() { + + ( # new shell is intentional + case "$1" in + ''|*[!0-9]*) + unset forced_sub_pkgrel + ;; + *) + forced_sub_pkgrel=$( + printf '%s' "$1" | \ + base64 -w0 + ) + shift + ;; + esac + current_repository="$1" + package="$2" + + if [ $# -eq 2 ]; then + # second form + repository="${package##*.}" + package="${package%.*}" + mod_git_revision="${package##*.}" + package="${package%.*}" + git_revision="${package##*.}" + package="${package%.*}" + else + git_revision="$3" + mod_git_revision="$4" + repository="$5" + fi + + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + printf '.' >&2 + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then + printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" + exit 2 + fi + # remove empty lines and unsupported architectures + sed -i ' + /^[^=]*=\s*$/d + /^\s*arch = /{ + / \(i686\|any\)$/!d + } + ' "${temp_dir}/SRCINFO" + + if [ ! -s "${temp_dir}/SRCINFO" ]; then + >&2 printf '"make_source_info" had empty output - eh, what?\n' + exit 2 + fi + printf '\n\n' >> "${temp_dir}/SRCINFO" + + printf '.' >&2 + pkgbase=$( + grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ + cut -d' ' -f3 + ) + if [ -z "${pkgbase}" ]; then + >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' + exit 2 + fi + + # add the package source + mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + printf '.' >&2 + + # now we encode everything in base64 + current_repository=$( + printf '%s' "${current_repository}" | \ + base64 -w0 + ) + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -w0 + ) + git_revision=$( + printf '%s' "${git_revision}" | \ + base64 -w0 + ) + mod_git_revision=$( + printf '%s' "${mod_git_revision}" | \ + base64 -w0 + ) + repository=$( + printf '%s' "${repository}" | \ + base64 -w0 + ) + + # add the build assignment(s) + { + archs=$( + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + grep -vxF 'any' | \ + sort -u + ) + if [ -z "${archs}" ]; then + echo 'any' + else + printf '%s\n' "${archs}" + fi + } | \ + while read -r arch; do + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' + printf ' FROM `architectures` JOIN `package_sources`' + printf ' WHERE `architectures`.`name` = from_base64("%s")' \ + "$( + printf '%s' "${arch}" | \ + base64 -w0 + )" + printf ' AND `package_sources`.`%s` = from_base64("%s")' \ + 'pkgbase' "${pkgbase}" \ + 'git_revision' "${git_revision}" \ + 'mod_git_revision' "${mod_git_revision}" + printf ';\n' + done > \ + "${temp_dir}/add-build-assignments-command" + + # TODO: correctly link between binary_packages and build_assignments using any_arch + + # shellcheck disable=SC2034 + # select any specific arch (which will be building the 'any' part of a split package) + any_arch=$( + { + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + sort -r | \ + grep -vxFm 1 'any' || \ + echo 'any' + } | \ + base64_encode_each + ) + + grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ + cut -d' ' -f3 | \ + while read -r pkgname; do + pkgname64=$( + printf '%s' "${pkgname}" | \ + base64 -w0 + ) + sed -n ' + /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ + :a + N + /\n$/{ + p + T + } + ba + } + ' "${temp_dir}/SRCINFO" | \ + sed ' + /^\S/d + s/^\s*// + ' > \ + "${temp_dir}/BINARYINFO.${pkgname64}" + + grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ + cut -d' ' -f3 | \ + while read -r arch; do + arch64=$( + printf '%s' "${arch}" | \ + base64 -w0 + ) + sed ' + s/^\(\S\+\)_'"${arch}"' = /\1 = / + ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ + "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" + done + done + find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ + while read -r _ pkgname arch; do + pkgver=$( + grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + pkgrel=$( + grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + epoch=$( + { + grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \ + echo 'epoch = 0' + } | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + provides=$( + grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + makedepends=$( + grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + checkdepends=$( + grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + rundepends=$( + grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + if [ -n "${forced_sub_pkgrel}" ]; then + sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")' + else + sub_pkgrel=$( + printf '(SELECT COALESCE(' + # do not add binary packages which are currently on the + # build-list or in $current_repository (beware of split + # packages!) + printf '(SELECT `sub_pkgrel` FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + printf ' `architectures`.`name`=from_base64("%s")' \ + "${arch}" + printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \ + "${current_repository}" + # max(sub_pkgrel)+1 + printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' + mysql_join_binary_packages_architectures + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then + # 'any' gets higher sub_pkgrel than any architecture + printf ' 1' + else + # not-'any' gets higher sub_pkgrel than same or 'any' architecture + printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \ + "${arch}" + fi + printf ')' + printf ',0))' + ) + fi + { + printf 'INSERT IGNORE INTO `binary_packages` (' + printf '`%s`,' \ + 'build_assignment' \ + 'repository' \ + 'architecture' \ + 'epoch' \ + 'pkgver' \ + 'pkgrel' \ + 'pkgname' \ + 'sub_pkgrel' \ + 'has_issues' \ + 'is_tested' \ + 'is_to_be_deleted' + printf ') SELECT ' + printf '`%s`.`id`,' \ + 'build_assignments' \ + 'repositories' \ + 'architectures' + printf 'from_base64("%s"),' \ + "${epoch}" \ + "${pkgver}" \ + "${pkgrel}" \ + "${pkgname}" + printf '%s,0,0,0 FROM' \ + "${sub_pkgrel}" + printf ' `%s` JOIN' \ + 'repositories' \ + 'architectures' \ + 'build_assignments' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE' + printf ' `%s`.`%s` = from_base64("%s") AND' \ + 'repositories' 'name' "${current_repository}" \ + 'architectures' 'name' "${arch}" \ + 'package_sources' 'pkgbase' "${pkgbase}" \ + 'package_sources' 'git_revision' "${git_revision}" \ + 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ + 'upstream_repositories' 'name' "${repository}" + printf ';\n' + } | \ + sed ' + s|,)|)|g + s| JOIN JOIN | JOIN | + s| AND;$|;| + ' >> \ + "${temp_dir}/add-binary-packages-command" + { + printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' + + printf 'INSERT INTO `provides` VALUES\n' + echo "${provides}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' (from_base64("%s"));\n' \ + "${pkgname}" + + printf 'INSERT INTO `rundepends` VALUES\n' + echo "${rundepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' ("base");\n' + + echo "${checkdepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + 1 s/^/INSERT INTO `checkdepends` VALUES \n/ + $ s/,$/;/ + ' + + printf 'INSERT INTO `makedepends` VALUES\n' + echo "${makedepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' ("base-devel");\n' + + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' SELECT (`name`) FROM `%s` UNION' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' | \ + sed 's| UNION$|;\n|' + + for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do + case "${link}" in + 'provides') + printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' + printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' + ;; + 'makedepends'|'checkdepends'|'rundepends') + printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' + printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' + printf ' `dependency_types` JOIN' + ;; + esac + printf ' `binary_packages`' + mysql_join_binary_packages_architectures + printf ' JOIN `install_targets`' + printf ' JOIN `%s`' "${link}" + printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" + printf ' WHERE' + if [ "${link}" = 'makedepends' ] || \ + [ "${link}" = 'checkdepends' ] || \ + [ "${link}" = 'rundepends' ]; then + printf ' `dependency_types`.`name` = "%s" AND' \ + "${link%depends}" + fi + printf ' `binary_packages`.`%s` = from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + # we do not want to match the sub_pkgrel: + # a) it is tedious to do so (because it may be calculated + # dynamically) + # b) it is not necessary to do so: if only the sub_pkgrel + # changed, the dependencies and provided install_targets + # should not have changed + printf ' `architectures`.`name` = from_base64("%s");\n' \ + "${arch}" + # the repository is of no relevance: it hardly matters for + # the dependencies + done + + printf 'DROP TABLE `%s`;\n' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' + } >> \ + "${temp_dir}/add-install-targets-command" + done + printf '.' >&2 + + { + if [ -s "${temp_dir}/add-build-assignments-command" ]; then + cat "${temp_dir}/add-build-assignments-command" + fi + if [ -s "${temp_dir}/add-binary-packages-command" ]; then + cat "${temp_dir}/add-binary-packages-command" + fi + if [ -s "${temp_dir}/add-install-targets-command" ]; then + cat "${temp_dir}/add-install-targets-command" + fi + } | \ + ${mysql_command} + printf '.' >&2 + + ) +} + +# mysql_sanity_check +# do a sanity check on the mysql database + +mysql_sanity_check() { + { + printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + mysql_join_binary_packages_architectures '' 'bp_arch' + mysql_join_build_assignments_architectures '' 'ba_arch' + printf ' WHERE `bp_arch`.`name`!="any"' + printf ' AND `ba_arch`.`name`="any";\n' + printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' + printf ' FROM `binary_packages` AS `a`' + printf ' JOIN `binary_packages` AS `b`' + printf ' ON `a`.`pkgname`=`b`.`pkgname`' + printf ' AND `a`.`repository`=`b`.`repository`' + printf ' AND `a`.`id`!=`b`.`id`' + mysql_join_binary_packages_repositories 'a' + printf ' WHERE `repositories`.`name`="build-list";\n' + printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' + printf ' FROM `binary_packages` AS `a`' + printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' + mysql_join_binary_packages_repositories 'a' 'arep' + mysql_join_binary_packages_repositories 'b' 'brep' + printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' + printf ' AND `%srep`.`name`="build-list"' \ + 'a' 'b' + printf ';\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + /^CONCAT("/d + s,^,, + s,$,, + ' + ( # new shell is intentional + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + for dir in $(ls_master_mirror 'i686'); do + ls_master_mirror "i686/${dir}" | \ + sed -n ' + s/\.pkg\.tar\.xz$// + T + s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ + s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ + s|^|'"${dir}"'/| + p + ' + done | \ + sort > \ + "${temp_dir}/master-mirror-listing" + + { + printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`is_on_master_mirror`' + } | \ + ${mysql_command} --batch | \ + sed ' + 1d + s,\t,/, + s,\t,-, + s,\t,:, + s,\t,-, + s,\t,., + s,\t,-, + ' | \ + sort > \ + "${temp_dir}/mysql-packages" + + diff -u \ + "${temp_dir}/master-mirror-listing" \ + "${temp_dir}/mysql-packages" + ) +} + +mysql_find_build_assignment_loops() { + new_loops=$( + { + printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' + printf ' FROM `dependencies`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'packages_dependency' + mysql_join_dependencies_binary_packages '' 'packages_dependent' + mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' + mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' + printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + y/\t/ / + ' | \ + tsort 2>&1 >/dev/null | \ + sed 's/^tsort:\s*//' | \ + { + loop=0 + while read -r id; do + if [ "x${id}" = 'x-: input contains a loop:' ]; then + loop=$((loop+1)) + continue + fi + if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then + >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}" + continue + fi + printf '(%s,%s),' "${loop}" "${id}" + done | \ + sed 's/,$//' + } + ) + { + printf 'DELETE FROM `build_dependency_loops`;\n' + if [ -n "${new_loops}" ]; then + printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \ + "${new_loops}" + fi + } | \ + ${mysql_command} +} + +# mysql_cleanup [dry] +# clean up left overs from mysql database +mysql_cleanup() { + local operator + if [ "$#" = '0' ]; then + operator='DELETE' + elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then + operator='SELECT COUNT(*)' + else + >&2 echo 'Unknown parameter' + >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".' + exit 2 + fi + { + # remove to-be-decided binary_packages + printf '%s ' \ + "${operator}" + if [ "${operator}" = 'DELETE' ]; then + printf '`binary_packages` ' + fi + printf 'FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="to-be-decided";\n' + # remove dependencies w/o binary_package or install_target + printf '%s FROM `dependencies` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`' + printf ') OR NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `install_targets` ' + printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`' + printf ');\n' + # remove install_target_providers w/o binary_package or install_target + printf '%s FROM `install_target_providers` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`' + printf ') OR NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `install_targets` ' + printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`' + printf ');\n' + # remove build_assignments w/o binary_package + printf '%s FROM `build_assignments` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ');\n' + # remove failed_builds with unbroken build_assignments + printf '%s ' \ + "${operator}" + if [ "${operator}" = 'DELETE' ]; then + printf '`failed_builds` ' + fi + printf 'FROM `failed_builds` ' + mysql_join_failed_builds_build_assignments + printf 'WHERE NOT `build_assignments`.`is_broken`' + printf ';\n' + # remove failed_builds w/o build_assignment + printf '%s FROM `failed_builds` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `build_assignments` ' + printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`' + printf ');\n' + # remove package_sources w/o build_assignment + printf '%s FROM `package_sources` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `build_assignments` ' + printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' + printf ');\n' + printf 'UPDATE `build_slaves`' + mysql_join_build_slaves_binary_packages + mysql_join_binary_packages_repositories + printf ' SET `currently_building`=NULL' + printf ' WHERE `repositories`.`name`!="build-list";\n' + } | \ + ${mysql_command} +} + +# mysql_query_has_pending_dependencies `build_assignment`.`id` +# print a mysql query giving wether dependencies are pending +mysql_query_has_pending_dependencies() { + printf 'EXISTS (' + printf 'SELECT * FROM `binary_packages` as `to_dos`' + mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' + mysql_join_binary_packages_dependencies 'to_dos' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'bin_deps' + mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' + printf ' WHERE' + printf ' `%s`.`name`="build-list" AND' \ + 'dep_repos' 'to_do_repos' + printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND' + printf ' `to_dos`.`build_assignment`=%s' \ + "$1" + printf ')' +} + +# mysql_query_is_part_of_loop `build_assignment`.`id` +# print a mysql query giving wether the package is part of a loop +mysql_query_is_part_of_loop() { + printf 'EXISTS (' + printf 'SELECT * FROM `build_dependency_loops`' + printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \ + "$1" + printf ')' +} + +# mysql_query_select_pkgbase_and_revision +# print the part of a mysql query giving: +# pkgbase git_revision mod_git_revision upstream_package_repository +mysql_query_select_pkgbase_and_revision() { + printf '`package_sources`.`%s`,' \ + 'pkgbase' \ + 'git_revision' \ + 'mod_git_revision' + printf '`upstream_repositories`.`name`' + printf ' FROM `build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories +} + +# mysql_repair_binary_packages_without_build_assignment +# try to generate valid build assignments to binary packages without +# a valid one yet +mysql_repair_binary_packages_without_build_assignment() { + { + printf 'SELECT ' + printf '`binary_packages`.`id`' + printf ',replace(to_base64(%s),"\\n","")' \ + '`binary_packages`.`pkgname`' \ + '`architectures`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + printf ' WHERE `binary_packages`.`build_assignment`<0' + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + while read -r id pkgname arch; do + pkgname=$( + printf '%s' "${pkgname}" | \ + base64 -d + ) + pkgbase=$( + curl -Ss "$( + printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ + "${pkgname}" + )" | \ + sed ' + s/^.*"results":\s*\[// + s/}\s*,\s*{/\n/g + ' | \ + grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ + tr ',' '\n' | \ + grep '"pkgbase":' | \ + cut -d'"' -f4 | \ + sort -u | \ + head -n1 + ) + if [ -z "${pkgbase}" ] && \ + { + printf 'SELECT count(*) FROM `package_sources`' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ + "$(printf '%s' "${pkgname}" | base64 -w0)" + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + grep -qvxF '0'; then + pkgbase="${pkgname}" + fi + if [ -z "${pkgbase}" ]; then + >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" + continue + fi + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -w0 + ) + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' + printf ' FROM `package_sources`' + printf ' JOIN `architectures`' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" + printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" + printf ' LIMIT 1;\n' + printf 'UPDATE `binary_packages`' + printf ' JOIN `build_assignments`' + mysql_join_binary_packages_build_assignments + printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ' WHERE `binary_packages`.`id`=%s' "${id}" + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" + done | \ + ${mysql_command} +} + +# mysql_remove_duplicate_binary_packages +# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel, +# having difference of 1 in sub_pkgrel + +mysql_remove_duplicate_build_order() { + { + printf 'CREATE TEMPORARY TABLE `ren`' + printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n' + printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)' + printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`' + printf ' FROM `binary_packages` as `old`' + printf ' JOIN `binary_packages` as `new` ON' + printf ' `old`.`%s`=`new`.`%s` AND' \ + 'pkgname' 'pkgname' \ + 'epoch' 'epoch' \ + 'pkgver' 'pkgver' \ + 'pkgrel' 'pkgrel' + printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' + mysql_join_binary_packages_repositories 'old' 'orep' + mysql_join_binary_packages_repositories 'new' 'nrep' + printf ' WHERE `orep`.`name`!="to-be-decided"' + printf ' AND `nrep`.`name`="to-be-decided";\n' + printf 'UPDATE IGNORE `dependencies`' + printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`' + printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n' + printf 'UPDATE IGNORE `install_target_providers`' + printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`' + printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n' + printf 'DELETE FROM `binary_packages`' + printf ' WHERE EXISTS (' + printf 'SELECT * FROM `ren`' + printf ' WHERE `ren`.`old`=`binary_packages`.`id`' + printf ');\n' + printf 'UPDATE IGNORE `binary_packages`' + printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`' + printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' + printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' + } | \ + ${mysql_command} +} + +# mysql_package_name_query +# print a mysql query of the full name of a package file + +mysql_package_name_query() { + printf 'CONCAT(' + printf '`binary_packages`.`pkgname`,"-",' + printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' + printf '`binary_packages`.`pkgver`,"-",' + printf '`binary_packages`.`pkgrel`,".",' + printf '`binary_packages`.`sub_pkgrel`,"-",' + printf '`architectures`.`name`,".pkg.tar.xz"' + printf ')' +} + +# mysql_join_*_* +# print 'JOIN' part of mysql query to connect the respective tables +# these functions take 2 optional arguments, acting as aliases for +# the tables + +# mysql_join__generic $table_a $column_a $table_b $column_b +# create mysql_join_${table_a}_${table_b}() function + +mysql_join__generic() { + eval "$( + printf 'mysql_join_%s_%s() {\n' "$1" "$3" + printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' + printf ' fi\n' + printf ' if [ -n "$1" ]; then\n' + printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" + printf ' else\n' + printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" + printf ' fi\n' + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" + printf ' else\n' + printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" + printf ' fi\n' + printf '}\n' + )" +} + +for link in \ + 'binary_packages:architecture:architectures' \ + 'binary_packages:repository:repositories' \ + 'binary_packages:build_assignment:build_assignments' \ + \ + 'build_assignments:architecture:architectures' \ + 'build_assignments:package_source:package_sources' \ + \ + 'build_dependency_loops:build_assignment:build_assignments' \ + 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ + \ + 'build_slaves:currently_building:build_assignments' \ + 'build_slaves:currently_building build_assignment:binary_packages' \ + \ + 'dependencies:depending_on:install_targets' \ + 'dependencies:dependent:binary_packages' \ + 'dependencies:dependency_type:dependency_types' \ + \ + 'failed_builds:reason:fail_reason' \ + 'failed_builds:build_assignment:build_assignments' \ + 'failed_builds:build_slave:build_slaves' \ + \ + 'install_target_providers:package:binary_packages' \ + 'install_target_providers:install_target:install_targets' \ + 'install_target_providers:install_target depending_on:dependencies' \ + \ + 'package_sources:upstream_package_repository:upstream_repositories' \ + \ + 'repositories:stability:repository_stabilities' \ + \ + 'upstream_repositories:git_repository:git_repositories'; do +# A join for these cannot be done, because it's not clear on what to join: +# 'repository_stability_relations:more_stable:repository_stabilities' +# 'repository_stability_relations:less_stable:repository_stabilities' + + table_b="${link##*:}" + table_a="${link%:*}" + column_b="${table_a##*:}" + table_a="${table_a%:*}" + column_a="${column_b% *}" + if [ "${column_a}" = "${column_b}" ]; then + column_b='id' + else + column_b="${column_b##* }" + fi + + mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" + mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" +done -- cgit v1.2.3-70-g09d2 From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'lib/mysql-functions') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-70-g09d2 From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:05:51 +0100 Subject: use "mysql -N" instead of cumbersome sedding --- bin/build-master-status | 5 ++--- bin/build-master-status-from-mysql | 46 ++++++++++---------------------------- bin/check-bugtracker | 5 ++--- bin/copy-to-build-support | 7 ++---- bin/db-update | 14 ++++-------- bin/get-assignment | 16 +++++-------- bin/get-package-updates | 6 ++--- bin/modify-package-state | 3 +-- bin/return-assignment | 20 +++++------------ bin/seed-build-list | 13 ++++------- bin/show-dependencies | 3 +-- bin/why-dont-you | 7 ++---- lib/common-functions | 7 ++---- lib/mysql-functions | 19 +++++----------- 14 files changed, 51 insertions(+), 120 deletions(-) (limited to 'lib/mysql-functions') diff --git a/bin/build-master-status b/bin/build-master-status index 9be7823..deb58da 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,10 +562,9 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d - 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index fa54177..d1985e7 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,11 +27,8 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" sort "${work_dir}/build-list" > \ @@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \ @@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" { @@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then # TODO: consider versions dependencies, too diff --git a/bin/check-bugtracker b/bin/check-bugtracker index f15757b..156f438 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,8 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' @@ -61,5 +60,5 @@ bug_list=$( tr '\n' ',' printf '"");\n' done - done | tee /dev/stderr | \ + done | \ mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 0b87107..e447596 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,11 +44,8 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${id}" ]; then continue diff --git a/bin/db-update b/bin/db-update index 534c245..78ff622 100755 --- a/bin/db-update +++ b/bin/db-update @@ -457,11 +457,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ sort -u @@ -496,11 +493,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ sort -u diff --git a/bin/get-assignment b/bin/get-assignment index bc67fab..964c6b7 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,11 +58,8 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' { # shellcheck disable=SC2154 @@ -157,8 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ -n "${currently_building}" ]; then @@ -214,9 +210,8 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ + mysql_run_query -N --batch --raw | \ sed ' - 1d y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ ' @@ -238,8 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index 11f384c..e855b19 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,8 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch } | \ sort -u ) @@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 8c5fac7..68b2384 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,8 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index ed29f34..a7e6623 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' @@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -283,9 +279,8 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ - grep -vxF '0'; then + mysql_run_query -N --raw --batch | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi @@ -441,10 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - ' + mysql_run_query -N --raw --batch } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index b9ae193..def9d56 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ @@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sort -u | \ - sed ' - y/\t/ / - / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d - ' | \ + tr '\t' ' ' | \ if ${update}; then # always block if locked exec 9> "${build_list_lock_file}" diff --git a/bin/show-dependencies b/bin/show-dependencies index 9e94fe3..ed7ca87 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,9 +10,8 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/\s.*$// ' | \ sort -u diff --git a/bin/why-dont-you b/bin/why-dont-you index b5d73c5..35b9419 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,11 +85,8 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' / NULL \S\+$/ b multi-dep diff --git a/lib/common-functions b/lib/common-functions index 740c266..ba434ad 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,11 +501,8 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ shrink_version 4 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 6cbc502..3e7e19f 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -668,9 +668,8 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - /^CONCAT("/d s,^,, s,$,, ' @@ -699,9 +698,8 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s,\t,/, s,\t,-, s,\t,:, @@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ { @@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-70-g09d2 From 7726164a7bc661bb4c0e5bfa6b81788b375a973b Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:30:27 +0100 Subject: lib/mysql-functions: mysql_run_query logs everything, now --- lib/mysql-functions | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 3e7e19f..d8ffeb5 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -25,7 +25,28 @@ base64_encode_each() { # mysql_run_query # wrapper function to query mysql mysql_run_query() { - ${mysql_command} "$@" + { + printf '%s %s: ' "$0" "${mysql_command} $*" + date + } \ + | tee -a "${work_dir}/mysql.stdin" \ + | tee -a "${work_dir}/mysql.stdout" \ + >> "${work_dir}/mysql.stderr" + tee -a "${work_dir}/mysql.stdin" \ + | ${mysql_command} "$@" \ + 2>> "${work_dir}/mysql.stderr" \ + | tee -a "${work_dir}/mysql.stdout" + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + tail -n 10000 "$s" + printf '%s %s done: ' "$0" "${mysql_command} $*" + date + } \ + | sponge "$s" + done } # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository -- cgit v1.2.3-70-g09d2 From 9ef68780beab5634b7b322bcb731ba5c9aec0350 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 19:03:27 +0100 Subject: lib/mysql-functions: mysql_run_query: only log if sane --- lib/mysql-functions | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index d8ffeb5..a84d74c 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -25,28 +25,32 @@ base64_encode_each() { # mysql_run_query # wrapper function to query mysql mysql_run_query() { - { - printf '%s %s: ' "$0" "${mysql_command} $*" - date - } \ - | tee -a "${work_dir}/mysql.stdin" \ - | tee -a "${work_dir}/mysql.stdout" \ - >> "${work_dir}/mysql.stderr" - tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ - 2>> "${work_dir}/mysql.stderr" \ - | tee -a "${work_dir}/mysql.stdout" - for s in \ - "${work_dir}/mysql.stdin" \ - "${work_dir}/mysql.stdout" \ - "${work_dir}/mysql.stderr"; do + if [ -s "${work_dir}/build-master-sanity" ]; then + ${mysql_command} "$@" + else { - tail -n 10000 "$s" - printf '%s %s done: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} $*" date } \ - | sponge "$s" - done + | tee -a "${work_dir}/mysql.stdin" \ + | tee -a "${work_dir}/mysql.stdout" \ + >> "${work_dir}/mysql.stderr" + tee -a "${work_dir}/mysql.stdin" \ + | ${mysql_command} "$@" \ + 2>> "${work_dir}/mysql.stderr" \ + | tee -a "${work_dir}/mysql.stdout" + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + tail -n 10000 "$s" + printf '%s %s done: ' "$0" "${mysql_command} $*" + date + } \ + | sponge "$s" + done + fi } # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository -- cgit v1.2.3-70-g09d2 From 8a24ee22bd08dff3f9610a837eaa0aacd7c6032a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Mar 2018 13:18:30 +0100 Subject: lib/mysql-functions: new joins for `repository_moves` --- lib/mysql-functions | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index a84d74c..9d705d2 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -1100,6 +1100,8 @@ for link in \ 'package_sources:upstream_package_repository:upstream_repositories' \ \ 'repositories:stability:repository_stabilities' \ + \ + 'repository_moves:upstream_package_repository:upstream_repositories' \ \ 'upstream_repositories:git_repository:git_repositories'; do # A join for these cannot be done, because it's not clear on what to join: -- cgit v1.2.3-70-g09d2 From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 16:30:10 +0100 Subject: make -N --raw --batch the default arguments to mysql, explicitely override them if required --- bin/build-master-status | 2 +- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 2 +- bin/copy-to-build-support | 2 +- bin/db-update | 4 ++-- bin/get-assignment | 8 ++++---- bin/get-package-updates | 4 ++-- bin/modify-package-state | 2 +- bin/return-assignment | 8 ++++---- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/why-dont-you | 2 +- lib/common-functions | 2 +- lib/mysql-functions | 18 +++++++++--------- 14 files changed, 40 insertions(+), 40 deletions(-) (limited to 'lib/mysql-functions') diff --git a/bin/build-master-status b/bin/build-master-status index deb58da..006750b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,7 +562,7 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7cf1b95..a72d396 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" @@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" @@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" @@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 156f438..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index e447596..b3decd4 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${id}" ]; then diff --git a/bin/db-update b/bin/db-update index 5efa220..5756494 100755 --- a/bin/db-update +++ b/bin/db-update @@ -467,7 +467,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ @@ -503,7 +503,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ diff --git a/bin/get-assignment b/bin/get-assignment index 964c6b7..2df3059 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' { @@ -154,7 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ -n "${currently_building}" ]; then @@ -210,7 +210,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw | \ + mysql_run_query | \ sed ' y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ @@ -233,7 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index e855b19..a95bd94 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -u ) @@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 68b2384..9b0571b 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index a7e6623..717a30e 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then @@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query -N --raw --batch + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -279,7 +279,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 @@ -436,7 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index def9d56..d49a8f3 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' @@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u | \ tr '\t' ' ' | \ if ${update}; then diff --git a/bin/show-dependencies b/bin/show-dependencies index ed7ca87..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/\s.*$// ' | \ @@ -177,7 +177,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/why-dont-you b/bin/why-dont-you index 35b9419..64dfa1b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' diff --git a/lib/common-functions b/lib/common-functions index 0f75cb8..c7a64fe 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 9d705d2..26d3db8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,17 +26,17 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then - ${mysql_command} "$@" + ${mysql_command} -N --raw --batch "$@" else { - printf '%s %s: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ | tee -a "${work_dir}/mysql.stdin" \ | tee -a "${work_dir}/mysql.stdout" \ >> "${work_dir}/mysql.stderr" tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ + | ${mysql_command} -N --raw --batch "$@" \ 2>> "${work_dir}/mysql.stderr" \ | tee -a "${work_dir}/mysql.stdout" for s in \ @@ -193,7 +193,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - mysql_run_query --html + mysql_run_query --html --column-names } # mysql_add_install_target $install_target @@ -693,7 +693,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,^,, s,$,, @@ -723,7 +723,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,\t,/, s,\t,-, @@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ @@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-70-g09d2 From f1bf6de03ad68e75dd2602a1744cc30a4a23b6f5 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 08:47:57 +0100 Subject: lib/mysql-functions: add comment --- lib/mysql-functions | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 26d3db8..97fc2f4 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,6 +26,8 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then + # If the build master is insane, the calling command should only check + # if the build master is still insane - we do not want to log that. ${mysql_command} -N --raw --batch "$@" else { -- cgit v1.2.3-70-g09d2 From 4a8a8b2408ebf2543bc9fb00165b502a12cef7a2 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 09:13:07 +0100 Subject: lib/mysql-functions: mysql_run_query: we save the query in a file and delete that file if the query succeeded --- lib/mysql-functions | 73 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 18 deletions(-) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 97fc2f4..d87b79e 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -25,33 +25,70 @@ base64_encode_each() { # mysql_run_query # wrapper function to query mysql mysql_run_query() { + local query_file if [ -s "${work_dir}/build-master-sanity" ]; then # If the build master is insane, the calling command should only check # if the build master is still insane - we do not want to log that. ${mysql_command} -N --raw --batch "$@" else - { - printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" - date - } \ - | tee -a "${work_dir}/mysql.stdin" \ - | tee -a "${work_dir}/mysql.stdout" \ - >> "${work_dir}/mysql.stderr" - tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} -N --raw --batch "$@" \ - 2>> "${work_dir}/mysql.stderr" \ - | tee -a "${work_dir}/mysql.stdout" - for s in \ - "${work_dir}/mysql.stdin" \ - "${work_dir}/mysql.stdout" \ - "${work_dir}/mysql.stderr"; do + # we save the query in a file and delete that file if the query succeeded + query_file=$(mktemp "${work_dir}/tmp.mysql-functions.query.$(date +'%Y-%m-%dT%T').XXXXXX") + cat > "${query_file}" + for i in {1..10}; do { - tail -n 10000 "$s" - printf '%s %s done: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ - | sponge "$s" + | tee -a "${work_dir}/mysql.stdin" \ + | tee -a "${work_dir}/mysql.stdout" \ + >> "${work_dir}/mysql.stderr" + cat "${query_file}" \ + >> "${work_dir}/mysql.stdin" + { + ${mysql_command} -N --raw --batch "$@" \ + < "${query_file}" \ + 2>> "${work_dir}/mysql.stderr" \ + && rm "${query_file}" + } \ + | tee -a "${work_dir}/mysql.stdout" + if ! [ -f "${query_file}" ]; then + # success! + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + tail -n 10000 "$s" + printf '%s %s done: ' "$0" "${mysql_command} $*" + date + } \ + | sponge "$s" + done + break + fi + for s in \ + "${work_dir}/mysql.stdin" \ + "${work_dir}/mysql.stdout" \ + "${work_dir}/mysql.stderr"; do + { + printf '%s %s FAILED: ' "$0" "${mysql_command} $*" + date + } \ + >> "$s" + done done + # a present query_file means there was an error + if [ -f "${query_file}" ]; then + >&2 printf 'I could not complete a mysql query!\n' + if [ ! -s "${work_dir}/build-master-sanity" ]; then + printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ + "${query_file##*/}" \ + | sponge "${irc_dir}/#archlinux-ports/in" + fi + echo 'A mysql query failed.' > \ + "${work_dir}/build-master-sanity" + return 2 + fi fi } -- cgit v1.2.3-70-g09d2 From 6bcf73307e3bad6488a2e36dbaeaea594df475c0 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Mar 2018 08:57:23 +0100 Subject: lib/mysql-functions: remove unnecessary functions --- lib/mysql-functions | 62 ----------------------------------------------------- 1 file changed, 62 deletions(-) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index d87b79e..7a377d5 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -142,50 +142,6 @@ mysql_add_package_source() { mysql_run_query } -# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository - -# shellcheck disable=SC2031,SC2086,SC2154 -mysql_add_binary_package() { - local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'INSERT IGNORE INTO binary_packages' - printf ' (' - printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted' - printf ') SELECT' - printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" - printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' - printf ' 0, 0, 0' - printf ' FROM' - printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' - printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' - printf ' WHERE' - printf ' `%s`.`name` = from_base64("%s") AND' \ - 'repositories' "${repository}" \ - 'architectures' "${architecture}" - printf ' `package_sources`.`%s` = from_base64("%s") AND' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ' `upstream_repositories`.`name` = from_base64("%s")' \ - "${upstream_package_repository}" - } | \ - sed ' - s|, )|)|g - s|, FROM| FROM|g - ' | \ - mysql_run_query -} - # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel # shellcheck disable=SC2031,SC2086,SC2154 @@ -235,24 +191,6 @@ mysql_show_binary_package() { mysql_run_query --html --column-names } -# mysql_add_install_target $install_target - -# shellcheck disable=2086 -mysql_add_install_target() { - local install_target - install_target=$( - printf "%s" "$1" | \ - base64 -w0 - ) - - { - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' VALUES (from_base64("%s"))' \ - "${install_target}" - } | \ - mysql_run_query -} - # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository # or # mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository -- cgit v1.2.3-70-g09d2 From b8f92b3a98ef521d15140fa4c28327e85f73d71e Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Mar 2018 09:02:07 +0100 Subject: lib/mysql-functions: small improvement --- lib/mysql-functions | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 7a377d5..aace33a 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -109,6 +109,9 @@ mysql_add_package_source() { fi uses_upstream=0 for repo in ${repo_names}; do + if [ "${repo}" = 'archlinux32' ]; then + continue + fi if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \ tar -t 2>/dev/null | \ sed 's,-x86_64/,-any/,' | \ -- cgit v1.2.3-70-g09d2 From 3d758f046fc29327d5ee57dcdadcb10f225d9dde Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 15:13:35 +0100 Subject: lib/mysql-functions: i -> _ --- lib/mysql-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index aace33a..2a6eda8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -34,7 +34,7 @@ mysql_run_query() { # we save the query in a file and delete that file if the query succeeded query_file=$(mktemp "${work_dir}/tmp.mysql-functions.query.$(date +'%Y-%m-%dT%T').XXXXXX") cat > "${query_file}" - for i in {1..10}; do + for _ in {1..10}; do { printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date -- cgit v1.2.3-70-g09d2 From e84d9650048d1dccf8d30ae456811a08ea88baaa Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 15:35:54 +0100 Subject: lib/mysql-functions: strip sub_pkgrel from extracted pkgrel --- lib/mysql-functions | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 2a6eda8..4e0ce93 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -395,6 +395,7 @@ mysql_generate_package_metadata() { pkgrel=$( grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ + sed 's/\.[0-9]\+$//' | \ base64_encode_each ) epoch=$( -- cgit v1.2.3-70-g09d2 From 88dbd44bcc25d3a59e721915b887daa39acc0b66 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 15:51:46 +0100 Subject: lib/mysql-functions mysql_generate_package_metadata: create dependency of $x-doc packages on $x-$pkgver and install_target_provider $x -> $x-$pkgver, $x-$pkgver-$pkgrel --- lib/mysql-functions | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'lib/mysql-functions') diff --git a/lib/mysql-functions b/lib/mysql-functions index 4e0ce93..b9ae4ab 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -535,6 +535,15 @@ mysql_generate_package_metadata() { s|^| (from_base64("| s|$|")),| ' + printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"),"-",from_base64("%s"))),\n' \ + "${pkgname}" \ + "${epoch}" \ + "${pkgver}" \ + "${pkgrel}" + printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \ + "${pkgname}" \ + "${epoch}" \ + "${pkgver}" printf ' (from_base64("%s"));\n' \ "${pkgname}" @@ -546,6 +555,14 @@ mysql_generate_package_metadata() { s|^| (from_base64("| s|$|")),| ' + if printf '%s' "${pkgname}" | \ + base64 -d | \ + grep -q -- '-doc$'; then + printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \ + "$(printf '%s' "${pkgname}" | base64 -d | sed 's/-doc$//' | base64 -w0)" \ + "${epoch}" \ + "${pkgver}" + fi printf ' ("base");\n' echo "${checkdepends}" | \ -- cgit v1.2.3-70-g09d2 From 6a69a4c9f2b38b10c26202847f8e64e47a0306fe Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 19 Mar 2018 22:10:52 +0100 Subject: #archlinux-ports -> #archlinux32 --- bin/build-master-status-from-mysql | 2 +- bin/get-package-updates | 2 +- bin/ii-connect | 6 +++--- bin/return-assignment | 4 ++-- bin/sanity-check | 4 ++-- lib/mysql-functions | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/mysql-functions') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 3918489..2c33708 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then printf 'girls: my database is dirty again ...\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" echo 'build master is insane' > \ "${work_dir}/build-master-sanity" fi diff --git a/bin/get-package-updates b/bin/get-package-updates index 1cf8782..d3cb140 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi exit 3 fi diff --git a/bin/ii-connect b/bin/ii-connect index 973b227..cf332c0 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -19,12 +19,12 @@ if tail -n1 "${irc_dir}/nickserv/out" 2> /dev/null | \ sponge "${irc_dir}/nickserv/in" fi -# join #archlinux-ports if not yet done +# join #archlinux32 if not yet done if ! grep ' buildmaster\.archlinux32\.org .* buildmaster$' "${irc_dir}/out" | \ tail -n1 | \ - grep -q ' #archlinux-ports '; then + grep -q ' #archlinux32 '; then { - echo '/j #archlinux-ports' + echo '/j #archlinux32' echo '/WHO buildmaster' } | \ sponge "${irc_dir}/in" diff --git a/bin/return-assignment b/bin/return-assignment index a61460e..417654b 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -222,7 +222,7 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux-ports/in" ]; then + if [ -p "${irc_dir}/#archlinux32/in" ]; then { printf '%s is broken (says %s).' \ "$1" \ @@ -235,7 +235,7 @@ if [ "$5" = 'ERROR' ]; then fi printf '\n' } | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi fi diff --git a/bin/sanity-check b/bin/sanity-check index b3e02aa..d98332c 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -22,7 +22,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -432,5 +432,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi diff --git a/lib/mysql-functions b/lib/mysql-functions index b9ae4ab..0b21bdf 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -83,7 +83,7 @@ mysql_run_query() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_file##*/}" \ - | sponge "${irc_dir}/#archlinux-ports/in" + | sponge "${irc_dir}/#archlinux32/in" fi echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" -- cgit v1.2.3-70-g09d2 From bd5173dc0c845e0e01de8aed01bd85b0e6646344 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 10:33:35 +0100 Subject: lib/common-functions: irc_say new --- bin/build-master-status-from-mysql | 2 +- bin/get-package-updates | 2 +- bin/return-assignment | 28 +++++++++++++--------------- bin/sanity-check | 4 ++-- lib/common-functions | 8 ++++++++ lib/mysql-functions | 2 +- 6 files changed, 26 insertions(+), 20 deletions(-) (limited to 'lib/mysql-functions') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 729375f..e4dd313 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then printf 'oh girls, my database is dirty again ...\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say echo 'build master is insane' > \ "${work_dir}/build-master-sanity" fi diff --git a/bin/get-package-updates b/bin/get-package-updates index d3cb140..a6c3f32 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi exit 3 fi diff --git a/bin/return-assignment b/bin/return-assignment index 417654b..c259f1c 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -222,21 +222,19 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux32/in" ]; then - { - printf '%s is broken (says %s).' \ - "$1" \ - "${slave}" - if [ -n "${rescheduled_packages}" ]; then - printf -- ' - I rescheduled:' - # shellcheck disable=SC2086 - printf ' %s,' ${rescheduled_packages} | \ - sed 's/,$/./' - fi - printf '\n' - } | \ - sponge "${irc_dir}/#archlinux32/in" - fi + { + printf '%s is broken (says %s).' \ + "$1" \ + "${slave}" + if [ -n "${rescheduled_packages}" ]; then + printf -- ' - I rescheduled:' + # shellcheck disable=SC2086 + printf ' %s,' ${rescheduled_packages} | \ + sed 's/,$/./' + fi + printf '\n' + } | \ + irc_say fi exit 0 diff --git a/bin/sanity-check b/bin/sanity-check index d98332c..9615ac7 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -22,7 +22,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -432,5 +432,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi diff --git a/lib/common-functions b/lib/common-functions index 1abe7bc..aa32409 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -1390,3 +1390,11 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { epoch="${epoch%%:*}" fi } + +# irc_say +# say content of stdin in irc channel +irc_say() { + if [ -p "${irc_dir}/#archlinux32/in" ]; then + sponge "${irc_dir}/#archlinux32/in" + fi +} diff --git a/lib/mysql-functions b/lib/mysql-functions index 0b21bdf..801eca2 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -83,7 +83,7 @@ mysql_run_query() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_file##*/}" \ - | sponge "${irc_dir}/#archlinux32/in" + | irc_say fi echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" -- cgit v1.2.3-70-g09d2