result of archlinux32 build master'"'"'s sanity check

#!/bin/sh # do some basic sanity checks # shellcheck disable=SC2119,SC2120 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" usage() { >&2 echo '' >&2 echo 'sanity-check [options] [checks]: check sanity of build master' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -q|--quiet: Only print errors found.' >&2 echo ' -r|--really-quiet: Do not print anything.' >&2 echo ' -w|--wait: If necessary, wait for lock blocking.' [ -z "$1" ] && exit 1 || exit "$1" } exit_code=0 i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then # shellcheck disable=SC2119 printf '\001ACTION goes insane.\001\n' | \ irc_say fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" echo 'SANITY CHECK FAILED' | \ tee -a "${tmp_dir}/messages" >&2 exit_code=2 } eval set -- "$( getopt -o hqrw \ --long help \ --long quiet \ --long really-quiet \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" block_flag='-n' silence=0 while true do case "$1" in -h|--help) usage 0 ;; -q|--quiet) silence=1 ;; -r|--really-quiet) silence=2 ;; -w|--wait) block_flag='' ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done exec 9> "${sanity_check_lock_file}" # shellcheck disable=SC2086 if ! verbose_flock ${block_flag} 9; then >&2 echo 'Sanity check skipped, cannot acquire lock.' exit fi execute_all_intentions # shellcheck disable=SC2016 repos=$( { printf 'SELECT DISTINCT `repositories`.`name`' printf ' FROM `repositories`' printf ' WHERE `repositories`.`is_on_master_mirror`;\n' } | \ mysql_run_query ) # shellcheck disable=SC2016 archs=$( { printf 'SELECT DISTINCT `architectures`.`name`' printf ' FROM `architectures`' printf ' WHERE EXISTS (' printf 'SELECT 1' printf ' FROM `repositories`' printf ' WHERE `repositories`.`architecture`=`architectures`.`id`' printf ' AND `repositories`.`is_on_master_mirror`' printf ');\n' } | \ mysql_run_query ) mysql_cleanup finish() { { printf '%s\n' \ '' \ '' \ 'result of archlinux32 build master'"'"'s sanity check' \ '' \ '' printf '%s
\n' "$(date)" sed 's|$|
|' "${tmp_dir}/messages" printf '%s\n' \ '' \ '' } > \ "${webserver_directory}/master-sanity.html" rm -rf --one-file-system "${tmp_dir}" } tmp_dir=$(mktemp -d 'tmp.sanity-check.XXXXXXXXXX' --tmpdir) touch "${tmp_dir}/messages" trap 'finish' EXIT if [ $# -eq 0 ]; then set -- git-repositories build-list master-mirror-availability mysql ssh-keys repos package-database track-state fi while [ $# -gt 0 ]; do case "$1" in master-mirror-availability) [ ${silence} -gt 0 ] || \ printf 'checking if master mirror can be reached ...' | \ tee -a "${tmp_dir}/messages" >&2 for host in $( printf '%s\n' \ "${master_mirror_sftp_command}" \ "${master_mirror_rsync_directory}" | \ sed ' s,^.*@$[^:/@]\+$$[:/].*$\?$,\1, t d ' | \ sort -u ); do if ! ping -c1 "${host}" >/dev/null 2>&1; then if [ ${silence} -le 1 ]; then printf '\nThe master mirror %s cannot be pinged.\n' \ "${host}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane # if the master mirror cannot be reached, further tests do not make any sense set -- "$1" fi done if ! ${master_mirror_rsync_command} ${master_mirror_rsync_directory} >/dev/null 2>&1; then if [ ${silence} -le 1 ]; then printf '\nThe master mirror %s cannot be reached via rsync.\n' \ "${host}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane # if the master mirror cannot be reached, further tests do not make any sense set -- "$1" fi if ! ${master_mirror_sftp_command} /dev/null 2>&1; then if [ ${silence} -le 1 ]; then printf '\nThe master mirror %s cannot be reached via sftp.\n' \ "${host}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane # if the master mirror cannot be reached, further tests do not make any sense set -- "$1" fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; git-repositories) [ ${silence} -gt 0 ] || \ printf 'checking git repositories ...' | \ tee -a "${tmp_dir}/messages" >&2 for repo in ${repo_names}; do eval 'repo_path="${repo_paths__'"${repo}"'}"' repo_revision=$( # shellcheck disable=SC2016 { printf 'SELECT `git_repositories`.`head`' printf ' FROM `git_repositories`' printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" } | \ mysql_run_query ) if ! obj_type=$(git -C "${repo_path}" cat-file -t "${repo_revision}" 2>/dev/null); then if [ ${silence} -le 1 ]; then printf '\nThe repository %s (%s) does not know the current revision %s.\n' \ "${repo}" "${repo_path}" "${repo_revision}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane elif [ "${obj_type}" != 'commit' ]; then if [ ${silence} -le 1 ]; then printf '\nThe repository %s (%s) knows the current revision %s, but it is not a commit, but a %s.\n' \ "${repo}" "${repo_path}" "${repo_revision}" "${obj_type}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi done [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; build-list) [ ${silence} -gt 0 ] || \ printf 'checking build-list ...' | \ tee -a "${tmp_dir}/messages" >&2 errors=$( # shellcheck disable=SC2016 { printf 'SELECT `architectures`.`name`,`package_sources`.`pkgbase`' printf ' FROM `package_sources`' mysql_join_package_sources_build_assignments mysql_join_build_assignments_architectures printf ' WHERE EXISTS(' printf 'SELECT 1' printf ' FROM `binary_packages`' mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' AND `binary_packages`.`build_assignment`=`build_assignments`.`id`' printf ');\n' } | \ mysql_run_query | \ sort | \ uniq -d ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages have duplicate build orders:\n%s\n' \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi errors=$( # shellcheck disable=SC2016 { printf 'SELECT `a`.`pkgname`' printf ' FROM `binary_packages` AS `a`' mysql_join_binary_packages_binary_packages_in_repositories 'a' 'air' mysql_join_binary_packages_in_repositories_repositories 'air' 'a_r' printf ' AND `a_r`.`name`="build-list"' printf ' JOIN `binary_packages` AS `b`' printf ' ON `a`.`pkgname`=`b`.`pkgname`' printf ' JOIN `architecture_compatibilities` AS `ac_a`' printf ' ON `ac_a`.`fully_compatible`' printf ' AND `ac_a`.`built_for`=`a`.`architecture`' printf ' JOIN `architecture_compatibilities` AS `ac_b`' printf ' ON `ac_b`.`fully_compatible`' printf ' AND `ac_b`.`built_for`=`b`.`architecture`' printf ' AND `ac_a`.`runs_on`=`ac_b`.`runs_on`' mysql_join_binary_packages_binary_packages_in_repositories 'b' 'bir' mysql_join_binary_packages_in_repositories_repositories 'bir' 'b_r' printf ' AND `b_r`.`name`="deletion-list";\n' } | \ mysql_run_query ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages appear on the build- and deletion-list:\n%s\n' \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; repos) [ ${silence} -gt 0 ] || \ printf 'checking repos on master mirror ...' | \ tee -a "${tmp_dir}/messages" >&2 errors=$( { # shellcheck disable=SC2086 for arch in ${archs}; do printf 'expected '"${arch}"' %s\n' ${repos} ls_master_mirror "${arch}" | \ sed 's|^|found '"${arch}"' |' done } | \ sort -k2,3 | \ uniq -uf1 ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following repos are missing or obsolete on the mirror:\n%s\n' \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; package-database) for arch in ${archs}; do for repo in ${repos}; do [ ${silence} -gt 0 ] || \ printf 'checking consistency of repository "%s/%s" on the master mirror ...' "${arch}" "${repo}" | \ tee -a "${tmp_dir}/messages" >&2 packages=$( ls_master_mirror "${arch}/${repo}" | \ grep '\.pkg\.$'"${package_compression_suffix_regex}"'$$\.sig$\?$' ) || true errors=$( echo "${packages}" | \ grep '\S' | \ sed ' s@^$.*\.pkg\.\('"${package_compression_suffix_regex}"'$\)$@package \1@ s@^$.*\.pkg\.\('"${package_compression_suffix_regex}"'$\)\.sig$@signature \1@ ' | \ sort -k2 | \ uniq -cf1 | \ grep -v '^\s*2\s' | \ awk '{print $2 " " $3}' ) || true if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages in %s/%s are missing a signature or vice versa:\n%s\n' \ "${arch}" \ "${repo}" \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi failsafe_rsync \ "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db.tar.gz" \ "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files.tar.gz" \ "${tmp_dir}/" errors=$( { tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \ sed -n ' /^%FILENAME%$/ { N s/^.*\n/in_database / p } ' echo "${packages}" | \ sed ' /\.pkg\.$'"${package_compression_suffix_regex}"'$$/ !d s/^/in_repository / ' | \ sort -u } | \ sort -k2 | \ uniq -uf1 ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages in %s/%s are missing from the database or vice versa:\n%s\n' \ "${arch}" \ "${repo}" \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi errors=$( { tar -tzf "${tmp_dir}/${repo}.files.tar.gz" 2>/dev/null | \ grep '/$' | \ sed ' s|/$|| s|^|in_database | ' echo "${packages}" | \ grep '\S' | \ sed ' s|-[^-]\+$|| s|^|in_repository | ' | \ sort -u } | \ sort -k2 | \ uniq -uf1 ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages in %s/%s are missing from the file-database or vice versa:\n%s\n' \ "${arch}" \ "${repo}" \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi find "${tmp_dir:?}" -mindepth 1 $ -not -name 'messages' $ -delete [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 done done ;; track-state) [ ${silence} -gt 0 ] || \ printf 'checking if all packages are tracked correctly ...' | \ tee -a "${tmp_dir}/messages" >&2 errors=$( { # shellcheck disable=SC2016 { printf 'SELECT "mysql",CONCAT(`r_a`.`name`,"/",`repositories`.`name`,"/",' mysql_package_name_query printf ') FROM `binary_packages`' mysql_join_binary_packages_binary_packages_in_repositories mysql_join_binary_packages_in_repositories_repositories printf ' AND `repositories`.`is_on_master_mirror`' mysql_join_binary_packages_architectures printf ' LEFT' mysql_join_binary_packages_compressions mysql_join_repositories_architectures '' 'r_a' } | \ mysql_run_query | \ tr '\t' ' ' for arch in ${archs}; do ls_master_mirror "${arch}" | \ while read -r repo; do ls_master_mirror "${arch}/${repo}" | \ sed ' /\.pkg\.$'"${package_compression_suffix_regex}"'$$/!d s,^,package-file '"${arch}"'/'"${repo}"'/, ' done done } | \ sort -k2 | \ uniq -uf1 ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe following packages from the master mirror are not tracked in the database or vice versa:\n%s\n' \ "${errors}" | \ tee -a "${tmp_dir}/messages" >&2 fi i_am_insane fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; mysql) [ ${silence} -gt 0 ] || \ printf 'checking mysql-sanity ...' | \ tee -a "${tmp_dir}/messages" >&2 mysql_sanity_check | \ sed ' s,^-.*$,\0, s,^+.*$,\0, s/$/
/ 1 i sanity of the buildmaster'"'"'s mysql database $ a ' | \ sponge "${webserver_directory}/mysql-sanity.html" if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then # shellcheck disable=SC2119 { printf 'girls, Tasos, my database is dirty again ...\n' printf 'dirty! girls, Tasos, my database - so dirty :-(\n' printf 'girls, Tasos, please have a look at my dirty database\n' } | \ shuf -n1 | \ irc_say fi if [ -s "${webserver_directory}/mysql-sanity.html" ]; then if [ ${silence} -le 1 ]; then printf '\nThere is something wrong with the database:\n' cat "${webserver_directory}/mysql-sanity.html" fi | \ tee -a "${tmp_dir}/messages" >&2 i_am_insane fi # hopefully, this gets rid of false positives :-) sleep 1 errors=$( find "${work_dir}" -mindepth 1 -maxdepth 1 \ -not -name 'tmp.mysql-functions.unimportant_query.*' \ -name 'tmp.mysql-functions.query.*' | \ parallel -j100 \ 'bash -c " sleep 5 test -s {} && '\ 'printf '"'"'%s:\n'"'"' {} && '\ 'sed '"'"'s/^/>> /'"'"' {} "' ) || \ true if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThere are pending mysql queries:\n%s\n' \ "${errors}" fi | \ tee -a "${tmp_dir}/messages" >&2 i_am_insane fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; ssh-keys) [ ${silence} -gt 0 ] || \ printf 'checking ssh-keys ...' | \ tee -a "${tmp_dir}/messages" >&2 # shellcheck disable=SC2016 { printf 'SELECT' printf ' CONCAT(' printf '"command=\\\"%s/bin/slave-build-connect ",' \ "${base_dir}" printf '`build_slaves`.`name`,' printf '"\\\" ",' printf '`ssh_keys`.`type`,' printf '" ",' printf '`ssh_keys`.`fingerprint`' printf ')' printf ' FROM' printf ' `build_slaves`' mysql_join_build_slaves_ssh_keys printf ' WHERE `build_slaves`.`access_allowed`' } | \ mysql_run_query | \ sort > \ "${tmp_dir}/ssh-keys.mysql" sed -n ' /^command/ { s/^$\(\S\+\s\+$\{4\}\)\S.*$/\1/ s/\s\+$// p } ' ~/".ssh/authorized_keys" | \ sort > \ "${tmp_dir}/ssh-keys.authorized_keys" errors=$( diff "${tmp_dir}/ssh-keys.mysql" "${tmp_dir}/ssh-keys.authorized_keys" ) || true if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then printf '\nThe ssh keys in the database and the file system differ:\n%s\n' \ "${errors}" fi | \ tee -a "${tmp_dir}/messages" >&2 i_am_insane fi [ ${silence} -gt 0 ] || \ echo ' passed.' | \ tee -a "${tmp_dir}/messages" >&2 ;; *) [ ${silence} -gt 1 ] || \ >&2 printf 'unknown sanity-check "%s".\n' "$1" exit 2 ;; esac shift done if [ ${exit_code} -ne 0 ]; then exit ${exit_code} fi # remove all saved successful queries - they did not break anything find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'successful.*' -delete if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" # shellcheck disable=SC2119 printf '\001ACTION resumes sanity.\001\n' | \ irc_say fi