#!/bin/sh
# report about status of build master
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
usage() {
>&2 echo ''
>&2 echo 'build-master-status: report about status of build master'
>&2 echo ''
>&2 echo 'possible options:'
>&2 echo ' -w|--web:'
>&2 echo ' Output to webserver instead of stdout.'
>&2 echo ' -h|--help:'
>&2 echo ' Show this help and exit.'
[ -z "$1" ] && exit 1 || exit "$1"
}
eval set -- "$(
getopt -o hw \
--long help \
--long web \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
web=false
while true
do
case "$1" in
-h|--help)
usage 0
;;
-w|--web)
web=true
;;
--)
shift
break
;;
*)
>&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
exit 42
;;
esac
shift
done
if [ $# -ne 0 ]; then
>&2 echo 'Too many arguments.'
usage
fi
if [ -s "${work_dir}/build-master-sanity" ]; then
>&2 echo 'Build master is not sane.'
exit
fi
tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
stable=$(
ls_master_mirror 'i686' | \
grep -v 'testing$\|staging$\|-unstable$' | \
while read -r dir; do
ls_master_mirror "i686/${dir}"
done | \
grep -c '\.pkg\.tar\.xz$'
)
tasks=$(
grep -c '^\S\+ \S\+ \S\+ \S\+$' \
"${work_dir}/build-list"
) || true
pending_packages=$(
grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \
tr ' ' '.' | \
while read -r package; do
generate_package_metadata "${package}" 2>&1 > /dev/null
cat "${work_dir}/package-infos/${package}.packages"
done |
wc -l
)
next_tasks=$(
{
cat "${work_dir}/build-list"
find "${work_dir}/package-states" -maxdepth 1 \
\( -name '*.broken' -o -name '*.blocked' \) \
-printf '%f\n' | \
sed '
s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
p
'
} | \
sort | \
uniq -u | \
while read -r package git_revision mod_git_revision repository; do
if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
fi
done | \
wc -l
)
staging=$(
find "${work_dir}/package-states" -name '*.done' \
-exec cat '{}' \; | \
sort -u | \
wc -l
)
testing=$(
find "${work_dir}/package-states" -name '*.testing' \
-exec cat '{}' \; | \
sort -u | \
wc -l
)
tested=$(
find "${work_dir}/package-states" -name '*.tested' \
-exec cat '{}' \; | \
sort -u | \
wc -l
)
{
find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
while read -r pkg rev mod_rev repo; do
if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then
echo "${pkg}"
fi
done
{
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' \
-exec cat '{}' \; | \
sort -u
find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\(\.[^.]\+\)\{4\}||' | \
sort -u
} | \
sort | \
uniq -d
} | \
sort -u > \
"${tmp_dir}/broken-packages-names"
broken=$(
wc -l < \
"${tmp_dir}/broken-packages-names"
)
blocked=$(
find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \
wc -l
)
locked=$(
find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \
wc -l
)
loops=$(
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' | \
wc -l
)
looped_packages=$(
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' \
-exec cat '{}' \; | \
sort -u | \
wc -l
)
{
printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \
"${stable}" \
"$((staging+testing+tested+pending_packages))"
printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \
"$((tasks-broken))" \
"${tasks}" \
"${pending_packages}" \
"${next_tasks}"
printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \
"$((testing+tested))" \
"${tested}" \
"${staging}"
printf 'There are %d broken package builds.\n' \
"${broken}"
if [ "${loops}" -ne 0 ]; then
printf 'There are %d loops containing %d package builds.\n' \
"${loops}" \
"${looped_packages}"
fi
if [ $((broken+testing+tested+staging)) -ne 0 ]; then
printf '%.1f%% of all packages are broken.\n' \
"$(
echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \
bc
)"
fi
if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then
printf '%.1f%% of the planned work has been done.\n' \
"$(
echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \
bc
)"
fi
} > \
"${tmp_dir}/build-master-status.html"
if ${web}; then
"${base_dir}/bin/calculate-dependent-packages"
{
printf '%s\n' \
'' \
'
' \
'Status of archlinux32 build master' \
'' \
'' \
''
sed 's|$|
|' "${tmp_dir}/build-master-status.html"
printf '%s\n' \
'
' \
'currently building packages:
' \
''
printf ''
printf '%s | ' \
'since (UTC)' \
'pkgname' \
'git revision' \
'modification git revision' \
'package repository' \
'build slave'
printf '
'
find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \
-printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \
-execdir sed '
:a
$!{
N
s/\n/, /
ba
}
' '{}' \; | \
sort -k1n,1 | \
sed '
s|^\S\+ ||
s|\.locked | |
s|\.\([^.]\+\)$| \1|
s|\.\([^.]\+\)$| \1|
s|\.\([^.]\+\)$| \1|
' | \
while read -r date time pkg rev mod_rev repo slaves; do
printf ''
printf '%s | ' \
"${date} ${time}" \
"${pkg}" \
"${rev}
" \
"$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")
" \
"${repo}" \
"${slaves}"
printf '
\n'
done
printf '%s\n' \
'
' \
'' \
''
} | \
sponge "${tmp_dir}/build-master-status.html"
end=$(($(date +%s)-7*24*60*60))
{
[ -f "${webserver_directory}/statistics" ] && \
cat "${webserver_directory}/statistics"
printf '%s ' \
"$(date +%s)" \
"${stable}" \
"${tasks}" \
"${pending_packages}" \
"${staging}" \
"${testing}" \
"${broken}" \
"${loops}" \
"${looped_packages}" \
"${locked}" \
"${blocked}" \
"${next_tasks}" \
"${tested}" | \
sed 's| $|\n|'
echo "${end}"
} | \
sort -k1nr,1 | \
sed -n "
/^${end}\$/q
p
" | \
tac > \
"${tmp_dir}/statistics"
find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \
-printf '%f build()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \
-printf '%f check()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \
-printf '%f prepare()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \
-printf '%f package()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \
-printf '%f source\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \
-printf '%f dependencies\n' \
\) -o \
\( \
-exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \
-printf '%f package-cache\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \
-printf '%f run-as-root\n' \
\) -o \
-printf '%f unknown\n' \
\) | \
sed '
s|\(\.[^.]\+\)\{3\} | |
' | \
sort -u | \
sed '
:a
$!N
s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/
ta
P
D
' | \
sort -k1,1 > \
"${tmp_dir}/broken-packages.reason"
{
printf '%s\n' \
'' \
'' \
'List of broken package builds' \
'' \
'' \
'' \
'build logs
' \
'' \
''
printf '%s | ' \
'package' \
'git revision' \
'modification git revision' \
'package repository' \
'compilations' \
'dependent' \
'build error' \
'blocked'
printf '
\n'
find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\.broken$||' | \
sort -k1,1 | \
join -j 1 - "${tmp_dir}/broken-packages.reason" | \
sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \
while read -r sf pkg rev mod_rev repo build_error; do
if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then
printf '1 '
else
printf '0 '
fi
printf '%s ' \
"${pkg}" \
"${rev}" \
"${mod_rev}" \
"${repo}" \
"$(wc -l < "${work_dir}/package-states/${sf}.broken")" \
"$(
# shellcheck disable=SC2010
ls -t "${webserver_directory}/build-logs/error" | \
grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$'
)" \
"$(
{
grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \
echo 'x '
} | \
cut -d' ' -f2
)" \
"${build_error}"
if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then
sed '
s|\s\(wait for \)|\n\1|g
' "${work_dir}/package-states/${sf}.blocked" | \
while read -r blocked_reason; do
if echo "${blocked_reason}" | \
grep -q '^wait for '; then
printf 'wait for '
echo "${blocked_reason}" | \
sed '
s|^wait for ||
s@\( and \| or \)@\n\1\n@
' | \
while read -r reason; do
if [ "FS#${reason#FS#}" = "${reason}" ]; then
printf '%s' \
"${reason#FS#}" \
"${reason}"
elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then
printf '%s' \
"${reason#FS32#}" \
"${reason}"
elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then
printf '%s' \
"${reason}" \
"${reason}"
elif [ "${reason% *}" != "${reason}" ]; then
printf '%s' \
"${reason}"
else
printf '%s' \
"${reason}"
fi
if read -r operator; then
printf ' %s ' "${operator}"
fi
done
else
echo "${blocked_reason}"
fi
done | \
tr '\n' ' '
else
printf ' '
fi
printf '\n'
done | \
sort -k6n,6 | \
while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do
if [ "${buildable}" -eq 0 ]; then
left='('
right=')'
else
unset left
unset right
fi
printf ''
mod_rev=$(
modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
)
build_error=$(
echo "${build_error}" | \
sed 's|,|, |g'
)
printf '%s | ' \
''"${left}${pkg}${right}"'' \
"${rev}
" \
"${mod_rev}
" \
"${repo}" \
''"${count}"'' \
"${dependent}" \
"${build_error}" \
"${reason}"
printf '
\n'
done
printf '%s\n' \
'
' \
'' \
''
} > \
"${tmp_dir}/broken-packages.html"
rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason"
{
printf '%s\n' \
'' \
'' \
'Todos in the build scripts' \
'' \
''
find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
-exec grep -nHF '' '{}' \; | \
awk '
{ print $0 }
/^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i}
' | \
sed -n '
s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/
T
N
s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/
:a
N
s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/
ta
s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/
p
' | \
tee "${tmp_dir}/todos" | \
sed '
:a
N
/\n$/!ba
s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|TODO #\2 - \1 (line \3):\n|
' | \
sed '
s|$|
|
'
printf '%s\n' \
'' \
''
} > \
"${tmp_dir}/todos.html"
if [ -s "${tmp_dir}/todos" ]; then
sed '
:a
N
/\n$/!ba
s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 |
s/\n$//
s/\n/\\n/g
' -i "${tmp_dir}/todos"
while read -r file line desc; do
printf '%s %s %s\n' \
"$(printf '%s' "${file}" | base64 -w0)" \
"$(printf '%s' "${line}" | base64 -w0)" \
"$(printf '%s' "${desc}" | base64 -w0)"
done < \
"${tmp_dir}/todos" | \
sponge "${tmp_dir}/todos"
# update todos
# shellcheck disable=SC2016
while read -r file line desc; do
printf 'UPDATE TABLE `todos`'
printf ' SET `todos`.`line`=from_base64("%s")' \
"${line}"
printf ' WHERE `todos`.`file`=from_base64("%s")' \
"${file}"
printf ' AND `todos`.`desc`=from_base64("%s");\n' \
"${desc}"
printf 'UPDATE TABLE `todos`'
printf ' SET `todos`.`desc`=from_base64("%s")' \
"${desc}"
printf ' WHERE `todos`.`file`=from_base64("%s")' \
"${file}"
printf ' AND `todos`.`line`=from_base64("%s");\n' \
"${line}"
done | \
${mysql_command}
# insert unfound todos
# shellcheck disable=SC2016
while read -r file line desc; do
printf ' ('
printf 'from_base64("%s"),' \
"${file}" \
"${line}" \
"${desc}" | \
sed 's/,$/),/'
done < \
"${tmp_dir}/todos" | \
sed '
1 i INSERT IGNORE INTO `todos` (`file`,`line`,`description`) VALUES
s/,$/;\n/
' | \
${mysql_command}
rm -f "${tmp_dir}/todos"
fi
{
printf '%s\n' \
'' \
'' \
'Blacklisted packages' \
'' \
'' \
'' \
''
printf ''
printf '%s | ' \
'package' \
'reason'
printf '
\n'
git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
tar -Ox | \
sed '
s@FS#\([0-9]\+\)@\0@
s@FS32#\([0-9]\+\)@\0@
/.#/!s/$/#/
s|\(.\)#|\1|
/^\s*#/{
s/^\s*#\s*//
s|\s*\( | \)|\1|
s/^//
}
s|^||
s|$| | |
'
printf '%s\n' \
' |
' \
'' \
''
} > \
"${tmp_dir}/blacklist.html"
{
printf '%s\n' \
'' \
'' \
'log of ssh connections from build slaves' \
'' \
'' \
''
printf ''
printf '%s | ' \
'time' \
'build slave' \
'command' \
'arguments'
printf '
\n'
if [ -r "${work_dir}/ssh-log" ]; then
tac "${work_dir}/ssh-log" | \
while read -r date time slave command arguments; do
printf ''
printf '%s | ' \
"${date} ${time}" \
"${slave}" \
"${command}" \
"${arguments}"
printf '
\n'
done
fi
printf '%s\n' \
'
' \
'' \
''
} > \
"${tmp_dir}/ssh-log.html"
find "${tmp_dir}" -maxdepth 1 -type f | \
while read -r file; do
cat "${file}" > \
"${webserver_directory}/${file##*/}"
done
else
cat "${tmp_dir}/build-master-status.html"
fi