#!/bin/sh
# report about status of build master
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"
usage() {
>&2 echo ''
>&2 echo 'build-master-status: report about status of build master'
>&2 echo ''
>&2 echo 'possible options:'
>&2 echo ' -w|--web:'
>&2 echo ' Output to webserver instead of stdout.'
>&2 echo ' -h|--help:'
>&2 echo ' Show this help and exit.'
[ -z "$1" ] && exit 1 || exit "$1"
}
eval set -- "$(
getopt -o hw \
--long help \
--long web \
-n "$(basename "$0")" -- "$@" || \
echo usage
)"
web=false
while true
do
case "$1" in
-h|--help)
usage 0
;;
-w|--web)
web=true
;;
--)
shift
break
;;
*)
>&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
exit 42
;;
esac
shift
done
if [ $# -ne 0 ]; then
>&2 echo 'Too many arguments.'
usage
fi
tmp_dir=$(mktemp -d)
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
stable=$(
ls_master_mirror 'i686' | \
grep -v 'testing$\|staging$\|-unstable$' | \
while read -r dir; do
ls_master_mirror "i686/${dir}"
done | \
grep -c '\.pkg\.tar\.xz$'
)
tasks=$(
grep -c '^\S\+ \S\+ \S\+ \S\+$' \
"${work_dir}/build-list"
) || true
pending_packages=$(
grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \
tr ' ' '.' | \
while read -r package; do
generate_package_metadata "${package}" 2>&1 > /dev/null
cat "${work_dir}/package-infos/${package}.packages"
done |
wc -l
)
next_tasks=$(
(
cat "${work_dir}/build-list"
find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed '
s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
p
'
) | \
sort | \
uniq -u | \
while read -r package git_revision mod_git_revision repository; do
if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
fi
done | \
wc -l
)
staging=$(
ls_master_mirror 'i686' | \
grep 'staging$' | \
while read -r dir; do
ls_master_mirror "i686/${dir}"
done | \
grep -c '\.pkg\.tar\.xz$'
) || true
testing=$(
ls_master_mirror 'i686' | \
grep 'testing$' | \
while read -r dir; do
ls_master_mirror "i686/${dir}"
done | \
grep -c '\.pkg\.tar\.xz$'
) || true
(
find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
while read -r pkg rev mod_rev repo; do
if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then
echo "${pkg}"
fi
done
(
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' \
-exec cat '{}' \; | \
sort -u
find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\(\.[^.]\+\)\{4\}||' | \
sort -u
) | \
sort | \
uniq -d
) | \
sort -u > \
"${tmp_dir}/broken-packages-names"
broken=$(
wc -l < \
"${tmp_dir}/broken-packages-names"
)
blocked=$(
find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \
wc -l
)
locked=$(
find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \
wc -l
)
loops=$(
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' | \
wc -l
)
looped_packages=$(
find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
-regex '.*/loop_[0-9]\+' \
-exec cat '{}' \; | \
sort -u | \
wc -l
)
{
printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \
"${stable}" \
"$((staging+testing+pending_packages))"
printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \
"$((tasks-broken))" \
"${tasks}" \
"${pending_packages}" \
"${next_tasks}"
printf 'There are %d testing and %d staging packages.\n' \
"${testing}" \
"${staging}"
printf 'There are %d broken package builds.\n' \
"${broken}"
if [ "${loops}" -ne 0 ]; then
printf 'There are %d loops containing %d package builds.\n' \
"${loops}" \
"${looped_packages}"
fi
if [ $((broken+testing+staging)) -ne 0 ]; then
printf '%.1f%% of all packages are broken.\n' \
"$(
echo "scale=10; 100*${broken}/(${broken}+${testing}+${staging})" | \
bc
)"
fi
if [ $((testing+staging+pending_packages-broken)) -ne 0 ]; then
printf '%.1f%% of the planned work has been done.\n' \
"$(
echo "scale=10; 100*(${testing}+${staging})/(${testing}+${staging}+${pending_packages}-${broken})" | \
bc
)"
fi
} > \
"${tmp_dir}/build-master-status.html"
if ${web}; then
"${base_dir}/bin/calculate-dependent-packages"
(
printf '%s\n' \
'' \
'
' \
'Status of archlinux32 build master' \
'' \
'' \
''
sed 's|$|
|' "${tmp_dir}/build-master-status.html"
printf '%s\n' \
'
' \
'currently building packages:
' \
''
printf ''
printf '%s | ' \
'since (UTC)' \
'pkgname' \
'git revision' \
'modification git revision' \
'package repository' \
'build slave'
printf '
'
find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' -printf '%T@ %TY-%Tm-%Td %TH:%TM | %f ' -execdir head -n1 {} \; | \
sort -k1n,1 | \
sed '
s|^\S\+ ||
s|$| |
|
s|\.locked ||
s|\.\([^.]\+\)$| | \1|
s|\.\([^.]\+\)$| | \1|
s|\.\([^.]\+\)$| | \1|
'
printf '%s\n' \
' |
' \
'' \
''
) | \
sponge "${tmp_dir}/build-master-status.html"
end=$(($(date +%s)-7*24*60*60))
(
[ -f "${webserver_directory}/statistics" ] && \
cat "${webserver_directory}/statistics"
printf '%s ' \
"$(date +%s)" \
"${stable}" \
"${tasks}" \
"${pending_packages}" \
"${staging}" \
"${testing}" \
"${broken}" \
"${loops}" \
"${looped_packages}" \
"${locked}" \
"${blocked}" \
"${next_tasks}" | \
sed 's| $|\n|'
echo "${end}"
) | \
sort -k1nr,1 | \
sed -n "
/^${end}\$/q
p
" | \
tac > \
"${tmp_dir}/statistics"
find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \
-printf '%f build()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \
-printf '%f check()\n' \
\) -o \
\( \
-exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \
-printf '%f source\n' \
\) -o \
\( \
-exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \
-printf '%f package-cache\n' \
\) -o \
-printf '%f unknown\n' \
\) | \
sed '
s|\(\.[^.]\+\)\{3\} | |
' | \
sort -u | \
sed '
:a
$!N
s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/
ta
P
D
' | \
sort -k1,1 > \
"${tmp_dir}/broken-packages.reason"
{
printf '%s\n' \
'' \
'' \
'List of broken package builds' \
'' \
'' \
'' \
'build logs
' \
'' \
''
printf '%s | ' \
'package' \
'git revision' \
'modification git revision' \
'package repository' \
'compilations' \
'dependent' \
'build error' \
'blocked'
printf '
\n'
find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \
sed 's|\.broken$||' | \
sort -k1,1 | \
join -j 1 - "${tmp_dir}/broken-packages.reason" | \
sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \
while read -r sf pkg rev mod_rev repo build_error; do
if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then
printf '1 '
else
printf '0 '
fi
printf '%s ' \
"${pkg}" \
"${rev}" \
"${mod_rev}" \
"${repo}" \
"$(wc -l < "${work_dir}/package-states/${sf}.broken")" \
"$(
# shellcheck disable=SC2010
ls -t "${webserver_directory}/build-logs/error" | \
grep -m1 "^$(str_to_regex "${sf}.")[^.]\+\.build-log\.gz\$"
)" \
"$(
(
grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \
echo 'x '
) | \
cut -d' ' -f2
)" \
"${build_error}"
if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then
while read -r blocked_reason; do
if echo "${blocked_reason}" | \
grep -q '^wait for '; then
printf 'wait for '
echo "${blocked_reason}" | \
sed '
s|^wait for ||
s@\( and \| or \)@\n\1\n@
' | \
while read -r reason; do
if [ "FS#${reason#FS#}" = "${reason}" ]; then
printf '%s' \
"${reason#FS#}" \
"${reason}"
elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then
printf '%s' \
"${reason}" \
"${reason}"
elif [ "${reason% *}" != "${reason}" ]; then
printf '%s' \
"${reason}"
else
printf '%s' \
"${reason}"
fi
if read -r operator; then
printf ' %s ' "${operator}"
fi
done
else
echo "${blocked_reason}"
fi
done < \
"${work_dir}/package-states/${sf}.blocked" | \
tr '\n' ' '
else
printf ' '
fi
printf '\n'
done | \
sort -k6n,6 | \
while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do
if [ "${buildable}" -eq 0 ]; then
left='('
right=')'
else
unset left
unset right
fi
printf ''
if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
mod_rev="${mod_rev}"
fi
build_error=$(
echo "${build_error}" | \
sed 's|,|, |g'
)
printf '%s | ' \
''"${left}${pkg}${right}"'' \
"${rev}
" \
"${mod_rev}
" \
"${repo}" \
''"${count}"'' \
"${dependent}" \
"${build_error}" \
"${reason}"
printf '
\n'
done
printf '%s\n' \
'
' \
'' \
''
} > \
"${tmp_dir}/broken-packages.html"
rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason"
{
printf '%s\n' \
'' \
'' \
'Todos in the build scripts' \
'' \
''
find "${base_dir}/bin/" "${base_dir}/conf/" -type f \
-exec sed -n '
/^\s*#\s*TODO:/{
s/^\s*#\s*TODO:\s*//
:a
N
s/\s*\n\s*#/\n/
ta
s/\n\n\+/\n/g
s/\n[^\n]*$/\n/
i {}
=
p
}' {} \; | \
sed '
:a
N
/\n$/!ba
s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n|\1 (line \2):\n|
' | \
sed '
s|$|
|
'
printf '%s\n' \
'' \
''
} > \
"${tmp_dir}/todos.html"
{
printf '%s\n' \
'' \
'' \
'Blacklisted packages' \
'' \
'' \
'' \
''
printf ''
printf '%s | ' \
'package' \
'reason'
printf '
\n'
git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
tar -Ox | \
sed '
s@FS#\([0-9]\+\)@\0@
/.#/!s/$/#/
s|\(.\)#|\1|
/^\s*#/{
s/^\s*#\s*//
s|\s*\( | \)|\1|
s/^//
}
s|^||
s|$| | |
'
printf '%s\n' \
' |
' \
'' \
''
} > \
"${tmp_dir}/blacklist.html"
find "${tmp_dir}" -maxdepth 1 -type f | \
while read -r file; do
cat "${file}" > \
"${webserver_directory}/${file##*/}"
done
else
cat "${tmp_dir}/build-master-status.html"
fi