From 2e6d97b8ddcd16f029673bc6f101fc3baf141e51 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sat, 24 Feb 2018 19:41:56 +0100 Subject: bin/common-functions -> lib/common-functions, bin/mysql-functions -> lib/mysql-functions --- bin/common-functions | 1378 -------------------------------------------------- bin/mysql-functions | 1098 ---------------------------------------- conf/default.conf | 8 +- lib/common-functions | 1378 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/mysql-functions | 1098 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 2480 insertions(+), 2480 deletions(-) delete mode 100755 bin/common-functions delete mode 100755 bin/mysql-functions create mode 100755 lib/common-functions create mode 100755 lib/mysql-functions diff --git a/bin/common-functions b/bin/common-functions deleted file mode 100755 index a84271b..0000000 --- a/bin/common-functions +++ /dev/null @@ -1,1378 +0,0 @@ -#!/bin/sh - -# contains functions used by more than one script - -# shellcheck disable=SC2039 - -# TODO: include link depenendencies in run-depends metadata - -# TODO: have full information (currently in files) in database - -# TODO: remove state files / metadata files - -if [ -z "${base_dir}" ]; then - # just to make shellcheck happy - . 'conf/default.conf' -fi - -# find_pkgbuilds package repository git_repository git_revision mod_git_revision -# find the PKGBUILD and modification of $package from $repository -# sets $PKGBUILD and $PKGBUILD_mod - -find_pkgbuilds() { - - local package="$1" - local repository="$2" - local git_repository="$3" - local git_revision="$4" - local mod_git_revision="$5" - - local repo_path - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - PKGBUILD=$( - git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sort | \ - tail -n1 - ) - - PKGBUILD_mod=$( - git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \ - tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null - ) || true - - if [ -z "${PKGBUILD}" ] && \ - [ -z "${PKGBUILD_mod}" ]; then - >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ - "${package}" \ - "${repository}" \ - "${git_repository}" \ - "${git_revision}" \ - "${mod_git_revision}" - return 1 - fi - -} - -# find_repository_with_commit commit -# find the repository which has $commit - -find_repository_with_commit() { - - local repository - - for repository in ${repo_names}; do - # shellcheck disable=SC2016 - if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then - echo "${repository}" - return 0 - fi - done - >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" - exit 1 - -} - -# find_git_repository_to_package_repository repository -# find the git repository which tracks the package repository $repository - -find_git_repository_to_package_repository() { - - local repository - local package_repository - local repo_path - - package_repository="$1" - - if [ "$1" = 'build-support' ]; then - echo 'packages' - return 0 - fi - - for repository in ${repo_names}; do - if [ "${repository}" = "archlinux32" ]; then - continue - fi - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ - tar -t --wildcards '*/repos' | \ - grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ - cut -d/ -f3 | \ - sed 's|-[^-]\+$||' | \ - sort -u | \ - grep -qxF "${package_repository}"; then - echo "${repository}" - return 0 - fi - done - >&2 echo "can't find git repository with package repository '$1'" - exit 1 - -} - -# generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# generate the meta data files of a package (dependencies, built packages, ...) - -generate_package_metadata() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - local file_prefix - local file - local PKGBUILD - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" - - if [ -e "${file_prefix}.builds" ] && \ - [ -e "${file_prefix}.build-depends" ] && \ - [ -e "${file_prefix}.run-depends" ] && \ - [ -e "${file_prefix}.groups" ] && \ - [ -e "${file_prefix}.packages" ]; then - return 0 - fi - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" - exit 1 - fi - if [ ! -s "${file_prefix}.SRCINFO" ]; then - >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" - exit 1 - fi - - # otherwise this just calls for trouble - sed -i ' - /^[^=]*=\s*$/d - s/_i686\(\s*=\)/\1/ - ' "${file_prefix}.SRCINFO" - - # extract "groups" = groups \cup provides - grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.groups" - - # extract "packages" = pkgname - grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.packages" - - # extract "builds" = provides \cup pkgname \cup groups - cat "${file_prefix}.groups" "${file_prefix}.packages" | \ - sort -u > \ - "${file_prefix}.builds" - - # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' 'base-devel' - sed -n "$( - printf '/^\t%s = /p\n' \ - 'depends' \ - 'makedepends' - )" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.build-depends" - - # extract "run-depends" = depends \cup \{ base \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' - sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.run-depends" - - rm "${file_prefix}.SRCINFO" - -} - -# delete_old_metadata -# delete old (=unneeded) meta data of packages - -delete_old_metadata() { - - local current_metadata - - current_metadata=$( - find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| - ' | \ - sort -u - ) - - ( # the new shell is intentional - # what we have - echo "${current_metadata}" - - # package-states should stay - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - ' | \ - sort -u | \ - sed 'p' - - # build-list items should stay - sed 'p' "${work_dir}/build-list" - - tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - echo "${current_metadata}" | \ - sort -k1,1 > \ - "${tmp_dir}/current-metadata" - - # the newest of the following should stay: - { - # deletion-list items - cat "${work_dir}/deletion-list" - # all packages in the repos - for repo in ${repo_names}; do - eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - sed ' - s|/$|| - /\//d - ' - done - } | \ - sort -u | \ - join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ - sort -k4,4 > \ - "${tmp_dir}/find-newest-revisions" - - uniq -uf3 < \ - "${tmp_dir}/find-newest-revisions" | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - - uniq -Df3 < \ - "${tmp_dir}/find-newest-revisions" | \ - uniq --group=append -f3 | \ - { - revs='' - mod_revs='' - opkg='' - orepo='' - while read -r rev mod_rev repo pkg; do - - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - - printf '%s %s %s %s\n' \ - "$( - printf '%s\n' ${revs} | \ - find_newest_of_git_revisions - )" \ - "$( - printf '%s\n' ${mod_revs} | \ - find_newest_of_git_revisions - )" \ - "${orepo}" \ - "${opkg}" - - revs='' - mod_revs='' - orepo='' - opkg='' - continue - fi - revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} ${rev} | \ - sort -u - ) - mod_revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} ${mod_rev} | \ - sort -u - ) - orepo="${repo}" - opkg="${pkg}" - done - } | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - ) | \ - sort | \ - uniq -u | \ - while read -r pkg rev mod_rev repo; do - rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* - done -} - -# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository -# print which (stable) repository a package belongs to - -repository_of_package() { - local package="$1" - local repository="${package##*.}" - package="${package%.*}" - local a32_rev="${package##*.}" - package="${package%.*.*}" - - case "${repository}" in - 'multilib') - if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ - tar -Ox | \ - grep -qFx "${package%.*.*.*}"; then - echo 'extra' - else - echo 'community' - fi - ;; - *) - echo "${repository}" - esac -} - -# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending -# print wether the specified package is an official package (print -# $ending) or a community package (print 'community-$ending') or a -# build-support package (print 'build-support') - -official_or_community() { - local prepo - prepo=$(repository_of_package "$1") - - if [ "${prepo}" = 'community' ]; then - echo 'community-'"$2" - elif [ "${prepo}" = 'build-support' ]; then - echo 'build-support' - else - echo "$2" - fi -} - -# ls_master_mirror $path -# list content of $path on the master mirror (via rsync) - -ls_master_mirror() { - - local path="$1" - - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${path}/" | \ - grep -v '\s\.$' | \ - awk '{print $5}' - -} - -# TODO: the actions of remove_old_package_versions should be done -# on basis of the information in the database - -# remove_old_package_versions $arch $repository $package_file - -# removes all older (not-newer) versions of $package_file -# in all repositories not-older (newer) than $repository - -# TODO: should remove all other version (also newer) from -# some repositories :-/ - -# A package is considered not newer if -# a) its version is not newer -# A package is considered older if -# b) its version is older or -# c) if it's "not newer" and its architecture is 'any' and different or -# d) if it's "not newer" and the other architecture is 'any' and different - -# this ensures an any package may replace arch-specific packages of the same version and vice versa - -remove_old_package_versions() { - - local arch="$1" - local repository="$2" - local package="$3" - - local pkgname - local epoch - local pkgver - local pkgrel - local sub_pkgrel - pkgname="${package%-*}" - pkgrel="${pkgname##*-}" - sub_pkgrel="${pkgrel##*.}" - if [ "${sub_pkgrel}" = "${pkgrel}" ]; then - sub_pkgrel='0' - else - pkgrel="${pkgrel%.*}" - fi - pkgname="${pkgname%-*}" - pkgver="${pkgname##*-}" - epoch="${pkgver%%:*}" - if [ "${epoch}" = "${pkgver}" ]; then - epoch='0' - else - pkgver="${pkgver#*:}" - fi - pkgname="${pkgname%-*}" - - # shellcheck disable=SC2016 - { - printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \ - "$( - printf '%s' "${repository}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${package}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${package}" | \ - sed ' - s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/ - ' | \ - base64 -w0 - )" - printf 'SELECT ' - printf '`binary_packages`.`id`,' - printf 'CONCAT(`repositories`.`name`,"/",' - mysql_package_name_query - printf '),' - # should we delete packages of identical version? - printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),' - printf 'CONCAT(' - printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' - printf '`binary_packages`.`pkgver`,"-",' - printf '`binary_packages`.`pkgrel`,".",' - printf '`binary_packages`.`sub_pkgrel`' - printf ')' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_binary_packages_architectures - printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' - printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' - # name must match - printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \ - "$(printf '%s' "${package%-*-*-*}" | base64 -w0)" - # repository, where package should be deleted, should be less stable - printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \ - "$(printf '%s' "${repository}" | base64 -w0)" - printf ';\n' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ - expand_version 4 | \ - sort -k4V,4 -k3r,3 | \ - shrink_version 4 | \ - sed -n ' - /^bogus /q - p - ' | \ - cut -d' ' -f1,2 >&2 - - # repositories in which older packages should be deleted - local delete_older_repositories - # repositories in which not-newer packages should be deleted - local delete_not_newer_repositories - - if echo "${standalone_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories='' - - elif echo "${staging_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories=$( - echo "${staging_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${testing_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${testing_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${stable_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${stable_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - else - - >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}" - return 1 - - fi - - ( # the new shell is intentional - tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - { - # the architecture of the package (any vs. i686) - package_arch="${package##*-}" - package_arch="${package_arch%%.*}" - if [ "${package_arch}" = 'any' ]; then - package_arch_regex_inverter='!' - else - unset package_arch_regex_inverter - fi - - for repo in ${delete_older_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|2 '"${arch} ${repo}"' | - / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{ - s|^2|0| - } - p - } - ' - done - for repo in ${delete_not_newer_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|0 '"${arch} ${repo}"' | - p - } - ' - done - echo "${package%-*}" | \ - sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|' - - # the generated list contains the following columns: - # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz - } | \ - expand_version 5 | \ - sort -k5V,5 -k1n,1 | \ - shrink_version 5 | \ - sed -n ' - /^1 %cut% %it% %here% /q - s/^[02] // - s/ \(\S\+\)$/-\1/ - p - ' | \ - sort -u > \ - "${tmp_dir}/packages-to-delete" - # this file contains a list of packages to be deleted, one on each line: - # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz - - cut -d' ' -f1,2 < \ - "${tmp_dir}/packages-to-delete" | \ - grep -vxF "${arch} ${repository}" | \ - sort -u > \ - "${tmp_dir}/repositories-to-modify" - - # fetch all databases being modified - while read -r del_arch del_repo; do - mkdir -p "${tmp_dir}/${del_arch}/${del_repo}" - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${tmp_dir}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - while read -r del_arch del_repo del_package _; do - if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then - # we do not repo-remove the package in the target repository - continue - fi - repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \ - "${del_package}" - done < \ - "${tmp_dir}/packages-to-delete" - - # upload modified databases - while read -r del_arch del_repo; do - ${master_mirror_rsync_command} \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - # shellcheck disable=SC2016 - sed ' - s/\.pkg\.tar\.xz$// - s/^\S\+ // - s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/ - s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/ - s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ - ' "${tmp_dir}/packages-to-delete" | \ - while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do - printf 'DELETE FROM `binary_packages` WHERE' - printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \ - 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \ - 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)" - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ - 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ - 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ - 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ - 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \ - sed 's/ AND$//' - printf ';\n' - done | \ - ${mysql_command} - - sed ' - s| \(\S\+\)$|-\1| - y| |/| - s|^|rm "| - s|$|"| - p - s|"$|.sig"| - ' "${tmp_dir}/packages-to-delete" | \ - ${master_mirror_sftp_command} - ) - -} - -# wait_some_time $minimum $diff -# wait between minimum and minimum+diff seconds (diff defaults to 30) - -wait_some_time() { - local minimum=$1 - local diff=$2 - local random - - if [ -z "${diff}" ]; then - diff=30 - fi - - random=$( - dd if='/dev/urandom' count=1 2> /dev/null | \ - cksum | \ - cut -d' ' -f1 - ) - - sleep $((minimum + random % diff)) -} - -# str_to_regex $string -# escape dots for use in regex - -str_to_regex() { - echo "$1" | \ - sed ' - s|[.[]|\\\0|g - ' -} - -# make_source_info $package $repository $git_revision $mod_git_revision $output -# create .SRCINFO from PKGBUILD within git repositories, output to $output - -make_source_info() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local mod_git_revision="$4" - local output="$5" - - local git_repo - local PKGBUILD - local PKGBUILD_mod - - git_repo=$(find_repository_with_commit "${git_revision}") - - if [ -z "${git_repo}" ]; then - return 1 - fi - - find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" - - ( # the new shell is intentional - - tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' - - { - cd "${tmp_dir}" - makepkg --printsrcinfo - cd .. - } | - if [ "${package%-i18n}-i18n" = "${package}" ]; then - sed ' - 1 a \\tdepends = '"${package%-i18n}"' - ' - else - cat - fi > \ - "${output}" - - ) - -} - -# recursively_umount_and_rm $dir -# umount all mountpoints in $dir which are also in $dir's -# filesystem, possibly also $dir itself and then -# rm -rf --one-file-system $dir - -recursively_umount_and_rm() { - local dir="$1" - - if [ -z "${dir}" ]; then - >&2 echo 'ERROR: recursively_umount_and_rm requires an argument' - exit 42 - fi - - find "${dir}" \ - -xdev -depth -type d \ - -exec 'mountpoint' '-q' '{}' ';' \ - -exec 'sudo' 'umount' '-l' '{}' ';' - rm -rf --one-file-system "${dir}" -} - -# mangle_pkgbuild $PKGBUILD [$sub_pkgrel] -# mangle $arch in PKBUILDs to contain i486, i586, i686 -# append $sub_pkgrel to the pkgrel - -mangle_pkgbuild() { - local PKGBUILD="$1" - local sub_pkgrel="$2" - - if [ -n "${sub_pkgrel}" ]; then - sub_pkgrel=".${sub_pkgrel}" - fi - - sed -i ' - /^arch=[^#]*any/!{ - /^arch=(/s/(/(i686 / - } - s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ - ' "${PKGBUILD}" -} - -# find_newest_of_git_revisions -# find newest git revision of the ones provided at stdin -# (assuming linear history) - -find_newest_of_git_revisions() { - local revisions - local repo - revisions=$(cat) - - if [ "$( - echo "${revisions}" | \ - wc -l - )" -eq 1 ]; then - - echo "${revisions}" - return - - fi - - repo=$( - find_repository_with_commit \ - "$( - echo "${revisions}" | \ - grep -xm1 '[0-9a-f]\{40\}' - )" - ) - - eval 'repo="${repo_paths__'"${repo}"'}"' - - echo "${revisions}" | \ - xargs -rn1 git -C "${repo}" rev-parse | \ - { - newest='' - while read -r current; do - if [ -z "${newest}" ] || \ - git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then - newest="${current}" - fi - done - echo "${newest}" - } -} - -# find_package_repository_to_package $package $git_repository $git_commit -# find the package repository a package from a given git repository -# belongs to - -find_package_repository_to_package() { - - local package="$1" - local git_repository="$2" - local git_commit="$3" - local repo_path - local repo - - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - if [ "${git_repository}" = 'archlinux32' ]; then - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- | \ - tar -t --wildcards "*/${package}/" | \ - cut -d/ -f1 | \ - sort -u - ) - else - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \ - tar -t | \ - cut -d/ -f3 | \ - grep -vxF '' | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -v -- '-i686$' | \ - sed 's|-[^-]\+$||' | \ - sort -u - ) - fi - - if [ -z "${repo}" ]; then - return 1 - fi - - if [ "$( - echo "${repo}" | \ - wc -l - )" -ne 1 ]; then - return 1 - fi - - echo "${repo}" - -} - -# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel -# extract files found in the svn/git source directories -# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly - -extract_source_directory() { - - local git_repo="$1" - local rev="$2" - local mod_rev="$3" - local output="$4" - local sub_pkgrel="$5" - - if [ -n "${PKGBUILD}" ]; then - eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ - tar -x --strip-components=3 -C "${output}" - fi - - if [ -n "${PKGBUILD_mod}" ]; then - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ - tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ - true - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ - tar -Ox "${PKGBUILD_mod}" >> \ - "${output}/PKGBUILD" - fi - - mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" - - # shellcheck disable=SC2016 - sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" - -} - -# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository -# return a list of packages on the build list which are (run- / build- / check-time) -# dependencies of the given package - -find_dependencies_on_build_list() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - - { - cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - awk '{print $1 "." $2 "." $3 "." $4}' < \ - "${work_dir}/build-list" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|\.builds| - ' | \ - xargs -r cat | \ - sort -u - } | \ - sort | \ - uniq -d - -} - -# download_sources_by_hash $package $repository $git_revision $git_mod_revision -# try to download all sources by their hash into the current directory -# returns 0 if any source was downloaded and 1 otherwise - -download_sources_by_hash() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local git_mod_revision="$4" - - local return_value=1 - local tmp_dir - local sum_type - local arch_suffix - - tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir) - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then - >&2 echo 'download_sources_by_hash: make_source_info failed.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - if ! [ -s "${tmp_dir}/.SRCINFO" ]; then - >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - for arch_suffix in '' '_i686'; do - for sum_type in 'sha256sum' 'sha512sum'; do - grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed 's|^.* = ||' | \ - cat -n > \ - "${tmp_dir}/sums" - grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed ' - s|^.* = || - s|::.*$|| - s|.*/|| - ' | \ - cat -n > \ - "${tmp_dir}/urls" - if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then - join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \ - "${tmp_dir}/joined" - while read -r sum file; do - if [ "${sum}" = 'SKIP' ]; then - continue - fi - if echo "${sum} ${file}" | \ - ${sum_type} -c > /dev/null 2>&1; then - # the correct source is already there - continue - fi - if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then - mv "${tmp_dir}/transfer" "${file}" - return_value=0 - fi - done < \ - "${tmp_dir}/joined" - fi - done - done - - rm -rf --one-file-system "${tmp_dir}" - return ${return_value} - -} - -# expand_version $column_num -# add "0:" to version in $colum_num-th column if no ":" is there (epoch) -# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash) - -expand_version() { - local column_num - column_num="$1" - - sed ' - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/ - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/ - ' -} - -# shrink_version $column_num -# remove "0:" from version in $colum_num-th column (epoch) -# remove "+0" from version in $colum_num-th column (git count/hash) - -shrink_version() { - local column_num - column_num="$1" - - sed ' - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/ - ' -} - -# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] - -# Return (to stdout) the biggest subset A of the packages in $omega whose -# runtime dependencies in $omega \cup $keep are also in A - -# $all_builds and $all_depends either point to an empty file - then they will get -# filled with cached data for subsequent calls - or to the same files of a previous -# call - -# If non-empty, $force contains packages which are assumed to match the above -# condition without checking. - -# The arguments are names of files with one $package.$revision.$mod_revision.$repository -# per line. - -find_biggest_subset_of_packages() { - - ( # the new shell is intentional - omega="$1" - keep="$2" - all_builds="$3" - all_depends="$4" - if [ $# -eq 4 ]; then - force='/dev/null' - elif [ $# -eq 5 ]; then - force="$5" - else - >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" - return 2 - fi - - if [ ! -s "${all_builds}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.builds | | - ' {} \; | \ - sort -k2,2 > \ - "${all_builds}" - fi - - if [ ! -s "${all_depends}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.run-depends | | - ' {} \; | \ - grep -v ' base$' | \ - sort -k2,2 > \ - "${all_depends}" - fi - - sort -u "${omega}" | \ - sponge "${omega}" - - temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - { - sort -u "${keep}" - cat "${force}" "${force}" - } | \ - sort | \ - uniq -u > \ - "${temp_dir}/keep.new" - touch "${temp_dir}/keep" - - while [ -s "${temp_dir}/keep.new" ]; do - cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ - sort -u | \ - sponge "${temp_dir}/keep" - - { - # we append all packages which are run-dependencies of keep-packages - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.run-depends:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_builds}" - - # we append all packages with run-dependencies on the keep-list - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.builds:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_depends}" - } | \ - sort -u | \ - join -1 1 -2 1 -o 2.1 - "${omega}" | \ - sort -u > \ - "${temp_dir}/keep.new" - - # "new" is only what has not been there before and what is not forced - cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ - sort | \ - uniq -u | \ - sponge "${temp_dir}/keep.new" - done - - cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ - sort | \ - uniq -u - - ) - -} - -# sort_square_bracket_content $file -# sort the content of [] in $file, print to stdout - -sort_square_bracket_content() { - local file - local line - local token - local token_list - local rest - file="$1" - - while read -r line; do - printf '%s ' "${line}" | \ - tr ' ' '\n' | \ - while read -r token; do - if echo "${token}" | \ - grep -qF '['; then - printf '%s[' "${token%[*}" - token="${token##*[}" - token_list="${token%,}" - while ! echo "${token_list}" | \ - grep -qF ']'; do - read -r token - token_list=$( - printf '%s\n' \ - "${token_list}" \ - "${token%,}" - ) - done - rest="]${token_list#*]}" - token_list="${token_list%%]*}" - token=$( - printf '%s' "${token_list}" | \ - sort | \ - sed 's|$|,|' - printf '%s' "${rest}" - ) - fi - printf '%s\n' "${token}" - done | \ - tr '\n' ' ' | \ - sed ' - s|, ]|]|g - s| $|| - ' - printf '\n' - done < \ - "${file}" -} - -# smoothen_namcap_log $file -# remove unneccesary differences from namcap-logs: -# - remove architecture specific information -# - sort lines -# - sort content of square brackets - -smoothen_namcap_log() { - local file - file="$1" - # shellcheck disable=SC2016 - sort_square_bracket_content "${file}" | \ - sed ' - # normalize architecture specific information - s|i[34567]86|$ARCH|g - s|x86\([-_]64\)\?|$ARCH|g - # remove haskell hashes - s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g - ' | \ - sort | \ - sponge "${file}" -} - -# print_list_of_archaic_packages $source1 $source2 ... -# print a list of packages which have not been touched for a while, -# but which are still in the pipeline, e.g. in $source1, $source2 or ... - -print_list_of_archaic_packages() { - for source in "$@"; do - case "${source}" in - 'testing') - # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ - -exec head -n1 {} \; | \ - "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin - # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated - # and no longer block other packages from being moved - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ - sed ' - s|\.testing$|| - ' - ;; - 'build-list') - while read -r pkg rev mod_rev repo; do - git_repo=$( - find_repository_with_commit "${rev}" - ) - eval repo_path='"${repo_paths__'"${git_repo}"'}"' - commit_date=$( - git -C "${repo_path}" show -s --format=%ct "${rev}" - ) - mod_commit_date=$( - git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" - ) - if [ "${mod_commit_date}" -gt "${commit_date}" ]; then - commit_date="${mod_commit_date}" - fi - # packages remaining longer than $max_package_age_build_list days on the build list - if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then - printf '%s %s %s %s\n' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" - fi - done < \ - "${work_dir}/build-list" - ;; - 'staging') - # packages remaining longer than $max_package_age_staging days in staging - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ - sed ' - s|\.done$|| - ' - ;; - *) - >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" - ;; - esac - done | \ - sort -u -} - -# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" -# print the given modification revision possibly with a html link to github - -modification_revision_link() { - local mod_rev="$1" - local repo="$2" - local pkg="$3" - - if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then - printf '%s\n' \ - "${mod_rev}" \ - "${repo}" \ - "${pkg}" \ - "${mod_rev}" - else - printf '%s\n' \ - "${mod_rev}" - fi -} - -# trigger_mirror_refreshs -# trigger a refresh of capable tier 1 mirrors (as backup for master mirror) - -trigger_mirror_refreshs() { - local tmp_file - - tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir) - date '+%s' > \ - "${tmp_file}" - ${master_mirror_rsync_command} \ - "${tmp_file}" \ - "${master_mirror_rsync_directory}/lastupdate" - rm "${tmp_file}" - for trigger_url in ${mirror_refresh_trigger_urls}; do - screen -S trigger-mirror-update -d -m curl -L "${trigger_url}" - done -} - -# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name -extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { - pkgname="$1" - pkgname="${pkgname%.pkg.tar.xz}" - arch="${pkgname##*-}" - pkgname="${pkgname%-*}" - sub_pkgrel="${pkgname##*-}" - pkgname="${pkgname%-*}" - pkgrel="${sub_pkgrel%.*}" - if [ "${pkgrel}" = "${sub_pkgrel}" ]; then - sub_pkgrel='0' - else - sub_pkgrel="${sub_pkgrel##*.}" - fi - epoch="${pkgname##*-}" - pkgname="${pkgname%-*}" - pkgver="${epoch#*:}" - if [ "${pkgver}" = "${epoch}" ]; then - epoch='0' - else - epoch="${epoch%%:*}" - fi -} diff --git a/bin/mysql-functions b/bin/mysql-functions deleted file mode 100755 index 185d9a9..0000000 --- a/bin/mysql-functions +++ /dev/null @@ -1,1098 +0,0 @@ -#!/bin/sh - -# contains functions used to access mysql db - -# shellcheck disable=SC2016,SC2039 - -if [ -z "${base_dir}" ]; then - # just to make shellcheck happy - . 'conf/default.conf' -fi - -# base64_encode_each encode each line of stdin with base64 - -base64_encode_each() { - local line - - while read -r line; do - printf '%s' \ - "${line}" | \ - base64 -w0 - printf '\n' - done -} - -# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository - -# shellcheck disable=SC2086 -mysql_add_package_source() { - local names='pkgbase git_revision mod_git_revision upstream_package_repository' - local values - local uses_upstream - local uses_modification - local repo - - if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then - uses_modification=1 - else - uses_modification=0 - fi - uses_upstream=0 - for repo in ${repo_names}; do - if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \ - tar -t 2>/dev/null | \ - sed 's,-x86_64/,-any/,' | \ - grep -qFx "$1/repos/$4-any/PKGBUILD"; then - uses_upstream=1 - fi - done - - for _ in ${names}; do - values="${values}$( - printf '%s' "$1" | \ - base64 -w0 - ) " - shift - done - values="${values% }" - - { - printf 'INSERT IGNORE INTO package_sources' - printf ' (' - printf '`%s`, ' ${names} - printf '`uses_upstream`,`uses_modification`' - printf ') SELECT' - printf ' from_base64("%s"), ' ${values% *} - printf ' `upstream_repositories`.`id`,%s,%s' \ - ${uses_upstream} ${uses_modification} - printf ' FROM `upstream_repositories`' - printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ - "${values##* }" - } | \ - ${mysql_command} -} - -# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository - -# shellcheck disable=SC2031,SC2086,SC2154 -mysql_add_binary_package() { - local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'INSERT IGNORE INTO binary_packages' - printf ' (' - printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted' - printf ') SELECT' - printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" - printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' - printf ' 0, 0, 0' - printf ' FROM' - printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' - printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' - printf ' WHERE' - printf ' `%s`.`name` = from_base64("%s") AND' \ - 'repositories' "${repository}" \ - 'architectures' "${architecture}" - printf ' `package_sources`.`%s` = from_base64("%s") AND' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ' `upstream_repositories`.`name` = from_base64("%s")' \ - "${upstream_package_repository}" - } | \ - sed ' - s|, )|)|g - s|, FROM| FROM|g - ' | \ - ${mysql_command} -} - -# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel - -# shellcheck disable=SC2031,SC2086,SC2154 -mysql_show_binary_package() { - local names='pkgname pkgver pkgrel sub_pkgrel' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'SELECT' - printf ' `%s`.`%s`,' \ - 'repositories' 'name' \ - 'binary_packages' 'pkgname' \ - 'package_sources' 'pkgver' \ - 'package_sources' 'pkgrel' \ - 'binary_packages' 'sub_pkgrel' \ - 'architectures' 'name' \ - 'package_sources' 'pkgbase' \ - 'package_sources' 'git_revision' \ - 'package_sources' 'mod_git_revision' \ - 'upstream_repositories' 'name' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_repositories - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'binary_packages' 'pkgname' "${pkgname}" \ - 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ - 'package_sources' 'pkgver' "${pkgver}" \ - 'package_sources' 'pkgrel' "${pkgrel}" - printf ';' - } | \ - sed ' - s|, FROM| FROM|g - s|AND;|;|g - ' | \ - ${mysql_command} --html -} - -# mysql_add_install_target $install_target - -# shellcheck disable=2086 -mysql_add_install_target() { - local install_target - install_target=$( - printf "%s" "$1" | \ - base64 -w0 - ) - - { - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' VALUES (from_base64("%s"))' \ - "${install_target}" - } | \ - ${mysql_command} -} - -# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository -# or -# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository -# if sub_pkgrel should be determined automatically -# and -# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository -# or -# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository -# if $sub_pkgrel should be forced - -# generate the meta data of a package (dependencies, built packages, ...) in the database - -mysql_generate_package_metadata() { - - ( # new shell is intentional - case "$1" in - ''|*[!0-9]*) - unset forced_sub_pkgrel - ;; - *) - forced_sub_pkgrel=$( - printf '%s' "$1" | \ - base64 -w0 - ) - shift - ;; - esac - current_repository="$1" - package="$2" - - if [ $# -eq 2 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - else - git_revision="$3" - mod_git_revision="$4" - repository="$5" - fi - - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - printf '.' >&2 - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" - exit 2 - fi - # remove empty lines and unsupported architectures - sed -i ' - /^[^=]*=\s*$/d - /^\s*arch = /{ - / \(i686\|any\)$/!d - } - ' "${temp_dir}/SRCINFO" - - if [ ! -s "${temp_dir}/SRCINFO" ]; then - >&2 printf '"make_source_info" had empty output - eh, what?\n' - exit 2 - fi - printf '\n\n' >> "${temp_dir}/SRCINFO" - - printf '.' >&2 - pkgbase=$( - grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ - cut -d' ' -f3 - ) - if [ -z "${pkgbase}" ]; then - >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' - exit 2 - fi - - # add the package source - mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" - printf '.' >&2 - - # now we encode everything in base64 - current_repository=$( - printf '%s' "${current_repository}" | \ - base64 -w0 - ) - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - git_revision=$( - printf '%s' "${git_revision}" | \ - base64 -w0 - ) - mod_git_revision=$( - printf '%s' "${mod_git_revision}" | \ - base64 -w0 - ) - repository=$( - printf '%s' "${repository}" | \ - base64 -w0 - ) - - # add the build assignment(s) - { - archs=$( - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - grep -vxF 'any' | \ - sort -u - ) - if [ -z "${archs}" ]; then - echo 'any' - else - printf '%s\n' "${archs}" - fi - } | \ - while read -r arch; do - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' - printf ' FROM `architectures` JOIN `package_sources`' - printf ' WHERE `architectures`.`name` = from_base64("%s")' \ - "$( - printf '%s' "${arch}" | \ - base64 -w0 - )" - printf ' AND `package_sources`.`%s` = from_base64("%s")' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ';\n' - done > \ - "${temp_dir}/add-build-assignments-command" - - # TODO: correctly link between binary_packages and build_assignments using any_arch - - # shellcheck disable=SC2034 - # select any specific arch (which will be building the 'any' part of a split package) - any_arch=$( - { - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - sort -r | \ - grep -vxFm 1 'any' || \ - echo 'any' - } | \ - base64_encode_each - ) - - grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ - cut -d' ' -f3 | \ - while read -r pkgname; do - pkgname64=$( - printf '%s' "${pkgname}" | \ - base64 -w0 - ) - sed -n ' - /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ - :a - N - /\n$/{ - p - T - } - ba - } - ' "${temp_dir}/SRCINFO" | \ - sed ' - /^\S/d - s/^\s*// - ' > \ - "${temp_dir}/BINARYINFO.${pkgname64}" - - grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ - cut -d' ' -f3 | \ - while read -r arch; do - arch64=$( - printf '%s' "${arch}" | \ - base64 -w0 - ) - sed ' - s/^\(\S\+\)_'"${arch}"' = /\1 = / - ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ - "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" - done - done - find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ - while read -r _ pkgname arch; do - pkgver=$( - grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - pkgrel=$( - grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - epoch=$( - { - grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \ - echo 'epoch = 0' - } | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - provides=$( - grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - makedepends=$( - grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - checkdepends=$( - grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - rundepends=$( - grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - if [ -n "${forced_sub_pkgrel}" ]; then - sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")' - else - sub_pkgrel=$( - printf '(SELECT COALESCE(' - # do not add binary packages which are currently on the - # build-list or in $current_repository (beware of split - # packages!) - printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_repositories - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - printf ' `architectures`.`name`=from_base64("%s")' \ - "${arch}" - printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \ - "${current_repository}" - # max(sub_pkgrel)+1 - printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - mysql_join_binary_packages_architectures - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then - # 'any' gets higher sub_pkgrel than any architecture - printf ' 1' - else - # not-'any' gets higher sub_pkgrel than same or 'any' architecture - printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \ - "${arch}" - fi - printf ')' - printf ',0))' - ) - fi - { - printf 'INSERT IGNORE INTO `binary_packages` (' - printf '`%s`,' \ - 'build_assignment' \ - 'repository' \ - 'architecture' \ - 'epoch' \ - 'pkgver' \ - 'pkgrel' \ - 'pkgname' \ - 'sub_pkgrel' \ - 'has_issues' \ - 'is_tested' \ - 'is_to_be_deleted' - printf ') SELECT ' - printf '`%s`.`id`,' \ - 'build_assignments' \ - 'repositories' \ - 'architectures' - printf 'from_base64("%s"),' \ - "${epoch}" \ - "${pkgver}" \ - "${pkgrel}" \ - "${pkgname}" - printf '%s,0,0,0 FROM' \ - "${sub_pkgrel}" - printf ' `%s` JOIN' \ - 'repositories' \ - 'architectures' \ - 'build_assignments' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'repositories' 'name' "${current_repository}" \ - 'architectures' 'name' "${arch}" \ - 'package_sources' 'pkgbase' "${pkgbase}" \ - 'package_sources' 'git_revision' "${git_revision}" \ - 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ - 'upstream_repositories' 'name' "${repository}" - printf ';\n' - } | \ - sed ' - s|,)|)|g - s| JOIN JOIN | JOIN | - s| AND;$|;| - ' >> \ - "${temp_dir}/add-binary-packages-command" - { - printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' - - printf 'INSERT INTO `provides` VALUES\n' - echo "${provides}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' (from_base64("%s"));\n' \ - "${pkgname}" - - printf 'INSERT INTO `rundepends` VALUES\n' - echo "${rundepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' ("base");\n' - - echo "${checkdepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - 1 s/^/INSERT INTO `checkdepends` VALUES \n/ - $ s/,$/;/ - ' - - printf 'INSERT INTO `makedepends` VALUES\n' - echo "${makedepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' ("base-devel");\n' - - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' SELECT (`name`) FROM `%s` UNION' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' | \ - sed 's| UNION$|;\n|' - - for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do - case "${link}" in - 'provides') - printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' - ;; - 'makedepends'|'checkdepends'|'rundepends') - printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' - printf ' `dependency_types` JOIN' - ;; - esac - printf ' `binary_packages`' - mysql_join_binary_packages_architectures - printf ' JOIN `install_targets`' - printf ' JOIN `%s`' "${link}" - printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" - printf ' WHERE' - if [ "${link}" = 'makedepends' ] || \ - [ "${link}" = 'checkdepends' ] || \ - [ "${link}" = 'rundepends' ]; then - printf ' `dependency_types`.`name` = "%s" AND' \ - "${link%depends}" - fi - printf ' `binary_packages`.`%s` = from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - # we do not want to match the sub_pkgrel: - # a) it is tedious to do so (because it may be calculated - # dynamically) - # b) it is not necessary to do so: if only the sub_pkgrel - # changed, the dependencies and provided install_targets - # should not have changed - printf ' `architectures`.`name` = from_base64("%s");\n' \ - "${arch}" - # the repository is of no relevance: it hardly matters for - # the dependencies - done - - printf 'DROP TABLE `%s`;\n' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' - } >> \ - "${temp_dir}/add-install-targets-command" - done - printf '.' >&2 - - { - if [ -s "${temp_dir}/add-build-assignments-command" ]; then - cat "${temp_dir}/add-build-assignments-command" - fi - if [ -s "${temp_dir}/add-binary-packages-command" ]; then - cat "${temp_dir}/add-binary-packages-command" - fi - if [ -s "${temp_dir}/add-install-targets-command" ]; then - cat "${temp_dir}/add-install-targets-command" - fi - } | \ - ${mysql_command} - printf '.' >&2 - - ) -} - -# mysql_sanity_check -# do a sanity check on the mysql database - -mysql_sanity_check() { - { - printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_build_assignments - mysql_join_binary_packages_architectures '' 'bp_arch' - mysql_join_build_assignments_architectures '' 'ba_arch' - printf ' WHERE `bp_arch`.`name`!="any"' - printf ' AND `ba_arch`.`name`="any";\n' - printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' - printf ' FROM `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b`' - printf ' ON `a`.`pkgname`=`b`.`pkgname`' - printf ' AND `a`.`repository`=`b`.`repository`' - printf ' AND `a`.`id`!=`b`.`id`' - mysql_join_binary_packages_repositories 'a' - printf ' WHERE `repositories`.`name`="build-list";\n' - printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' - printf ' FROM `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' - mysql_join_binary_packages_repositories 'a' 'arep' - mysql_join_binary_packages_repositories 'b' 'brep' - printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' - printf ' AND `%srep`.`name`="build-list"' \ - 'a' 'b' - printf ';\n' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - /^CONCAT("/d - s,^,, - s,$,, - ' - ( # new shell is intentional - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - for dir in $(ls_master_mirror 'i686'); do - ls_master_mirror "i686/${dir}" | \ - sed -n ' - s/\.pkg\.tar\.xz$// - T - s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ - s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ - s|^|'"${dir}"'/| - p - ' - done | \ - sort > \ - "${temp_dir}/master-mirror-listing" - - { - printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`is_on_master_mirror`' - } | \ - ${mysql_command} --batch | \ - sed ' - 1d - s,\t,/, - s,\t,-, - s,\t,:, - s,\t,-, - s,\t,., - s,\t,-, - ' | \ - sort > \ - "${temp_dir}/mysql-packages" - - diff -u \ - "${temp_dir}/master-mirror-listing" \ - "${temp_dir}/mysql-packages" - ) -} - -mysql_find_build_assignment_loops() { - new_loops=$( - { - printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' - printf ' FROM `dependencies`' - mysql_join_dependencies_install_target_providers - mysql_join_install_target_providers_binary_packages '' 'packages_dependency' - mysql_join_dependencies_binary_packages '' 'packages_dependent' - mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' - mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' - printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ - tsort 2>&1 >/dev/null | \ - sed 's/^tsort:\s*//' | \ - { - loop=0 - while read -r id; do - if [ "x${id}" = 'x-: input contains a loop:' ]; then - loop=$((loop+1)) - continue - fi - if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then - >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}" - continue - fi - printf '(%s,%s),' "${loop}" "${id}" - done | \ - sed 's/,$//' - } - ) - { - printf 'DELETE FROM `build_dependency_loops`;\n' - if [ -n "${new_loops}" ]; then - printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \ - "${new_loops}" - fi - } | \ - ${mysql_command} -} - -# mysql_cleanup [dry] -# clean up left overs from mysql database -mysql_cleanup() { - local operator - if [ "$#" = '0' ]; then - operator='DELETE' - elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then - operator='SELECT COUNT(*)' - else - >&2 echo 'Unknown parameter' - >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".' - exit 2 - fi - { - # remove to-be-decided binary_packages - printf '%s ' \ - "${operator}" - if [ "${operator}" = 'DELETE' ]; then - printf '`binary_packages` ' - fi - printf 'FROM `binary_packages`' - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="to-be-decided";\n' - # remove dependencies w/o binary_package or install_target - printf '%s FROM `dependencies` ' \ - "${operator}" - printf 'WHERE NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `binary_packages` ' - printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`' - printf ') OR NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `install_targets` ' - printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`' - printf ');\n' - # remove install_target_providers w/o binary_package or install_target - printf '%s FROM `install_target_providers` ' \ - "${operator}" - printf 'WHERE NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `binary_packages` ' - printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`' - printf ') OR NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `install_targets` ' - printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`' - printf ');\n' - # remove build_assignments w/o binary_package - printf '%s FROM `build_assignments` ' \ - "${operator}" - printf 'WHERE NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `binary_packages` ' - printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ');\n' - # remove failed_builds with unbroken build_assignments - printf '%s ' \ - "${operator}" - if [ "${operator}" = 'DELETE' ]; then - printf '`failed_builds` ' - fi - printf 'FROM `failed_builds` ' - mysql_join_failed_builds_build_assignments - printf 'WHERE NOT `build_assignments`.`is_broken`' - printf ';\n' - # remove failed_builds w/o build_assignment - printf '%s FROM `failed_builds` ' \ - "${operator}" - printf 'WHERE NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `build_assignments` ' - printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`' - printf ');\n' - # remove package_sources w/o build_assignment - printf '%s FROM `package_sources` ' \ - "${operator}" - printf 'WHERE NOT EXISTS ' - printf '(' - printf 'SELECT * FROM `build_assignments` ' - printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' - printf ');\n' - printf 'UPDATE `build_slaves`' - mysql_join_build_slaves_binary_packages - mysql_join_binary_packages_repositories - printf ' SET `currently_building`=NULL' - printf ' WHERE `repositories`.`name`!="build-list";\n' - } | \ - ${mysql_command} -} - -# mysql_query_has_pending_dependencies `build_assignment`.`id` -# print a mysql query giving wether dependencies are pending -mysql_query_has_pending_dependencies() { - printf 'EXISTS (' - printf 'SELECT * FROM `binary_packages` as `to_dos`' - mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' - mysql_join_binary_packages_dependencies 'to_dos' - mysql_join_dependencies_install_target_providers - mysql_join_install_target_providers_binary_packages '' 'bin_deps' - mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' - printf ' WHERE' - printf ' `%s`.`name`="build-list" AND' \ - 'dep_repos' 'to_do_repos' - printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND' - printf ' `to_dos`.`build_assignment`=%s' \ - "$1" - printf ')' -} - -# mysql_query_is_part_of_loop `build_assignment`.`id` -# print a mysql query giving wether the package is part of a loop -mysql_query_is_part_of_loop() { - printf 'EXISTS (' - printf 'SELECT * FROM `build_dependency_loops`' - printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \ - "$1" - printf ')' -} - -# mysql_query_select_pkgbase_and_revision -# print the part of a mysql query giving: -# pkgbase git_revision mod_git_revision upstream_package_repository -mysql_query_select_pkgbase_and_revision() { - printf '`package_sources`.`%s`,' \ - 'pkgbase' \ - 'git_revision' \ - 'mod_git_revision' - printf '`upstream_repositories`.`name`' - printf ' FROM `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories -} - -# mysql_repair_binary_packages_without_build_assignment -# try to generate valid build assignments to binary packages without -# a valid one yet -mysql_repair_binary_packages_without_build_assignment() { - { - printf 'SELECT ' - printf '`binary_packages`.`id`' - printf ',replace(to_base64(%s),"\\n","")' \ - '`binary_packages`.`pkgname`' \ - '`architectures`.`name`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - printf ' WHERE `binary_packages`.`build_assignment`<0' - } | \ - ${mysql_command} --raw --batch | \ - sed '1d' | \ - while read -r id pkgname arch; do - pkgname=$( - printf '%s' "${pkgname}" | \ - base64 -d - ) - pkgbase=$( - curl -Ss "$( - printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ - "${pkgname}" - )" | \ - sed ' - s/^.*"results":\s*\[// - s/}\s*,\s*{/\n/g - ' | \ - grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ - tr ',' '\n' | \ - grep '"pkgbase":' | \ - cut -d'"' -f4 | \ - sort -u | \ - head -n1 - ) - if [ -z "${pkgbase}" ] && \ - { - printf 'SELECT count(*) FROM `package_sources`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "${pkgname}" | base64 -w0)" - } | \ - ${mysql_command} --raw --batch | \ - sed '1d' | \ - grep -qvxF '0'; then - pkgbase="${pkgname}" - fi - if [ -z "${pkgbase}" ]; then - >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" - continue - fi - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' - printf ' FROM `package_sources`' - printf ' JOIN `architectures`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" - printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" - printf ' LIMIT 1;\n' - printf 'UPDATE `binary_packages`' - printf ' JOIN `build_assignments`' - mysql_join_binary_packages_build_assignments - printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' WHERE `binary_packages`.`id`=%s' "${id}" - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" - done | \ - ${mysql_command} -} - -# mysql_remove_duplicate_binary_packages -# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel, -# having difference of 1 in sub_pkgrel - -mysql_remove_duplicate_build_order() { - { - printf 'CREATE TEMPORARY TABLE `ren`' - printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n' - printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)' - printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`' - printf ' FROM `binary_packages` as `old`' - printf ' JOIN `binary_packages` as `new` ON' - printf ' `old`.`%s`=`new`.`%s` AND' \ - 'pkgname' 'pkgname' \ - 'epoch' 'epoch' \ - 'pkgver' 'pkgver' \ - 'pkgrel' 'pkgrel' - printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - mysql_join_binary_packages_repositories 'old' 'orep' - mysql_join_binary_packages_repositories 'new' 'nrep' - printf ' WHERE `orep`.`name`!="to-be-decided"' - printf ' AND `nrep`.`name`="to-be-decided";\n' - printf 'UPDATE IGNORE `dependencies`' - printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`' - printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n' - printf 'UPDATE IGNORE `install_target_providers`' - printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`' - printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n' - printf 'DELETE FROM `binary_packages`' - printf ' WHERE EXISTS (' - printf 'SELECT * FROM `ren`' - printf ' WHERE `ren`.`old`=`binary_packages`.`id`' - printf ');\n' - printf 'UPDATE IGNORE `binary_packages`' - printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`' - printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' - printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' - } | \ - ${mysql_command} -} - -# mysql_package_name_query -# print a mysql query of the full name of a package file - -mysql_package_name_query() { - printf 'CONCAT(' - printf '`binary_packages`.`pkgname`,"-",' - printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' - printf '`binary_packages`.`pkgver`,"-",' - printf '`binary_packages`.`pkgrel`,".",' - printf '`binary_packages`.`sub_pkgrel`,"-",' - printf '`architectures`.`name`,".pkg.tar.xz"' - printf ')' -} - -# mysql_join_*_* -# print 'JOIN' part of mysql query to connect the respective tables -# these functions take 2 optional arguments, acting as aliases for -# the tables - -# mysql_join__generic $table_a $column_a $table_b $column_b -# create mysql_join_${table_a}_${table_b}() function - -mysql_join__generic() { - eval "$( - printf 'mysql_join_%s_%s() {\n' "$1" "$3" - printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" - printf ' if [ -n "$2" ]; then\n' - printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' - printf ' fi\n' - printf ' if [ -n "$1" ]; then\n' - printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" - printf ' else\n' - printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" - printf ' fi\n' - printf ' if [ -n "$2" ]; then\n' - printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" - printf ' else\n' - printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" - printf ' fi\n' - printf '}\n' - )" -} - -for link in \ - 'binary_packages:architecture:architectures' \ - 'binary_packages:repository:repositories' \ - 'binary_packages:build_assignment:build_assignments' \ - \ - 'build_assignments:architecture:architectures' \ - 'build_assignments:package_source:package_sources' \ - \ - 'build_dependency_loops:build_assignment:build_assignments' \ - 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ - \ - 'build_slaves:currently_building:build_assignments' \ - 'build_slaves:currently_building build_assignment:binary_packages' \ - \ - 'dependencies:depending_on:install_targets' \ - 'dependencies:dependent:binary_packages' \ - 'dependencies:dependency_type:dependency_types' \ - \ - 'failed_builds:reason:fail_reason' \ - 'failed_builds:build_assignment:build_assignments' \ - 'failed_builds:build_slave:build_slaves' \ - \ - 'install_target_providers:package:binary_packages' \ - 'install_target_providers:install_target:install_targets' \ - 'install_target_providers:install_target depending_on:dependencies' \ - \ - 'package_sources:upstream_package_repository:upstream_repositories' \ - \ - 'repositories:stability:repository_stabilities' \ - \ - 'upstream_repositories:git_repository:git_repositories'; do -# A join for these cannot be done, because it's not clear on what to join: -# 'repository_stability_relations:more_stable:repository_stabilities' -# 'repository_stability_relations:less_stable:repository_stabilities' - - table_b="${link##*:}" - table_a="${link%:*}" - column_b="${table_a##*:}" - table_a="${table_a%:*}" - column_a="${column_b% *}" - if [ "${column_a}" = "${column_b}" ]; then - column_b='id' - else - column_b="${column_b##* }" - fi - - mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" - mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" -done diff --git a/conf/default.conf b/conf/default.conf index 84c0e4f..18f5d73 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -14,10 +14,10 @@ else base_dir=$(printf '%s/..' "$(dirname "$(readlink -f "$0")")") fi -# shellcheck source=bin/common-functions -. "${base_dir}/bin/common-functions" -# shellcheck source=bin/mysql-functions -. "${base_dir}/bin/mysql-functions" +# shellcheck source=lib/common-functions +. "${base_dir}/lib/common-functions" +# shellcheck source=lib/mysql-functions +. "${base_dir}/lib/mysql-functions" work_dir="${base_dir}/work" diff --git a/lib/common-functions b/lib/common-functions new file mode 100755 index 0000000..a84271b --- /dev/null +++ b/lib/common-functions @@ -0,0 +1,1378 @@ +#!/bin/sh + +# contains functions used by more than one script + +# shellcheck disable=SC2039 + +# TODO: include link depenendencies in run-depends metadata + +# TODO: have full information (currently in files) in database + +# TODO: remove state files / metadata files + +if [ -z "${base_dir}" ]; then + # just to make shellcheck happy + . 'conf/default.conf' +fi + +# find_pkgbuilds package repository git_repository git_revision mod_git_revision +# find the PKGBUILD and modification of $package from $repository +# sets $PKGBUILD and $PKGBUILD_mod + +find_pkgbuilds() { + + local package="$1" + local repository="$2" + local git_repository="$3" + local git_revision="$4" + local mod_git_revision="$5" + + local repo_path + eval 'repo_path="${repo_paths__'"${git_repository}"'}"' + + PKGBUILD=$( + git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \ + tar -t 2> /dev/null | \ + grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \ + grep -v -- '-i686/PKGBUILD$' | \ + grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ + sort | \ + tail -n1 + ) + + PKGBUILD_mod=$( + git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \ + tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null + ) || true + + if [ -z "${PKGBUILD}" ] && \ + [ -z "${PKGBUILD_mod}" ]; then + >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ + "${package}" \ + "${repository}" \ + "${git_repository}" \ + "${git_revision}" \ + "${mod_git_revision}" + return 1 + fi + +} + +# find_repository_with_commit commit +# find the repository which has $commit + +find_repository_with_commit() { + + local repository + + for repository in ${repo_names}; do + # shellcheck disable=SC2016 + if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then + echo "${repository}" + return 0 + fi + done + >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" + exit 1 + +} + +# find_git_repository_to_package_repository repository +# find the git repository which tracks the package repository $repository + +find_git_repository_to_package_repository() { + + local repository + local package_repository + local repo_path + + package_repository="$1" + + if [ "$1" = 'build-support' ]; then + echo 'packages' + return 0 + fi + + for repository in ${repo_names}; do + if [ "${repository}" = "archlinux32" ]; then + continue + fi + eval 'repo_path="${repo_paths__'"${repository}"'}"' + if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ + tar -t --wildcards '*/repos' | \ + grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ + cut -d/ -f3 | \ + sed 's|-[^-]\+$||' | \ + sort -u | \ + grep -qxF "${package_repository}"; then + echo "${repository}" + return 0 + fi + done + >&2 echo "can't find git repository with package repository '$1'" + exit 1 + +} + +# generate_package_metadata $package $git_revision $mod_git_revision $repository +# or +# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository +# generate the meta data files of a package (dependencies, built packages, ...) + +generate_package_metadata() { + + local package="$1" + local git_revision="$2" + local mod_git_revision="$3" + local repository="$4" + local file_prefix + local file + local PKGBUILD + + if [ $# -eq 1 ]; then + # second form + repository="${package##*.}" + package="${package%.*}" + mod_git_revision="${package##*.}" + package="${package%.*}" + git_revision="${package##*.}" + package="${package%.*}" + fi + + file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" + + if [ -e "${file_prefix}.builds" ] && \ + [ -e "${file_prefix}.build-depends" ] && \ + [ -e "${file_prefix}.run-depends" ] && \ + [ -e "${file_prefix}.groups" ] && \ + [ -e "${file_prefix}.packages" ]; then + return 0 + fi + + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then + printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" + exit 1 + fi + if [ ! -s "${file_prefix}.SRCINFO" ]; then + >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" + exit 1 + fi + + # otherwise this just calls for trouble + sed -i ' + /^[^=]*=\s*$/d + s/_i686\(\s*=\)/\1/ + ' "${file_prefix}.SRCINFO" + + # extract "groups" = groups \cup provides + grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u > \ + "${file_prefix}.groups" + + # extract "packages" = pkgname + grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u > \ + "${file_prefix}.packages" + + # extract "builds" = provides \cup pkgname \cup groups + cat "${file_prefix}.groups" "${file_prefix}.packages" | \ + sort -u > \ + "${file_prefix}.builds" + + # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" + { + { + printf 'all_depend = %s\n' 'base' 'base-devel' + sed -n "$( + printf '/^\t%s = /p\n' \ + 'depends' \ + 'makedepends' + )" "${file_prefix}.SRCINFO" + } | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u + sed 'p' "${file_prefix}.builds" + } | \ + sort | \ + uniq -u > \ + "${file_prefix}.build-depends" + + # extract "run-depends" = depends \cup \{ base \} \setminus "builds" + { + { + printf 'all_depend = %s\n' 'base' + sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" + } | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u + sed 'p' "${file_prefix}.builds" + } | \ + sort | \ + uniq -u > \ + "${file_prefix}.run-depends" + + rm "${file_prefix}.SRCINFO" + +} + +# delete_old_metadata +# delete old (=unneeded) meta data of packages + +delete_old_metadata() { + + local current_metadata + + current_metadata=$( + find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ + sed ' + s|\.[^.]\+$|| + s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| + ' | \ + sort -u + ) + + ( # the new shell is intentional + # what we have + echo "${current_metadata}" + + # package-states should stay + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ + sed ' + s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| + ' | \ + sort -u | \ + sed 'p' + + # build-list items should stay + sed 'p' "${work_dir}/build-list" + + tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + echo "${current_metadata}" | \ + sort -k1,1 > \ + "${tmp_dir}/current-metadata" + + # the newest of the following should stay: + { + # deletion-list items + cat "${work_dir}/deletion-list" + # all packages in the repos + for repo in ${repo_names}; do + eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ + tar -t | \ + sed ' + s|/$|| + /\//d + ' + done + } | \ + sort -u | \ + join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ + sort -k4,4 > \ + "${tmp_dir}/find-newest-revisions" + + uniq -uf3 < \ + "${tmp_dir}/find-newest-revisions" | \ + awk '{print $4 " " $1 " " $2 " " $3}' | \ + sed 'p' + + uniq -Df3 < \ + "${tmp_dir}/find-newest-revisions" | \ + uniq --group=append -f3 | \ + { + revs='' + mod_revs='' + opkg='' + orepo='' + while read -r rev mod_rev repo pkg; do + + if [ -z "${rev}" ] && \ + [ -z "${mod_rev}" ] && \ + [ -z "${repo}" ] && \ + [ -z "${pkg}" ]; then + + printf '%s %s %s %s\n' \ + "$( + printf '%s\n' ${revs} | \ + find_newest_of_git_revisions + )" \ + "$( + printf '%s\n' ${mod_revs} | \ + find_newest_of_git_revisions + )" \ + "${orepo}" \ + "${opkg}" + + revs='' + mod_revs='' + orepo='' + opkg='' + continue + fi + revs=$( + # shellcheck disable=SC2086 + printf '%s\n' ${revs} ${rev} | \ + sort -u + ) + mod_revs=$( + # shellcheck disable=SC2086 + printf '%s\n' ${mod_revs} ${mod_rev} | \ + sort -u + ) + orepo="${repo}" + opkg="${pkg}" + done + } | \ + awk '{print $4 " " $1 " " $2 " " $3}' | \ + sed 'p' + ) | \ + sort | \ + uniq -u | \ + while read -r pkg rev mod_rev repo; do + rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* + done +} + +# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository +# print which (stable) repository a package belongs to + +repository_of_package() { + local package="$1" + local repository="${package##*.}" + package="${package%.*}" + local a32_rev="${package##*.}" + package="${package%.*.*}" + + case "${repository}" in + 'multilib') + if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ + tar -Ox | \ + grep -qFx "${package%.*.*.*}"; then + echo 'extra' + else + echo 'community' + fi + ;; + *) + echo "${repository}" + esac +} + +# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending +# print wether the specified package is an official package (print +# $ending) or a community package (print 'community-$ending') or a +# build-support package (print 'build-support') + +official_or_community() { + local prepo + prepo=$(repository_of_package "$1") + + if [ "${prepo}" = 'community' ]; then + echo 'community-'"$2" + elif [ "${prepo}" = 'build-support' ]; then + echo 'build-support' + else + echo "$2" + fi +} + +# ls_master_mirror $path +# list content of $path on the master mirror (via rsync) + +ls_master_mirror() { + + local path="$1" + + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/${path}/" | \ + grep -v '\s\.$' | \ + awk '{print $5}' + +} + +# TODO: the actions of remove_old_package_versions should be done +# on basis of the information in the database + +# remove_old_package_versions $arch $repository $package_file + +# removes all older (not-newer) versions of $package_file +# in all repositories not-older (newer) than $repository + +# TODO: should remove all other version (also newer) from +# some repositories :-/ + +# A package is considered not newer if +# a) its version is not newer +# A package is considered older if +# b) its version is older or +# c) if it's "not newer" and its architecture is 'any' and different or +# d) if it's "not newer" and the other architecture is 'any' and different + +# this ensures an any package may replace arch-specific packages of the same version and vice versa + +remove_old_package_versions() { + + local arch="$1" + local repository="$2" + local package="$3" + + local pkgname + local epoch + local pkgver + local pkgrel + local sub_pkgrel + pkgname="${package%-*}" + pkgrel="${pkgname##*-}" + sub_pkgrel="${pkgrel##*.}" + if [ "${sub_pkgrel}" = "${pkgrel}" ]; then + sub_pkgrel='0' + else + pkgrel="${pkgrel%.*}" + fi + pkgname="${pkgname%-*}" + pkgver="${pkgname##*-}" + epoch="${pkgver%%:*}" + if [ "${epoch}" = "${pkgver}" ]; then + epoch='0' + else + pkgver="${pkgver#*:}" + fi + pkgname="${pkgname%-*}" + + # shellcheck disable=SC2016 + { + printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \ + "$( + printf '%s' "${repository}" | \ + base64 -w0 + )" \ + "$( + printf '%s' "${package}" | \ + base64 -w0 + )" \ + "$( + printf '%s' "${package}" | \ + sed ' + s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/ + ' | \ + base64 -w0 + )" + printf 'SELECT ' + printf '`binary_packages`.`id`,' + printf 'CONCAT(`repositories`.`name`,"/",' + mysql_package_name_query + printf '),' + # should we delete packages of identical version? + printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),' + printf 'CONCAT(' + printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' + printf '`binary_packages`.`pkgver`,"-",' + printf '`binary_packages`.`pkgrel`,".",' + printf '`binary_packages`.`sub_pkgrel`' + printf ')' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' + printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' + # name must match + printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \ + "$(printf '%s' "${package%-*-*-*}" | base64 -w0)" + # repository, where package should be deleted, should be less stable + printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \ + "$(printf '%s' "${repository}" | base64 -w0)" + printf ';\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + /^\S\+\sCONCAT(/d + y/\t/ / + ' | \ + expand_version 4 | \ + sort -k4V,4 -k3r,3 | \ + shrink_version 4 | \ + sed -n ' + /^bogus /q + p + ' | \ + cut -d' ' -f1,2 >&2 + + # repositories in which older packages should be deleted + local delete_older_repositories + # repositories in which not-newer packages should be deleted + local delete_not_newer_repositories + + if echo "${standalone_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories="${repository}" + delete_not_newer_repositories='' + + elif echo "${staging_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories="${repository}" + delete_not_newer_repositories=$( + echo "${staging_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + elif echo "${testing_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories=$( + printf '%s\n' "${staging_package_repositories}" "${repository}" + ) + delete_not_newer_repositories=$( + echo "${testing_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + elif echo "${stable_package_repositories}" | \ + grep -qxF "${repository}"; then + + delete_older_repositories=$( + printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}" + ) + delete_not_newer_repositories=$( + echo "${stable_package_repositories}" | \ + grep -vxF "${repository}" + ) || true + + else + + >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}" + return 1 + + fi + + ( # the new shell is intentional + tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + { + # the architecture of the package (any vs. i686) + package_arch="${package##*-}" + package_arch="${package_arch%%.*}" + if [ "${package_arch}" = 'any' ]; then + package_arch_regex_inverter='!' + else + unset package_arch_regex_inverter + fi + + for repo in ${delete_older_repositories}; do + ls_master_mirror "${arch}/${repo}" | \ + sed -n ' + /\.pkg\.tar\.xz$/!d + s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| + /^'"$(str_to_regex "${package%-*-*-*}")"' / { + s|^|2 '"${arch} ${repo}"' | + / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{ + s|^2|0| + } + p + } + ' + done + for repo in ${delete_not_newer_repositories}; do + ls_master_mirror "${arch}/${repo}" | \ + sed -n ' + /\.pkg\.tar\.xz$/!d + s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| + /^'"$(str_to_regex "${package%-*-*-*}")"' / { + s|^|0 '"${arch} ${repo}"' | + p + } + ' + done + echo "${package%-*}" | \ + sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|' + + # the generated list contains the following columns: + # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz + } | \ + expand_version 5 | \ + sort -k5V,5 -k1n,1 | \ + shrink_version 5 | \ + sed -n ' + /^1 %cut% %it% %here% /q + s/^[02] // + s/ \(\S\+\)$/-\1/ + p + ' | \ + sort -u > \ + "${tmp_dir}/packages-to-delete" + # this file contains a list of packages to be deleted, one on each line: + # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz + + cut -d' ' -f1,2 < \ + "${tmp_dir}/packages-to-delete" | \ + grep -vxF "${arch} ${repository}" | \ + sort -u > \ + "${tmp_dir}/repositories-to-modify" + + # fetch all databases being modified + while read -r del_arch del_repo; do + mkdir -p "${tmp_dir}/${del_arch}/${del_repo}" + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \ + "${tmp_dir}/${del_arch}/${del_repo}/" + done < \ + "${tmp_dir}/repositories-to-modify" + + while read -r del_arch del_repo del_package _; do + if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then + # we do not repo-remove the package in the target repository + continue + fi + repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \ + "${del_package}" + done < \ + "${tmp_dir}/packages-to-delete" + + # upload modified databases + while read -r del_arch del_repo; do + ${master_mirror_rsync_command} \ + "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \ + "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \ + "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/" + done < \ + "${tmp_dir}/repositories-to-modify" + + # shellcheck disable=SC2016 + sed ' + s/\.pkg\.tar\.xz$// + s/^\S\+ // + s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/ + s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/ + s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ + ' "${tmp_dir}/packages-to-delete" | \ + while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do + printf 'DELETE FROM `binary_packages` WHERE' + printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \ + 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \ + 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)" + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \ + sed 's/ AND$//' + printf ';\n' + done | \ + ${mysql_command} + + sed ' + s| \(\S\+\)$|-\1| + y| |/| + s|^|rm "| + s|$|"| + p + s|"$|.sig"| + ' "${tmp_dir}/packages-to-delete" | \ + ${master_mirror_sftp_command} + ) + +} + +# wait_some_time $minimum $diff +# wait between minimum and minimum+diff seconds (diff defaults to 30) + +wait_some_time() { + local minimum=$1 + local diff=$2 + local random + + if [ -z "${diff}" ]; then + diff=30 + fi + + random=$( + dd if='/dev/urandom' count=1 2> /dev/null | \ + cksum | \ + cut -d' ' -f1 + ) + + sleep $((minimum + random % diff)) +} + +# str_to_regex $string +# escape dots for use in regex + +str_to_regex() { + echo "$1" | \ + sed ' + s|[.[]|\\\0|g + ' +} + +# make_source_info $package $repository $git_revision $mod_git_revision $output +# create .SRCINFO from PKGBUILD within git repositories, output to $output + +make_source_info() { + + local package="$1" + local repository="$2" + local git_revision="$3" + local mod_git_revision="$4" + local output="$5" + + local git_repo + local PKGBUILD + local PKGBUILD_mod + + git_repo=$(find_repository_with_commit "${git_revision}") + + if [ -z "${git_repo}" ]; then + return 1 + fi + + find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" + + ( # the new shell is intentional + + tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' + + { + cd "${tmp_dir}" + makepkg --printsrcinfo + cd .. + } | + if [ "${package%-i18n}-i18n" = "${package}" ]; then + sed ' + 1 a \\tdepends = '"${package%-i18n}"' + ' + else + cat + fi > \ + "${output}" + + ) + +} + +# recursively_umount_and_rm $dir +# umount all mountpoints in $dir which are also in $dir's +# filesystem, possibly also $dir itself and then +# rm -rf --one-file-system $dir + +recursively_umount_and_rm() { + local dir="$1" + + if [ -z "${dir}" ]; then + >&2 echo 'ERROR: recursively_umount_and_rm requires an argument' + exit 42 + fi + + find "${dir}" \ + -xdev -depth -type d \ + -exec 'mountpoint' '-q' '{}' ';' \ + -exec 'sudo' 'umount' '-l' '{}' ';' + rm -rf --one-file-system "${dir}" +} + +# mangle_pkgbuild $PKGBUILD [$sub_pkgrel] +# mangle $arch in PKBUILDs to contain i486, i586, i686 +# append $sub_pkgrel to the pkgrel + +mangle_pkgbuild() { + local PKGBUILD="$1" + local sub_pkgrel="$2" + + if [ -n "${sub_pkgrel}" ]; then + sub_pkgrel=".${sub_pkgrel}" + fi + + sed -i ' + /^arch=[^#]*any/!{ + /^arch=(/s/(/(i686 / + } + s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ + ' "${PKGBUILD}" +} + +# find_newest_of_git_revisions +# find newest git revision of the ones provided at stdin +# (assuming linear history) + +find_newest_of_git_revisions() { + local revisions + local repo + revisions=$(cat) + + if [ "$( + echo "${revisions}" | \ + wc -l + )" -eq 1 ]; then + + echo "${revisions}" + return + + fi + + repo=$( + find_repository_with_commit \ + "$( + echo "${revisions}" | \ + grep -xm1 '[0-9a-f]\{40\}' + )" + ) + + eval 'repo="${repo_paths__'"${repo}"'}"' + + echo "${revisions}" | \ + xargs -rn1 git -C "${repo}" rev-parse | \ + { + newest='' + while read -r current; do + if [ -z "${newest}" ] || \ + git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then + newest="${current}" + fi + done + echo "${newest}" + } +} + +# find_package_repository_to_package $package $git_repository $git_commit +# find the package repository a package from a given git repository +# belongs to + +find_package_repository_to_package() { + + local package="$1" + local git_repository="$2" + local git_commit="$3" + local repo_path + local repo + + eval 'repo_path="${repo_paths__'"${git_repository}"'}"' + + if [ "${git_repository}" = 'archlinux32' ]; then + repo=$( + git -C "${repo_path}" archive "${git_commit}" -- | \ + tar -t --wildcards "*/${package}/" | \ + cut -d/ -f1 | \ + sort -u + ) + else + repo=$( + git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \ + tar -t | \ + cut -d/ -f3 | \ + grep -vxF '' | \ + grep -v 'staging\|testing\|-unstable' | \ + grep -v -- '-i686$' | \ + sed 's|-[^-]\+$||' | \ + sort -u + ) + fi + + if [ -z "${repo}" ]; then + return 1 + fi + + if [ "$( + echo "${repo}" | \ + wc -l + )" -ne 1 ]; then + return 1 + fi + + echo "${repo}" + +} + +# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel +# extract files found in the svn/git source directories +# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly + +extract_source_directory() { + + local git_repo="$1" + local rev="$2" + local mod_rev="$3" + local output="$4" + local sub_pkgrel="$5" + + if [ -n "${PKGBUILD}" ]; then + eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ + tar -x --strip-components=3 -C "${output}" + fi + + if [ -n "${PKGBUILD_mod}" ]; then + git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ + tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ + true + git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ + tar -Ox "${PKGBUILD_mod}" >> \ + "${output}/PKGBUILD" + fi + + mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" + + # shellcheck disable=SC2016 + sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" + +} + +# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository +# return a list of packages on the build list which are (run- / build- / check-time) +# dependencies of the given package + +find_dependencies_on_build_list() { + + local package="$1" + local git_revision="$2" + local mod_git_revision="$3" + local repository="$4" + + generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" + + { + cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" + awk '{print $1 "." $2 "." $3 "." $4}' < \ + "${work_dir}/build-list" | \ + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|\.builds| + ' | \ + xargs -r cat | \ + sort -u + } | \ + sort | \ + uniq -d + +} + +# download_sources_by_hash $package $repository $git_revision $git_mod_revision +# try to download all sources by their hash into the current directory +# returns 0 if any source was downloaded and 1 otherwise + +download_sources_by_hash() { + + local package="$1" + local repository="$2" + local git_revision="$3" + local git_mod_revision="$4" + + local return_value=1 + local tmp_dir + local sum_type + local arch_suffix + + tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir) + + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then + >&2 echo 'download_sources_by_hash: make_source_info failed.' + rm -rf --one-file-system "${tmp_dir}" + return 1 + fi + + if ! [ -s "${tmp_dir}/.SRCINFO" ]; then + >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.' + rm -rf --one-file-system "${tmp_dir}" + return 1 + fi + + for arch_suffix in '' '_i686'; do + for sum_type in 'sha256sum' 'sha512sum'; do + grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ + sed 's|^.* = ||' | \ + cat -n > \ + "${tmp_dir}/sums" + grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ + sed ' + s|^.* = || + s|::.*$|| + s|.*/|| + ' | \ + cat -n > \ + "${tmp_dir}/urls" + if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then + join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \ + "${tmp_dir}/joined" + while read -r sum file; do + if [ "${sum}" = 'SKIP' ]; then + continue + fi + if echo "${sum} ${file}" | \ + ${sum_type} -c > /dev/null 2>&1; then + # the correct source is already there + continue + fi + if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then + mv "${tmp_dir}/transfer" "${file}" + return_value=0 + fi + done < \ + "${tmp_dir}/joined" + fi + done + done + + rm -rf --one-file-system "${tmp_dir}" + return ${return_value} + +} + +# expand_version $column_num +# add "0:" to version in $colum_num-th column if no ":" is there (epoch) +# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash) + +expand_version() { + local column_num + column_num="$1" + + sed ' + /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/ + /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/ + ' +} + +# shrink_version $column_num +# remove "0:" from version in $colum_num-th column (epoch) +# remove "+0" from version in $colum_num-th column (git count/hash) + +shrink_version() { + local column_num + column_num="$1" + + sed ' + s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ + s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/ + ' +} + +# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] + +# Return (to stdout) the biggest subset A of the packages in $omega whose +# runtime dependencies in $omega \cup $keep are also in A + +# $all_builds and $all_depends either point to an empty file - then they will get +# filled with cached data for subsequent calls - or to the same files of a previous +# call + +# If non-empty, $force contains packages which are assumed to match the above +# condition without checking. + +# The arguments are names of files with one $package.$revision.$mod_revision.$repository +# per line. + +find_biggest_subset_of_packages() { + + ( # the new shell is intentional + omega="$1" + keep="$2" + all_builds="$3" + all_depends="$4" + if [ $# -eq 4 ]; then + force='/dev/null' + elif [ $# -eq 5 ]; then + force="$5" + else + >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" + return 2 + fi + + if [ ! -s "${all_builds}" ]; then + find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ + -exec sed ' + s|^|{} | + s|^\S\+/|| + s|\.builds | | + ' {} \; | \ + sort -k2,2 > \ + "${all_builds}" + fi + + if [ ! -s "${all_depends}" ]; then + find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ + -exec sed ' + s|^|{} | + s|^\S\+/|| + s|\.run-depends | | + ' {} \; | \ + grep -v ' base$' | \ + sort -k2,2 > \ + "${all_depends}" + fi + + sort -u "${omega}" | \ + sponge "${omega}" + + temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + { + sort -u "${keep}" + cat "${force}" "${force}" + } | \ + sort | \ + uniq -u > \ + "${temp_dir}/keep.new" + touch "${temp_dir}/keep" + + while [ -s "${temp_dir}/keep.new" ]; do + cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ + sort -u | \ + sponge "${temp_dir}/keep" + + { + # we append all packages which are run-dependencies of keep-packages + # to the keep-list + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|.run-depends| + ' "${temp_dir}/keep" | \ + xargs -r grep -HF '' | \ + sed ' + s|^.*/|| + s|\.run-depends:| | + ' | \ + sort -u | \ + sort -k2,2 | \ + uniq -f1 | \ + join -1 2 -2 2 -o 2.1 - "${all_builds}" + + # we append all packages with run-dependencies on the keep-list + # to the keep-list + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|.builds| + ' "${temp_dir}/keep" | \ + xargs -r grep -HF '' | \ + sed ' + s|^.*/|| + s|\.builds:| | + ' | \ + sort -u | \ + sort -k2,2 | \ + uniq -f1 | \ + join -1 2 -2 2 -o 2.1 - "${all_depends}" + } | \ + sort -u | \ + join -1 1 -2 1 -o 2.1 - "${omega}" | \ + sort -u > \ + "${temp_dir}/keep.new" + + # "new" is only what has not been there before and what is not forced + cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ + sort | \ + uniq -u | \ + sponge "${temp_dir}/keep.new" + done + + cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ + sort | \ + uniq -u + + ) + +} + +# sort_square_bracket_content $file +# sort the content of [] in $file, print to stdout + +sort_square_bracket_content() { + local file + local line + local token + local token_list + local rest + file="$1" + + while read -r line; do + printf '%s ' "${line}" | \ + tr ' ' '\n' | \ + while read -r token; do + if echo "${token}" | \ + grep -qF '['; then + printf '%s[' "${token%[*}" + token="${token##*[}" + token_list="${token%,}" + while ! echo "${token_list}" | \ + grep -qF ']'; do + read -r token + token_list=$( + printf '%s\n' \ + "${token_list}" \ + "${token%,}" + ) + done + rest="]${token_list#*]}" + token_list="${token_list%%]*}" + token=$( + printf '%s' "${token_list}" | \ + sort | \ + sed 's|$|,|' + printf '%s' "${rest}" + ) + fi + printf '%s\n' "${token}" + done | \ + tr '\n' ' ' | \ + sed ' + s|, ]|]|g + s| $|| + ' + printf '\n' + done < \ + "${file}" +} + +# smoothen_namcap_log $file +# remove unneccesary differences from namcap-logs: +# - remove architecture specific information +# - sort lines +# - sort content of square brackets + +smoothen_namcap_log() { + local file + file="$1" + # shellcheck disable=SC2016 + sort_square_bracket_content "${file}" | \ + sed ' + # normalize architecture specific information + s|i[34567]86|$ARCH|g + s|x86\([-_]64\)\?|$ARCH|g + # remove haskell hashes + s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g + ' | \ + sort | \ + sponge "${file}" +} + +# print_list_of_archaic_packages $source1 $source2 ... +# print a list of packages which have not been touched for a while, +# but which are still in the pipeline, e.g. in $source1, $source2 or ... + +print_list_of_archaic_packages() { + for source in "$@"; do + case "${source}" in + 'testing') + # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ + -exec head -n1 {} \; | \ + "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin + # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated + # and no longer block other packages from being moved + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ + sed ' + s|\.testing$|| + ' + ;; + 'build-list') + while read -r pkg rev mod_rev repo; do + git_repo=$( + find_repository_with_commit "${rev}" + ) + eval repo_path='"${repo_paths__'"${git_repo}"'}"' + commit_date=$( + git -C "${repo_path}" show -s --format=%ct "${rev}" + ) + mod_commit_date=$( + git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" + ) + if [ "${mod_commit_date}" -gt "${commit_date}" ]; then + commit_date="${mod_commit_date}" + fi + # packages remaining longer than $max_package_age_build_list days on the build list + if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then + printf '%s %s %s %s\n' \ + "${pkg}" \ + "${rev}" \ + "${mod_rev}" \ + "${repo}" + fi + done < \ + "${work_dir}/build-list" + ;; + 'staging') + # packages remaining longer than $max_package_age_staging days in staging + find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ + sed ' + s|\.done$|| + ' + ;; + *) + >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" + ;; + esac + done | \ + sort -u +} + +# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" +# print the given modification revision possibly with a html link to github + +modification_revision_link() { + local mod_rev="$1" + local repo="$2" + local pkg="$3" + + if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then + printf '%s\n' \ + "${mod_rev}" \ + "${repo}" \ + "${pkg}" \ + "${mod_rev}" + else + printf '%s\n' \ + "${mod_rev}" + fi +} + +# trigger_mirror_refreshs +# trigger a refresh of capable tier 1 mirrors (as backup for master mirror) + +trigger_mirror_refreshs() { + local tmp_file + + tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir) + date '+%s' > \ + "${tmp_file}" + ${master_mirror_rsync_command} \ + "${tmp_file}" \ + "${master_mirror_rsync_directory}/lastupdate" + rm "${tmp_file}" + for trigger_url in ${mirror_refresh_trigger_urls}; do + screen -S trigger-mirror-update -d -m curl -L "${trigger_url}" + done +} + +# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name +extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { + pkgname="$1" + pkgname="${pkgname%.pkg.tar.xz}" + arch="${pkgname##*-}" + pkgname="${pkgname%-*}" + sub_pkgrel="${pkgname##*-}" + pkgname="${pkgname%-*}" + pkgrel="${sub_pkgrel%.*}" + if [ "${pkgrel}" = "${sub_pkgrel}" ]; then + sub_pkgrel='0' + else + sub_pkgrel="${sub_pkgrel##*.}" + fi + epoch="${pkgname##*-}" + pkgname="${pkgname%-*}" + pkgver="${epoch#*:}" + if [ "${pkgver}" = "${epoch}" ]; then + epoch='0' + else + epoch="${epoch%%:*}" + fi +} diff --git a/lib/mysql-functions b/lib/mysql-functions new file mode 100755 index 0000000..185d9a9 --- /dev/null +++ b/lib/mysql-functions @@ -0,0 +1,1098 @@ +#!/bin/sh + +# contains functions used to access mysql db + +# shellcheck disable=SC2016,SC2039 + +if [ -z "${base_dir}" ]; then + # just to make shellcheck happy + . 'conf/default.conf' +fi + +# base64_encode_each encode each line of stdin with base64 + +base64_encode_each() { + local line + + while read -r line; do + printf '%s' \ + "${line}" | \ + base64 -w0 + printf '\n' + done +} + +# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository + +# shellcheck disable=SC2086 +mysql_add_package_source() { + local names='pkgbase git_revision mod_git_revision upstream_package_repository' + local values + local uses_upstream + local uses_modification + local repo + + if git -C "${repo_paths__archlinux32}" archive "$3" -- "$4/$1" >/dev/null 2>&1; then + uses_modification=1 + else + uses_modification=0 + fi + uses_upstream=0 + for repo in ${repo_names}; do + if eval 'git -C "${repo_paths__'"${repo}"'}" archive "$2" -- "$1/repos/$4-*/PKGBUILD" 2>/dev/null' | \ + tar -t 2>/dev/null | \ + sed 's,-x86_64/,-any/,' | \ + grep -qFx "$1/repos/$4-any/PKGBUILD"; then + uses_upstream=1 + fi + done + + for _ in ${names}; do + values="${values}$( + printf '%s' "$1" | \ + base64 -w0 + ) " + shift + done + values="${values% }" + + { + printf 'INSERT IGNORE INTO package_sources' + printf ' (' + printf '`%s`, ' ${names} + printf '`uses_upstream`,`uses_modification`' + printf ') SELECT' + printf ' from_base64("%s"), ' ${values% *} + printf ' `upstream_repositories`.`id`,%s,%s' \ + ${uses_upstream} ${uses_modification} + printf ' FROM `upstream_repositories`' + printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ + "${values##* }" + } | \ + ${mysql_command} +} + +# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository + +# shellcheck disable=SC2031,SC2086,SC2154 +mysql_add_binary_package() { + local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository' + local name + for name in ${names}; do + eval 'local '"${name}" + eval "${name}"'=$( + printf "%s" "$1" | + base64 -w0 + )' + shift + done + + { + printf 'INSERT IGNORE INTO binary_packages' + printf ' (' + printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted' + printf ') SELECT' + printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" + printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' + printf ' 0, 0, 0' + printf ' FROM' + printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' + printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' + printf ' WHERE' + printf ' `%s`.`name` = from_base64("%s") AND' \ + 'repositories' "${repository}" \ + 'architectures' "${architecture}" + printf ' `package_sources`.`%s` = from_base64("%s") AND' \ + 'pkgbase' "${pkgbase}" \ + 'git_revision' "${git_revision}" \ + 'mod_git_revision' "${mod_git_revision}" + printf ' `upstream_repositories`.`name` = from_base64("%s")' \ + "${upstream_package_repository}" + } | \ + sed ' + s|, )|)|g + s|, FROM| FROM|g + ' | \ + ${mysql_command} +} + +# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel + +# shellcheck disable=SC2031,SC2086,SC2154 +mysql_show_binary_package() { + local names='pkgname pkgver pkgrel sub_pkgrel' + local name + for name in ${names}; do + eval 'local '"${name}" + eval "${name}"'=$( + printf "%s" "$1" | + base64 -w0 + )' + shift + done + + { + printf 'SELECT' + printf ' `%s`.`%s`,' \ + 'repositories' 'name' \ + 'binary_packages' 'pkgname' \ + 'package_sources' 'pkgver' \ + 'package_sources' 'pkgrel' \ + 'binary_packages' 'sub_pkgrel' \ + 'architectures' 'name' \ + 'package_sources' 'pkgbase' \ + 'package_sources' 'git_revision' \ + 'package_sources' 'mod_git_revision' \ + 'upstream_repositories' 'name' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE' + printf ' `%s`.`%s` = from_base64("%s") AND' \ + 'binary_packages' 'pkgname' "${pkgname}" \ + 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ + 'package_sources' 'pkgver' "${pkgver}" \ + 'package_sources' 'pkgrel' "${pkgrel}" + printf ';' + } | \ + sed ' + s|, FROM| FROM|g + s|AND;|;|g + ' | \ + ${mysql_command} --html +} + +# mysql_add_install_target $install_target + +# shellcheck disable=2086 +mysql_add_install_target() { + local install_target + install_target=$( + printf "%s" "$1" | \ + base64 -w0 + ) + + { + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' VALUES (from_base64("%s"))' \ + "${install_target}" + } | \ + ${mysql_command} +} + +# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository +# or +# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository +# if sub_pkgrel should be determined automatically +# and +# mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository +# or +# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository +# if $sub_pkgrel should be forced + +# generate the meta data of a package (dependencies, built packages, ...) in the database + +mysql_generate_package_metadata() { + + ( # new shell is intentional + case "$1" in + ''|*[!0-9]*) + unset forced_sub_pkgrel + ;; + *) + forced_sub_pkgrel=$( + printf '%s' "$1" | \ + base64 -w0 + ) + shift + ;; + esac + current_repository="$1" + package="$2" + + if [ $# -eq 2 ]; then + # second form + repository="${package##*.}" + package="${package%.*}" + mod_git_revision="${package##*.}" + package="${package%.*}" + git_revision="${package##*.}" + package="${package%.*}" + else + git_revision="$3" + mod_git_revision="$4" + repository="$5" + fi + + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + printf '.' >&2 + if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then + printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" + exit 2 + fi + # remove empty lines and unsupported architectures + sed -i ' + /^[^=]*=\s*$/d + /^\s*arch = /{ + / \(i686\|any\)$/!d + } + ' "${temp_dir}/SRCINFO" + + if [ ! -s "${temp_dir}/SRCINFO" ]; then + >&2 printf '"make_source_info" had empty output - eh, what?\n' + exit 2 + fi + printf '\n\n' >> "${temp_dir}/SRCINFO" + + printf '.' >&2 + pkgbase=$( + grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ + cut -d' ' -f3 + ) + if [ -z "${pkgbase}" ]; then + >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' + exit 2 + fi + + # add the package source + mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + printf '.' >&2 + + # now we encode everything in base64 + current_repository=$( + printf '%s' "${current_repository}" | \ + base64 -w0 + ) + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -w0 + ) + git_revision=$( + printf '%s' "${git_revision}" | \ + base64 -w0 + ) + mod_git_revision=$( + printf '%s' "${mod_git_revision}" | \ + base64 -w0 + ) + repository=$( + printf '%s' "${repository}" | \ + base64 -w0 + ) + + # add the build assignment(s) + { + archs=$( + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + grep -vxF 'any' | \ + sort -u + ) + if [ -z "${archs}" ]; then + echo 'any' + else + printf '%s\n' "${archs}" + fi + } | \ + while read -r arch; do + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' + printf ' FROM `architectures` JOIN `package_sources`' + printf ' WHERE `architectures`.`name` = from_base64("%s")' \ + "$( + printf '%s' "${arch}" | \ + base64 -w0 + )" + printf ' AND `package_sources`.`%s` = from_base64("%s")' \ + 'pkgbase' "${pkgbase}" \ + 'git_revision' "${git_revision}" \ + 'mod_git_revision' "${mod_git_revision}" + printf ';\n' + done > \ + "${temp_dir}/add-build-assignments-command" + + # TODO: correctly link between binary_packages and build_assignments using any_arch + + # shellcheck disable=SC2034 + # select any specific arch (which will be building the 'any' part of a split package) + any_arch=$( + { + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + sort -r | \ + grep -vxFm 1 'any' || \ + echo 'any' + } | \ + base64_encode_each + ) + + grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ + cut -d' ' -f3 | \ + while read -r pkgname; do + pkgname64=$( + printf '%s' "${pkgname}" | \ + base64 -w0 + ) + sed -n ' + /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ + :a + N + /\n$/{ + p + T + } + ba + } + ' "${temp_dir}/SRCINFO" | \ + sed ' + /^\S/d + s/^\s*// + ' > \ + "${temp_dir}/BINARYINFO.${pkgname64}" + + grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ + cut -d' ' -f3 | \ + while read -r arch; do + arch64=$( + printf '%s' "${arch}" | \ + base64 -w0 + ) + sed ' + s/^\(\S\+\)_'"${arch}"' = /\1 = / + ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ + "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" + done + done + find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ + while read -r _ pkgname arch; do + pkgver=$( + grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + pkgrel=$( + grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + epoch=$( + { + grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \ + echo 'epoch = 0' + } | \ + cut -d' ' -f3 | \ + base64_encode_each + ) + provides=$( + grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + makedepends=$( + grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + checkdepends=$( + grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + rundepends=$( + grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + if [ -n "${forced_sub_pkgrel}" ]; then + sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")' + else + sub_pkgrel=$( + printf '(SELECT COALESCE(' + # do not add binary packages which are currently on the + # build-list or in $current_repository (beware of split + # packages!) + printf '(SELECT `sub_pkgrel` FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + printf ' `architectures`.`name`=from_base64("%s")' \ + "${arch}" + printf ' AND `repositories`.`name` IN ("build-list",from_base64("%s"))),' \ + "${current_repository}" + # max(sub_pkgrel)+1 + printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' + mysql_join_binary_packages_architectures + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then + # 'any' gets higher sub_pkgrel than any architecture + printf ' 1' + else + # not-'any' gets higher sub_pkgrel than same or 'any' architecture + printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \ + "${arch}" + fi + printf ')' + printf ',0))' + ) + fi + { + printf 'INSERT IGNORE INTO `binary_packages` (' + printf '`%s`,' \ + 'build_assignment' \ + 'repository' \ + 'architecture' \ + 'epoch' \ + 'pkgver' \ + 'pkgrel' \ + 'pkgname' \ + 'sub_pkgrel' \ + 'has_issues' \ + 'is_tested' \ + 'is_to_be_deleted' + printf ') SELECT ' + printf '`%s`.`id`,' \ + 'build_assignments' \ + 'repositories' \ + 'architectures' + printf 'from_base64("%s"),' \ + "${epoch}" \ + "${pkgver}" \ + "${pkgrel}" \ + "${pkgname}" + printf '%s,0,0,0 FROM' \ + "${sub_pkgrel}" + printf ' `%s` JOIN' \ + 'repositories' \ + 'architectures' \ + 'build_assignments' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE' + printf ' `%s`.`%s` = from_base64("%s") AND' \ + 'repositories' 'name' "${current_repository}" \ + 'architectures' 'name' "${arch}" \ + 'package_sources' 'pkgbase' "${pkgbase}" \ + 'package_sources' 'git_revision' "${git_revision}" \ + 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ + 'upstream_repositories' 'name' "${repository}" + printf ';\n' + } | \ + sed ' + s|,)|)|g + s| JOIN JOIN | JOIN | + s| AND;$|;| + ' >> \ + "${temp_dir}/add-binary-packages-command" + { + printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' + + printf 'INSERT INTO `provides` VALUES\n' + echo "${provides}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' (from_base64("%s"));\n' \ + "${pkgname}" + + printf 'INSERT INTO `rundepends` VALUES\n' + echo "${rundepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' ("base");\n' + + echo "${checkdepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + 1 s/^/INSERT INTO `checkdepends` VALUES \n/ + $ s/,$/;/ + ' + + printf 'INSERT INTO `makedepends` VALUES\n' + echo "${makedepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + s|^| (from_base64("| + s|$|")),| + ' + printf ' ("base-devel");\n' + + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' SELECT (`name`) FROM `%s` UNION' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' | \ + sed 's| UNION$|;\n|' + + for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do + case "${link}" in + 'provides') + printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' + printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' + ;; + 'makedepends'|'checkdepends'|'rundepends') + printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' + printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' + printf ' `dependency_types` JOIN' + ;; + esac + printf ' `binary_packages`' + mysql_join_binary_packages_architectures + printf ' JOIN `install_targets`' + printf ' JOIN `%s`' "${link}" + printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" + printf ' WHERE' + if [ "${link}" = 'makedepends' ] || \ + [ "${link}" = 'checkdepends' ] || \ + [ "${link}" = 'rundepends' ]; then + printf ' `dependency_types`.`name` = "%s" AND' \ + "${link%depends}" + fi + printf ' `binary_packages`.`%s` = from_base64("%s") AND' \ + 'epoch' "${epoch}" \ + 'pkgver' "${pkgver}" \ + 'pkgrel' "${pkgrel}" \ + 'pkgname' "${pkgname}" + # we do not want to match the sub_pkgrel: + # a) it is tedious to do so (because it may be calculated + # dynamically) + # b) it is not necessary to do so: if only the sub_pkgrel + # changed, the dependencies and provided install_targets + # should not have changed + printf ' `architectures`.`name` = from_base64("%s");\n' \ + "${arch}" + # the repository is of no relevance: it hardly matters for + # the dependencies + done + + printf 'DROP TABLE `%s`;\n' \ + 'provides' \ + 'makedepends' \ + 'checkdepends' \ + 'rundepends' + } >> \ + "${temp_dir}/add-install-targets-command" + done + printf '.' >&2 + + { + if [ -s "${temp_dir}/add-build-assignments-command" ]; then + cat "${temp_dir}/add-build-assignments-command" + fi + if [ -s "${temp_dir}/add-binary-packages-command" ]; then + cat "${temp_dir}/add-binary-packages-command" + fi + if [ -s "${temp_dir}/add-install-targets-command" ]; then + cat "${temp_dir}/add-install-targets-command" + fi + } | \ + ${mysql_command} + printf '.' >&2 + + ) +} + +# mysql_sanity_check +# do a sanity check on the mysql database + +mysql_sanity_check() { + { + printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + mysql_join_binary_packages_architectures '' 'bp_arch' + mysql_join_build_assignments_architectures '' 'ba_arch' + printf ' WHERE `bp_arch`.`name`!="any"' + printf ' AND `ba_arch`.`name`="any";\n' + printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' + printf ' FROM `binary_packages` AS `a`' + printf ' JOIN `binary_packages` AS `b`' + printf ' ON `a`.`pkgname`=`b`.`pkgname`' + printf ' AND `a`.`repository`=`b`.`repository`' + printf ' AND `a`.`id`!=`b`.`id`' + mysql_join_binary_packages_repositories 'a' + printf ' WHERE `repositories`.`name`="build-list";\n' + printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' + printf ' FROM `binary_packages` AS `a`' + printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' + mysql_join_binary_packages_repositories 'a' 'arep' + mysql_join_binary_packages_repositories 'b' 'brep' + printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' + printf ' AND `%srep`.`name`="build-list"' \ + 'a' 'b' + printf ';\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + /^CONCAT("/d + s,^,, + s,$,, + ' + ( # new shell is intentional + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + for dir in $(ls_master_mirror 'i686'); do + ls_master_mirror "i686/${dir}" | \ + sed -n ' + s/\.pkg\.tar\.xz$// + T + s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ + s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ + s|^|'"${dir}"'/| + p + ' + done | \ + sort > \ + "${temp_dir}/master-mirror-listing" + + { + printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`is_on_master_mirror`' + } | \ + ${mysql_command} --batch | \ + sed ' + 1d + s,\t,/, + s,\t,-, + s,\t,:, + s,\t,-, + s,\t,., + s,\t,-, + ' | \ + sort > \ + "${temp_dir}/mysql-packages" + + diff -u \ + "${temp_dir}/master-mirror-listing" \ + "${temp_dir}/mysql-packages" + ) +} + +mysql_find_build_assignment_loops() { + new_loops=$( + { + printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' + printf ' FROM `dependencies`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'packages_dependency' + mysql_join_dependencies_binary_packages '' 'packages_dependent' + mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' + mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' + printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + y/\t/ / + ' | \ + tsort 2>&1 >/dev/null | \ + sed 's/^tsort:\s*//' | \ + { + loop=0 + while read -r id; do + if [ "x${id}" = 'x-: input contains a loop:' ]; then + loop=$((loop+1)) + continue + fi + if ! printf '%s' "${id}" | tr '\n' ' ' | grep -q '^[0-9]\+$'; then + >&2 printf 'ERROR: non-numeric id "%s"\n' "${id}" + continue + fi + printf '(%s,%s),' "${loop}" "${id}" + done | \ + sed 's/,$//' + } + ) + { + printf 'DELETE FROM `build_dependency_loops`;\n' + if [ -n "${new_loops}" ]; then + printf 'INSERT INTO `build_dependency_loops` (`loop`,`build_assignment`) VALUES %s;\n' \ + "${new_loops}" + fi + } | \ + ${mysql_command} +} + +# mysql_cleanup [dry] +# clean up left overs from mysql database +mysql_cleanup() { + local operator + if [ "$#" = '0' ]; then + operator='DELETE' + elif [ "$#" = '1' ] && [ "x$1" = 'xdry' ]; then + operator='SELECT COUNT(*)' + else + >&2 echo 'Unknown parameter' + >&2 echo 'Call "mysql_clean_up" or "mysql_clean_up dry".' + exit 2 + fi + { + # remove to-be-decided binary_packages + printf '%s ' \ + "${operator}" + if [ "${operator}" = 'DELETE' ]; then + printf '`binary_packages` ' + fi + printf 'FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="to-be-decided";\n' + # remove dependencies w/o binary_package or install_target + printf '%s FROM `dependencies` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `dependencies`.`dependent`=`binary_packages`.`id`' + printf ') OR NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `install_targets` ' + printf 'WHERE `dependencies`.`depending_on`=`install_targets`.`id`' + printf ');\n' + # remove install_target_providers w/o binary_package or install_target + printf '%s FROM `install_target_providers` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `install_target_providers`.`package`=`binary_packages`.`id`' + printf ') OR NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `install_targets` ' + printf 'WHERE `install_target_providers`.`install_target`=`install_targets`.`id`' + printf ');\n' + # remove build_assignments w/o binary_package + printf '%s FROM `build_assignments` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `binary_packages` ' + printf 'WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ');\n' + # remove failed_builds with unbroken build_assignments + printf '%s ' \ + "${operator}" + if [ "${operator}" = 'DELETE' ]; then + printf '`failed_builds` ' + fi + printf 'FROM `failed_builds` ' + mysql_join_failed_builds_build_assignments + printf 'WHERE NOT `build_assignments`.`is_broken`' + printf ';\n' + # remove failed_builds w/o build_assignment + printf '%s FROM `failed_builds` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `build_assignments` ' + printf 'WHERE `build_assignments`.`id`=`failed_builds`.`build_assignment`' + printf ');\n' + # remove package_sources w/o build_assignment + printf '%s FROM `package_sources` ' \ + "${operator}" + printf 'WHERE NOT EXISTS ' + printf '(' + printf 'SELECT * FROM `build_assignments` ' + printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' + printf ');\n' + printf 'UPDATE `build_slaves`' + mysql_join_build_slaves_binary_packages + mysql_join_binary_packages_repositories + printf ' SET `currently_building`=NULL' + printf ' WHERE `repositories`.`name`!="build-list";\n' + } | \ + ${mysql_command} +} + +# mysql_query_has_pending_dependencies `build_assignment`.`id` +# print a mysql query giving wether dependencies are pending +mysql_query_has_pending_dependencies() { + printf 'EXISTS (' + printf 'SELECT * FROM `binary_packages` as `to_dos`' + mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' + mysql_join_binary_packages_dependencies 'to_dos' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'bin_deps' + mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' + printf ' WHERE' + printf ' `%s`.`name`="build-list" AND' \ + 'dep_repos' 'to_do_repos' + printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND' + printf ' `to_dos`.`build_assignment`=%s' \ + "$1" + printf ')' +} + +# mysql_query_is_part_of_loop `build_assignment`.`id` +# print a mysql query giving wether the package is part of a loop +mysql_query_is_part_of_loop() { + printf 'EXISTS (' + printf 'SELECT * FROM `build_dependency_loops`' + printf ' WHERE `build_dependency_loops`.`build_assignment`=%s' \ + "$1" + printf ')' +} + +# mysql_query_select_pkgbase_and_revision +# print the part of a mysql query giving: +# pkgbase git_revision mod_git_revision upstream_package_repository +mysql_query_select_pkgbase_and_revision() { + printf '`package_sources`.`%s`,' \ + 'pkgbase' \ + 'git_revision' \ + 'mod_git_revision' + printf '`upstream_repositories`.`name`' + printf ' FROM `build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories +} + +# mysql_repair_binary_packages_without_build_assignment +# try to generate valid build assignments to binary packages without +# a valid one yet +mysql_repair_binary_packages_without_build_assignment() { + { + printf 'SELECT ' + printf '`binary_packages`.`id`' + printf ',replace(to_base64(%s),"\\n","")' \ + '`binary_packages`.`pkgname`' \ + '`architectures`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + printf ' WHERE `binary_packages`.`build_assignment`<0' + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + while read -r id pkgname arch; do + pkgname=$( + printf '%s' "${pkgname}" | \ + base64 -d + ) + pkgbase=$( + curl -Ss "$( + printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ + "${pkgname}" + )" | \ + sed ' + s/^.*"results":\s*\[// + s/}\s*,\s*{/\n/g + ' | \ + grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ + tr ',' '\n' | \ + grep '"pkgbase":' | \ + cut -d'"' -f4 | \ + sort -u | \ + head -n1 + ) + if [ -z "${pkgbase}" ] && \ + { + printf 'SELECT count(*) FROM `package_sources`' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ + "$(printf '%s' "${pkgname}" | base64 -w0)" + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + grep -qvxF '0'; then + pkgbase="${pkgname}" + fi + if [ -z "${pkgbase}" ]; then + >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" + continue + fi + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -w0 + ) + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' + printf ' FROM `package_sources`' + printf ' JOIN `architectures`' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" + printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" + printf ' LIMIT 1;\n' + printf 'UPDATE `binary_packages`' + printf ' JOIN `build_assignments`' + mysql_join_binary_packages_build_assignments + printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ' WHERE `binary_packages`.`id`=%s' "${id}" + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" + done | \ + ${mysql_command} +} + +# mysql_remove_duplicate_binary_packages +# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel, +# having difference of 1 in sub_pkgrel + +mysql_remove_duplicate_build_order() { + { + printf 'CREATE TEMPORARY TABLE `ren`' + printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n' + printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)' + printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`' + printf ' FROM `binary_packages` as `old`' + printf ' JOIN `binary_packages` as `new` ON' + printf ' `old`.`%s`=`new`.`%s` AND' \ + 'pkgname' 'pkgname' \ + 'epoch' 'epoch' \ + 'pkgver' 'pkgver' \ + 'pkgrel' 'pkgrel' + printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' + mysql_join_binary_packages_repositories 'old' 'orep' + mysql_join_binary_packages_repositories 'new' 'nrep' + printf ' WHERE `orep`.`name`!="to-be-decided"' + printf ' AND `nrep`.`name`="to-be-decided";\n' + printf 'UPDATE IGNORE `dependencies`' + printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`' + printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n' + printf 'UPDATE IGNORE `install_target_providers`' + printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`' + printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n' + printf 'DELETE FROM `binary_packages`' + printf ' WHERE EXISTS (' + printf 'SELECT * FROM `ren`' + printf ' WHERE `ren`.`old`=`binary_packages`.`id`' + printf ');\n' + printf 'UPDATE IGNORE `binary_packages`' + printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`' + printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' + printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' + } | \ + ${mysql_command} +} + +# mysql_package_name_query +# print a mysql query of the full name of a package file + +mysql_package_name_query() { + printf 'CONCAT(' + printf '`binary_packages`.`pkgname`,"-",' + printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' + printf '`binary_packages`.`pkgver`,"-",' + printf '`binary_packages`.`pkgrel`,".",' + printf '`binary_packages`.`sub_pkgrel`,"-",' + printf '`architectures`.`name`,".pkg.tar.xz"' + printf ')' +} + +# mysql_join_*_* +# print 'JOIN' part of mysql query to connect the respective tables +# these functions take 2 optional arguments, acting as aliases for +# the tables + +# mysql_join__generic $table_a $column_a $table_b $column_b +# create mysql_join_${table_a}_${table_b}() function + +mysql_join__generic() { + eval "$( + printf 'mysql_join_%s_%s() {\n' "$1" "$3" + printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' + printf ' fi\n' + printf ' if [ -n "$1" ]; then\n' + printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" + printf ' else\n' + printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" + printf ' fi\n' + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" + printf ' else\n' + printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" + printf ' fi\n' + printf '}\n' + )" +} + +for link in \ + 'binary_packages:architecture:architectures' \ + 'binary_packages:repository:repositories' \ + 'binary_packages:build_assignment:build_assignments' \ + \ + 'build_assignments:architecture:architectures' \ + 'build_assignments:package_source:package_sources' \ + \ + 'build_dependency_loops:build_assignment:build_assignments' \ + 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ + \ + 'build_slaves:currently_building:build_assignments' \ + 'build_slaves:currently_building build_assignment:binary_packages' \ + \ + 'dependencies:depending_on:install_targets' \ + 'dependencies:dependent:binary_packages' \ + 'dependencies:dependency_type:dependency_types' \ + \ + 'failed_builds:reason:fail_reason' \ + 'failed_builds:build_assignment:build_assignments' \ + 'failed_builds:build_slave:build_slaves' \ + \ + 'install_target_providers:package:binary_packages' \ + 'install_target_providers:install_target:install_targets' \ + 'install_target_providers:install_target depending_on:dependencies' \ + \ + 'package_sources:upstream_package_repository:upstream_repositories' \ + \ + 'repositories:stability:repository_stabilities' \ + \ + 'upstream_repositories:git_repository:git_repositories'; do +# A join for these cannot be done, because it's not clear on what to join: +# 'repository_stability_relations:more_stable:repository_stabilities' +# 'repository_stability_relations:less_stable:repository_stabilities' + + table_b="${link##*:}" + table_a="${link%:*}" + column_b="${table_a##*:}" + table_a="${table_a%:*}" + column_a="${column_b% *}" + if [ "${column_a}" = "${column_b}" ]; then + column_b='id' + else + column_b="${column_b##* }" + fi + + mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" + mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" +done -- cgit v1.2.3-70-g09d2