index : builder | |
Archlinux32 build system | gitolite user |
summaryrefslogtreecommitdiff |
-rwxr-xr-x | bin/common-functions | 1378 |
diff --git a/bin/common-functions b/bin/common-functions deleted file mode 100755 index a84271b..0000000 --- a/bin/common-functions +++ /dev/null @@ -1,1378 +0,0 @@ -#!/bin/sh - -# contains functions used by more than one script - -# shellcheck disable=SC2039 - -# TODO: include link depenendencies in run-depends metadata - -# TODO: have full information (currently in files) in database - -# TODO: remove state files / metadata files - -if [ -z "${base_dir}" ]; then - # just to make shellcheck happy - . 'conf/default.conf' -fi - -# find_pkgbuilds package repository git_repository git_revision mod_git_revision -# find the PKGBUILD and modification of $package from $repository -# sets $PKGBUILD and $PKGBUILD_mod - -find_pkgbuilds() { - - local package="$1" - local repository="$2" - local git_repository="$3" - local git_revision="$4" - local mod_git_revision="$5" - - local repo_path - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - PKGBUILD=$( - git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sort | \ - tail -n1 - ) - - PKGBUILD_mod=$( - git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \ - tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null - ) || true - - if [ -z "${PKGBUILD}" ] && \ - [ -z "${PKGBUILD_mod}" ]; then - >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ - "${package}" \ - "${repository}" \ - "${git_repository}" \ - "${git_revision}" \ - "${mod_git_revision}" - return 1 - fi - -} - -# find_repository_with_commit commit -# find the repository which has $commit - -find_repository_with_commit() { - - local repository - - for repository in ${repo_names}; do - # shellcheck disable=SC2016 - if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then - echo "${repository}" - return 0 - fi - done - >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" - exit 1 - -} - -# find_git_repository_to_package_repository repository -# find the git repository which tracks the package repository $repository - -find_git_repository_to_package_repository() { - - local repository - local package_repository - local repo_path - - package_repository="$1" - - if [ "$1" = 'build-support' ]; then - echo 'packages' - return 0 - fi - - for repository in ${repo_names}; do - if [ "${repository}" = "archlinux32" ]; then - continue - fi - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ - tar -t --wildcards '*/repos' | \ - grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ - cut -d/ -f3 | \ - sed 's|-[^-]\+$||' | \ - sort -u | \ - grep -qxF "${package_repository}"; then - echo "${repository}" - return 0 - fi - done - >&2 echo "can't find git repository with package repository '$1'" - exit 1 - -} - -# generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# generate the meta data files of a package (dependencies, built packages, ...) - -generate_package_metadata() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - local file_prefix - local file - local PKGBUILD - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" - - if [ -e "${file_prefix}.builds" ] && \ - [ -e "${file_prefix}.build-depends" ] && \ - [ -e "${file_prefix}.run-depends" ] && \ - [ -e "${file_prefix}.groups" ] && \ - [ -e "${file_prefix}.packages" ]; then - return 0 - fi - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" - exit 1 - fi - if [ ! -s "${file_prefix}.SRCINFO" ]; then - >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" - exit 1 - fi - - # otherwise this just calls for trouble - sed -i ' - /^[^=]*=\s*$/d - s/_i686\(\s*=\)/\1/ - ' "${file_prefix}.SRCINFO" - - # extract "groups" = groups \cup provides - grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.groups" - - # extract "packages" = pkgname - grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.packages" - - # extract "builds" = provides \cup pkgname \cup groups - cat "${file_prefix}.groups" "${file_prefix}.packages" | \ - sort -u > \ - "${file_prefix}.builds" - - # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' 'base-devel' - sed -n "$( - printf '/^\t%s = /p\n' \ - 'depends' \ - 'makedepends' - )" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.build-depends" - - # extract "run-depends" = depends \cup \{ base \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' - sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.run-depends" - - rm "${file_prefix}.SRCINFO" - -} - -# delete_old_metadata -# delete old (=unneeded) meta data of packages - -delete_old_metadata() { - - local current_metadata - - current_metadata=$( - find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| - ' | \ - sort -u - ) - - ( # the new shell is intentional - # what we have - echo "${current_metadata}" - - # package-states should stay - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - ' | \ - sort -u | \ - sed 'p' - - # build-list items should stay - sed 'p' "${work_dir}/build-list" - - tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - echo "${current_metadata}" | \ - sort -k1,1 > \ - "${tmp_dir}/current-metadata" - - # the newest of the following should stay: - { - # deletion-list items - cat "${work_dir}/deletion-list" - # all packages in the repos - for repo in ${repo_names}; do - eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - sed ' - s|/$|| - /\//d - ' - done - } | \ - sort -u | \ - join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ - sort -k4,4 > \ - "${tmp_dir}/find-newest-revisions" - - uniq -uf3 < \ - "${tmp_dir}/find-newest-revisions" | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - - uniq -Df3 < \ - "${tmp_dir}/find-newest-revisions" | \ - uniq --group=append -f3 | \ - { - revs='' - mod_revs='' - opkg='' - orepo='' - while read -r rev mod_rev repo pkg; do - - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - - printf '%s %s %s %s\n' \ - "$( - printf '%s\n' ${revs} | \ - find_newest_of_git_revisions - )" \ - "$( - printf '%s\n' ${mod_revs} | \ - find_newest_of_git_revisions - )" \ - "${orepo}" \ - "${opkg}" - - revs='' - mod_revs='' - orepo='' - opkg='' - continue - fi - revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} ${rev} | \ - sort -u - ) - mod_revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} ${mod_rev} | \ - sort -u - ) - orepo="${repo}" - opkg="${pkg}" - done - } | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - ) | \ - sort | \ - uniq -u | \ - while read -r pkg rev mod_rev repo; do - rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* - done -} - -# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository -# print which (stable) repository a package belongs to - -repository_of_package() { - local package="$1" - local repository="${package##*.}" - package="${package%.*}" - local a32_rev="${package##*.}" - package="${package%.*.*}" - - case "${repository}" in - 'multilib') - if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ - tar -Ox | \ - grep -qFx "${package%.*.*.*}"; then - echo 'extra' - else - echo 'community' - fi - ;; - *) - echo "${repository}" - esac -} - -# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending -# print wether the specified package is an official package (print -# $ending) or a community package (print 'community-$ending') or a -# build-support package (print 'build-support') - -official_or_community() { - local prepo - prepo=$(repository_of_package "$1") - - if [ "${prepo}" = 'community' ]; then - echo 'community-'"$2" - elif [ "${prepo}" = 'build-support' ]; then - echo 'build-support' - else - echo "$2" - fi -} - -# ls_master_mirror $path -# list content of $path on the master mirror (via rsync) - -ls_master_mirror() { - - local path="$1" - - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${path}/" | \ - grep -v '\s\.$' | \ - awk '{print $5}' - -} - -# TODO: the actions of remove_old_package_versions should be done -# on basis of the information in the database - -# remove_old_package_versions $arch $repository $package_file - -# removes all older (not-newer) versions of $package_file -# in all repositories not-older (newer) than $repository - -# TODO: should remove all other version (also newer) from -# some repositories :-/ - -# A package is considered not newer if -# a) its version is not newer -# A package is considered older if -# b) its version is older or -# c) if it's "not newer" and its architecture is 'any' and different or -# d) if it's "not newer" and the other architecture is 'any' and different - -# this ensures an any package may replace arch-specific packages of the same version and vice versa - -remove_old_package_versions() { - - local arch="$1" - local repository="$2" - local package="$3" - - local pkgname - local epoch - local pkgver - local pkgrel - local sub_pkgrel - pkgname="${package%-*}" - pkgrel="${pkgname##*-}" - sub_pkgrel="${pkgrel##*.}" - if [ "${sub_pkgrel}" = "${pkgrel}" ]; then - sub_pkgrel='0' - else - pkgrel="${pkgrel%.*}" - fi - pkgname="${pkgname%-*}" - pkgver="${pkgname##*-}" - epoch="${pkgver%%:*}" - if [ "${epoch}" = "${pkgver}" ]; then - epoch='0' - else - pkgver="${pkgver#*:}" - fi - pkgname="${pkgname%-*}" - - # shellcheck disable=SC2016 - { - printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \ - "$( - printf '%s' "${repository}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${package}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${package}" | \ - sed ' - s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/ - ' | \ - base64 -w0 - )" - printf 'SELECT ' - printf '`binary_packages`.`id`,' - printf 'CONCAT(`repositories`.`name`,"/",' - mysql_package_name_query - printf '),' - # should we delete packages of identical version? - printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),' - printf 'CONCAT(' - printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),' - printf '`binary_packages`.`pkgver`,"-",' - printf '`binary_packages`.`pkgrel`,".",' - printf '`binary_packages`.`sub_pkgrel`' - printf ')' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_binary_packages_architectures - printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' - printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' - # name must match - printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \ - "$(printf '%s' "${package%-*-*-*}" | base64 -w0)" - # repository, where package should be deleted, should be less stable - printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \ - "$(printf '%s' "${repository}" | base64 -w0)" - printf ';\n' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ - expand_version 4 | \ - sort -k4V,4 -k3r,3 | \ - shrink_version 4 | \ - sed -n ' - /^bogus /q - p - ' | \ - cut -d' ' -f1,2 >&2 - - # repositories in which older packages should be deleted - local delete_older_repositories - # repositories in which not-newer packages should be deleted - local delete_not_newer_repositories - - if echo "${standalone_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories='' - - elif echo "${staging_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories=$( - echo "${staging_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${testing_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${testing_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${stable_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${stable_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - else - - >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}" - return 1 - - fi - - ( # the new shell is intentional - tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - { - # the architecture of the package (any vs. i686) - package_arch="${package##*-}" - package_arch="${package_arch%%.*}" - if [ "${package_arch}" = 'any' ]; then - package_arch_regex_inverter='!' - else - unset package_arch_regex_inverter - fi - - for repo in ${delete_older_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|2 '"${arch} ${repo}"' | - / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{ - s|^2|0| - } - p - } - ' - done - for repo in ${delete_not_newer_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|0 '"${arch} ${repo}"' | - p - } - ' - done - echo "${package%-*}" | \ - sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|' - - # the generated list contains the following columns: - # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz - } | \ - expand_version 5 | \ - sort -k5V,5 -k1n,1 | \ - shrink_version 5 | \ - sed -n ' - /^1 %cut% %it% %here% /q - s/^[02] // - s/ \(\S\+\)$/-\1/ - p - ' | \ - sort -u > \ - "${tmp_dir}/packages-to-delete" - # this file contains a list of packages to be deleted, one on each line: - # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz - - cut -d' ' -f1,2 < \ - "${tmp_dir}/packages-to-delete" | \ - grep -vxF "${arch} ${repository}" | \ - sort -u > \ - "${tmp_dir}/repositories-to-modify" - - # fetch all databases being modified - while read -r del_arch del_repo; do - mkdir -p "${tmp_dir}/${del_arch}/${del_repo}" - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${tmp_dir}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - while read -r del_arch del_repo del_package _; do - if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then - # we do not repo-remove the package in the target repository - continue - fi - repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \ - "${del_package}" - done < \ - "${tmp_dir}/packages-to-delete" - - # upload modified databases - while read -r del_arch del_repo; do - ${master_mirror_rsync_command} \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - # shellcheck disable=SC2016 - sed ' - s/\.pkg\.tar\.xz$// - s/^\S\+ // - s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/ - s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/ - s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ - ' "${tmp_dir}/packages-to-delete" | \ - while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do - printf 'DELETE FROM `binary_packages` WHERE' - printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \ - 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \ - 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)" - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ - 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ - 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ - 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ - 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \ - sed 's/ AND$//' - printf ';\n' - done | \ - ${mysql_command} - - sed ' - s| \(\S\+\)$|-\1| - y| |/| - s|^|rm "| - s|$|"| - p - s|"$|.sig"| - ' "${tmp_dir}/packages-to-delete" | \ - ${master_mirror_sftp_command} - ) - -} - -# wait_some_time $minimum $diff -# wait between minimum and minimum+diff seconds (diff defaults to 30) - -wait_some_time() { - local minimum=$1 - local diff=$2 - local random - - if [ -z "${diff}" ]; then - diff=30 - fi - - random=$( - dd if='/dev/urandom' count=1 2> /dev/null | \ - cksum | \ - cut -d' ' -f1 - ) - - sleep $((minimum + random % diff)) -} - -# str_to_regex $string -# escape dots for use in regex - -str_to_regex() { - echo "$1" | \ - sed ' - s|[.[]|\\\0|g - ' -} - -# make_source_info $package $repository $git_revision $mod_git_revision $output -# create .SRCINFO from PKGBUILD within git repositories, output to $output - -make_source_info() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local mod_git_revision="$4" - local output="$5" - - local git_repo - local PKGBUILD - local PKGBUILD_mod - - git_repo=$(find_repository_with_commit "${git_revision}") - - if [ -z "${git_repo}" ]; then - return 1 - fi - - find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" - - ( # the new shell is intentional - - tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' - - { - cd "${tmp_dir}" - makepkg --printsrcinfo - cd .. - } | - if [ "${package%-i18n}-i18n" = "${package}" ]; then - sed ' - 1 a \\tdepends = '"${package%-i18n}"' - ' - else - cat - fi > \ - "${output}" - - ) - -} - -# recursively_umount_and_rm $dir -# umount all mountpoints in $dir which are also in $dir's -# filesystem, possibly also $dir itself and then -# rm -rf --one-file-system $dir - -recursively_umount_and_rm() { - local dir="$1" - - if [ -z "${dir}" ]; then - >&2 echo 'ERROR: recursively_umount_and_rm requires an argument' - exit 42 - fi - - find "${dir}" \ - -xdev -depth -type d \ - -exec 'mountpoint' '-q' '{}' ';' \ - -exec 'sudo' 'umount' '-l' '{}' ';' - rm -rf --one-file-system "${dir}" -} - -# mangle_pkgbuild $PKGBUILD [$sub_pkgrel] -# mangle $arch in PKBUILDs to contain i486, i586, i686 -# append $sub_pkgrel to the pkgrel - -mangle_pkgbuild() { - local PKGBUILD="$1" - local sub_pkgrel="$2" - - if [ -n "${sub_pkgrel}" ]; then - sub_pkgrel=".${sub_pkgrel}" - fi - - sed -i ' - /^arch=[^#]*any/!{ - /^arch=(/s/(/(i686 / - } - s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ - ' "${PKGBUILD}" -} - -# find_newest_of_git_revisions -# find newest git revision of the ones provided at stdin -# (assuming linear history) - -find_newest_of_git_revisions() { - local revisions - local repo - revisions=$(cat) - - if [ "$( - echo "${revisions}" | \ - wc -l - )" -eq 1 ]; then - - echo "${revisions}" - return - - fi - - repo=$( - find_repository_with_commit \ - "$( - echo "${revisions}" | \ - grep -xm1 '[0-9a-f]\{40\}' - )" - ) - - eval 'repo="${repo_paths__'"${repo}"'}"' - - echo "${revisions}" | \ - xargs -rn1 git -C "${repo}" rev-parse | \ - { - newest='' - while read -r current; do - if [ -z "${newest}" ] || \ - git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then - newest="${current}" - fi - done - echo "${newest}" - } -} - -# find_package_repository_to_package $package $git_repository $git_commit -# find the package repository a package from a given git repository -# belongs to - -find_package_repository_to_package() { - - local package="$1" - local git_repository="$2" - local git_commit="$3" - local repo_path - local repo - - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - if [ "${git_repository}" = 'archlinux32' ]; then - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- | \ - tar -t --wildcards "*/${package}/" | \ - cut -d/ -f1 | \ - sort -u - ) - else - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \ - tar -t | \ - cut -d/ -f3 | \ - grep -vxF '' | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -v -- '-i686$' | \ - sed 's|-[^-]\+$||' | \ - sort -u - ) - fi - - if [ -z "${repo}" ]; then - return 1 - fi - - if [ "$( - echo "${repo}" | \ - wc -l - )" -ne 1 ]; then - return 1 - fi - - echo "${repo}" - -} - -# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel -# extract files found in the svn/git source directories -# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly - -extract_source_directory() { - - local git_repo="$1" - local rev="$2" - local mod_rev="$3" - local output="$4" - local sub_pkgrel="$5" - - if [ -n "${PKGBUILD}" ]; then - eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ - tar -x --strip-components=3 -C "${output}" - fi - - if [ -n "${PKGBUILD_mod}" ]; then - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ - tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ - true - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ - tar -Ox "${PKGBUILD_mod}" >> \ - "${output}/PKGBUILD" - fi - - mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" - - # shellcheck disable=SC2016 - sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" - -} - -# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository -# return a list of packages on the build list which are (run- / build- / check-time) -# dependencies of the given package - -find_dependencies_on_build_list() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - - { - cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - awk '{print $1 "." $2 "." $3 "." $4}' < \ - "${work_dir}/build-list" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|\.builds| - ' | \ - xargs -r cat | \ - sort -u - } | \ - sort | \ - uniq -d - -} - -# download_sources_by_hash $package $repository $git_revision $git_mod_revision -# try to download all sources by their hash into the current directory -# returns 0 if any source was downloaded and 1 otherwise - -download_sources_by_hash() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local git_mod_revision="$4" - - local return_value=1 - local tmp_dir - local sum_type - local arch_suffix - - tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir) - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then - >&2 echo 'download_sources_by_hash: make_source_info failed.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - if ! [ -s "${tmp_dir}/.SRCINFO" ]; then - >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - for arch_suffix in '' '_i686'; do - for sum_type in 'sha256sum' 'sha512sum'; do - grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed 's|^.* = ||' | \ - cat -n > \ - "${tmp_dir}/sums" - grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed ' - s|^.* = || - s|::.*$|| - s|.*/|| - ' | \ - cat -n > \ - "${tmp_dir}/urls" - if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then - join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \ - "${tmp_dir}/joined" - while read -r sum file; do - if [ "${sum}" = 'SKIP' ]; then - continue - fi - if echo "${sum} ${file}" | \ - ${sum_type} -c > /dev/null 2>&1; then - # the correct source is already there - continue - fi - if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then - mv "${tmp_dir}/transfer" "${file}" - return_value=0 - fi - done < \ - "${tmp_dir}/joined" - fi - done - done - - rm -rf --one-file-system "${tmp_dir}" - return ${return_value} - -} - -# expand_version $column_num -# add "0:" to version in $colum_num-th column if no ":" is there (epoch) -# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash) - -expand_version() { - local column_num - column_num="$1" - - sed ' - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/ - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/ - ' -} - -# shrink_version $column_num -# remove "0:" from version in $colum_num-th column (epoch) -# remove "+0" from version in $colum_num-th column (git count/hash) - -shrink_version() { - local column_num - column_num="$1" - - sed ' - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/ - ' -} - -# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] - -# Return (to stdout) the biggest subset A of the packages in $omega whose -# runtime dependencies in $omega \cup $keep are also in A - -# $all_builds and $all_depends either point to an empty file - then they will get -# filled with cached data for subsequent calls - or to the same files of a previous -# call - -# If non-empty, $force contains packages which are assumed to match the above -# condition without checking. - -# The arguments are names of files with one $package.$revision.$mod_revision.$repository -# per line. - -find_biggest_subset_of_packages() { - - ( # the new shell is intentional - omega="$1" - keep="$2" - all_builds="$3" - all_depends="$4" - if [ $# -eq 4 ]; then - force='/dev/null' - elif [ $# -eq 5 ]; then - force="$5" - else - >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" - return 2 - fi - - if [ ! -s "${all_builds}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.builds | | - ' {} \; | \ - sort -k2,2 > \ - "${all_builds}" - fi - - if [ ! -s "${all_depends}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.run-depends | | - ' {} \; | \ - grep -v ' base$' | \ - sort -k2,2 > \ - "${all_depends}" - fi - - sort -u "${omega}" | \ - sponge "${omega}" - - temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - { - sort -u "${keep}" - cat "${force}" "${force}" - } | \ - sort | \ - uniq -u > \ - "${temp_dir}/keep.new" - touch "${temp_dir}/keep" - - while [ -s "${temp_dir}/keep.new" ]; do - cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ - sort -u | \ - sponge "${temp_dir}/keep" - - { - # we append all packages which are run-dependencies of keep-packages - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.run-depends:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_builds}" - - # we append all packages with run-dependencies on the keep-list - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.builds:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_depends}" - } | \ - sort -u | \ - join -1 1 -2 1 -o 2.1 - "${omega}" | \ - sort -u > \ - "${temp_dir}/keep.new" - - # "new" is only what has not been there before and what is not forced - cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ - sort | \ - uniq -u | \ - sponge "${temp_dir}/keep.new" - done - - cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ - sort | \ - uniq -u - - ) - -} - -# sort_square_bracket_content $file -# sort the content of [] in $file, print to stdout - -sort_square_bracket_content() { - local file - local line - local token - local token_list - local rest - file="$1" - - while read -r line; do - printf '%s ' "${line}" | \ - tr ' ' '\n' | \ - while read -r token; do - if echo "${token}" | \ - grep -qF '['; then - printf '%s[' "${token%[*}" - token="${token##*[}" - token_list="${token%,}" - while ! echo "${token_list}" | \ - grep -qF ']'; do - read -r token - token_list=$( - printf '%s\n' \ - "${token_list}" \ - "${token%,}" - ) - done - rest="]${token_list#*]}" - token_list="${token_list%%]*}" - token=$( - printf '%s' "${token_list}" | \ - sort | \ - sed 's|$|,|' - printf '%s' "${rest}" - ) - fi - printf '%s\n' "${token}" - done | \ - tr '\n' ' ' | \ - sed ' - s|, ]|]|g - s| $|| - ' - printf '\n' - done < \ - "${file}" -} - -# smoothen_namcap_log $file -# remove unneccesary differences from namcap-logs: -# - remove architecture specific information -# - sort lines -# - sort content of square brackets - -smoothen_namcap_log() { - local file - file="$1" - # shellcheck disable=SC2016 - sort_square_bracket_content "${file}" | \ - sed ' - # normalize architecture specific information - s|i[34567]86|$ARCH|g - s|x86\([-_]64\)\?|$ARCH|g - # remove haskell hashes - s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g - ' | \ - sort | \ - sponge "${file}" -} - -# print_list_of_archaic_packages $source1 $source2 ... -# print a list of packages which have not been touched for a while, -# but which are still in the pipeline, e.g. in $source1, $source2 or ... - -print_list_of_archaic_packages() { - for source in "$@"; do - case "${source}" in - 'testing') - # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ - -exec head -n1 {} \; | \ - "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin - # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated - # and no longer block other packages from being moved - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ - sed ' - s|\.testing$|| - ' - ;; - 'build-list') - while read -r pkg rev mod_rev repo; do - git_repo=$( - find_repository_with_commit "${rev}" - ) - eval repo_path='"${repo_paths__'"${git_repo}"'}"' - commit_date=$( - git -C "${repo_path}" show -s --format=%ct "${rev}" - ) - mod_commit_date=$( - git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" - ) - if [ "${mod_commit_date}" -gt "${commit_date}" ]; then - commit_date="${mod_commit_date}" - fi - # packages remaining longer than $max_package_age_build_list days on the build list - if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then - printf '%s %s %s %s\n' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" - fi - done < \ - "${work_dir}/build-list" - ;; - 'staging') - # packages remaining longer than $max_package_age_staging days in staging - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ - sed ' - s|\.done$|| - ' - ;; - *) - >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" - ;; - esac - done | \ - sort -u -} - -# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" -# print the given modification revision possibly with a html link to github - -modification_revision_link() { - local mod_rev="$1" - local repo="$2" - local pkg="$3" - - if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then - printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \ - "${mod_rev}" \ - "${repo}" \ - "${pkg}" \ - "${mod_rev}" - else - printf '%s\n' \ - "${mod_rev}" - fi -} - -# trigger_mirror_refreshs -# trigger a refresh of capable tier 1 mirrors (as backup for master mirror) - -trigger_mirror_refreshs() { - local tmp_file - - tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir) - date '+%s' > \ - "${tmp_file}" - ${master_mirror_rsync_command} \ - "${tmp_file}" \ - "${master_mirror_rsync_directory}/lastupdate" - rm "${tmp_file}" - for trigger_url in ${mirror_refresh_trigger_urls}; do - screen -S trigger-mirror-update -d -m curl -L "${trigger_url}" - done -} - -# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name -extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { - pkgname="$1" - pkgname="${pkgname%.pkg.tar.xz}" - arch="${pkgname##*-}" - pkgname="${pkgname%-*}" - sub_pkgrel="${pkgname##*-}" - pkgname="${pkgname%-*}" - pkgrel="${sub_pkgrel%.*}" - if [ "${pkgrel}" = "${sub_pkgrel}" ]; then - sub_pkgrel='0' - else - sub_pkgrel="${sub_pkgrel##*.}" - fi - epoch="${pkgname##*-}" - pkgname="${pkgname%-*}" - pkgver="${epoch#*:}" - if [ "${pkgver}" = "${epoch}" ]; then - epoch='0' - else - epoch="${epoch%%:*}" - fi -} |