#!/bin/sh # check for packages that need to be built # shellcheck disable=SC2119,SC2120 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" # TODO: keep database clean in case of abort # shellcheck disable=SC2016 usage() { >&2 echo '' >&2 echo 'get-package-updates: check for packages that need to be built,' >&2 echo ' and build a list in the proper build order' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -d|--date $datetime:' >&2 echo ' Pull latest commit before $datetime' >&2 echo ' (yyyy-mm-ddThh:mm:ss). Conflicts -n.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -i|--ignore-insanity:' >&2 echo ' Do not abort when insane.' >&2 echo ' -m|--max-upstream-packages $number:' >&2 echo ' Do not update more than $number upstream packages.' >&2 echo ' Do not update git head of state repository.' >&2 echo ' USE WITH CAUTION: This may remove packages, if they' >&2 echo ' were moved. Always do a final run without -m.' >&2 echo ' -n|--no-pull: Do not pull git repos, merely reorder build list.' >&2 echo ' Conflicts -d.' >&2 echo ' -r|--recent-modifications:' >&2 echo ' Use the latest commit for the modifications' >&2 echo ' git repository (e.g. ignore -d for it).' >&2 echo ' Requires -d.' >&2 echo ' -w|--wait: If necessary, wait for lock blocking.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o d:him:nrw \ --long date: \ --long help \ --long ignore-insanity \ --long max-upstream-packages: \ --long no-pull \ --long recent-modifications \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" block_flag='-n' date_time='' ignore_insanity=false max_upstream_packages='' pull=true recent_modifications=false while true do case "$1" in -d|--date) shift date_time="$1" ;; -h|--help) usage 0 ;; -i|--ignore-insanity) ignore_insanity=true ;; -m|--max-upstream-packages) shift max_upstream_packages="$1" ;; -n|--no-pull) pull=false ;; -r|--recent-modifications) recent_modifications=true ;; -w|--wait) block_flag='' ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi if [ -n "${date_time}" ] && ! ${pull}; then >&2 printf -- '-d and -n are mutually exclusive.\n' usage fi if ${recent_modifications} && [ -z "${date_time}" ]; then >&2 printf -- '-r requires -d.\n' usage fi if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' if ! ${ignore_insanity}; then exit fi fi # delete_package arch package repository # mark $arch/$package for deletion # shellcheck disable=SC3043 delete_package() { local architecture="$1" local pkgbase="$2" local repository="$3" >&2 printf 'delete_package %s %s %s\n' "${architecture}" "${pkgbase}" "${repository}" # shellcheck disable=SC2016 query_delete_packages=$( printf '`architectures` AS `d_a`' printf ' JOIN `architecture_compatibilities` AS `a_c`' printf ' ON `a_c`.`runs_on`=`d_a`.`id`' printf ' AND `d_a`.`name`=from_base64("%s")' \ "$(printf '%s' "${architecture}" | base64 -w0)" printf ' JOIN `build_assignments`' printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`' # "any" references all architectures, but this is not represented # in `architecture_compatibilities`: If a package is not buildable # for "any", this means literally, that it is not buildable for # the _least_ architecture (e.g. it is not generic) printf ' OR `d_a`.`name`="any")' mysql_join_build_assignments_package_sources printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgbase}" | base64 -w0)" mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories mysql_join_package_sources_upstream_repositories printf ' AND `upstream_repositories`.`name` LIKE from_base64("%s")' \ "$(printf '%s' "${repository}" | base64 -w0)" ) # shellcheck disable=SC2016 { # packages from the build-list/to-be-decided go straight to the deletion-list # this happens in two steps, because we need to create one item per # target architecture printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (' printf '`package`,' printf '`repository`,' printf '`last_moved`,' printf '`is_to_be_deleted`' printf ') SELECT' printf ' `binary_packages`.`id`,' printf '`d_r`.`id`,' printf 'NOW(),' printf '1' printf ' FROM' printf ' %s' "${query_delete_packages}" mysql_join_binary_packages_in_repositories_repositories # shellcheck disable=SC2154 printf ' AND `repositories`.`stability` in (%s,%s)' \ "${repository_stability_ids__unbuilt}" \ "${repository_stability_ids__virtual}" mysql_join_build_assignments_architectures '' 'ba_a' printf ' JOIN `repositories` AS `d_r`' printf ' ON (' # arch-specific build_assignments must match exactly printf '`d_r`.`architecture`=`ba_a`.`id`' # "any" build_assignments build for all architectures printf ' OR `ba_a`.`name`="any"' printf ')' # shellcheck disable=SC2154 printf ' AND `d_r`.`stability`=%s;\n' \ "${repository_stability_ids__forbidden}" printf 'COMMIT;\n' printf 'DELETE `binary_packages_in_repositories`' printf ' FROM %s' "${query_delete_packages}" mysql_join_binary_packages_in_repositories_repositories # shellcheck disable=SC2154 printf ' AND `repositories`.`stability` in (%s,%s);\n' \ "${repository_stability_ids__unbuilt}" \ "${repository_stability_ids__virtual}" printf 'COMMIT;\n' # other packages are marked as `is_to_be_deleted` printf 'UPDATE %s' "${query_delete_packages}" printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n' } | \ mysql_run_query } something_new=false for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' if [ "${repo}" = 'archlinux32' ]; then branch='master' else branch='main' fi # Update git repositories (upstream state and our package customizations). if [ -d "${repo_path}/.git" ]; then git -C "${repo_path}" pull --ff-only else git -C "${repo_path}" fetch origin "${branch}:${branch}" fi || \ true # read previous git revision numbers from database. # shellcheck disable=SC2016 eval "old_repo_revisions__${repo}='$( { printf 'SELECT `git_repositories`.`head`' printf ' FROM `git_repositories`' printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" } | \ mysql_run_query )'" # determine new git revision if ${pull}; then if ${recent_modifications} && \ [ "${repo}" = 'archlinux32' ] || \ [ -z "${date_time}" ]; then eval "new_repo_revisions__${repo}='$( git -C "${repo_path}" rev-parse HEAD )'" else new_rev=$( git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD ) eval 'old_rev="${old_repo_revisions__'"${repo}"'}"' # do not go backwards in time # shellcheck disable=SC2154 if ! git -C "${repo_path}" merge-base --is-ancestor "${old_rev}" "${new_rev}"; then new_rev="${old_rev}" fi eval "new_repo_revisions__${repo}='${new_rev}'" fi else eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"' fi if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then something_new=true fi done if ${pull} && \ ! ${something_new}; then >&2 echo 'Nothing changed.' exit fi # Create a lock file for build list. exec 9> "${build_list_lock_file}" # shellcheck disable=SC2086 if ! verbose_flock ${block_flag} 9; then >&2 echo 'come back (shortly) later - I cannot lock build list.' exit fi exec 8> "${sanity_check_lock_file}" # shellcheck disable=SC2086 if ! verbose_flock -s ${block_flag} 8; then >&2 echo 'come back (shortly) later - sanity-check running.' exit fi cleanup() { mysql_cleanup rm -rf --one-file-system "${tmp_dir:?}" } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap cleanup EXIT # shellcheck disable=SC2119 mysql_cleanup echo 'Check modified packages from the last update, and put them to the build list.' # Check modified packages from the last update, and put them to the build list. # If a package is updated, but already on the rebuild list, then just update the git revision number. # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. { directories=$( # shellcheck disable=SC2154 git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \ | tar -t \ | cut -d/ -f1 \ | grep -vF -- '-testing-' \ | grep -vF -- '-staging-' \ | sort -u ) # shellcheck disable=SC2016 { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,' printf '`package_sources`.`git_revision`,' printf '`upstream_repositories`.`name`' printf ' FROM `package_sources`' mysql_join_package_sources_upstream_repositories mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`' } \ | mysql_run_query \ | tr '\t' ' ' \ | sort -u \ > "${tmp_dir}/mysql-packages" for directory in ${directories}; do # shellcheck disable=SC2154 git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- "${directory}" \ | tar -Ox \ | grep -v '^\S\+\( \S\+\)\1\1$' \ | cut -d' ' -f1,4 \ | sed ' s@$@ '"${directory%-*}"'@ ' done \ | grep -v '^lib32-' \ | sort -u \ > "${tmp_dir}/upstream-packages" diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \ | grep '^>' \ | awk '{print $3 " " $2 " " $4}' \ | sort -k2,2 -k3,3 \ | uniq -uf1 \ | while read -r git_revision pkgbase repository; do # shellcheck disable=SC2154 if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then mod_git_revision="${new_repo_revisions__archlinux32}" else mod_git_revision='0000000000000000000000000000000000000000' fi printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${mod_git_revision}" done \ | if [ -n "${max_upstream_packages}" ]; then head -n"${max_upstream_packages}" else cat fi # shellcheck disable=SC2154 git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" --name-status \ | tr '\t/' ' ' \ | grep '^[AM]' \ | cut -d' ' -f2,3 \ | grep -v '^blacklist ' \ | while read -r repository pkgbase; do git_revision=$( # shellcheck disable=SC2046 git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- $( printf '%s\n' "${directories}" \ | grep '^'"${repository}"'-' ) \ | tar -Ox \ | sort -k1,1 \ > "${tmp_dir}/git-revisions" echo "${pkgbase}" \ | join -1 1 -2 1 -o 1.4 "${tmp_dir}/git-revisions" - ) if [ -z "${git_revision}" ]; then git_revision='0000000000000000000000000000000000000000' fi if [ ${#git_revision} != 40 ] \ || printf '%s\n' "${git_revision}" \ | grep -vq '[0-9]'; then >&2 printf 'invalid git revision "%s"\n' "${git_revision}" exit 1 fi printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${new_repo_revisions__archlinux32}" done } \ | sort -u \ | sort -k1,1 -k2,2 \ > "${tmp_dir}/modified-packages" errors=$( awk '{print $3 " " $4 " " $1 " " $2}' \ < "${tmp_dir}/modified-packages" \ | uniq -Df2 ) if [ -n "${errors}" ]; then >&2 printf 'Some packages are scheduled with different versions - this should not happen:\n' >&2 printf '%s\n' "${errors}" exit 1 fi echo 'Mark to be deleted packages for removal' cut -d' ' -f1,3 "${tmp_dir}/mysql-packages" \ | sort -u \ | sponge "${tmp_dir}/mysql-packages" { cut -d' ' -f1,3 "${tmp_dir}/upstream-packages" git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" \ | tar -t \ | grep -v '^blacklist/' \ | tr '/' ' ' \ | awk '{print $2 " " $1}' } \ | sort -u \ | sponge "${tmp_dir}/upstream-packages" diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \ | grep '^<' \ | cut -d' ' -f2,3 \ | while read -r pkgbase repository; do delete_package 'any' "${pkgbase}" "${repository}" done echo 'Insert updated packages' while read -r pkgbase repository git_revision mod_git_revision; do # shellcheck disable=SC2016 { # delete old binary packages which are not yet built or on the # deletion list mysql_query_delete_packages \ '`package_sources`.`pkgbase`=from_base64("'"$( printf '%s' "${pkgbase}" \ | base64 -w0 )"'")' \ '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')' # remove is-to-be-deleted marker from old binary packages printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${pkgbase}" \ | base64 -w0 )" printf ' AND `upstream_repositories`.`name`=from_base64("%s");\n' \ "$( printf '%s' "${repository}" \ | base64 -w0 )" } \ | mysql_run_query # shellcheck disable=SC2154 >&2 printf '%s ' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" >&2 printf '\n' done \ < "${tmp_dir}/modified-packages" echo 'remove blacklisted packages' { git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' \ | tar -t 'blacklist' \ | sed ' s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@ t d ' \ | sort -k2,2 \ | join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' \ | sed ' s@^[AM]\sblacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@ t d ' } \ | sort -u \ | while read -r arch pkgbase; do delete_package "${arch}" "${pkgbase}" '%' done echo 'Done - mark decisions as final.' # shellcheck disable=SC2016 { # save blacklist into database printf 'CREATE TEMPORARY TABLE `blacklist` (`arch` VARCHAR(16), `pkgbase` VARCHAR(64), `reason` TEXT);\n' git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ tar -x --to-command 'sed "s@^@$TAR_FILENAME @"' 'blacklist' | \ sed ' s@^blacklist/\([^/[:space:]]\+\)/\S\+/\([^/[:space:]]\+\) @\1 \2 @ t d ' | \ while read -r arch pkgbase reason; do printf '(from_base64("%s"),from_base64("%s"),from_base64("%s")),\n' \ "$(printf '%s' "${arch}" | base64 -w0)" \ "$(printf '%s' "${pkgbase}" | base64 -w0)" \ "$(printf '%s' "${reason}" | base64 -w0)" done | \ sed ' 1 i INSERT IGNORE INTO `blacklist` (`arch`,`pkgbase`,`reason`) VALUES $ s/,$/;/ ' printf 'UPDATE `build_assignments`' printf ' SET `build_assignments`.`is_black_listed`=NULL;\n' printf 'UPDATE `blacklist`' printf ' JOIN `architectures`' printf ' ON `architectures`.`name`=`blacklist`.`arch`' printf ' JOIN `package_sources`' printf ' ON `blacklist`.`pkgbase`=`package_sources`.`pkgbase`' mysql_join_package_sources_build_assignments printf ' JOIN `architecture_compatibilities`' printf ' ON `build_assignments`.`architecture`=`architecture_compatibilities`.`built_for`' printf ' AND (' printf '`architectures`.`id`=`architecture_compatibilities`.`runs_on`' # shellcheck disable=SC2154 printf ' OR `architectures`.`id`=%s' \ "${architecture_ids__any}" printf ')' printf ' SET `build_assignments`.`is_black_listed`=`blacklist`.`reason`;\n' printf 'DROP TEMPORARY TABLE `blacklist`;\n' printf 'COMMIT;\n' # update hashes of repositories in mysql database for repo in ${repo_names}; do if [ -n "${max_upstream_packages}" ] \ && [ "${repo}" = 'state' ]; then continue fi printf 'UPDATE `git_repositories`' printf ' SET `git_repositories`.`head`=from_base64("%s")' \ "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}"' | base64 -w0)" printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" done # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages printf ' SET `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \ "${repository_ids__any_to_be_decided}" } | \ mysql_run_query echo 'Aftermath - sort versions.' mysql_sort_versions echo 'Aftermath - find assignment loops.' # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". mysql_find_build_assignment_loops echo 'Aftermath - remove duplicate binary_packages.' # remove duplicate binary_packages from "build-list" mysql_query_remove_old_binary_packages_from_build_list | \ mysql_run_query 'unimportant'