| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/get-package-updates | 451 |
diff --git a/bin/get-package-updates b/bin/get-package-updates index 8e7e4ef..cc9e03e 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -21,7 +21,12 @@ usage() { >&2 echo ' (yyyy-mm-ddThh:mm:ss). Conflicts -n.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -i|--ignore-insanity:' - >&2 echo ' Do not abort when insane.' + >&2 echo ' Do not abort when insane.' + >&2 echo ' -m|--max-upstream-packages $number:' + >&2 echo ' Do not update more than $number upstream packages.' + >&2 echo ' Do not update git head of state repository.' + >&2 echo ' USE WITH CAUTION: This may remove packages, if they' + >&2 echo ' were moved. Always do a final run without -m.' >&2 echo ' -n|--no-pull: Do not pull git repos, merely reorder build list.' >&2 echo ' Conflicts -d.' >&2 echo ' -r|--recent-modifications:' @@ -33,10 +38,11 @@ usage() { } eval set -- "$( - getopt -o d:hinrw \ + getopt -o d:him:nrw \ --long date: \ --long help \ --long ignore-insanity \ + --long max-upstream-packages: \ --long no-pull \ --long recent-modifications \ --long wait \ @@ -47,6 +53,7 @@ eval set -- "$( block_flag='-n' date_time='' ignore_insanity=false +max_upstream_packages='' pull=true recent_modifications=false @@ -63,6 +70,10 @@ do -i|--ignore-insanity) ignore_insanity=true ;; + -m|--max-upstream-packages) + shift + max_upstream_packages="$1" + ;; -n|--no-pull) pull=false ;; @@ -106,16 +117,22 @@ if [ -s "${work_dir}/build-master-sanity" ]; then fi fi -# delete_package arch package +# delete_package arch package repository # mark $arch/$package for deletion + +# shellcheck disable=SC3043 delete_package() { + local architecture="$1" + local pkgbase="$2" + local repository="$3" + >&2 printf 'delete_package %s %s %s\n' "${architecture}" "${pkgbase}" "${repository}" # shellcheck disable=SC2016 query_delete_packages=$( printf '`architectures` AS `d_a`' printf ' JOIN `architecture_compatibilities` AS `a_c`' printf ' ON `a_c`.`runs_on`=`d_a`.`id`' printf ' AND `d_a`.`name`=from_base64("%s")' \ - "$(printf '%s' "$1" | base64 -w0)" + "$(printf '%s' "${architecture}" | base64 -w0)" printf ' JOIN `build_assignments`' printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`' # "any" references all architectures, but this is not represented @@ -125,9 +142,12 @@ delete_package() { printf ' OR `d_a`.`name`="any")' mysql_join_build_assignments_package_sources printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "$2" | base64 -w0)" + "$(printf '%s' "${pkgbase}" | base64 -w0)" mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories + mysql_join_package_sources_upstream_repositories + printf ' AND `upstream_repositories`.`name` LIKE from_base64("%s")' \ + "$(printf '%s' "${repository}" | base64 -w0)" ) # shellcheck disable=SC2016 { @@ -165,6 +185,8 @@ delete_package() { printf 'COMMIT;\n' + # no need to remove the binary_package, because it is now on the + # deletion-lists printf 'DELETE `binary_packages_in_repositories`' printf ' FROM %s' "${query_delete_packages}" mysql_join_binary_packages_in_repositories_repositories @@ -186,11 +208,16 @@ something_new=false for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' - # Update git repositories (official packages, community packages and the repository of package customizations). + if [ "${repo}" = 'archlinux32' ]; then + branch='master' + else + branch='main' + fi + # Update git repositories (upstream state and our package customizations). if [ -d "${repo_path}/.git" ]; then git -C "${repo_path}" pull --ff-only else - git -C "${repo_path}" fetch origin master:master + git -C "${repo_path}" fetch origin "${branch}:${branch}" fi || \ true # read previous git revision numbers from database. @@ -238,18 +265,9 @@ if ${pull} && \ exit fi -# Create a lock file for build list. - -exec 9> "${build_list_lock_file}" -# shellcheck disable=SC2086 -if ! verbose_flock ${block_flag} 9; then - >&2 echo 'come back (shortly) later - I cannot lock build list.' - exit -fi - -exec 8> "${sanity_check_lock_file}" +exec 9> "${sanity_check_lock_file}" # shellcheck disable=SC2086 -if ! verbose_flock -s ${block_flag} 8; then +if ! verbose_flock -s ${block_flag} 9; then >&2 echo 'come back (shortly) later - sanity-check running.' exit fi @@ -272,227 +290,204 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -# shellcheck disable=SC2016 -{ - printf 'SELECT DISTINCT' - printf ' `package_sources`.`pkgbase`,' - printf '`upstream_repositories`.`name`,' - printf 'IF(`build_assignments`.`architecture`=%s,"any","x86_64")' \ - "${architecture_ids__any}" - printf ' FROM `binary_packages`' - mysql_join_binary_packages_binary_packages_in_repositories - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \ - "${repository_ids__any_build_list}" -} | \ - mysql_run_query | \ - sed ' - s@^\(\S*\)\t\(\S*\)\t\(\S*\)$@s,^\\(.\\t\1/repos/\2-\3/\\)[^/]\\+$,\\1PKGBUILD,\ns,^\\(.\\t\2/\1/\\)[^/]\\+$,\\1PKGBUILD,@ - ' > \ - "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" - { - # trigger rebuild of packages removed from blacklist + directories=$( + # shellcheck disable=SC2154 + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \ + | tar -t \ + | cut -d/ -f1 \ + | grep -vF -- '-testing-' \ + | grep -vF -- '-staging-' \ + | sort -u + ) # shellcheck disable=SC2016 { - printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`,`git_repositories`.`name`' + printf 'SELECT DISTINCT `package_sources`.`pkgbase`,' + printf '`package_sources`.`git_revision`,' + printf '`upstream_repositories`.`name`' printf ' FROM `package_sources`' mysql_join_package_sources_upstream_repositories - mysql_join_upstream_repositories_git_repositories - printf ' WHERE `package_sources`.`pkgbase` IN (' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_binary_packages_in_repositories + printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`' + } \ + | mysql_run_query \ + | tr '\t' ' ' \ + | sort -u \ + > "${tmp_dir}/mysql-packages" + for directory in ${directories}; do # shellcheck disable=SC2154 - git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - sed ' - s@^D\tblacklist/[^/]\+/[^/]\+/@@ - t - d - ' | \ - base64_encode_each | \ - sed ' - s/^/from_base64("/ - s/$/")/ - ' | \ - tr '\n' ',' - printf '"");\n' - } | \ - mysql_run_query | \ - while read -r pkgbase repository git_repository; do - eval 'git_rev="${new_repo_revisions__'"${git_repository}"'}"' - eval 'git_dir="${repo_paths__'"${git_repository}"'}"' - # TODO: maybe, we want to exclude our own packages here, too? But why - # would they be black listed in the first place? - # shellcheck disable=SC2154 - if git -C "${git_dir}" archive "${git_rev}" -- "${pkgbase}/repos" \ - | tar -tf - \ - | grep -q -- '-\(any\|x86_64\)/PKGBUILD$'; then - printf 'A %s %s %s\n' \ - "${pkgbase}" \ - "${git_rev}" \ - "${repository}" - fi - done - # actual updates/removes - for repo in ${repo_names}; do - eval repo_path='"${repo_paths__'"${repo}"'}"' - eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' - eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"' - # if old revision unknown, mimic "git diff"-output + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- "${directory}" \ + | tar -Ox \ + | grep -v '^\S\+\( \S\+\)\1\1$' \ + | grep ' [0-9a-f]\{40\}$' \ + | cut -d' ' -f1,4 \ + | sed ' + s@$@ '"${directory%-*}"'@ + ' + done \ + | grep -v '^lib32-' \ + | grep -vF '03fd4a0a1cee6144848a5e5de6e03f0319e9d118' \ + | sort -u \ + > "${tmp_dir}/upstream-packages" + diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \ + | grep '^>' \ + | awk '{print $3 " " $2 " " $4}' \ + | sort -k2,2 -k3,3 \ + | uniq -uf1 \ + | while read -r git_revision pkgbase repository; do # shellcheck disable=SC2154 - if [ "${old_repo_revision}" = "NONE" ]; then - git -C "${repo_path}" archive --format=tar HEAD | \ - tar -t | \ - sed 's|^|A\t|' + if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then + mod_git_revision="${new_repo_revisions__archlinux32}" else - git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" - fi | \ - # rename any file to "PKGBUILD" for packages on the build-list - sed -f "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" | \ - # only track changes in PKGBUILDs - grep '/PKGBUILD$' | \ - if [ "${repo}" = "archlinux32" ]; then - # modify the directory structure from the modifiaction-repository - # to the one of an original source repository - # shellcheck disable=SC2016 - sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ - while read -r pkg_repo rest; do - repo=$(find_git_repository_to_package_repository "${pkg_repo}") - eval 'printf '"'"'%s %s\n'"'" \ - "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \ - "'${rest}'" - done - else - sed "s|^|${new_repo_revision} |" - fi | \ - grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - # ignore i686 - grep -- '-\(any\|x86_64\)/PKGBUILD$' | \ - # ignore staging and testing - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' - done | \ - sort -u | \ - sed ' - s|^D\s|0 \0| - t - s|^[AM]\s|1 \0| - t - s|^|2 | - ' | \ - sort -k1,1 | \ - sed 's|^[012] ||' -} | \ - while read -r mode package git_revision repository; do - if [ "${mode}" = 'D' ]; then - # deleted PKGBUILD - # shellcheck disable=SC2154 - git_revision="${new_repo_revisions__archlinux32}" - found_package=false - # we need to test archlinux32 last, because otherwise - # find_package_repository_to_package might look in the wrong git - # repository of a package w/o upstream - for repository in ${repo_names} archlinux32; do - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if [ "${repository}" = "archlinux32" ]; then - if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep -q '^[^/]\+/'"$(str_to_regex "${package}")"'/PKGBUILD$'; then - found_package=true - fi - else - # shellcheck disable=SC2154 - if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos"' 2> /dev/null | \ - tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \ - cut -d/ -f3 | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -vq -- '-i686$'; then - eval 'git_revision="${new_repo_revisions__'"${repository}"'}"' - found_package=true - break - fi - fi - done - if ${found_package}; then - mode='M' - repository=$( - find_package_repository_to_package "${package}" "${repository}" "${git_revision}" - ) - else - delete_package 'any' "${package}" - continue - fi + mod_git_revision='0000000000000000000000000000000000000000' fi - if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - # shellcheck disable=SC2016 - { - # delete old binary packages which are not yet built or on the - # deletion list - mysql_query_delete_packages \ - '`package_sources`.`pkgbase`=from_base64("'"$( - printf '%s' "${package}" | \ - base64 -w0 - )"'")' \ - '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')' - # remove is-to-be-deleted marker from old binary packages - printf 'UPDATE `binary_packages_in_repositories`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ - "$( - printf '%s' "${package}" | \ - base64 -w0 - )" - } | \ - mysql_run_query - printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages" - # shellcheck disable=SC2154 - printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2 -# TODO: sometimes we schedule the identical package twice (once "normal" -# and once, because we found the deleted package elsewhere) - this could -# be avoided - mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" - printf '\n' >&2 - continue + printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${mod_git_revision}" + done \ + | if [ -n "${max_upstream_packages}" ]; then + head -n"${max_upstream_packages}" + else + cat + fi + # shellcheck disable=SC2154 + git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" --name-status \ + | tr '\t/' ' ' \ + | grep '^[AM]' \ + | cut -d' ' -f2,3 \ + | grep -v '^blacklist ' \ + | while read -r repository pkgbase; do + git_revision=$( + # shellcheck disable=SC2046 + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- $( + printf '%s\n' "${directories}" \ + | grep '^'"${repository}"'-' + ) \ + | tar -Ox \ + | sort -k1,1 \ + > "${tmp_dir}/git-revisions" + echo "${pkgbase}" \ + | join -1 1 -2 1 -o 1.4 "${tmp_dir}/git-revisions" - + ) + if [ -z "${git_revision}" ]; then + git_revision='0000000000000000000000000000000000000000' fi - - >&2 echo "unknown git diff mode '${mode}'" - exit 1 + if [ ${#git_revision} != 40 ] \ + || printf '%s\n' "${git_revision}" \ + | grep -vq '[0-9]'; then + >&2 printf 'invalid git revision "%s"\n' "${git_revision}" + exit 1 + fi + printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${new_repo_revisions__archlinux32}" done +} \ +| sort -u \ +| sort -k1,1 -k2,2 \ +> "${tmp_dir}/modified-packages" + +errors=$( + awk '{print $3 " " $4 " " $1 " " $2}' \ + < "${tmp_dir}/modified-packages" \ + | uniq -Df2 +) +if [ -n "${errors}" ]; then + >&2 printf 'Some packages are scheduled with different versions - this should not happen:\n' + >&2 printf '%s\n' "${errors}" + exit 1 +fi -if [ -s "${tmp_dir}/modified-packages" ] || ! ${pull}; then - echo 'Delete black-listed packages.' - if [ -s "${tmp_dir}/modified-packages" ]; then - sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages" +echo 'Mark to be deleted packages for removal' + +cut -d' ' -f1,3 "${tmp_dir}/mysql-packages" \ +| sort -u \ +| sponge "${tmp_dir}/mysql-packages" + +{ + cut -d' ' -f1,3 "${tmp_dir}/upstream-packages" + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" \ + | tar -t \ + | grep -v '^blacklist/' \ + | tr '/' ' ' \ + | awk '{print $2 " " $1}' +} \ +| sort -u \ +| sponge "${tmp_dir}/upstream-packages" + +diff "${tmp_dir}/mysql-packages" "${tmp_dir}/upstream-packages" \ +| grep '^<' \ +| cut -d' ' -f2,3 \ +| while read -r pkgbase repository; do + delete_package 'any' "${pkgbase}" "${repository}" +done + +echo 'Insert updated packages' + +while read -r pkgbase repository git_revision mod_git_revision; do + if test "$repository" = "kde-unstable"; then + continue fi - # extract black-listed packages - git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - tar -t 'blacklist' | \ - sed ' - s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@ - t - d - ' | \ - expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" | \ - if [ -s "${tmp_dir}/modified-packages" ]; then - sort -k2,2 | \ - join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - - sed ' - /^lib32-/ s/^/any / - t - d - ' "${tmp_dir}/modified-packages" - else - cat - fi | \ - sort -u | \ - while read -r arch pkgbase; do - delete_package "${arch}" "${pkgbase}" - done - rm "${tmp_dir}/architecture-compatibilities" -fi + if test "$repository" = "gnome-unstable"; then + continue + fi + # shellcheck disable=SC2016 + { + # delete old binary packages which are not yet built or on the + # deletion list + mysql_query_delete_packages \ + '`package_sources`.`pkgbase`=from_base64("'"$( + printf '%s' "${pkgbase}" \ + | base64 -w0 + )"'")' \ + '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')' + # remove is-to-be-deleted marker from old binary packages + printf 'UPDATE `binary_packages_in_repositories`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ + "$( + printf '%s' "${pkgbase}" \ + | base64 -w0 + )" + printf ' AND `upstream_repositories`.`name`=from_base64("%s");\n' \ + "$( + printf '%s' "${repository}" \ + | base64 -w0 + )" + } \ + | mysql_run_query + # shellcheck disable=SC2154 + >&2 printf '%s ' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + >&2 printf '\n' +done \ +< "${tmp_dir}/modified-packages" + +echo 'remove blacklisted packages' + +{ + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' \ + | tar -t 'blacklist' \ + | sed ' + s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@ + t + d + ' \ + | sort -k2,2 \ + | join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - + git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' \ + | sed ' + s@^[AM]\sblacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1 \2@ + t + d + ' +} \ +| sort -u \ +| while read -r arch pkgbase; do + delete_package "${arch}" "${pkgbase}" '%' +done echo 'Done - mark decisions as final.' @@ -539,6 +534,10 @@ echo 'Done - mark decisions as final.' # update hashes of repositories in mysql database for repo in ${repo_names}; do + if [ -n "${max_upstream_packages}" ] \ + && [ "${repo}" = 'state' ]; then + continue + fi printf 'UPDATE `git_repositories`' printf ' SET `git_repositories`.`head`=from_base64("%s")' \ "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}"' | base64 -w0)" |