| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/get-package-updates | 379 | ||||
| -rwxr-xr-x | lib/common-functions | 1 |
diff --git a/bin/get-package-updates b/bin/get-package-updates index 9835bea..f83aa4c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -182,6 +182,25 @@ delete_package() { mysql_run_query } +# check_if_package_should_be_deleted $pkgbase $repository +check_if_package_should_be_deleted() { + if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then + return 1 + fi + if git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \ + | tar -t \ + | tr '/' ' ' \ + | cut -d' ' -f1,2 \ + | sort -k2,2 \ + | join -1 2 -2 1 - <(echo "${pkgbase}") \ + | cut -d' ' -f1 \ + | sed 's@-[^-]\+$@@' \ + | grep -qxF "${repository}"; then + return 1 + fi + return 0 +} + something_new=false for repo in ${repo_names}; do @@ -277,229 +296,161 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -# shellcheck disable=SC2016 { - printf 'SELECT DISTINCT' - printf ' `package_sources`.`pkgbase`,' - printf '`upstream_repositories`.`name`,' - printf 'IF(`build_assignments`.`architecture`=%s,"any","x86_64")' \ - "${architecture_ids__any}" - printf ' FROM `binary_packages`' - mysql_join_binary_packages_binary_packages_in_repositories - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \ - "${repository_ids__any_build_list}" -} | \ - mysql_run_query | \ - sed ' - s@^\(\S*\)\t\(\S*\)\t\(\S*\)$@s,^\\(.\\t\1/repos/\2-\3/\\)[^/]\\+$,\\1PKGBUILD,\ns,^\\(.\\t\2/\1/\\)[^/]\\+$,\\1PKGBUILD,@ - ' > \ - "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" - -{ - # trigger rebuild of packages removed from blacklist - # shellcheck disable=SC2016 - { - # TODO: this needs to be properly fixed - printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`' - printf ' FROM `package_sources`' - mysql_join_package_sources_upstream_repositories - printf ' WHERE `package_sources`.`pkgbase` IN (' - # shellcheck disable=SC2154 - git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - sed ' - s@^D\tblacklist/[^/]\+/[^/]\+/@@ - t - d - ' | \ - base64_encode_each | \ - sed ' - s/^/from_base64("/ - s/$/")/ - ' | \ - tr '\n' ',' - printf '"");\n' - } | \ - echo mysql_run_query | \ - while read -r pkgbase repository; do - break - eval 'git_rev="${new_repo_revisions__'"${git_repository}"'}"' - eval 'git_dir="${repo_paths__'"${git_repository}"'}"' - # TODO: maybe, we want to exclude our own packages here, too? But why - # would they be black listed in the first place? - # shellcheck disable=SC2154 - if git -C "${git_dir}" archive "${git_rev}" -- "${pkgbase}/repos" \ - | tar -tf - \ - | grep -q -- '-\(any\|x86_64\)/PKGBUILD$'; then - printf 'A %s %s %s\n' \ - "${pkgbase}" \ - "${git_rev}" \ - "${repository}" - fi - done - # actual updates/removes - for repo in ${repo_names}; do - eval repo_path='"${repo_paths__'"${repo}"'}"' - eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' - eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"' - # if old revision unknown, mimic "git diff"-output - # shellcheck disable=SC2154 - if [ "${old_repo_revision}" = "NONE" ]; then - git -C "${repo_path}" archive --format=tar HEAD | \ - tar -t | \ - sed 's|^|A\t|' - else - git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" - fi | \ - # rename any file to "PKGBUILD" for packages on the build-list - sed -f "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" | \ - # only track changes in PKGBUILDs - grep '/PKGBUILD$' | \ - if [ "${repo}" = "archlinux32" ]; then - # modify the directory structure from the modifiaction-repository - # to the one of an original source repository - # shellcheck disable=SC2016 - sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ - while read -r pkg_repo rest; do - repo=$(find_git_repository_to_package_repository "${pkg_repo}") - eval 'printf '"'"'%s %s\n'"'" \ - "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \ - "'${rest}'" - done - else - sed "s|^|${new_repo_revision} |" - fi | \ - grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - # ignore i686 - grep -- '-\(any\|x86_64\)/PKGBUILD$' | \ - # ignore staging and testing - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' - done | \ - sort -u | \ - sed ' - s|^D\s|0 \0| - t - s|^[AM]\s|1 \0| - t - s|^|2 | - ' | \ - sort -k1,1 | \ - sed 's|^[012] ||' -} | \ - while read -r mode package git_revision repository; do - if [ "${mode}" = 'D' ]; then - # deleted PKGBUILD - # shellcheck disable=SC2154 - git_revision="${new_repo_revisions__archlinux32}" - found_package=false - # we need to test archlinux32 last, because otherwise - # find_package_repository_to_package might look in the wrong git - # repository of a package w/o upstream - for repository in ${repo_names} archlinux32; do - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if [ "${repository}" = "archlinux32" ]; then - if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep -q '^[^/]\+/'"$(str_to_regex "${package}")"'/PKGBUILD$'; then - found_package=true - fi - else - # shellcheck disable=SC2154 - if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos"' 2> /dev/null | \ - tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \ - cut -d/ -f3 | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -vq -- '-i686$'; then - eval 'git_revision="${new_repo_revisions__'"${repository}"'}"' - found_package=true - break - fi - fi - done - if ${found_package}; then - mode='M' - repository=$( - find_package_repository_to_package "${package}" "${repository}" "${git_revision}" - ) - else - delete_package 'any' "${package}" + directories=$( + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \ + | tar -t \ + | cut -d/ -f1 \ + | grep -vF -- '-testing-' \ + | grep -vF -- '-staging-' \ + | sort -u + ) + diff <( + { + printf 'SELECT `package_sources`.`pkgbase`,' + printf '`package_sources`.`git_revision`,' + printf '`upstream_repositories`.`name`' + printf ' FROM `package_sources`' + mysql_join_package_sources_upstream_repositories + } \ + | mysql_run_query \ + | tr '\t' ' ' \ + | sort -u + ) <( + for directory in ${directories}; do + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- "${directory}" \ + | tar -Ox \ + | cut -d' ' -f1,4 \ + | sed ' + s@$@ '"${directory%-*}"'@ + ' + done \ + | sort -u + ) \ + | grep '^[<>]' \ + | awk '{print $1 " " $3 " " $2 " " $4}' \ + | sort -k3,3 -k1r,1 \ + | uniq -uf2 \ + | while read -r mode git_revision pkgbase repository; do + if [ "${mode}" = '<' ]; then + if check_if_package_should_be_deleted "${pkgbase}" "${repository}"; then + delete_package 'any' "${pkgbase}" continue + else + mode='>' fi + if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then + mod_git_revision="${new_repo_revisions__archlinux32}" + else + mod_git_revision='0000000000000000000000000000000000000000' fi - if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - # shellcheck disable=SC2016 - { - # delete old binary packages which are not yet built or on the - # deletion list - mysql_query_delete_packages \ - '`package_sources`.`pkgbase`=from_base64("'"$( - printf '%s' "${package}" | \ - base64 -w0 - )"'")' \ - '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')' - # remove is-to-be-deleted marker from old binary packages - printf 'UPDATE `binary_packages_in_repositories`' - mysql_join_binary_packages_in_repositories_binary_packages - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ - "$( - printf '%s' "${package}" | \ - base64 -w0 - )" - } | \ - mysql_run_query - printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages" - # shellcheck disable=SC2154 - printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2 -# TODO: sometimes we schedule the identical package twice (once "normal" -# and once, because we found the deleted package elsewhere) - this could -# be avoided - mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" - printf '\n' >&2 + printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${mod_git_revision}" + done + git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}^^" "${new_repo_revisions__archlinux32}" --name-status \ + | tr '\t/' ' ' \ + | cut -d' ' -f1,2,3 \ + | sed ' + s/^[AM]/>/ + t + s/^D/</ + t + d + ' \ + | while read -r mode repository pkgbase; do + if check_if_package_should_be_deleted "${pkgbase}" "${repository}"; then + delete_package 'any' "${pkgbase}" continue + else + mode='>' fi - - >&2 echo "unknown git diff mode '${mode}'" - exit 1 + git_revision=$( + git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- $( + printf '%s\n' "${directories}" \ + | grep '^'"${repository}"'-' + ) \ + | tar -Ox \ + | sort -k1,1 \ + | join -1 1 -2 1 -o 1.4 - <(echo "${pkgbase}") + ) + if [ -z "${git_revision}" ]; then + git_revision='0000000000000000000000000000000000000000' + fi + if [ ${#git_revision} != 40 ] \ + || printf '%s\n' "${git_revision}" \ + | grep -vq '[0-9]'; then + >&2 printf 'invalid git revision "%s"\n' + exit 1 + fi + printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${new_repo_revisions__archlinux32}" done - -if [ -s "${tmp_dir}/modified-packages" ] || ! ${pull}; then - echo 'Delete black-listed packages.' - if [ -s "${tmp_dir}/modified-packages" ]; then - sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages" - fi - # extract black-listed packages - git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - tar -t 'blacklist' | \ - sed ' - s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@ - t - d - ' | \ - expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" | \ - if [ -s "${tmp_dir}/modified-packages" ]; then - sort -k2,2 | \ - join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - - sed ' - /^lib32-/ s/^/any / - t - d - ' "${tmp_dir}/modified-packages" - else - cat - fi | \ - sort -u | \ - while read -r arch pkgbase; do - delete_package "${arch}" "${pkgbase}" - done - rm "${tmp_dir}/architecture-compatibilities" +} \ +| sort -u \ +| sort -k1,1 -k2,2 \ +> "${tmp_dir}/modified-packages" + +errors=$( + awk '{print $3 " " $4 " " $1 " " $2}' \ + < "${tmp_dir}/modified-packages" \ + | uniq -Df2 +) +if [ -n "${errors}" ]; then + >&2 printf 'Some packages are scheduled with different versions - this should not happen:\n' + >&2 printf '%s\n' "${errors}" + exit 1 fi +# TODO: do not use head, here +grep -v '^lib32-' "${tmp_dir}/modified-packages" \ +| head -n3 \ +| sponge "${tmp_dir}/modified-packages" + +while read -r pkgbase repository git_revision mod_git_revision; do + # shellcheck disable=SC2016 + { + # delete old binary packages which are not yet built or on the + # deletion list + mysql_query_delete_packages \ + '`package_sources`.`pkgbase`=from_base64("'"$( + printf '%s' "${pkgbase}" \ + | base64 -w0 + )"'")' \ + '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')' + # remove is-to-be-deleted marker from old binary packages + printf 'UPDATE `binary_packages_in_repositories`' + mysql_join_binary_packages_in_repositories_binary_packages + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ + "$( + printf '%s' "${pkgbase}" \ + | base64 -w0 + )" + } \ + | mysql_run_query + # shellcheck disable=SC2154 + >&2 printf '%s ' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + >&2 printf '\n' +done \ +< "${tmp_dir}/modified-packages" + +# extract black-listed packages +git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' \ +| tar -t 'blacklist' \ +| sed ' + s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@ + t + d +' \ +| expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \ +| sort -k2,2 \ +| join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - \ +| sort -u \ +| while read -r arch pkgbase; do + delete_package "${arch}" "${pkgbase}" +done + echo 'Done - mark decisions as final.' # shellcheck disable=SC2016 diff --git a/lib/common-functions b/lib/common-functions index 02fd36b..bf5d99c 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -1048,4 +1048,5 @@ expand_blacklist_architectures() { sort -k1,1 \ | join -1 1 -2 1 -o 1.2,2.2 "$1" - \ | sort -u + rm "$1" } |