| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/seed-build-list | 309 |
diff --git a/bin/seed-build-list b/bin/seed-build-list index 61b187f..832641a 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -2,6 +2,17 @@ # shellcheck disable=SC2119,SC2120 +# seed the build list from various sources +# +# This uses the following schedule: +# - find scheduleworthy packages by pkgname/pkgbase (diff to mirror, +# broken dependencies, explicite regex) and their architecture +# - find pkgbases of those packages (ask upstream, if necessary) +# - remove all ignored packages (seed-igonre, explicitely ignored, +# blacklist) +# - run mysql_generate_package_metadata on each of them once (ignoring +# the architecture for now) + # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" @@ -9,10 +20,6 @@ # TODO: -a|--auto schedules too much - or is the scheduling really /that/ broken? -# TODO: -u and -m should be merged - -# TODO: the blacklist is currently being ignored - why? - # shellcheck disable=SC2016 usage() { >&2 echo '' @@ -29,23 +36,20 @@ usage() { >&2 echo ' Do not exit if mysql_generate_package_metadata() failed.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' - >&2 echo ' -i|--ignore $pkgbase:' - >&2 echo ' Do not update the given package.' + >&2 echo ' -i|--ignore $pkgbase|$arch/$pkgbase:' + >&2 echo ' Do not update the given package (for the given $arch).' >&2 echo ' -j|--jostle' >&2 echo ' Give new build assignments the highest priority.' >&2 echo ' -m|--mirror $url:' - >&2 echo ' Schedule all packages, that are newer on the given' - >&2 echo ' x86_64 mirror - except packages in' - >&2 echo ' conf/seed-ignore-packages.' + >&2 echo ' Schedule all packages, that are newer (this includes' + >&2 echo ' packages currently unavailable on archlinux32) on the' + >&2 echo ' given x86_64 mirror - except packages on the blacklist' + >&2 echo ' or in conf/seed-ignore-packages.' >&2 echo ' -n|--no-action:' >&2 echo ' Do not actually update build-list, just print it.' >&2 echo ' -p|--package $pkg_regex:' >&2 echo ' Reschedule packages with matching pkgname or pkgbase.' >&2 echo ' Note, that these packages must be known to the database.' - >&2 echo ' -u|--undelete $url: ' - >&2 echo ' Schedule all former deletion-list packages which do not' - >&2 echo ' belong on the deletion-list anymore (e.g. can be built)' - >&2 echo ' and which are available on the given x86_64 mirror.' >&2 echo ' -w|--wait:' >&2 echo ' Wait for lock if necessary.' [ -z "$1" ] && exit 1 || exit "$1" @@ -56,7 +60,7 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir) trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT eval set -- "$( - getopt -o afhi:jm:np:u:w \ + getopt -o afhi:jm:np:w \ --long auto \ --long force \ --long help \ @@ -65,7 +69,6 @@ eval set -- "$( --long mirror: \ --long no-action \ --long package: \ - --long undelete: \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage @@ -77,11 +80,6 @@ jostle=false update=true wait_for_lock='-n' -if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then - sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \ - "${tmp_dir}/ignore-packages" -fi - while true do case "$1" in @@ -96,8 +94,13 @@ do ;; -i|--ignore) shift - printf '3\t%s\n' "$1" >> \ - "${tmp_dir}/ignore-packages" + if [ -z "${1%%*/*}" ]; then + printf '%s\n' "$1" \ + | tr '/' '\t' + else + printf 'any\t%s\n' "$1" + fi \ + >> "${tmp_dir}/ignore-packages" ;; -j|--jostle) jostle=true @@ -115,11 +118,6 @@ do printf '%s\n' "$1" >> \ "${tmp_dir}/package-regexes" ;; - -u|--undelete) - shift - printf '%s\n' "$1" >> \ - "${tmp_dir}/undelete-mirrors" - ;; -w|--wait) wait_for_lock='' ;; @@ -142,7 +140,6 @@ fi if [ ! -s "${tmp_dir}/mirrors" ] && \ [ ! -s "${tmp_dir}/package-regexes" ] && \ - [ ! -s "${tmp_dir}/undelete-mirrors" ] && \ ! ${auto}; then # nothing to do >&2 echo 'No options given to do anything.' @@ -178,7 +175,30 @@ repos=$( printf '%s\n' 'multilib' ) -# TODO: translate lib32-* packages instead of ignoring them for -u and -m +if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then + sed ' + s@/@\t@ + t + s/^/any\t/ + ' "${base_dir}/conf/seed-ignore-packages" >> \ + "${tmp_dir}/ignore-packages" +fi + +# shellcheck disable=SC2154 +git -C "${repo_paths__archlinux32}" archive "${repo_heads__archlinux32}" -- blacklist \ +| tar -t \ +| sed ' + s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@ + t + d +' \ +>> "${tmp_dir}/ignore-packages" + +expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \ +< "${tmp_dir}/ignore-packages" \ +| sponge "${tmp_dir}/ignore-packages" + +# TODO: translate lib32-* packages instead of ignoring them for -m # harvest pkgnames from mirror delta if [ -s "${tmp_dir}/mirrors" ]; then @@ -196,7 +216,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then N s/^.*\n// /^lib32-/d - s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/ + s/^\(.*-\)x86_64\.pkg\.tar\.xz$/\1i486\n\1i686\n\1pentium4/ + s/^\(.*-any\)\.pkg\.tar\.xz$/\1/ ' | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/ @@ -212,12 +233,14 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures mysql_join_binary_packages_binary_packages_in_repositories - printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`' + mysql_join_binary_packages_in_repositories_repositories + printf ' WHERE `repositories`.`is_on_master_mirror`' + printf ' OR `repositories`.`name`="build-list"' } | \ mysql_run_query | \ sed ' - s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)$/ours \2 \4 \1/ - s/^\(.* \)any\(\.\S\+ \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/ + s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)\.pkg\.tar\.xz$/ours \2 \4 \1/ + s/^\(.* \)any\( \S\+\)$/\0\n\1i486\2\n\1i686\2\n\1pentium4\2/ ' } | \ expand_version 2 | \ @@ -227,48 +250,13 @@ if [ -s "${tmp_dir}/mirrors" ]; then sed -n ' s/^theirs \(\S\+ \)// T - s/^\(\S\+ \)// - T p ' | \ - if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then - grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")" + if [ -r "${tmp_dir}/ignore-packages" ]; then + grep -vxFf "${tmp_dir}/ignore-packages" else cat fi | \ - sed 's/^/1\t/' | \ - sort -u >> \ - "${tmp_dir}/pkgnames" -fi - -# harvest pkgnames from undelete-mirrors -if [ -s "${tmp_dir}/undelete-mirrors" ]; then - while read -r mirror; do - if [ -z "${mirror}" ]; then - continue - fi - for repo in ${repos}; do - curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ - tar -Oxz --wildcards '*/desc' | \ - sed ' - /^%FILENAME%$/!d - N - s/^.*\n// - /^lib32-/d - s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2\n\1pentium4\2/ - ' | \ - sed ' - s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/ - ' - done - done < \ - "${tmp_dir}/undelete-mirrors" | \ - if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then - grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")" - else - cat - fi | \ - sed 's/^/2\t/' | \ sort -u >> \ "${tmp_dir}/pkgnames" fi @@ -294,14 +282,15 @@ if [ -s "${tmp_dir}/package-regexes" ]; then printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n' } | \ mysql_run_query | \ + tr '\t' ' ' | \ sed -n ' - /^pkgbase\s/ { - s/^\S\+\s/3\t/ + /^pkgbase / { + s/^\S\+ /any / w /dev/stdout d } - /^pkgname\s/ { - s/^\S\+\s/3\t/ + /^pkgname / { + s/^\S\+ /any / w /dev/stderr d } @@ -315,14 +304,18 @@ fi # create pkgbases to given pkgnames if [ -s "${tmp_dir}/pkgnames" ]; then printf 'CREATE TEMPORARY TABLE `pkgnames` (' - printf '`priority` SMALLINT,' + printf '`architecture` VARCHAR(16),' printf '`pkgname` VARCHAR(64),' - printf 'UNIQUE KEY `pkgname`(`pkgname`)' + printf 'UNIQUE KEY `content`(`architecture`,`pkgname`)' printf ');\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \ + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` COLUMNS TERMINATED BY " " (`architecture`,`pkgname`);\n' \ "${tmp_dir}/pkgnames" - printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`' + printf 'SELECT DISTINCT' + printf ' "pkgbase",' + printf '`pkgnames`.`architecture`,' + printf '`package_sources`.`pkgbase`,' + printf '`upstream_repositories`.`name`' printf ' FROM `pkgnames`' printf ' JOIN `binary_packages`' printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`' @@ -331,7 +324,10 @@ fi mysql_join_package_sources_upstream_repositories printf ';\n' - printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`' + printf 'SELECT DISTINCT' + printf ' "pkgname",' + printf '`pkgnames`.`architecture`,' + printf '`pkgnames`.`pkgname`' printf ' FROM `package_sources`' mysql_join_package_sources_upstream_repositories mysql_join_package_sources_build_assignments @@ -344,6 +340,8 @@ fi fi # auto-detect pkgbases if ${auto}; then + # TODO: check, if this still works or if it relied on the `priority` magic + >&2 echo 'autoscheduling needs testing / inspection - be prepared to schedule too many packages this way!' # schedule any package, that: # 1) is not on the build-list currently and # 2) has some dependency which is not provided by any package which @@ -413,7 +411,7 @@ fi # 1) printf ' WHERE `least_stable_bp`.`is_on_build_list`;' - printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name`' + printf 'SELECT DISTINCT "pkgbase",`ba_a`.`name`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`' printf ' FROM `least_stable_bp`' printf ' JOIN `binary_packages`' printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`' @@ -422,6 +420,7 @@ fi printf ' AND `dependency_types`.`relevant_for_binary_packages`' mysql_join_dependencies_versions '' 'd_v' mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_architectures '' 'ba_a' mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories # some dependencies are not provided by the least stable packages @@ -456,14 +455,15 @@ fi fi } | \ mysql_run_query | \ + tr '\t' ' ' | \ sed -n ' - /^pkgbase\s/ { - s/^\S\+\s// + /^pkgbase / { + s/^\S\+ // w /dev/stderr d } - /^pkgname\s/ { - s/^\S\+\s// + /^pkgname / { + s/^\S\+ // w /dev/stdout d } @@ -486,7 +486,7 @@ done # pkgnames -> pkgbases (with help from upstream) if [ -s "${tmp_dir}/pkgnames" ]; then - while read -r priority pkgname; do + while read -r architecture pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ tr ',' '\n' @@ -512,11 +512,11 @@ if [ -s "${tmp_dir}/pkgnames" ]; then ' ) if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then - printf '%s\t%s\n' "${priority}" "${pkgname}" + printf '%s %s\n' "${architecture}" "${pkgname}" continue fi - printf '%s\t%s\t%s\n' \ - "${priority}" \ + printf '%s %s %s\n' \ + "${architecture}" \ "${pkgbase}" \ "${repo}" >> \ "${tmp_dir}/pkgbases" @@ -531,112 +531,30 @@ if [ -s "${tmp_dir}/pkgnames" ]; then exit 2 fi -# now we (re)schedule all pkgbases which are: -# - not explicitely ignored (priority 3) <- explicitely scheduled -# - not explicitely/implicitely ignored, but not currently available -# (priority 2) <- from undelete-mirror-delta -# - neither ignored nor to-be-deleted (priority 1) <- from mirror-delta or auto-rebuild +sort -u "${tmp_dir}/pkgbases" \ +| sort -k3,3 \ +| sponge "${tmp_dir}/pkgbases" + +# get the current HEADs # shellcheck disable=SC2016 { - printf 'CREATE TEMPORARY TABLE `ignore_packages` (' - printf '`priority` SMALLINT,' - printf '`pkgbase` VARCHAR(64),' - printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)' - printf ');\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \ - "${tmp_dir}/ignore-packages" - - printf 'CREATE TEMPORARY TABLE `pkgbases` (' - printf '`priority` SMALLINT,' - printf '`pkgbase` VARCHAR(64),' - printf '`repo` VARCHAR(64),' - printf 'UNIQUE KEY `pkgbase`(`pkgbase`)' - printf ');\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \ - "${tmp_dir}/pkgbases" - - printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)' - printf ' SELECT 2,`sub_q`.`pkgbase`' - printf ' FROM (' - printf 'SELECT' - printf ' `package_sources`.`pkgbase`,' - printf 'GROUP_CONCAT(' - printf 'DISTINCT' - printf ' `build_assignments`.`architecture`' - printf ' ORDER BY `build_assignments`.`architecture`' - printf ') AS `architectures`' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_binary_packages_in_repositories - mysql_join_binary_packages_in_repositories_repositories - printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL' - printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"' - printf ' OR `repositories`.`is_on_master_mirror`' - printf ' OR `binary_packages_in_repositories`.`repository`=%s' \ - "${repository_ids__any_build_list}" - printf ' GROUP BY `package_sources`.`pkgbase`' - printf ') AS `sub_q`' - # shellcheck disable=SC2154 - printf ' WHERE `sub_q`.`architectures` IN ("%s","%s","%s,%s")' \ - "${architecture_ids__any}" \ - "${non_any_architecture_ids}" \ - "${architecture_ids__any}" \ - "${non_any_architecture_ids}" - printf ';\n' - - printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)' - printf ' SELECT 1,`package_sources`.`pkgbase`' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_binary_packages_in_repositories - printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`' - printf ';\n' - - printf 'SELECT ' - printf '`pkgbases`.`pkgbase`,' - printf '`git_repositories`.`head`,' - printf '(' - printf 'SELECT `al32`.`head`' - printf ' FROM `git_repositories` AS `al32`' - printf ' WHERE `al32`.`name`="archlinux32"' - printf '),' - printf '`pkgbases`.`repo`' - printf ' FROM `pkgbases`' - printf ' JOIN `upstream_repositories`' - printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`' + printf 'SELECT' + printf ' `upstream_repositories`.`name`,' + printf '`git_repositories`.`head`' + printf ' FROM `upstream_repositories`' mysql_join_upstream_repositories_git_repositories - printf ' WHERE NOT EXISTS (' - printf 'SELECT 1' - printf ' FROM `ignore_packages`' - printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`' - printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`' - printf ') AND (' - printf '`pkgbases`.`priority`!=2' - printf ' OR NOT EXISTS (' - printf 'SELECT 1' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_binary_packages_in_repositories - printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`' - mysql_join_binary_packages_in_repositories_repositories - printf ' AND (' - printf '`repositories`.`is_on_master_mirror`' - printf ' OR `repositories`.`name`="build-list"' - printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`' - printf ')' - printf ');\n' -} | \ - mysql_run_query | \ - tr '\t' ' ' | \ - sort -u > \ - "${tmp_dir}/update-list" +} \ +| mysql_run_query \ +| tr '\t' ' ' \ +| sort -k1,1 \ +| join -1 3 -2 1 -o 1.1,1.2,1.3,2.2 "${tmp_dir}/pkgbases" - \ +| sponge "${tmp_dir}/pkgbases" if ${update}; then - while read -r pkgbase git_rev mod_git_rev repo; do + cut -d' ' -f 2,3,4 "${tmp_dir}/pkgbases" \ + | sort -u \ + | while read -r pkgbase repo git_rev; do success=false # shellcheck disable=SC2154 for gr_r in \ @@ -644,9 +562,9 @@ if ${update}; then "${repo_heads__packages}:core" \ "${repo_heads__packages}:extra" \ "${repo_heads__community}:community"; do - printf '%s ' "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}" | \ + printf '%s ' "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}" | \ sed 's/ $/\n/' - if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${mod_git_rev}" "${gr_r#*:}"; then + if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}"; then success=true break fi @@ -655,13 +573,12 @@ if ${update}; then ! ${ignore_mysql_generate_package_metadata_errors}; then exit 2 fi - done < \ - "${tmp_dir}/update-list" + done cut -d' ' -f1 < \ - "${tmp_dir}/update-list" | \ + "${tmp_dir}/pkgbases" | \ sort -u | \ - sponge "${tmp_dir}/update-list" + sponge "${tmp_dir}/pkgbases" # shellcheck disable=SC2016 { @@ -670,7 +587,7 @@ if ${update}; then printf 'UNIQUE KEY `pkgbase`(`pkgbase`)' printf ');\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \ - "${tmp_dir}/update-list" + "${tmp_dir}/pkgbases" printf 'DELETE `d_bpir`' printf ' FROM `pkgbases`' @@ -723,5 +640,5 @@ if ${update}; then mysql_query_remove_old_binary_packages_from_build_list | \ mysql_run_query else - cat "${tmp_dir}/update-list" + cat "${tmp_dir}/pkgbases" fi |