From db5448138b313407d8062c04ed6df458ff64c66f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 3 Sep 2018 14:22:18 +0200 Subject: lib/mysql-functions: solve a few todos in mysql_generate_package_metadata() - we now have versioned install targets! --- lib/mysql-functions | 524 ++++++++++++++++++++++------------------------------ 1 file changed, 216 insertions(+), 308 deletions(-) (limited to 'lib') diff --git a/lib/mysql-functions b/lib/mysql-functions index 6c8154c..40843e3 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -210,6 +210,8 @@ mysql_add_package_source() { mysql_generate_package_metadata() { + mysql_load_min_and_max_versions + ( # new shell is intentional current_repository_id="$1" package="$2" @@ -260,9 +262,6 @@ mysql_generate_package_metadata() { mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO" ) - # TODO: architectures should be identified by id, not "(SELECT ...)" - - # add the build assignment(s) { sed -n ' s/^\tarch = // @@ -272,342 +271,251 @@ mysql_generate_package_metadata() { sort -u | \ grep -vxF 'any' || \ echo 'any' - } | \ - base64_encode_each | \ - while read -r arch; do - printf 'INSERT INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT %s,`architectures`.`id`,NULL,0,0' \ - "${package_source_id}" - printf ' FROM `architectures`' - printf ' WHERE `architectures`.`name` = from_base64("%s")' \ - "${arch}" - printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`build_assignments`.`id`);\n' - printf 'SELECT "%s",LAST_INSERT_ID();\n' \ - "${arch}" - done | \ - mysql_run_query > \ - "${temp_dir}/build-assignments.id" + } > \ + "${temp_dir}/architectures" # select any specific arch (which will be building the 'any' part of a split package) any_arch=$( - cut -f1 < \ - "${temp_dir}/build-assignments.id" | \ - sort | \ - head -n1 | \ - tr -d '\n' | \ - base64 -w0 + tail -n1 "${temp_dir}/architectures" ) # iterate over all pkgnames grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ cut -d' ' -f3 | \ while read -r pkgname; do - pkgname64=$( - printf '%s' "${pkgname}" | \ - base64 -w0 - ) sed -n ' /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/,/^$/ { + s/^pkgname/\t\0/ /^\S/d + /^\s*$/d s/^\s*// p } ' "${temp_dir}/SRCINFO" > \ - "${temp_dir}/BINARYINFO.${pkgname64}" + "${temp_dir}/SRCINFO.tmp" - # iterate over all archs - grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ - cut -d' ' -f3 | \ - while read -r arch; do - arch64=$( - printf '%s' "${arch}" | \ - base64 -w0 - ) - { - { - # this binary package is either built by the build_assignment - # with the identical arch (if existent) or by the one - # with arch=$any_arch - grep "^${arch64}"'\s' "${temp_dir}/build-assignments.id" || \ - grep "^${any_arch}"'\s' "${temp_dir}/build-assignments.id" - } | \ - sed ' - s/^.*\s/build-assignment-id = / - ' - sed ' - s/^\(\S\+\)_'"${arch}"' = /\1 = / - ' "${temp_dir}/BINARYINFO.${pkgname64}" - } > \ - "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" - done - done - - # TODO: Data should be loaded by "LOAD DATA LOCAL INFILE ...". This - # has three advantages: 1st, we separate data from the commands - # (aesthetic), 2nd, we can prepare/check the data to load "offline", - # 3rd, we are less sensitive to errors in the middle of the commit. - # For this we need some intermediate files which are loaded into - # temporary tables: - # - pkgbase -> build_assignment (multiple archs possible!) - # - build_assignment -> pkgname (split packages possible!) - # - pkgname -> dependency - # - pkgname -> provides - - # now each "ARCHINFO $pkgname $arch" file represents one binary package - find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ - while read -r _ pkgname arch; do - build_assignment_id=$( - grep '^build-assignment-id = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 - ) - pkgver=$( - grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - tr -d '\n' | \ - base64 -w0 - ) - pkgrel=$( - grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/\.[0-9]\+$//' | \ - tr -d '\n' | \ - base64 -w0 - ) epoch=$( { - grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \ + grep -m1 '^epoch = ' "${temp_dir}/SRCINFO.tmp" || \ echo 'epoch = 0' } | \ - cut -d' ' -f3 | \ - tr -d '\n' | \ - base64 -w0 - ) - # TODO: allow versioned install_targets - provides=$( - grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - makedepends=$( - grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - checkdepends=$( - grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each + cut -d' ' -f3 ) - rundepends=$( - grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each + pkgver=$( + grep -m1 '^pkgver = ' "${temp_dir}/SRCINFO.tmp" | \ + cut -d' ' -f3 ) - sub_pkgrel=$( - printf '(SELECT COALESCE(' - # do not add binary packages which are currently on the - # build-list or in $current_repository_id (beware of split - # packages!) - printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_binary_packages_in_repositories - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - printf ' `architectures`.`name`=from_base64("%s")' \ - "${arch}" - printf ' AND `binary_packages_in_repositories`.`repository` IN (%s,%s)),' \ - "${repository_ids__any_build_list}" \ - "${current_repository_id}" - # max(sub_pkgrel)+1 - printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - mysql_join_binary_packages_architectures - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then - # 'any' gets higher sub_pkgrel than any architecture - printf ' 1' - else - # not-'any' gets higher sub_pkgrel than same or 'any' architecture - printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \ - "${arch}" - fi - printf ')' - printf ',0))' + pkgrel=$( + grep -m1 '^pkgrel = ' "${temp_dir}/SRCINFO.tmp" | \ + cut -d' ' -f3 ) - { # TODO: identify arch by id - printf 'INSERT IGNORE INTO `binary_packages` (' - printf '`%s`,' \ - 'build_assignment' \ - 'architecture' \ - 'epoch' \ - 'pkgver' \ - 'pkgrel' \ - 'pkgname' \ - 'sub_pkgrel' \ - 'has_issues' \ - 'is_tested' - printf ') SELECT ' - printf '%s,' \ - "${build_assignment_id}" - printf '`%s`.`id`,' \ - 'architectures' - printf 'from_base64("%s"),' \ - "${epoch}" \ - "${pkgver}" \ - "${pkgrel}" \ - "${pkgname}" - printf '%s,0,0 FROM `architectures`' \ - "${sub_pkgrel}" - printf ' WHERE' - printf ' `architectures`.`name` = from_base64("%s")' \ - "${arch}" - printf ';\n' - printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (`package`,`repository`,`is_to_be_deleted`)' - printf ' SELECT LAST_INSERT_ID(),%s,0;\n' \ - "${current_repository_id}" - printf 'COMMIT;\n' - } | \ - sed ' - s|,)|)|g - s| JOIN WHERE | WHERE | - s| AND;$|;| - ' >> \ - "${temp_dir}/add-binary-packages-command" - { - printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' - - printf 'INSERT INTO `provides` VALUES\n' - echo "${provides}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"),"-",from_base64("%s"))),\n' \ - "${pkgname}" \ - "${epoch}" \ - "${pkgver}" \ - "${pkgrel}" - printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \ - "${pkgname}" \ - "${epoch}" \ - "${pkgver}" - printf ' (from_base64("%s"));\n' \ - "${pkgname}" - - printf 'INSERT INTO `rundepends` VALUES\n' - echo "${rundepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' ("base");\n' - echo "${checkdepends}" | \ - sort -u | \ - grep -vxF '' | \ + # iterate over all archs + grep '^arch = ' "${temp_dir}/SRCINFO.tmp" | \ + cut -d' ' -f3 | \ + while read -r arch; do + build_arch=$( + # this binary package is either built by the build_assignment + # with the identical arch (if existent) or by the one + # with arch=$any_arch + grep -xF "${arch}" "${temp_dir}/architectures" || \ + echo "${any_arch}" + ) sed ' - s|^| (from_base64("| - s|$|")),| - 1 s/^/INSERT INTO `checkdepends` VALUES \n/ - $ s/,$/;/ - ' - - printf 'INSERT INTO `makedepends` VALUES\n' - echo "${makedepends}" | \ - sort -u | \ - grep -vxF '' | \ + s/^\(\S\+\)_'"${arch}"' = /\1 = / + s/^pkgname = \(\S\+\)$/\0\nprovides = \1='"${epoch}"':'"${pkgver}"'\ndepends = base\nmakedepends = base-devel/ + ' "${temp_dir}/SRCINFO.tmp" | \ + sed ' + s/^arch = \S\+$/arch/ + t + s/^\(groups\|provides\) = /provides\t/ + t mangle_version + s/^makedepends = /makedepends\t/ + t mangle_version + s/^checkdepends = /checkdepends\t/ + t mangle_version + s/^depends = /rundepends\t/ + t mangle_version + d + :mangle_version + s/\(=\|<\|<=\|>=\|>\)\([^[:space:]-]\+\)$/\t\1\t\2/ + t split_epoch + h + s/$/\t>=\t'"${min_version}"'/ + s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/ + p + g + s/$/\t<=\t'"${max_version}"'/ + :split_epoch + s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/ + t + s/\s\S\+$/\t0\0/ + ' | \ sed ' - s|^| (from_base64("| - s|$|")),| + s/^/'"$( + printf '%s\t' \ + "${pkgname}" \ + "${arch}" \ + "${epoch}" \ + "${pkgver}" \ + "${pkgrel}" \ + "${build_arch}" + )"'/ ' - printf ' ("base-devel");\n' - - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' SELECT (`name`) FROM `%s` UNION' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' | \ - sed 's| UNION$|;\n|' - - for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do - case "${link}" in - 'provides') - printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' - ;; - 'makedepends'|'checkdepends'|'rundepends') - printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' - printf ' `dependency_types` JOIN' - ;; - esac - printf ' `binary_packages`' - mysql_join_binary_packages_architectures - printf ' JOIN `install_targets`' - printf ' JOIN `%s`' "${link}" - printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" - printf ' WHERE' - if [ "${link}" = 'makedepends' ] || \ - [ "${link}" = 'checkdepends' ] || \ - [ "${link}" = 'rundepends' ]; then - printf ' `dependency_types`.`name` = "%s" AND' \ - "${link%depends}" - fi - # TODO: identify binary_packages by id - printf ' `binary_packages`.`%s` = from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - # we do not want to match the sub_pkgrel: - # a) it is tedious to do so (because it may be calculated - # dynamically) - # b) it is not necessary to do so: if only the sub_pkgrel - # changed, the dependencies and provided install_targets - # should not have changed - printf ' `architectures`.`name` = from_base64("%s");\n' \ - "${arch}" - # the repository is of no relevance: it hardly matters for - # the dependencies done - - printf 'DROP TABLE `%s`;\n' \ - 'provides' \ - 'makedepends' \ - 'checkdepends' \ - 'rundepends' - } >> \ - "${temp_dir}/add-install-targets-command" - done + rm "${temp_dir}/SRCINFO.tmp" + done | \ + sort -u > \ + "${temp_dir}/database-input" { - if [ -s "${temp_dir}/add-binary-packages-command" ]; then - cat "${temp_dir}/add-binary-packages-command" - fi - if [ -s "${temp_dir}/add-install-targets-command" ]; then - cat "${temp_dir}/add-install-targets-command" - fi + printf 'CREATE TEMPORARY TABLE `links`(' + printf '`pkgname` VARCHAR(64),' + printf '`architecture` VARCHAR(16),' + printf '`epoch` MEDIUMINT,' + printf '`pkgver` VARCHAR(64),' + printf '`pkgrel` MEDIUMINT,' + printf '`build_architecture` VARCHAR(16),' + printf '`type` VARCHAR(16),' + printf '`install_target_name` VARCHAR(128),' + printf '`version_relation` VARCHAR(2),' + printf '`version_epoch` MEDIUMINT,' + printf '`version_version` VARCHAR(64)' + printf ');\n' + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `links`;\n' \ + "${temp_dir}/database-input" + + printf 'SET @sub_pkgrel=(' + printf 'SELECT MAX(`binary_packages`.`sub_pkgrel`)' + printf ' FROM `links`' + printf ' JOIN `architectures`' + printf ' ON `architectures`.`name`=`links`.`architecture`' + mysql_join_architectures_binary_packages + printf ' AND `binary_packages`.`%s`=`links`.`%s`' \ + 'pkgname' 'pkgname' \ + 'epoch' 'epoch' \ + 'pkgver' 'pkgver' \ + 'pkgrel' 'pkgrel' + printf ');\n' + + printf 'INSERT IGNORE INTO `build_assignments` (' + printf '`package_source`,' + printf '`architecture`' + printf ') SELECT' + printf ' %s,`architectures`.`id`' \ + "${package_source_id}" + printf ' FROM `links`' + printf ' JOIN `architectures`' + printf ' ON `architectures`.`name`=`links`.`build_architecture`' + printf ' WHERE `links`.`type`=`arch`;\n' + + printf 'INSERT IGNORE INTO `binary_packages` (' + printf '`build_assignment`,' + printf '`pkgname`,' + printf '`epoch`,' + printf '`pkgver`,' + printf '`pkgrel`,' + printf '`sub_pkgrel`,' + printf '`architecture`' + printf ') SELECT' + printf ' `build_assignments`.`id`,' + printf '`links`.`%s`,' \ + 'pkgname' \ + 'epoch' \ + 'pkgver' \ + 'pkgrel' + printf '@sub_pkgrel,' + printf '`architectures`.`id`' + printf ' FROM `links`' + printf ' JOIN `architectures`' + printf ' ON `links`.`architecture`=`architectures`.`name`' + printf ' JOIN `architectures` AS `ba_a`' + printf ' ON `ba_a`.`name`=`links`.`build_architecture`' + mysql_join_architectures_build_assignments 'ba_a' + printf ' WHERE `links`.`type`=`arch`;\n' + + printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (' + printf ' `package`' + printf ' `repository`' + printf ') SELECT' + printf ' `binary_packages`.`id`,' + printf '%s' \ + "${current_repository_id}" + printf ' FROM `links`' + printf ' JOIN `architectures`' + printf ' ON `architectures`.`name`=`links`.`architecture`' + mysql_join_architectures_binary_packages + printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel' + printf ' AND `binary_packages`.`%s`=`links`.`%s`' \ + 'pkgname' 'pkgname' \ + 'epoch' 'epoch' \ + 'pkgver' 'pkgver' \ + 'pkgrel' 'pkgrel' + printf ';\n' + + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' SELECT `links`.`install_target_name`' + printf ' FROM `links`;\n' + + printf 'INSERT IGNORE INTO `versions` (`epoch`,`version`)' + printf ' SELECT' + printf ' `links`.`version_epoch`,' + printf '`links`.`version_version`' + printf ' FROM `links`;\n' + + for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do + case "${link}" in + 'provides') + printf 'INSERT IGNORE INTO `install_target_providers` (' + printf '`package`,' + printf '`install_target`,' + printf '`version`' + printf ') SELECT' + printf ' `binary_packages`.`id`,' + printf '`install_targets`.`id`,' + printf '`versions`.`id`' + ;; + 'makedepends'|'checkdepends'|'rundepends') + printf 'INSERT IGNORE INTO `dependencies` (' + printf '`dependent`,' + printf '`depending_on`,' + printf '`dependency_type`,' + printf '`version`,' + printf '`version_relation`,' + printf ') SELECT' + printf ' `binary_packages`.`id`,' + printf '`install_targets`.`id`,' + printf '`dependency_types`.`id`,' + printf '`versions`.`id`,' + printf '`links`.`version_relation`,' + ;; + esac + printf ' FROM `links`' + printf ' JOIN `architectures`' + printf ' ON `architectures`.`name`=`links`.`architecture`' + mysql_join_architectures_binary_packages + printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel' + printf ' AND `binary_packages`.`%s`=`links`.`%s`' \ + 'pkgname' 'pkgname' \ + 'epoch' 'epoch' \ + 'pkgver' 'pkgver' \ + 'pkgrel' 'pkgrel' + printf ' JOIN `install_targets`' + printf ' ON `install_targets`.`name`=`links`.`install_target_name`' + printf ' JOIN `versions`' + printf ' ON `versions`.`epoch`=`links`.`install_target_epoch`' + printf ' AND `versions`.`version`=`links`.`install_target_version`' + if [ "${link}" = 'makedepends' ] || \ + [ "${link}" = 'checkdepends' ] || \ + [ "${link}" = 'rundepends' ]; then + printf ' JOIN `dependency_types`' + printf ' ON `dependency_types`.`name`="%s"' \ + "${link%depends}" + fi + printf ' WHERE `links`.`type`="%s";\n' \ + "${link}" + done + printf 'DROP TEMPORARY TABLE `links`;\n' } | \ mysql_run_query -- cgit v1.2.3-70-g09d2