| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/bootstrap-mysql | 522 | ||||
| -rwxr-xr-x | bin/build-master-status | 695 | ||||
| -rwxr-xr-x | bin/build-master-status-from-mysql | 152 | ||||
| -rwxr-xr-x | bin/build-packages | 126 | ||||
| -rwxr-xr-x | bin/calculate-dependent-packages | 184 | ||||
| -rwxr-xr-x | bin/check-bugtracker | 79 | ||||
| -rwxr-xr-x | bin/check-opcodes | 2 | ||||
| -rwxr-xr-x | bin/cleanup | 70 | ||||
| -rwxr-xr-x | bin/common-functions | 1348 | ||||
| -rwxr-xr-x | bin/copy-to-build-support | 187 | ||||
| -rwxr-xr-x | bin/db-update | 875 | ||||
| -rwxr-xr-x | bin/delete-packages | 407 | ||||
| -rwxr-xr-x | bin/get-assignment | 306 | ||||
| -rwxr-xr-x | bin/get-package-updates | 566 | ||||
| -rwxr-xr-x | bin/ii-connect | 13 | ||||
| -rwxr-xr-x | bin/interpret-mail | 34 | ||||
| -rwxr-xr-x | bin/modify-package-state | 212 | ||||
| -rwxr-xr-x | bin/mysql-functions | 647 | ||||
| -rwxr-xr-x | bin/ping-from-slave | 25 | ||||
| -rwxr-xr-x | bin/ping-to-master | 39 | ||||
| -rwxr-xr-x | bin/prioritize-build-list | 19 | ||||
| -rwxr-xr-x | bin/repo-copy | 92 | ||||
| -rwxr-xr-x | bin/return-assignment | 342 | ||||
| -rwxr-xr-x | bin/sanity-check | 186 | ||||
| -rwxr-xr-x | bin/seed-build-list | 455 | ||||
| -rwxr-xr-x | bin/show-dependencies | 638 | ||||
| -rwxr-xr-x | bin/slave-build-connect | 9 | ||||
| -rwxr-xr-x | bin/strict-bashism-check | 39 | ||||
| -rwxr-xr-x | bin/why-dont-you | 465 |
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 46678f7..86fa645 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: this should become some sort of mysql dump to bootstrap the +# tables and stored functions only - when the database is gone, we're +# pretty much screwed anyway :-D + tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT @@ -32,6 +36,8 @@ done | \ sort -k2,2 > \ "${tmp_dir}/master-mirror-listing" +# TODO: include hash of binary package in mysql database + if [ ! "$1" = 'slim' ]; then tables=$( printf '%s\n' \ @@ -61,6 +67,7 @@ if [ ! "$1" = 'slim' ]; then ' sub_pkgrel MEDIUMINT' \ ' has_issues BIT' \ ' is_tested BIT' \ + ' is_to_be_deleted BIT' \ ' pkgname VARCHAR(64)' \ ' architecture SMALLINT :architectures' \ ' UNIQUE content build_assignment sub_pkgrel pkgname architecture repository' \ @@ -81,7 +88,9 @@ if [ ! "$1" = 'slim' ]; then ' package_source BIGINT :package_sources' \ ' architecture SMALLINT :architectures' \ ' is_blocked VARCHAR(128) NULL' \ + ' is_black_listed TEXT NULL' \ ' is_broken BIT' \ + ' priority SMALLINT' \ ' UNIQUE content package_source architecture' \ 'build_dependency_loops BIGINT' \ ' loop MEDIUMINT' \ @@ -99,6 +108,8 @@ if [ ! "$1" = 'slim' ]; then ' git_revision VARCHAR(40)' \ ' mod_git_revision VARCHAR(40)' \ ' upstream_package_repository SMALLINT :upstream_repositories' \ + ' uses_upstream BIT' \ + ' uses_modification BIT' \ ' UNIQUE content pkgbase git_revision mod_git_revision' \ 'upstream_repositories SMALLINT' \ ' name VARCHAR(64)' \ @@ -122,9 +133,40 @@ if [ ! "$1" = 'slim' ]; then ' build_assignment BIGINT :build_assignments' \ ' date DATETIME' \ ' reason SMALLINT :fail_reasons' \ - ' log_file VARCHAR(512)' + ' log_file VARCHAR(512)' \ + 'todos MEDIUMINT' \ + ' file VARCHAR(64)' \ + ' line MEDIUMINT' \ + ' description VARCHAR(512)' + 'todo_links MEDIUMINT' \ + ' dependent MEDIUMINT' \ + ' depending_on MEDIUMINT' \ + 'repository_stability_relations MEDIUMINT' \ + ' more_stable MEDIUMINT :repository_stabilities' \ + ' less_stable MEDIUMINT :repository_stabilities' \ + 'repository_moves MEDIUMINT' \ + ' from_repository MEDIUMINT :repositories' \ + ' to_repository MEDIUMINT :repositories' \ + ' upstream_package_repository SMALLINT :upstream_repositories' \ + ' UNIQUE source from_repository upstream_package_repository' \ + 'statistics BIGINT' \ + ' date DATETIME' \ + ' stable_packages_count MEDIUMINT' \ + ' pending_tasks_count MEDIUMINT' \ + ' pending_packages_count MEDIUMINT' \ + ' staging_packages_count MEDIUMINT' \ + ' testing_packages_count MEDIUMINT' \ + ' tested_packages_count MEDIUMINT' \ + ' broken_tasks_count MEDIUMINT' \ + ' dependency_loops_count MEDIUMINT' \ + ' dependency_looped_tasks_count MEDIUMINT' \ + ' locked_tasks_count MEDIUMINT' \ + ' blocked_tasks_count MEDIUMINT' \ + ' next_tasks_count MEDIUMINT' \ + ' UNIQUE date' ) + # shellcheck disable=SC2016 { printf '%s\n' \ '/*!40014 SET UNIQUE_CHECKS=0 */;' \ @@ -137,14 +179,11 @@ if [ ! "$1" = 'slim' ]; then while read -r table size; do case "${turn}" in 'drop') - # shellcheck disable=SC2016 printf 'DROP TABLE IF EXISTS `%s`;\n' "${table}" ;; 'create'|'link') if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf 'CREATE TABLE `%s` (\n' "${table}" - # shellcheck disable=SC2016 printf ' `id` %s NOT NULL AUTO_INCREMENT,\n' "${size}" fi echo "${tables}" | \ @@ -167,16 +206,14 @@ if [ ! "$1" = 'slim' ]; then if [ -z "${rest}" ]; then rest="${type}" fi - # shellcheck disable=SC2016 printf ' UNIQUE KEY `%s` (' "${type}" - # shellcheck disable=SC2016,SC2086 + # shellcheck disable=SC2086 printf '`%s`,' ${rest} | \ sed 's|,$||' printf ')' ;; *) if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf ' `%s` %s' "${column}" "${type}" if ! echo "${rest}" | \ grep -qwF 'NULL'; then @@ -187,7 +224,6 @@ if [ ! "$1" = 'slim' ]; then tr ' ' '\n' | \ sed -n 's/^://;T;p' | \ while read -r link; do - # shellcheck disable=SC2016 printf 'ALTER TABLE `%s` ADD FOREIGN KEY (`%s`) REFERENCES `%s` (`id`);\n' \ "${table}" \ "${column}" \ @@ -201,7 +237,6 @@ if [ ! "$1" = 'slim' ]; then fi done if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf ' PRIMARY KEY (`id`));\n' fi ;; @@ -214,28 +249,318 @@ if [ ! "$1" = 'slim' ]; then printf '%s\n' \ '/*!40014 SET UNIQUE_CHECKS=1 */;' \ '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;' + + # show_broken_packages_and_dependencies + # gives a list of broken packages and their dependencies if they're + # still on the build- or deletion-list + printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n' + printf 'BEGIN\n' + printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ + 'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `build_assignments`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + printf ' AND `build_assignments`.`is_broken`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list";\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)' + printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `new_bp`.`build_assignment`' + printf ' FROM `broken_packages_and_dependencies_old`' + mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp' + mysql_join_binary_packages_dependencies 'old_bp' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'new_bp' + mysql_join_binary_packages_repositories 'new_bp' 'new_repo' + printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'SELECT ' + mysql_query_select_pkgbase_and_revision + printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + + # calculate_dependencies_of_package_upto_first_built_one + # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy` + printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n' + printf 'BEGIN\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `binary_packages`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)' + printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `install_target_providers`.`package`' + printf ' FROM `relevant_binary_packages_copy`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + printf ';\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + + # calculate_maximal_moveable_set + # stores results in `moveable_binary_packages` and `replaced_binary_packages` + # Give a maximal list of packages to be moved, while implementing the + # condition from db-update: + + # Every package which is replaced[1], must have its provided install_targets: + # a) provided by another moved or not-replaced package or + # b) not required by any not-replaced package. + + # Every package being moved needs to have all dependencies + # installable in the target repository. + + # TODO: [1] A "replaced" package may also be in a different repository + # e.g. if a-2 is moved from [staging] to [testing] and there is only + # a-1 in [core], then this will be "replaced" by a-2 on a system + # running on [testing] repositories. + + printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' + printf 'BEGIN\n' + # variables to store count of changed rows + printf 'DECLARE row_count_saved INT DEFAULT 0;\n' + for copy in '' '_copy' '_copy2'; do + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ + 'moveable' "${copy}" \ + 'replaced' "${copy}" + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \ + "${copy}" + if [ "${copy}" = '_copy2' ]; then + continue + fi + printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \ + "${copy}" + done + # these packages are considered for moving: + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' + # correct stability: "testing"/"staging" - as chosen + printf ' WHERE `repository_stabilities`.`name` = `from_stability`' + printf ' AND (' + printf '`from_stability`="staging"' + # "testing" packages must be tested + printf ' OR `binary_packages`.`is_tested`' + printf ')' + # no open issues + printf ' AND NOT `binary_packages`.`has_issues`' + printf ';\n' + # these packages are considered for being replaced: + # for each moved package + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' + printf ' SELECT `r_bp`.`id`,`m_bp`.`id`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories 'm_bp' 'm_r' + mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' + mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' + mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' + mysql_join_upstream_repositories_repository_moves 'm_ur' + printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' + # in its target repository + printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' + mysql_join_repositories_binary_packages 'r_r' 'r_bp' + # all packages with identical names + printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n' + # now we delete all unreplaceable and unmoveable packages from the respective + # list until no further changes are required + printf 'REPEAT\n' + # create copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" + done + done + # a package is not moveable if its dependencies are not provided ... + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`' + printf ' FROM `replaced_binary_packages`' + printf ' RIGHT JOIN `moveable_binary_packages`' + printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + # ... by a not-deleted, "more stable" package already in place or ... + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' + printf ')' + # ... by a moved package + printf ') AND NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' + printf 'SET @row_count_saved = ROW_COUNT();\n' + + # refresh copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" + done + done + + # packages which should not be replaced: ones providing something that is: + # a) still needed by a not-replaced package x "less stable" than the target repository and + # b) not provided by: + # 1) a moved package or + # 2) any current, not-replaced package in a repository more stable than x's repository + # + # Note, that this is not 100% clean from an academic point of view: + # It might require _downgrading_ of a package to keep fulfilling a dependency + # and it might require to do so _without_ any chance for the user to notice, + # because there may be more "dependencies" in the database than in the package files. + # + # However, in practice both should not happen. + # + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' + printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' + printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`' + mysql_join_binary_packages_install_target_providers 'repl_bp' + mysql_join_install_target_providers_dependencies + printf ' AND NOT EXISTS (' + # dependencies of replaced packages don't matter + printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`' + printf ')' + mysql_join_dependencies_dependency_types + # consider only runtime dependencies + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_dependencies_binary_packages '' 'req_bp' + # we need to check wether req_bp's dependency is (un)critical + mysql_join_binary_packages_repositories 'repl_bp' 'repl_r' + mysql_join_binary_packages_repositories 'req_bp' 'req_r' + # dependent package is "less stable" than dependency + printf ' JOIN `repository_stability_relations` AS `repl_rr`' + printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`' + printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`' + # a) ^ + printf ' WHERE NOT EXISTS (' + # no moved package ... + printf 'SELECT *' + printf ' FROM `moveable_binary_packages_copy`' + mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp' + # ... provides the same + printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 1) ^ + printf ') AND NOT EXISTS (' + # no current package ... + printf 'SELECT *' + printf ' FROM `binary_packages` AS `subst_bp`' + mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp' + # ... in a repository ... + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + # ... more stable ... + printf ' JOIN `repository_stability_relations` AS `subst_rr`' + printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`' + # ... than x's repository ... + printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`' + printf ' AND NOT EXISTS (' + # ... and which is not replaced ... + printf 'SELECT * FROM `replaced_binary_packages_copy2`' + printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`' + printf ')' + # ... and provides the same + printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 2) ^ + printf ');\n' + printf 'UNTIL row_count_saved=0 AND ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + for table in 'moveable' 'replaced'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \ + "${table}" "${copy}" + done + done + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE ROUTINE' 'buildmaster.*' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ - 'SHOW VIEW' 'buildmaster.*' \ + 'EXECUTE' 'buildmaster.*' \ + 'RELOAD' '*.*' \ 'SELECT' 'buildmaster.*' \ - 'UPDATE' 'buildmaster.*' \ - 'RELOAD' '*.*' - printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \ + 'SELECT' 'mysql.proc' \ 'SHOW VIEW' 'buildmaster.*' \ - 'SELECT' 'buildmaster.*' + 'UPDATE' 'buildmaster.*' + printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ + 'SELECT' 'buildmaster.*' \ + 'SHOW VIEW' 'buildmaster.*' printf 'FLUSH PRIVILEGES;\n' } | \ - mysql -u root -p buildmaster + mysql_run_query -u root -p fi +# shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `architectures` (`name`) VALUES ' printf '("%s"),' \ 'any' 'i686' | \ sed 's|,$||' printf ';\n' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `fail_reasons` (`identifier`,`name`,`severity`) VALUES ' printf '(from_base64("%s"),"%s",%s),' \ "$( @@ -276,7 +601,6 @@ fi )" 'unknown' 100 | \ sed 's|,$||' printf ';\n' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `git_repositories` (`name`,`url`,`directory`,`head`) VALUES' { for repo in ${repo_names}; do @@ -298,8 +622,8 @@ fi base64 -w0 )" \ "$( - base64_encode_each < \ - "${work_dir}/${repo}.revision" + git -C "${repo_path}" rev-parse HEAD | \ + base64_encode_each )" | \ sed 's|,$|),|' done @@ -314,9 +638,7 @@ fi 'extra:packages' \ 'multilib:packages' \ 'community:community'; do - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `upstream_repositories` (`name`,`git_repository`) SELECT\n' - # shellcheck disable=SC2016 printf ' from_base64("%s"),`id` FROM `git_repositories` WHERE `name` = from_base64("%s");\n' \ "$( printf '%s' "${repo%:*}" | \ @@ -328,7 +650,6 @@ fi )" done - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `build_slaves` (`name`,`ssh_key`,`operator`,`last_connection`) VALUES' { sed -n ' @@ -338,10 +659,13 @@ fi ' ~/.ssh/authorized_keys | \ while read -r name key; do case "${name}" in - 'nlopc'*|'rechenknecht'|'buildknecht'*) + 'nlopc'*|'rechenknecht') operator='deep42thought' ;; - 'eurobuild') + 'buildknecht'*) + operator='deep42thought/vollzornbrot' + ;; + 'eurobuild3') operator='abaumann' ;; *) @@ -368,20 +692,69 @@ fi } | \ sed 's|,;|;|' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES' { printf '\n ("%s",%s),' \ - 'stable' '"Packages:Stable"' \ - 'testing' '"Packages:Testing"' \ + 'stable' '"Packages: Stable"' \ + 'testing' '"Packages: Testing"' \ 'staging' 'NULL' \ 'standalone' 'NULL' \ - 'unbuilt' '"Packages:Build-list"' \ + 'unbuilt' '"Packages: Build-list"' \ 'forbidden' 'NULL' printf ';\n' } | \ sed 's|,;|;|' + { + printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)' + printf ' SELECT `ms`.`id`,`ls`.`id`' + printf ' FROM `repository_stabilities` AS `ms` JOIN `repository_stabilities` AS `ls`' + printf ' WHERE ' + printf '(`ms`.`name`="%s" AND `ls`.`name`="%s") OR ' \ + 'stable' 'stable' \ + 'stable' 'testing' \ + 'stable' 'staging' \ + 'stable' 'standalone' \ + 'stable' 'unbuilt' \ + 'stable' 'forbidden' \ + 'testing' 'testing' \ + 'testing' 'staging' \ + 'testing' 'standalone' \ + 'testing' 'unbuilt' \ + 'testing' 'forbidden' \ + 'staging' 'staging' \ + 'staging' 'standalone' \ + 'staging' 'unbuilt' \ + 'staging' 'forbidden' \ + 'unbuilt' 'forbidden' \ + 'standalone' 'standalone' + printf ';\n' + } | \ + sed 's| OR ;|;|' + + { + printf 'INSERT IGNORE INTO `repository_moves` (`from_repository`,`to_repository`,`upstream_package_repository`)' + printf ' SELECT `f`.`id`,`t`.`id`,`u`.`id`' + printf ' FROM' + printf ' `repositories` AS `%s` JOIN' \ + 'f' 't' + printf ' `upstream_repositories` AS `u`' + printf ' WHERE ' + printf '(`f`.`name`="%s" AND `t`.`name`="%s" AND `u`.`name`="%s") OR ' \ + 'staging' 'testing' 'core' \ + 'staging' 'testing' 'extra' \ + 'staging' 'testing' 'multilib' \ + 'community-staging' 'community-testing' 'community' \ + 'community-staging' 'community-testing' 'multilib' \ + 'testing' 'core' 'core' \ + 'testing' 'extra' 'extra' \ + 'testing' 'extra' 'multilib' \ + 'community-testing' 'community' 'community' \ + 'community-testing' 'community' 'multilib' + printf ';\n' + } | \ + sed 's| OR ;|;|' + for repo in \ 'core:stable:AQ==' \ 'extra:stable:AQ==' \ @@ -393,9 +766,7 @@ fi 'community-staging:staging:AQ==' \ 'build-list:unbuilt:AA==' \ 'deletion-list:forbidden:AA=='; do - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `repositories` (`name`,`stability`,`is_on_master_mirror`) SELECT' - # shellcheck disable=SC2016 printf ' from_base64("%s"),`id`,from_base64("%s") FROM `repository_stabilities` WHERE `name`=from_base64("%s");\n' \ "$( printf '%s' "${repo}" | \ @@ -413,80 +784,18 @@ fi )" done - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES' { printf '\n ("%s",%s,%s),' \ - 'build' '1' '0' \ + 'make' '1' '0' \ + 'check' '0' '0' \ + 'link' '0' '1' \ 'run' '1' '1' printf ';\n' } | \ sed 's|,;|;|' } | \ - mysql buildmaster - -find "${work_dir}/package-states" \ - -maxdepth 1 \ - -mindepth 1 \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed ' - 1!d - s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@ - s@^\S*/@@ - s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 / - s/ \([0-9]\+\) / \1.0 / - s/ [0-9]\+\.\([0-9]\+\) / \1 / - ' {} \; | \ - while read -r state_file sub_pkgrel arch package_file; do - printf '%s ' "${state_file}" - repository=$( - grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \ - cut -d' ' -f1 - ) - if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then - printf 'not found exactly once on the master mirror.\n' - >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}" - continue - fi - mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}" - printf ' ok\n' - done - -tr ' ' '.' < \ - "${work_dir}/build-list" | \ - while read -r state_file; do - mysql_generate_package_metadata "${state_file}" - done - -# shellcheck disable=SC2016 -find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ - sed ' - s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3| - ' | \ - while read -r state_file pkgbase git_revision mod_git_revision repository; do - printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' - printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ - "$( - tr -d '\n' < \ - "${state_file}" | \ - base64 -w0 - )" - printf ' WHERE' - printf ' `package_sources`.`%s`=from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" - printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${repository}" | base64 -w0)" - done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -520,10 +829,9 @@ grep '^\('"$( "${tmp_dir}/new-stable-packages" if [ -s "${tmp_dir}/new-stable-packages" ]; then + # shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'CREATE TEMPORARY TABLE `stable_packages` (' - # shellcheck disable=SC2016 printf '`%s` %s,' \ 'pkgname' 'VARCHAR(64)' \ 'epoch' 'MEDIUMINT' \ @@ -533,9 +841,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then 'architecture' 'VARCHAR(16)' \ 'repository' 'VARCHAR(64)' \ 'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT' - # shellcheck disable=SC2016 printf 'PRIMARY KEY (`build_assignment`));\n' - # shellcheck disable=SC2016 sed ' 1~10 ! b not_start s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/ @@ -546,29 +852,25 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then :end s/,$/;/ ' "${tmp_dir}/new-stable-packages" - # shellcheck disable=SC2016 - printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`)' + printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' printf ' SELECT ' - # shellcheck disable=SC2016 printf '`stable_packages`.`%s`,' \ 'pkgname' \ 'epoch' \ 'pkgver' \ 'pkgrel' \ 'sub_pkgrel' - # shellcheck disable=SC2016 printf '`%s`.`id`,' \ 'architectures' \ 'repositories' - # shellcheck disable=SC2016 - printf -- '-`build_assignment`,0,1 FROM `stable_packages`' - # shellcheck disable=SC2016 + printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ 'repositories' 'repository' 'repositories' \ 'architectures' 'architecture' 'architectures' printf ';\n' - # shellcheck disable=SC2016 printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi + +mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index 4d47976..c93a9aa 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -5,37 +5,31 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: replace by build-master-status-from-mysql + usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' >&2 echo '' >&2 echo 'possible options:' - >&2 echo ' -w|--web:' - >&2 echo ' Output to webserver instead of stdout.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o hw \ + getopt -o h \ --long help \ - --long web \ -n "$(basename "$0")" -- "$@" || \ echo usage )" -web=false - while true do case "$1" in -h|--help) usage 0 ;; - -w|--web) - web=true - ;; --) shift break @@ -61,538 +55,177 @@ fi tmp_dir=$(mktemp -d 'tmp.build-master-status.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -stable=$( - ls_master_mirror 'i686' | \ - grep -v 'testing$\|staging$\|-unstable$' | \ - while read -r dir; do - ls_master_mirror "i686/${dir}" - done | \ - grep -c '\.pkg\.tar\.xz$' -) -tasks=$( - grep -c '^\S\+ \S\+ \S\+ \S\+$' \ - "${work_dir}/build-list" -) || true -pending_packages=$( - grep '^\S\+ \S\+ \S\+ \S\+$' "${work_dir}/build-list" | \ - tr ' ' '.' | \ - while read -r package; do - generate_package_metadata "${package}" 2>&1 > /dev/null - cat "${work_dir}/package-infos/${package}.packages" - done | - wc -l -) -next_tasks=$( - { - cat "${work_dir}/build-list" - find "${work_dir}/package-states" -maxdepth 1 \ - \( -name '*.broken' -o -name '*.blocked' \) \ - -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - p - ' - } | \ - sort | \ - uniq -u | \ - while read -r package git_revision mod_git_revision repository; do - if [ -z "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then - echo "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - fi - done | \ - wc -l -) -staging=$( - find "${work_dir}/package-states" -name '*.done' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) -testing=$( - find "${work_dir}/package-states" -name '*.testing' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) -tested=$( - find "${work_dir}/package-states" -name '*.tested' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) { - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ - while read -r pkg rev mod_rev repo; do - if [ -z "$(find_dependencies_on_build_list "${pkg}" "${rev}" "${mod_rev}" "${repo}")" ]; then - echo "${pkg}" - fi - done - { - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u - find "${work_dir}/package-states/" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}||' | \ - sort -u - } | \ - sort | \ - uniq -d -} | \ - sort -u > \ - "${tmp_dir}/broken-packages-names" -broken=$( - wc -l < \ - "${tmp_dir}/broken-packages-names" -) -blocked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.blocked' | \ - wc -l -) -locked=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' | \ - wc -l -) -loops=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' | \ - wc -l -) -looped_packages=$( - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat '{}' \; | \ - sort -u | \ - wc -l -) - -{ - printf 'The mirror master contains %d stable packages (vs. ca. %d planned).\n' \ - "${stable}" \ - "$((staging+testing+tested+pending_packages))" - printf 'The build list contains %d tasks (incl. broken: %d, leading to %d packages), of which %s can be built immediately.\n' \ - "$((tasks-broken))" \ - "${tasks}" \ - "${pending_packages}" \ - "${next_tasks}" - printf 'There are %d testing (of which are %s tested) and %d staging packages.\n' \ - "$((testing+tested))" \ - "${tested}" \ - "${staging}" - printf 'There are %d broken package builds.\n' \ - "${broken}" - if [ "${loops}" -ne 0 ]; then - printf 'There are %d loops containing %d package builds.\n' \ - "${loops}" \ - "${looped_packages}" - fi - if [ $((broken+testing+tested+staging)) -ne 0 ]; then - printf '%.1f%% of all packages are broken.\n' \ - "$( - echo "scale=10; 100*${broken}/(${broken}+${testing}+${tested}+${staging})" | \ - bc - )" - fi - if [ $((testing+tested+staging+pending_packages-broken)) -ne 0 ]; then - printf '%.1f%% of the planned work has been done.\n' \ - "$( - echo "scale=10; 100*(${testing}+${staging})/(${testing}+${tested}+${staging}+${pending_packages}-${broken})" | \ - bc - )" - fi -} > \ - "${tmp_dir}/build-master-status.html" - -if ${web}; then - "${base_dir}/bin/calculate-dependent-packages" - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Status of archlinux32 build master</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' - sed 's|$|<br>|' "${tmp_dir}/build-master-status.html" - printf '%s\n' \ - '<br>' \ - 'currently building packages:<br>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'since (UTC)' \ - 'pkgname' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'build slave' - printf '</tr>' - find "${work_dir}/package-states" -maxdepth 1 -name '*.locked' \ - -printf '%T@ %TY-%Tm-%Td %TH:%TM %f ' \ - -execdir sed ' - :a - $!{ - N - s/\n/, / - ba - } - ' '{}' \; | \ - sort -k1n,1 | \ - sed ' - s|^\S\+ || - s|\.locked | | - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - s|\.\([^.]\+\)$| \1| - ' | \ - while read -r date time pkg rev mod_rev repo slaves; do - printf '<tr>' - printf '<td>%s</td>' \ - "${date} ${time}" \ - "${pkg}" \ - "<p style=\"font-size:8px\">${rev}</p>" \ - "<p style=\"font-size:8px\">$(modification_revision_link "${mod_rev}" "${repo}" "${pkg}")</p>" \ - "${repo}" \ - "${slaves}" - printf '</tr>\n' - done - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } | \ - sponge "${tmp_dir}/build-master-status.html" - end=$(($(date +%s)-7*24*60*60)) - { - [ -f "${webserver_directory}/statistics" ] && \ - cat "${webserver_directory}/statistics" - printf '%s ' \ - "$(date +%s)" \ - "${stable}" \ - "${tasks}" \ - "${pending_packages}" \ - "${staging}" \ - "${testing}" \ - "${broken}" \ - "${loops}" \ - "${looped_packages}" \ - "${locked}" \ - "${blocked}" \ - "${next_tasks}" \ - "${tested}" | \ - sed 's| $|\n|' - echo "${end}" - } | \ - sort -k1nr,1 | \ - sed -n " - /^${end}\$/q + printf '%s\n' \ + '<html>' \ + '<head>' \ + '<title>Todos in the build scripts</title>' \ + '</head>' \ + '<body>' + find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ + -exec grep -nHF '' '{}' \; | \ + awk ' + { print $0 } + /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} + ' | \ + sed -n ' + s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ + T + N + s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ + :a + N + s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ + ta + s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ p - " | \ - tac > \ - "${tmp_dir}/statistics" - - find "${build_log_directory}/error" -maxdepth 1 -type f -name '*.build-log.gz' \( \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in build()\.$' {} \; \ - -printf '%f build()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in check()\.$' {} \; \ - -printf '%f check()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in prepare()\.$' {} \; \ - -printf '%f prepare()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: A failure occurred in package\(_\S\+\)\?()\.$' {} \; \ - -printf '%f package()\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Could not download sources\.$' {} \; \ - -printf '%f source\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.$' {} \; \ - -printf '%f dependencies\n' \ - \) -o \ - \( \ - -exec zgrep -q 'error: failed to commit transaction (invalid or corrupted package)$' {} \; \ - -printf '%f package-cache\n' \ - \) -o \ - \( \ - -exec zgrep -q '^==> ERROR: Running makepkg as root is not allowed as it can cause permanent,' {} \; \ - -printf '%f run-as-root\n' \ - \) -o \ - -printf '%f unknown\n' \ - \) | \ - sed ' - s|\(\.[^.]\+\)\{3\} | | ' | \ - sort -u | \ + tee "${tmp_dir}/todos" | \ sed ' :a - $!N - s/^\(\S\+\) \([^\n]\+\)\n\1 /\1 \2,/ - ta - P - D + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n| ' | \ - sort -k1,1 > \ - "${tmp_dir}/broken-packages.reason" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>List of broken package builds</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' \ - '<a href="build-logs/">build logs</a><br>' \ - '<table>' \ - '<tr>' - printf '<th>%s</th>' \ - 'package' \ - 'git revision' \ - 'modification git revision' \ - 'package repository' \ - 'compilations' \ - 'dependent' \ - 'build error' \ - 'blocked' - printf '</tr>\n' - find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\.broken$||' | \ - sort -k1,1 | \ - join -j 1 - "${tmp_dir}/broken-packages.reason" | \ - sed 's|^\(\(.\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\) \(\S\+\)$|\1 \2 \3 \4 \5 \6|' | \ - while read -r sf pkg rev mod_rev repo build_error; do - if grep -qxF "${pkg}" "${tmp_dir}/broken-packages-names"; then - printf '1 ' - else - printf '0 ' - fi - printf '%s ' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" \ - "$(wc -l < "${work_dir}/package-states/${sf}.broken")" \ - "$( - # shellcheck disable=SC2010 - ls -t "${webserver_directory}/build-logs/error" | \ - grep -m1 '^'"$(str_to_regex "${sf}.")"'[^.]\+\.build-log\.gz$' - )" \ - "$( - { - grep -m1 "^$(str_to_regex "${sf}") " "${work_dir}/dependent-count" || \ - echo 'x ' - } | \ - cut -d' ' -f2 - )" \ - "${build_error}" - if [ -f "${work_dir}/package-states/${sf}.blocked" ]; then - sed ' - s|\s\(wait for \)|\n\1|g - ' "${work_dir}/package-states/${sf}.blocked" | \ - while read -r blocked_reason; do - if echo "${blocked_reason}" | \ - grep -q '^wait for '; then - printf 'wait for ' - echo "${blocked_reason}" | \ - sed ' - s|^wait for || - s@\( and \| or \)@\n\1\n@ - ' | \ - while read -r reason; do - if [ "FS#${reason#FS#}" = "${reason}" ]; then - printf '<a href="https://bugs.archlinux.org/task/%s">%s</a>' \ - "${reason#FS#}" \ - "${reason}" - elif [ "FS32#${reason#FS32#}" = "${reason}" ]; then - printf '<a href="https://bugs.archlinux32.org/index.php?do=details&task_id=%s">%s</a>' \ - "${reason#FS32#}" \ - "${reason}" - elif grep -q "^$(str_to_regex "${reason}") " "${work_dir}/build-list"; then - printf '<a href="graphs/%s.png">%s</a>' \ - "${reason}" \ - "${reason}" - elif [ "${reason% *}" != "${reason}" ]; then - printf '%s' \ - "${reason}" - else - printf '<font color="red">%s</font>' \ - "${reason}" - fi - if read -r operator; then - printf ' %s ' "${operator}" - fi - done - else - echo "${blocked_reason}" - fi - done | \ - tr '\n' ' ' - else - printf ' ' - fi - printf '\n' - done | \ - sort -k6n,6 | \ - while read -r buildable pkg rev mod_rev repo count log_file dependent build_error reason; do - if [ "${buildable}" -eq 0 ]; then - left='(' - right=')' - else - unset left - unset right - fi - printf '<tr>' - mod_rev=$( - modification_revision_link "${mod_rev}" "${repo}" "${pkg}" - ) - build_error=$( - echo "${build_error}" | \ - sed 's|,|, |g' - ) - printf '<td>%s</td>' \ - '<a href="graphs/'"${pkg}"'.png">'"${left}${pkg}${right}"'</a>' \ - "<p style=\"font-size:8px\">${rev}</p>" \ - "<p style=\"font-size:8px\">${mod_rev}</p>" \ - "${repo}" \ - '<a href="build-logs/error/'"${log_file}"'">'"${count}"'</a>' \ - "${dependent}" \ - "${build_error}" \ - "${reason}" - printf '</tr>\n' - done - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/broken-packages.html" - - rm -f "${tmp_dir}/broken-packages-names" "${tmp_dir}/broken-packages.reason" - + sed ' + s|$|<br>| + ' + printf '%s\n' \ + '</body>' \ + '</html>' +} > \ + "${tmp_dir}/todos.html" + +if [ -s "${tmp_dir}/todos" ]; then + sed ' + :a + N + /\n$/!ba + s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|\1 \3 | + s/\n$// + s/\n/\\n/g + ' -i "${tmp_dir}/todos" + while read -r file line desc; do + printf '%s %s %s\n' \ + "$(printf '%s' "${file}" | base64 -w0)" \ + "$(printf '%s' "${line}" | base64 -w0)" \ + "$(printf '%s' "${desc}" | base64 -w0)" + done < \ + "${tmp_dir}/todos" | \ + sponge "${tmp_dir}/todos" + # update todos + # shellcheck disable=SC2016 + while read -r file line desc; do + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`line`=from_base64("%s")' \ + "${line}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`description`=from_base64("%s");\n' \ + "${desc}" + + printf 'UPDATE IGNORE `todos`' + printf ' SET `todos`.`description`=from_base64("%s")' \ + "${desc}" + printf ' WHERE `todos`.`file`=from_base64("%s")' \ + "${file}" + printf ' AND `todos`.`line`=from_base64("%s");\n' \ + "${line}" + done < \ + "${tmp_dir}/todos" | \ + mysql_run_query + # insert unfound todos + # shellcheck disable=SC2016 { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Todos in the build scripts</title>' \ - '</head>' \ - '<body>' - find "${base_dir}/bin/" "${base_dir}/conf/" -type f \ - -exec grep -nHF '' '{}' \; | \ - awk ' - { print $0 } - /^[^:]+:[0-9]+:\s*#\s*TODO:/{print ++i} - ' | \ - sed -n ' - s/^\([^:]\+\):\([0-9]\+\):\s*#\s*TODO:\s*/\1\n\2\n/ - T - N - s/\n\(.*\)\n\([0-9]\+\)$/\n\2\n\1/ - :a - N - s/\n[^:\n]\+:[0-9]\+:[ \t]*#[ \t]*\(\S[^\n]*\)$/\n\1/ - ta - s/\n[^:\n]\+:[0-9]\+:[^\n]*$/\n/ - p - ' | \ + printf 'SHOW CREATE TABLE `todos`' | \ + mysql_run_query | \ sed ' - :a - N - /\n$/!ba - s|^[^\n]*/\([^/\n]\+/[^/\n]\+\)\n\([0-9]\+\)\n\([0-9]\+\)\n|<a href="#TODO\2" name="TODO\2">TODO #\2</a> - <a href="https://github.com/archlinux32/builder/blob/master/\1#L\3">\1 (line \3)</a>:\n| - ' | \ - sed ' - s|$|<br>| + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' - printf '%s\n' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/todos.html" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>Blacklisted packages</title>' \ - '<link rel="stylesheet" type="text/css" href="/static/style.css">' \ - '</head>' \ - '<body>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'package' \ - 'reason' - printf '</tr>\n' - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \ - tar -Ox | \ + printf ';\n' + printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' + while read -r file line desc; do + printf '(' + printf 'from_base64("%s"),' \ + "${file}" \ + "${line}" \ + "${desc}" | \ + sed 's/,$/),/' + done < \ + "${tmp_dir}/todos" | \ sed ' - s@FS#\([0-9]\+\)@<a href="https://bugs.archlinux.org/task/\1">\0</a>@ - s@FS32#\([0-9]\+\)@<a href="https://bugs.archlinux32.org/index.php?do=details\&task_id=\1">\0</a>@ - /.#/!s/$/#/ - s|\(.\)#|\1</td><td>| - /^\s*#/{ - s/^\s*#\s*// - s|\s*\(</td><td>\)|</font></s>\1| - s/^/<s><font color="#808080">/ - } - s|^|<tr><td>| - s|$|</td></tr>| + s/,$// ' - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/blacklist.html" - - { - printf '%s\n' \ - '<html>' \ - '<head>' \ - '<title>log of ssh connections from build slaves</title>' \ - '</head>' \ - '<body>' \ - '<table>' - printf '<tr>' - printf '<th>%s</th>' \ - 'time' \ - 'build slave' \ - 'command' \ - 'arguments' - printf '</tr>\n' - if [ -r "${work_dir}/ssh-log" ]; then - tac "${work_dir}/ssh-log" | \ - while read -r date time slave command arguments; do - printf '<tr>' - printf '<td>%s</td>' \ - "${date} ${time}" \ - "${slave}" \ - "${command}" \ - "${arguments}" - printf '</tr>\n' - done - fi - printf '%s\n' \ - '</table>' \ - '</body>' \ - '</html>' - } > \ - "${tmp_dir}/ssh-log.html" + printf ';\n' + printf 'INSERT IGNORE INTO `todos` (`file`,`line`,`description`) ' + printf 'SELECT `td`.`file`,`td`.`line`,`td`.`description` ' + printf 'FROM `td` ' + printf 'WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');\n' + + printf 'DELETE FROM `todos` WHERE NOT EXISTS (' + printf 'SELECT * FROM `td`' + printf ' AND `td`.`%s`=`todos`.`%s`' \ + 'file' 'file' \ + 'line' 'line' \ + 'description' 'description' | \ + sed 's/^ AND / WHERE /' + printf ');' + printf 'DROP TABLE `td`;\n' + printf 'DELETE FROM `todo_links` WHERE NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`depending_on`' + printf ') OR NOT EXISTS (' + printf 'SELECT * FROM `todos` ' + printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' + printf ');\n' + } | \ + mysql_run_query + rm -f "${tmp_dir}/todos" +fi - find "${tmp_dir}" -maxdepth 1 -type f | \ - while read -r file; do - cat "${file}" > \ - "${webserver_directory}/${file##*/}" - done +{ + printf '%s\n' \ + '<html>' \ + '<head>' \ + '<title>log of ssh connections from build slaves</title>' \ + '</head>' \ + '<body>' \ + '<table>' + printf '<tr>' + printf '<th>%s</th>' \ + 'time' \ + 'build slave' \ + 'command' \ + 'arguments' + printf '</tr>\n' + if [ -r "${work_dir}/ssh-log" ]; then + tac "${work_dir}/ssh-log" | \ + while read -r date time slave command arguments; do + printf '<tr>' + printf '<td>%s</td>' \ + "${date} ${time}" \ + "${slave}" \ + "${command}" \ + "${arguments}" + printf '</tr>\n' + done + fi + printf '%s\n' \ + '</table>' \ + '</body>' \ + '</html>' +} > \ + "${tmp_dir}/ssh-log.html" -else - cat "${tmp_dir}/build-master-status.html" -fi +find "${tmp_dir}" -maxdepth 1 -type f | \ + while read -r file; do + cat "${file}" > \ + "${webserver_directory}/${file##*/}" + done diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index a0c2b4f..962b12e 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -5,30 +5,23 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: add all (necessary) features from build-master-status + +tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir) +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + +# do not block if locked +exec 9> "${sanity_check_lock_file}" +if ! flock -n 9; then + >&2 echo 'Mysql-Sanity check skipped, cannot acquire lock.' + exit +fi + +# shellcheck disable=SC2119 +mysql_cleanup + { mysql_sanity_check || true - { - # shellcheck disable=SC2016 - { - printf 'SELECT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`' - printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' - printf ' WHERE `repositories`.`name`="build-list"' - } | \ - ${mysql_command} --batch | \ - sed ' - 1d - y/\t/ / - s/^/+ / - ' - sed 's/^/- /' "${work_dir}/build-list" - } | \ - sort -k2 -k1,1 | \ - uniq -uf1 } | \ sed ' s,^-.*$,<font color="#FF0000">\0</font>, @@ -38,3 +31,118 @@ $ a </body></html> ' | \ sponge "${webserver_directory}/mysql-sanity.html" + +if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ + [ ! -s "${work_dir}/build-master-sanity" ]; then + printf 'girls, my database is dirty again ...\n' | \ + irc_say + echo 'build master is insane' > \ + "${work_dir}/build-master-sanity" +fi + +if [ ! -s "${work_dir}/build-master-sanity" ]; then + # shellcheck disable=SC2016 + { + printf 'INSERT IGNORE INTO `statistics` (' + printf '`%s`,' \ + 'date' \ + 'stable_packages_count' \ + 'pending_tasks_count' \ + 'pending_packages_count' \ + 'staging_packages_count' \ + 'testing_packages_count' \ + 'tested_packages_count' \ + 'broken_tasks_count' \ + 'dependency_loops_count' \ + 'dependency_looped_tasks_count' \ + 'locked_tasks_count' \ + 'blocked_tasks_count' \ + 'next_tasks_count' | \ + sed 's/,$//' + printf ') VALUES (' + # date + printf 'NOW(),' + # stable_packages_count + printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="stable"),' + # pending_tasks_count + printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"),' + # pending_packages_count + printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"),' + # staging_packages_count + printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="staging"),' + # testing_packages_count + printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="testing"' + printf ' AND NOT `binary_packages`.`is_tested`),' + # tested_packages_count + printf '(SELECT COUNT(DISTINCT `binary_packages`.`id`) FROM' + printf ' `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`="testing"' + printf ' AND `binary_packages`.`is_tested`),' + # broken_tasks_count + printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_assignments`.`is_broken`),' + # dependency_loops_count + printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`loop`) FROM' + printf ' `build_dependency_loops`),' + # dependency_looped_tasks_count + printf '(SELECT COUNT(DISTINCT `build_dependency_loops`.`build_assignment`) FROM' + printf ' `build_dependency_loops`),' + # locked_tasks_count + printf '(SELECT COUNT(DISTINCT `build_slaves`.`currently_building`) FROM' + printf ' `build_slaves`' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"),' + # blocked_tasks_count + printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_assignments`.`is_blocked` IS NOT NULL),' + # next_tasks_count + printf '(SELECT COUNT(DISTINCT `build_assignments`.`id`) FROM' + printf ' `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `dependencies`' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' WHERE `prov_r`.`name`="build-list"' + printf ' AND `dependencies`.`dependent`=`binary_packages`.`id`' + printf '))' + printf ');\n' + } | \ + mysql_run_query +fi diff --git a/bin/build-packages b/bin/build-packages index 81afa7f..02334e9 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -7,8 +7,7 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -# TODO: report back in regular intervals, so the build master can -# abort if the package is outdated and/or someone else finished it. +# TODO: build other 'architectures', too (pentium4, i486) # shellcheck disable=SC2016 usage() { @@ -220,6 +219,11 @@ while [ "${count}" -ne 0 ]; do if [ "${mod_git_revision}" = 'work-tree' ]; then mod_git_revision=$( # we can't just create an empty index-file with mktemp, because git doesn't like it + find . \ + -mindepth 1 \ + -maxdepth 1 \ + -name 'tmp.build-packages.git.*' \ + -exec rm -rf --one-file-system {} \; tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT export GIT_INDEX_FILE="${tmp_subdir}/index.new" @@ -251,8 +255,18 @@ while [ "${count}" -ne 0 ]; do fi cd "${base_dir}" recursively_umount_and_rm "${tmp_dir}" + flock -u 9 || true exit "${err}" } + find "${work_dir}" \ + -mindepth 1 \ + -maxdepth 1 \ + -name 'tmp.build-packages.??????' \ + -printf '%p\n' | \ + while read -r old_tmp_dir; do + find "${old_tmp_dir}" -xdev -exec chmod 777 {} \; + rm -rf --one-file-system "${old_tmp_dir}" + done tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX") trap bail_out EXIT @@ -295,7 +309,19 @@ while [ "${count}" -ne 0 ]; do # we can't improve anything continue fi - tar -xz --overwrite -f "${source_name}" --exclude PKGBUILD --strip-components=1 || true + # shellcheck disable=SC2046 + tar -xz --overwrite \ + -f "${source_name}" \ + --exclude PKGBUILD \ + $( + if [ -n "${PKGBUILD_mod}" ]; then + git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \ + tar -t | \ + sed 's/^/--exclude /' + fi + ) \ + --strip-components=1 \ + || true fi if echo "${straw}" | \ @@ -321,19 +347,39 @@ while [ "${count}" -ne 0 ]; do if echo "${straw}" | \ grep -qF ':clean_chroot:'; then - parameters='-c' + outerParameters='-c' else - parameters='' + outerParameters='' fi if echo "${straw}" | \ grep -qF ':without_check:'; then - parameters="${parameters} -- -- --nocheck" + innerParameters='--nocheck' + else + innerParameters='' + fi + + if echo "${straw}" | \ + grep -qF ':with_/dev/fuse:'; then + middleParameters='-d /dev/fuse' + else + middleParameters='' fi find . -maxdepth 1 -type f \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' \) -exec \ rm {} \; + echo 'building' > "${tmp_dir}/.ping-build-master" + if [ -z "${forced_package}" ]; then + # we get a lock on "${work_dir}/ping-build-master.lock", + # if we release that lock, ping-to-master should stop _immediately_ + exec 9> "${work_dir}/ping-build-master.lock" + if ! flock -n 9; then + >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.' + exit 2 + fi + "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" & + fi >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) ...' \ "$(date +'%Y-%m-%d %T')" \ "${package}" \ @@ -344,7 +390,7 @@ while [ "${count}" -ne 0 ]; do # by piping the log, we don't see anything in the terminal, # but all ways to duplicate the logs seem pretty elaborate # shellcheck disable=SC2024,SC2086 - if sudo "${build_command}" ${parameters} > \ + if sudo "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \ "$( date -u --iso-8601=seconds | \ cut -d+ -f1 @@ -353,6 +399,7 @@ while [ "${count}" -ne 0 ]; do >&2 printf ' ok.\n' tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX") find . -maxdepth 1 -type f -name '*-debug-*.pkg.tar.xz*' -delete + echo 'post-build' > "${tmp_dir}/.ping-build-master" >&2 printf 'signing package(s)\n' find . -maxdepth 1 -type f -name '*.pkg.tar.xz' \ -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \ @@ -390,12 +437,50 @@ while [ "${count}" -ne 0 ]; do >&2 printf ' failed. Next ...\n' done done + >&2 printf 'searching for provided libraries\n' + find "${tar_content_dir}" -maxdepth 1 \ + -name '*.pkg.tar.xz' \ + -printf '%p\n' | \ + while read -r pkgfile; do + pacman -Qqlp "${pkgfile}" | \ + sed -n ' + s,^.*/,, + /\.so\(\..\+\)\?$/p + ' > \ + "${pkgfile}.so.provides" + done + >&2 printf 'searching for required libraries\n' + package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX") + find "${tar_content_dir}" -maxdepth 1 \ + -name '*.pkg.tar.xz' | \ + while read -r pkgfile; do + if printf '%s\n' "${pkgfile}" | \ + grep -vq -- '-any\.pkg\.tar\.xz$'; then + # we do not check "any" packages for linked libraries + # (why do they have them in the first place?) + mkdir "${package_content_dir}/${pkgfile##*/}" + tar -C "${package_content_dir}/${pkgfile##*/}" -xJf "${pkgfile}" 2>/dev/null + find "${package_content_dir}/${pkgfile##*/}" \ + -name 'opt' -prune , \ + -exec objdump -x '{}' \; 2>/dev/null | \ + grep -w 'NEEDED' | \ + awk '{print $2}' | \ + sed ' + /\.c32$/d + s,^.*/,, + ' + find "${package_content_dir:?}/${pkgfile##*/}" -xdev -exec chmod 777 '{}' \; + rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}" + fi | \ + sort -u > \ + "${pkgfile}.so.needs" + done + >&2 printf 'running namcap ...' if [ "${repository}" = 'multilib' ]; then x86_64_build_command='multilib-build' else x86_64_build_command='extra-x86_64-build' fi - >&2 printf 'running namcap ...' # this is a little hack: makepkg receives '--version', but namcap is run nevertheless # (and it only works with devtools32, because they are running namcap on *.pkg.tar.xz in the base directory, too) sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \ @@ -426,14 +511,31 @@ while [ "${count}" -ne 0 ]; do fi done >&2 printf ' ok.\n' + echo 'uploading' > "${tmp_dir}/.ping-build-master" if ${upload_to_build_master}; then - find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir gzip '{}' \; + find "${tar_content_dir}/" -maxdepth 1 \ + \( \ + -name '*.pkg.tar.xz-namcap.log' -o \ + -name '*.pkg.tar.xz.so.needs' -o \ + -name '*.pkg.tar.xz.so.provides' \ + \) \ + -execdir gzip '{}' \; else - find "${tar_content_dir}/" -maxdepth 1 -name '*.pkg.tar.xz-namcap.log' -execdir grep -HF '' '{}' \; + find "${tar_content_dir}/" -maxdepth 1 \ + -name '*.pkg.tar.xz-namcap.log' \ + -execdir grep -HF '' '{}' \; fi # shellcheck disable=SC2046 tar -cf 'package.tar' -C "${tar_content_dir}" -- $( - find "${tar_content_dir}/" -maxdepth 1 \( -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' \) -printf '%f\n' + find "${tar_content_dir}/" -maxdepth 1 \ + \( \ + -name '*.pkg.tar.xz' -o \ + -name '*.pkg.tar.xz.sig' -o \ + -name '*.pkg.tar.xz-namcap.log.gz' -o \ + -name '*.pkg.tar.xz.so.needs.gz' -o \ + -name '*.pkg.tar.xz.so.provides.gz' \ + \) \ + -printf '%f\n' ) while ${upload_to_build_master}; do err=0 @@ -474,6 +576,7 @@ while [ "${count}" -ne 0 ]; do success=true break fi + echo 'failure' > "${tmp_dir}/.ping-build-master" >&2 printf ' failed.\n' done @@ -528,6 +631,7 @@ while [ "${count}" -ne 0 ]; do # clean up tmp_dir cd "${base_dir}" recursively_umount_and_rm "${tmp_dir}" + flock -u 9 || true trap - EXIT continue diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages deleted file mode 100755 index bfb87a1..0000000 --- a/bin/calculate-dependent-packages +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/sh - -# shellcheck source=conf/default.conf -. "${0%/*}/../conf/default.conf" - -if [ -s "${work_dir}/build-master-sanity" ]; then - >&2 echo 'Build master is not sane.' - exit -fi - -tmp_dir=$(mktemp -d 'tmp.calculate-dependent-packages.XXXXXXXXXX' --tmpdir) -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - -sort -k1,1 -u "${work_dir}/build-list" > \ - "${tmp_dir}/build-list" - -while read -r pkg rev mod_rev repo; do - generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${repo}" -done < \ - "${tmp_dir}/build-list" - -mkdir "${tmp_dir}/loops" -find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' | \ - while read -r loop; do - sort -u "${loop}" | \ - join -j 1 - "${tmp_dir}/build-list" | \ - tr ' ' '.' > \ - "${tmp_dir}/loops/${loop##*/}" - done - -touch "${tmp_dir}/dependent-count" - -tr ' ' '.' < \ - "${tmp_dir}/build-list" | \ - sponge "${tmp_dir}/build-list" - -sums='' - -while [ -s "${tmp_dir}/build-list" ] && [ "${sums}" != "$(sha512sum "${tmp_dir}/dependent-count")" ]; do - - sums=$( - sha512sum "${tmp_dir}/dependent-count" - ) - - { - sed 's|^|? |' "${tmp_dir}/build-list" - sed 'p' "${tmp_dir}/dependent-count" - } | \ - sort -k2,2 | \ - uniq -uf1 | \ - cut -d' ' -f2 | \ - sponge "${tmp_dir}/build-list" - - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.build-depends| - ' "${tmp_dir}/build-list" | \ - xargs -r grep -hvx 'base\|base-devel' | \ - sort -u > \ - "${tmp_dir}/build-list.build-depends" - - sed ' - s|^\S\+ |'"${work_dir}"'/package-infos/| - s|$|.build-depends| - ' "${tmp_dir}/dependent-count" | \ - xargs -r grep -Hvx 'base\|base-devel' | \ - sed ' - s|^.*/|| - s|\.build-depends:| | - ' | \ - sort -k2,2 > \ - "${tmp_dir}/dependent-count.build-depends" - - rm -f "${tmp_dir}/dependent-count.new" - touch "${tmp_dir}/dependent-count.new" - - while read -r sf; do - if [ -n "$( - { - sort -u "${work_dir}/package-infos/${sf}.builds" - cat "${tmp_dir}/build-list.build-depends" - } | \ - sort | \ - uniq -d - )" ]; then - continue - fi - - count="/$( - sort -u "${work_dir}/package-infos/${sf}.builds" | \ - join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \ - sort -u | \ - join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \ - tr '/ ' '\n' | \ - grep -vxF '' | \ - sort -u | \ - tr '\n' '/' - )" - printf '%s %s\n' \ - "${count}" \ - "${sf}" >> \ - "${tmp_dir}/dependent-count.new" - find "${tmp_dir}/loops" -type f -maxdepth 1 \ - -exec grep -qxF "${sf}" {} \; \ - -exec rm {} \; - done < \ - "${tmp_dir}/build-list" - - find "${tmp_dir}/loops" -maxdepth 1 | \ - while read -r loop; do - if [ ! -f "${loop}" ]; then - continue - fi - if [ -n "$( - { - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${loop}" | \ - xargs -r cat | \ - sort -u - { - sed 'p' "${loop}" - cat "${tmp_dir}/build-list" - } | \ - sort | \ - uniq -u | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.build-depends| - ' | \ - xargs -r grep -hvx 'base\|base-devel' | \ - sort -u - } | \ - sort | \ - uniq -d - )" ]; then - continue - fi - - count="/$( - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${loop}" | \ - xargs -r cat | \ - sort -u | \ - join -1 1 -2 2 -o 2.1 - "${tmp_dir}/dependent-count.build-depends" | \ - sort -u | \ - join -1 1 -2 2 -o 2.1,2.2 - "${tmp_dir}/dependent-count" | \ - tr '/ ' '\n' | \ - grep -vxF '' | \ - sort -u | \ - tr '\n' '/' - )" - while read -r sf; do - printf '%s %s\n' "${count}" "${sf}" >> \ - "${tmp_dir}/dependent-count.new" - find "${tmp_dir}/loops" -maxdepth 1 -type f -not -name "${loop##*/}" \ - -exec grep -qxF "${sf}" {} \; \ - -exec rm {} \; - done < \ - "${loop}" - - rm "${loop}" - done - - cat "${tmp_dir}/dependent-count" "${tmp_dir}/dependent-count.new" | \ - sort -k2,2 | \ - sponge "${tmp_dir}/dependent-count" - -done - -while read -r count pkg; do - count=$( - echo "${count}" | \ - tr '/' '\n' | \ - grep -cvxF '' - ) || true - printf '%s %s\n' "${pkg}" "${count}" -done < \ - "${tmp_dir}/dependent-count" | \ - sponge "${work_dir}/dependent-count" diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 46e78db..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -1,33 +1,64 @@ #!/bin/sh # check the bug tracker for packages in testing and community-testing -# with issues and mark these packages as "testing" if they are currently -# marked as "tested" +# with issues and mark these packages as "has_issues" if they are faulty +# and vice versa # shellcheck disable=SC2039 # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir) -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT +bug_list=$( + curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \ + sed -n ' + 1d + s/^[^,]\+,"// + T + s/^\([^"]\+\)"\(,[^,]\+\)\{2\},"\([^"]\+\)".*$/"\1" "\3"/ + T + p + ' +) -receive_buglist 'Testing' | \ - tr ' ,;' '\n' | \ - sed -n ' - s/^\[// - T - s/]$// - T - p - ' | \ - sort -u > \ - "${tmp_dir}/faulty-packages" - -find "${work_dir}/package-states" -name '*.tested' -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|^\(.*\)\(\.[^.]\+\)\{3\}$|\1 \0| - ' | \ - sort -k1,1 | \ - join -1 1 -2 1 -o 1.2 - "${tmp_dir}/faulty-packages" | \ - "${base_dir}/bin/modify-package-state" -n --faulty /dev/stdin +# shellcheck disable=SC2016 +{ + printf 'SELECT `repository_stabilities`.`id`,`repository_stabilities`.`bugtracker_category`' + printf ' FROM `repository_stabilities`' + printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' +} | \ + mysql_run_query | \ + while read -r stability_id category; do + for has_issues in '1:' '0:NOT '; do + printf 'UPDATE `binary_packages`' + mysql_join_binary_packages_repositories + printf ' SET `has_issues`=%s' \ + "${has_issues%:*}" + printf ' WHERE `repositories`.`stability`=%s' \ + "${stability_id}" + printf ' AND `binary_packages`.`pkgname` %sIN (' \ + "${has_issues#*:}" + printf '%s\n' "${bug_list}" | \ + sed -n ' + s/^"'"$(str_to_regex "${category}")"'" // + T + :a + /\[.*]/ { + s/^[^[]*\[// + T + h + s/].*$// + p + x + ba + } + ' | \ + base64_encode_each | \ + sed ' + s/^/from_base64("/ + s/$/")/ + ' | \ + tr '\n' ',' + printf '"");\n' + done + done | \ + mysql_run_query diff --git a/bin/check-opcodes b/bin/check-opcodes index c7ab4a9..bd96cc6 100755 --- a/bin/check-opcodes +++ b/bin/check-opcodes @@ -101,7 +101,7 @@ log "Checking for architecture: $ARCH ($OPCODE_ARGS)" bsdtar --no-fflags -x -C $tmp_dir -f $PACKAGE # shellcheck disable=SC2044 -for absfile in $(find $tmp_dir -name '*.so*' -type f); do +for absfile in $(find $tmp_dir -regextype grep -regex '.*\.so\(\.[0-9.]\+\)\?' -type f); do file=$(basename $absfile) log "Checking shared library: $file" readelf -a $absfile > $tmp_dir/$file.elf diff --git a/bin/cleanup b/bin/cleanup index 78daece..c9a8d2b 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: clean database, too + # we only clean if run interactive or if no one is logged in if ! tty -s && \ [ -n "$(users)" ]; then @@ -17,54 +19,24 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# remove blocked/broken/locked markes of packages not on the buildlist anymore - -{ - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5| - ' - awk '{ - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked" - }' "${work_dir}/build-list" -} | \ - sort -k3 | \ - uniq -uf2 | \ - grep '^state ' | \ - awk '{print $2}' | \ - sed "s|^|${work_dir}/package-states/|" | \ - xargs -rn1 rm - # remove logs where package is not broken/locked anymore { - find "${build_log_directory}/error" -maxdepth 1 -type f -printf '1 %f %f\n' | \ - sed 's|\.[^. ]\+\.build-log\.gz$||' - find "${work_dir}/package-states" -maxdepth 1 \( \ - -name '*.broken' -o \ - -name '*.done' -o \ - -name '*.locked' -o \ - -name '*.testing' \ - \) -printf '0 0 %f\n' | \ - sed 's|\.[^.]\+$||' + find "${build_log_directory}/error" -maxdepth 1 -type f -printf 'file %f\n' + # shellcheck disable=SC2016 + printf 'SELECT "mysql",`failed_builds`.`log_file` FROM `failed_builds`;\n' | \ + mysql_run_query | \ + tr '\t' ' ' } | \ - sort -k3,3 -k1,2 | \ - uniq --group=prepend -f2 | \ - while read -r num file _; do - if [ "${num}" = '0' ]; then - while read -r line; do - if [ -z "${line}" ]; then - break - fi - done - continue - fi - if [ -z "${num}${file}" ]; then - continue - fi - rm -f "${build_log_directory}/error/${file}" + sort -k2,2 -k1,1 | \ + uniq -uf 1 | \ + sed -n ' + s/^file // + T + p + ' | \ + while read -r file; do + rm "${build_log_directory}/error/${file}" done # only keep 10 newest logs per failed package @@ -97,14 +69,4 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \ -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \ -delete -# remove old package meta data -delete_old_metadata - -# remove dependency graphs of packages on the deletion list -sed ' - s|^|'"${webserver_directory}"'/graphs/| - s|$|.png| -' "${work_dir}/deletion-list" | \ - xargs -rn1 rm -f - exit 0 diff --git a/bin/common-functions b/bin/common-functions deleted file mode 100755 index d5b6eaa..0000000 --- a/bin/common-functions +++ /dev/null @@ -1,1348 +0,0 @@ -#!/bin/sh - -# contains functions used by more than one script - -# TODO: include link depenendencies in run-depends metadata - -# shellcheck disable=SC2039 - -if [ -z "${base_dir}" ]; then - # just to make shellcheck happy - . 'conf/default.conf' -fi - -# find_pkgbuilds package repository git_repository git_revision mod_git_revision -# find the PKGBUILD and modification of $package from $repository -# sets $PKGBUILD and $PKGBUILD_mod - -find_pkgbuilds() { - - local package="$1" - local repository="$2" - local git_repository="$3" - local git_revision="$4" - local mod_git_revision="$5" - - local repo_path - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - PKGBUILD=$( - git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sort | \ - tail -n1 - ) - - PKGBUILD_mod=$( - git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \ - tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null - ) || true - - if [ -z "${PKGBUILD}" ] && \ - [ -z "${PKGBUILD_mod}" ]; then - >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ - "${package}" \ - "${repository}" \ - "${git_repository}" \ - "${git_revision}" \ - "${mod_git_revision}" - return 1 - fi - -} - -# find_repository_with_commit commit -# find the repository which has $commit - -find_repository_with_commit() { - - local repository - - for repository in ${repo_names}; do - # shellcheck disable=SC2016 - if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then - echo "${repository}" - return 0 - fi - done - >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" - exit 1 - -} - -# find_git_repository_to_package_repository repository -# find the git repository which tracks the package repository $repository - -find_git_repository_to_package_repository() { - - local repository - local package_repository - local repo_path - - package_repository="$1" - - if [ "$1" = 'build-support' ]; then - echo 'packages' - return 0 - fi - - for repository in ${repo_names}; do - if [ "${repository}" = "archlinux32" ]; then - continue - fi - eval 'repo_path="${repo_paths__'"${repository}"'}"' - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \ - tar -t --wildcards '*/repos' | \ - grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \ - cut -d/ -f3 | \ - sed 's|-[^-]\+$||' | \ - sort -u | \ - grep -qxF "${package_repository}"; then - echo "${repository}" - return 0 - fi - done - >&2 echo "can't find git repository with package repository '$1'" - exit 1 - -} - -# generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# generate the meta data files of a package (dependencies, built packages, ...) - -generate_package_metadata() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - local file_prefix - local file - local PKGBUILD - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" - - if [ -e "${file_prefix}.builds" ] && \ - [ -e "${file_prefix}.build-depends" ] && \ - [ -e "${file_prefix}.run-depends" ] && \ - [ -e "${file_prefix}.groups" ] && \ - [ -e "${file_prefix}.packages" ]; then - return 0 - fi - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" - exit 1 - fi - if [ ! -s "${file_prefix}.SRCINFO" ]; then - >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" - exit 1 - fi - - # otherwise this just calls for trouble - sed -i ' - /^[^=]*=\s*$/d - s/_i686\(\s*=\)/\1/ - ' "${file_prefix}.SRCINFO" - - # extract "groups" = groups \cup provides - grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.groups" - - # extract "packages" = pkgname - grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.packages" - - # extract "builds" = provides \cup pkgname \cup groups - cat "${file_prefix}.groups" "${file_prefix}.packages" | \ - sort -u > \ - "${file_prefix}.builds" - - # extract "build-depends" = makedepends \cup checkdepends \cup depends \cup \{ base, base-devel \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' 'base-devel' - sed -n "$( - printf '/^\t%s = /p\n' \ - 'depends' \ - 'makedepends' \ - 'checkdepends' - )" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.build-depends" - - # extract "run-depends" = depends \cup \{ base \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' - sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.run-depends" - - rm "${file_prefix}.SRCINFO" - -} - -# delete_old_metadata -# delete old (=unneeded) meta data of packages - -delete_old_metadata() { - - local current_metadata - - current_metadata=$( - find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| - ' | \ - sort -u - ) - - ( # the new shell is intentional - # what we have - echo "${current_metadata}" - - # package-states should stay - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - ' | \ - sort -u | \ - sed 'p' - - # build-list items should stay - sed 'p' "${work_dir}/build-list" - - tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - echo "${current_metadata}" | \ - sort -k1,1 > \ - "${tmp_dir}/current-metadata" - - # the newest of the following should stay: - { - # deletion-list items - cat "${work_dir}/deletion-list" - # all packages in the repos - for repo in ${repo_names}; do - eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - sed ' - s|/$|| - /\//d - ' - done - } | \ - sort -u | \ - join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ - sort -k4,4 > \ - "${tmp_dir}/find-newest-revisions" - - uniq -uf3 < \ - "${tmp_dir}/find-newest-revisions" | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - - uniq -Df3 < \ - "${tmp_dir}/find-newest-revisions" | \ - uniq --group=append -f3 | \ - { - revs='' - mod_revs='' - opkg='' - orepo='' - while read -r rev mod_rev repo pkg; do - - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - - printf '%s %s %s %s\n' \ - "$( - printf '%s\n' ${revs} | \ - find_newest_of_git_revisions - )" \ - "$( - printf '%s\n' ${mod_revs} | \ - find_newest_of_git_revisions - )" \ - "${orepo}" \ - "${opkg}" - - revs='' - mod_revs='' - orepo='' - opkg='' - continue - fi - revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} ${rev} | \ - sort -u - ) - mod_revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} ${mod_rev} | \ - sort -u - ) - orepo="${repo}" - opkg="${pkg}" - done - } | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - ) | \ - sort | \ - uniq -u | \ - while read -r pkg rev mod_rev repo; do - rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* - done -} - -# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository -# print which (stable) repository a package belongs to - -repository_of_package() { - local package="$1" - local repository="${package##*.}" - package="${package%.*}" - local a32_rev="${package##*.}" - package="${package%.*.*}" - - case "${repository}" in - 'multilib') - if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ - tar -Ox | \ - grep -qFx "${package%.*.*.*}"; then - echo 'extra' - else - echo 'community' - fi - ;; - *) - echo "${repository}" - esac -} - -# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending -# print wether the specified package is an official package (print -# $ending) or a community package (print 'community-$ending') or a -# build-suppor package (print 'build-support') - -official_or_community() { - local prepo - prepo=$(repository_of_package "$1") - - if [ "${prepo}" = 'community' ]; then - echo 'community-'"$2" - elif [ "${prepo}" = 'build-support' ]; then - echo 'build-support' - else - echo "$2" - fi -} - -# ls_master_mirror $path -# list content of $path on the master mirror (via rsync) - -ls_master_mirror() { - - local path="$1" - - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${path}/" | \ - grep -v '\s\.$' | \ - awk '{print $5}' - -} - -# remove_old_package_versions $arch $repository $package_file - -# removes all older (not-newer) versions of $package_file -# in all repositories not-older (newer) than $repository - -# TODO: should remove all other version (also newer) from -# some repositories :-/ - -# A package is considered not newer if -# a) its version is not newer -# A package is considered older if -# b) its version is older or -# c) if it's "not newer" and its architecture is 'any' and different or -# d) if it's "not newer" and the other architecture is 'any' and different - -# this ensures an any package may replace arch-specific packages of the same version and vice versa - -remove_old_package_versions() { - - local arch="$1" - local repository="$2" - local package="$3" - - # repositories in which older packages should be deleted - local delete_older_repositories - # repositories in which not-newer packages should be deleted - local delete_not_newer_repositories - - if echo "${standalone_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories='' - - elif echo "${staging_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories="${repository}" - delete_not_newer_repositories=$( - echo "${staging_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${testing_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${testing_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - elif echo "${stable_package_repositories}" | \ - grep -qxF "${repository}"; then - - delete_older_repositories=$( - printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}" - ) - delete_not_newer_repositories=$( - echo "${stable_package_repositories}" | \ - grep -vxF "${repository}" - ) || true - - else - - >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}" - return 1 - - fi - - ( # the new shell is intentional - tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - { - # the architecture of the package (any vs. i686) - package_arch="${package##*-}" - package_arch="${package_arch%%.*}" - if [ "${package_arch}" = 'any' ]; then - package_arch_regex_inverter='!' - else - unset package_arch_regex_inverter - fi - - for repo in ${delete_older_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|2 '"${arch} ${repo}"' | - / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{ - s|^2|0| - } - p - } - ' - done - for repo in ${delete_not_newer_repositories}; do - ls_master_mirror "${arch}/${repo}" | \ - sed -n ' - /\.pkg\.tar\.xz$/!d - s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2| - /^'"$(str_to_regex "${package%-*-*-*}")"' / { - s|^|0 '"${arch} ${repo}"' | - p - } - ' - done - echo "${package%-*}" | \ - sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|' - - # the generated list contains the following columns: - # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz - } | \ - expand_version 5 | \ - sort -k5V,5 -k1n,1 | \ - shrink_version 5 | \ - sed -n ' - /^1 %cut% %it% %here% /q - s/^[02] // - s/ \(\S\+\)$/-\1/ - p - ' | \ - sort -u > \ - "${tmp_dir}/packages-to-delete" - # this file contains a list of packages to be deleted, one on each line: - # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz - - cut -d' ' -f1,2 < \ - "${tmp_dir}/packages-to-delete" | \ - grep -vxF "${arch} ${repository}" | \ - sort -u > \ - "${tmp_dir}/repositories-to-modify" - - # fetch all databases being modified - while read -r del_arch del_repo; do - mkdir -p "${tmp_dir}/${del_arch}/${del_repo}" - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${tmp_dir}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - while read -r del_arch del_repo del_package _; do - if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then - # we do not repo-remove the package in the target repository - continue - fi - repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \ - "${del_package}" - done < \ - "${tmp_dir}/packages-to-delete" - - # upload modified databases - while read -r del_arch del_repo; do - ${master_mirror_rsync_command} \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \ - "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \ - "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/" - done < \ - "${tmp_dir}/repositories-to-modify" - - # shellcheck disable=SC2016 - sed ' - s/\.pkg\.tar\.xz$// - s/^\S\+ // - s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/ - s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/ - s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ - ' "${tmp_dir}/packages-to-delete" | \ - while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do - printf 'DELETE FROM `binary_packages` WHERE' - printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \ - 'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \ - 'repository' 'repositories' 'repositories' 'repositories' "$(printf '%s' "${repo}" | base64 -w0)" - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ - 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ - 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ - 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ - 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \ - sed 's/ AND$//' - printf ';\n' - done | \ - ${mysql_command} - - sed ' - s| \(\S\+\)$|-\1| - y| |/| - s|^|rm "| - s|$|"| - p - s|"$|.sig"| - ' "${tmp_dir}/packages-to-delete" | \ - ${master_mirror_sftp_command} - ) - -} - -# wait_some_time $minimum $diff -# wait between minimum and minimum+diff seconds (diff defaults to 30) - -wait_some_time() { - local minimum=$1 - local diff=$2 - local random - - if [ -z "${diff}" ]; then - diff=30 - fi - - random=$( - dd if='/dev/urandom' count=1 2> /dev/null | \ - cksum | \ - cut -d' ' -f1 - ) - - sleep $((minimum + random % diff)) -} - -# str_to_regex $string -# escape dots for use in regex - -str_to_regex() { - echo "$1" | \ - sed ' - s|[.[]|\\\0|g - ' -} - -# make_source_info $package $repository $git_revision $mod_git_revision $output -# create .SRCINFO from PKGBUILD within git repositories, output to $output - -make_source_info() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local mod_git_revision="$4" - local output="$5" - - local git_repo - local PKGBUILD - local PKGBUILD_mod - - git_repo=$(find_repository_with_commit "${git_revision}") - - if [ -z "${git_repo}" ]; then - return 1 - fi - - find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" - - ( # the new shell is intentional - - tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' - - { - cd "${tmp_dir}" - makepkg --printsrcinfo - cd .. - } | - if [ "${package%-i18n}-i18n" = "${package}" ]; then - sed ' - 1 a \\tdepends = '"${package%-i18n}"' - ' - else - cat - fi > \ - "${output}" - - ) - -} - -# recursively_umount_and_rm $dir -# umount all mountpoints in $dir which are also in $dir's -# filesystem, possibly also $dir itself and then -# rm -rf --one-file-system $dir - -recursively_umount_and_rm() { - local dir="$1" - - if [ -z "${dir}" ]; then - >&2 echo 'ERROR: recursively_umount_and_rm requires an argument' - exit 42 - fi - - find "${dir}" \ - -xdev -depth -type d \ - -exec 'mountpoint' '-q' '{}' ';' \ - -exec 'sudo' 'umount' '-l' '{}' ';' - rm -rf --one-file-system "${dir}" -} - -# mangle_pkgbuild $PKGBUILD [$sub_pkgrel] -# mangle $arch in PKBUILDs to contain i486, i586, i686 -# append $sub_pkgrel to the pkgrel - -mangle_pkgbuild() { - local PKGBUILD="$1" - local sub_pkgrel="$2" - - if [ -n "${sub_pkgrel}" ]; then - sub_pkgrel=".${sub_pkgrel}" - fi - - sed -i ' - /^arch=[^#]*any/!{ - /^arch=(/s/(/(i686 / - } - s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ - ' "${PKGBUILD}" -} - -# find_newest_of_git_revisions -# find newest git revision of the ones provided at stdin -# (assuming linear history) - -find_newest_of_git_revisions() { - local revisions - local repo - revisions=$(cat) - - if [ "$( - echo "${revisions}" | \ - wc -l - )" -eq 1 ]; then - - echo "${revisions}" - return - - fi - - repo=$( - find_repository_with_commit \ - "$( - echo "${revisions}" | \ - grep -xm1 '[0-9a-f]\{40\}' - )" - ) - - eval 'repo="${repo_paths__'"${repo}"'}"' - - echo "${revisions}" | \ - xargs -rn1 git -C "${repo}" rev-parse | \ - { - newest='' - while read -r current; do - if [ -z "${newest}" ] || \ - git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then - newest="${current}" - fi - done - echo "${newest}" - } -} - -# find_package_repository_to_package $package $git_repository $git_commit -# find the package repository a package from a given git repository -# belongs to - -find_package_repository_to_package() { - - local package="$1" - local git_repository="$2" - local git_commit="$3" - local repo_path - local repo - - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - if [ "${git_repository}" = 'archlinux32' ]; then - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- | \ - tar -t --wildcards "*/${package}/" | \ - cut -d/ -f1 | \ - sort -u - ) - else - repo=$( - git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \ - tar -t | \ - cut -d/ -f3 | \ - grep -vxF '' | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -v -- '-i686$' | \ - sed 's|-[^-]\+$||' | \ - sort -u - ) - fi - - if [ -z "${repo}" ]; then - return 1 - fi - - if [ "$( - echo "${repo}" | \ - wc -l - )" -ne 1 ]; then - return 1 - fi - - echo "${repo}" - -} - -# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel -# extract files found in the svn/git source directories -# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly - -extract_source_directory() { - - local git_repo="$1" - local rev="$2" - local mod_rev="$3" - local output="$4" - local sub_pkgrel="$5" - - if [ -n "${PKGBUILD}" ]; then - eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ - tar -x --strip-components=3 -C "${output}" - fi - - if [ -n "${PKGBUILD_mod}" ]; then - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ - tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ - true - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ - tar -Ox "${PKGBUILD_mod}" >> \ - "${output}/PKGBUILD" - fi - - mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}" - - # shellcheck disable=SC2016 - sed -i '/^\$Id\$$/d' "${output}/PKGBUILD" - -} - -# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository -# return a list of packages on the build list which are (run- / build- / check-time) -# dependencies of the given package - -find_dependencies_on_build_list() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - - { - cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - awk '{print $1 "." $2 "." $3 "." $4}' < \ - "${work_dir}/build-list" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|\.builds| - ' | \ - xargs -r cat | \ - sort -u - } | \ - sort | \ - uniq -d - -} - -# download_sources_by_hash $package $repository $git_revision $git_mod_revision -# try to download all sources by their hash into the current directory -# returns 0 if any source was downloaded and 1 otherwise - -download_sources_by_hash() { - - local package="$1" - local repository="$2" - local git_revision="$3" - local git_mod_revision="$4" - - local return_value=1 - local tmp_dir - local sum_type - local arch_suffix - - tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir) - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then - >&2 echo 'download_sources_by_hash: make_source_info failed.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - if ! [ -s "${tmp_dir}/.SRCINFO" ]; then - >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.' - rm -rf --one-file-system "${tmp_dir}" - return 1 - fi - - for arch_suffix in '' '_i686'; do - for sum_type in 'sha256sum' 'sha512sum'; do - grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed 's|^.* = ||' | \ - cat -n > \ - "${tmp_dir}/sums" - grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \ - sed ' - s|^.* = || - s|::.*$|| - s|.*/|| - ' | \ - cat -n > \ - "${tmp_dir}/urls" - if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then - join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \ - "${tmp_dir}/joined" - while read -r sum file; do - if [ "${sum}" = 'SKIP' ]; then - continue - fi - if echo "${sum} ${file}" | \ - ${sum_type} -c > /dev/null 2>&1; then - # the correct source is already there - continue - fi - if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then - mv "${tmp_dir}/transfer" "${file}" - return_value=0 - fi - done < \ - "${tmp_dir}/joined" - fi - done - done - - rm -rf --one-file-system "${tmp_dir}" - return ${return_value} - -} - -# expand_version $column_num -# add "0:" to version in $colum_num-th column if no ":" is there (epoch) -# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash) - -expand_version() { - local column_num - column_num="$1" - - sed ' - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/ - /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/ - ' -} - -# shrink_version $column_num -# remove "0:" from version in $colum_num-th column (epoch) -# remove "+0" from version in $colum_num-th column (git count/hash) - -shrink_version() { - local column_num - column_num="$1" - - sed ' - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ - s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/ - ' -} - -# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ] - -# Return (to stdout) the biggest subset A of the packages in $omega whose -# runtime dependencies in $omega \cup $keep are also in A - -# $all_builds and $all_depends either point to an empty file - then they will get -# filled with cached data for subsequent calls - or to the same files of a previous -# call - -# If non-empty, $force contains packages which are assumed to match the above -# condition without checking. - -# The arguments are names of files with one $package.$revision.$mod_revision.$repository -# per line. - -find_biggest_subset_of_packages() { - - ( # the new shell is intentional - omega="$1" - keep="$2" - all_builds="$3" - all_depends="$4" - if [ $# -eq 4 ]; then - force='/dev/null' - elif [ $# -eq 5 ]; then - force="$5" - else - >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#" - return 2 - fi - - if [ ! -s "${all_builds}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.builds | | - ' {} \; | \ - sort -k2,2 > \ - "${all_builds}" - fi - - if [ ! -s "${all_depends}" ]; then - find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \ - -exec sed ' - s|^|{} | - s|^\S\+/|| - s|\.run-depends | | - ' {} \; | \ - grep -v ' base$' | \ - sort -k2,2 > \ - "${all_depends}" - fi - - sort -u "${omega}" | \ - sponge "${omega}" - - temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - { - sort -u "${keep}" - cat "${force}" "${force}" - } | \ - sort | \ - uniq -u > \ - "${temp_dir}/keep.new" - touch "${temp_dir}/keep" - - while [ -s "${temp_dir}/keep.new" ]; do - cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ - sort -u | \ - sponge "${temp_dir}/keep" - - { - # we append all packages which are run-dependencies of keep-packages - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.run-depends:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_builds}" - - # we append all packages with run-dependencies on the keep-list - # to the keep-list - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${temp_dir}/keep" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.builds:| | - ' | \ - sort -u | \ - sort -k2,2 | \ - uniq -f1 | \ - join -1 2 -2 2 -o 2.1 - "${all_depends}" - } | \ - sort -u | \ - join -1 1 -2 1 -o 2.1 - "${omega}" | \ - sort -u > \ - "${temp_dir}/keep.new" - - # "new" is only what has not been there before and what is not forced - cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ - sort | \ - uniq -u | \ - sponge "${temp_dir}/keep.new" - done - - cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ - sort | \ - uniq -u - - ) - -} - -# sort_square_bracket_content $file -# sort the content of [] in $file, print to stdout - -sort_square_bracket_content() { - local file - local line - local token - local token_list - local rest - file="$1" - - while read -r line; do - printf '%s ' "${line}" | \ - tr ' ' '\n' | \ - while read -r token; do - if echo "${token}" | \ - grep -qF '['; then - printf '%s[' "${token%[*}" - token="${token##*[}" - token_list="${token%,}" - while ! echo "${token_list}" | \ - grep -qF ']'; do - read -r token - token_list=$( - printf '%s\n' \ - "${token_list}" \ - "${token%,}" - ) - done - rest="]${token_list#*]}" - token_list="${token_list%%]*}" - token=$( - printf '%s' "${token_list}" | \ - sort | \ - sed 's|$|,|' - printf '%s' "${rest}" - ) - fi - printf '%s\n' "${token}" - done | \ - tr '\n' ' ' | \ - sed ' - s|, ]|]|g - s| $|| - ' - printf '\n' - done < \ - "${file}" -} - -# smoothen_namcap_log $file -# remove unneccesary differences from namcap-logs: -# - remove architecture specific information -# - sort lines -# - sort content of square brackets - -smoothen_namcap_log() { - local file - file="$1" - # shellcheck disable=SC2016 - sort_square_bracket_content "${file}" | \ - sed ' - # normalize architecture specific information - s|i[34567]86|$ARCH|g - s|x86\([-_]64\)\?|$ARCH|g - # remove haskell hashes - s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g - ' | \ - sort | \ - sponge "${file}" -} - -# print_list_of_archaic_packages $source1 $source2 ... -# print a list of packages which have not been touched for a while, -# but which are still in the pipeline, e.g. in $source1, $source2 or ... - -print_list_of_archaic_packages() { - for source in "$@"; do - case "${source}" in - 'testing') - # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32 - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \ - -exec head -n1 {} \; | \ - "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin - # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated - # and no longer block other packages from being moved - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \ - sed ' - s|\.testing$|| - ' - ;; - 'build-list') - while read -r pkg rev mod_rev repo; do - git_repo=$( - find_repository_with_commit "${rev}" - ) - eval repo_path='"${repo_paths__'"${git_repo}"'}"' - commit_date=$( - git -C "${repo_path}" show -s --format=%ct "${rev}" - ) - mod_commit_date=$( - git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}" - ) - if [ "${mod_commit_date}" -gt "${commit_date}" ]; then - commit_date="${mod_commit_date}" - fi - # packages remaining longer than $max_package_age_build_list days on the build list - if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then - printf '%s %s %s %s\n' \ - "${pkg}" \ - "${rev}" \ - "${mod_rev}" \ - "${repo}" - fi - done < \ - "${work_dir}/build-list" - ;; - 'staging') - # packages remaining longer than $max_package_age_staging days in staging - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \ - sed ' - s|\.done$|| - ' - ;; - *) - >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}" - ;; - esac - done | \ - sort -u -} - -# receive_buglist $category -# receive the buglist from bugs.archlinux32.org for $category (e.g. 'Testing') -# and print it to stdout - -receive_buglist() { - local category - category="$1" - curl -LSs 'https://bugs.archlinux32.org/index.php?export_list=Export%20Tasklist' | \ - sed ' - 1 d - /^[^,]\+,"Packages: '"${category}"'",/ ! d - s/^\([^,]\+,\)\{3\}// - s/^\("[^"]\+"\|[^"][^,]*\),"// - s/".*$// - ' -} - -# next_sub_pkgrel $package $git_revision $mod_git_revision $repository -# giv out the next sub-pkgrel of the given package - -next_sub_pkgrel() { - ( # new shell is intentional - package="$1" - git_revision="$2" - mod_git_revision="$3" - repository="$4" - git_repo=$(find_git_repository_to_package_repository "${repository}") - - temp_dir=$(mktemp -d 'tmp.common-functions.next_sub_pkgrel.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" - extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${temp_dir}" '0' - - version=$( - cd "${temp_dir}" - makepkg --printsrcinfo | \ - sed -n ' - /^\s*\(epoch\|pkg\(name\|ver\|rel\)\) = /{s|^\s\+||;p} - ' | \ - sed ' - /^pkgname = /{ - s/^.*= // - w pkgnames - d - } - s|^epoch = \(.*\)$|1 \1:| - s|^pkgver = \(.*\)$|2 \1-| - s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1| - ' | \ - sort -k1n,1 | \ - sed ' - s|^[0-9] || - :a - N - s|\n[0-9] \(\S\+\)$|\1| - ta - ' - ) - sub_pkgrel=$( - ls_master_mirror 'i686/*/' | \ - sed -n "$( - sed ' - s/$/-'"${version}"'/ - s/\./\\./g - s/^/^/ - s/$/\\(\\.\\([0-9]\\+\\)\\)\\?-[^-]\\+$/ - s|^.*$|/\0/{ s/\0/\\2/; s/^$/0/; p; b; }| - ' "${temp_dir}/pkgnames" - )" | \ - sort -n | \ - tail -n1 - ) - if [ -z "${sub_pkgrel}" ]; then - echo '0' - else - echo "$((sub_pkgrel+1))" - fi - ) -} - -# modification_revision_link "${mod_rev}" "${repo}" "${pkg}" -# print the given modification revision possibly with a html link to github - -modification_revision_link() { - local mod_rev="$1" - local repo="$2" - local pkg="$3" - - if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then - printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \ - "${mod_rev}" \ - "${repo}" \ - "${pkg}" \ - "${mod_rev}" - else - printf '%s\n' \ - "${mod_rev}" - fi -} - -# trigger_mirror_refreshs -# trigger a refresh of capable tier 1 mirrors (as backup for master mirror) - -trigger_mirror_refreshs() { - local tmp_file - - tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir) - date '+%s' > \ - "${tmp_file}" - ${master_mirror_rsync_command} \ - "${tmp_file}" \ - "${master_mirror_rsync_directory}/lastupdate" - rm "${tmp_file}" - for trigger_url in ${mirror_refresh_trigger_urls}; do - screen -S trigger-mirror-update -d -m curl -L "${trigger_url}" - done -} diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support new file mode 100755 index 0000000..404a2b9 --- /dev/null +++ b/bin/copy-to-build-support @@ -0,0 +1,187 @@ +#!/bin/sh + +# copy the given package(s) into build-support + +# shellcheck source=conf/default.conf +. "${0%/*}/../conf/default.conf" + +# shellcheck disable=SC2016 +usage() { + >&2 echo '' + >&2 echo 'copy-to-build-support [options] package-list:' + >&2 echo ' copy the packages listed in package-list into [build-support]' + >&2 echo '' + >&2 echo 'possible options:' + >&2 echo ' -h|--help:' + >&2 echo ' Show this help and exit.' + >&2 echo ' -w|--wait:' + >&2 echo ' Wait for lock if necessary.' + [ -z "$1" ] && exit 1 || exit "$1" +} + +eval set -- "$( + getopt -o hw \ + --long help \ + --long wait \ + -n "$(basename "$0")" -- "$@" || \ + echo usage + )" + +wait_for_lock='-n' + +while true +do + case "$1" in + -h|--help) + usage 0 + ;; + -w|--wait) + wait_for_lock='' + ;; + --) + shift + break + ;; + *) + >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' + exit 42 + ;; + esac + shift +done + +if [ "$#" -ne 1 ]; then + >&2 echo 'No package-list was given.' + usage +fi + +exec 9> "${sanity_check_lock_file}" +flock -s ${wait_for_lock} 9 + +exec 8> "${package_database_lock_file}" +flock ${wait_for_lock} 8 + +tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX") +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + +to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' +sed -n ' + s/.\+/\0 \0/ + T + s/\.pkg\.tar\.xz$// + s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ + s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ + s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ + p +' "$1" | \ + while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do + + # shellcheck disable=SC2016 + id=$( + { + printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" + printf ' `architectures`.`name`=from_base64("%s")' \ + "$(printf '%s' "${architecture}" | base64 -w0)" + printf ' LIMIT 1;\n' + } | \ + mysql_run_query | \ + tr '\t' ' ' + ) + if [ -z "${id}" ]; then + continue + fi + repository="${id#* }" + id="${id%% *}" + + printf '%s\n' "${package}" >> \ + "${tmp_dir}/packages" + for suffix in '' '.sig'; do + printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \ + "${repository}" \ + "${package}" \ + "${suffix}" \ + "${package}" \ + "${suffix}" + done >> \ + "${tmp_dir}/sftp-command" + printf '%s/i686/%s/%s\n' \ + "${master_mirror_rsync_directory}" \ + "${repository}" \ + "${package}" | \ + sed ' + p + s/$/.sig/ + ' >> \ + "${tmp_dir}/to-copy" + + # shellcheck disable=SC2016 + { + printf 'INSERT IGNORE INTO `binary_packages`' + printf ' (`repository`' + # shellcheck disable=SC2086 + printf ',`%s`' ${to_copy} + printf ')' + printf ' SELECT' + printf ' (SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="build-support")' + # shellcheck disable=SC2086 + printf ',`binary_packages`.`%s`' ${to_copy} + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + printf ' WHERE' + printf ' `binary_packages`.`id`=%s;\n' \ + "${id}" + } >> \ + "${tmp_dir}/mysql-command" + done + +${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \ + "${master_mirror_rsync_directory}/i686/build-support/build-support.files."* \ + "${tmp_dir}/" + +if [ -s "${tmp_dir}/to-copy" ]; then + mkdir "${tmp_dir}/transit/" + # shellcheck disable=SC2046 + ${master_mirror_rsync_command} \ + $(cat "${tmp_dir}/to-copy") \ + "${tmp_dir}/transit/" + repo-add "${tmp_dir}/build-support.db.tar.gz" \ + "${tmp_dir}/transit/"*".pkg.tar.xz" +fi + +if [ -s "${tmp_dir}/sftp-command" ]; then + ${master_mirror_sftp_command} < \ + "${tmp_dir}/sftp-command" +fi + +${master_mirror_rsync_command} \ + "${tmp_dir}/build-support.db."* \ + "${tmp_dir}/build-support.files."* \ + "${master_mirror_rsync_directory}/i686/build-support/" + +if [ -s "${tmp_dir}/mysql-command" ]; then + mysql_run_query < \ + "${tmp_dir}/mysql-command" +fi + +if [ -s "${tmp_dir}/packages" ]; then + while read -r package; do + remove_old_package_versions 'i686' 'build-support' "${package}" + done < \ + "${tmp_dir}/packages" +fi + +if [ -w "$1" ]; then + cat "${tmp_dir}/packages" > \ + "$1" +fi diff --git a/bin/db-update b/bin/db-update index fe45f41..6722f24 100755 --- a/bin/db-update +++ b/bin/db-update @@ -4,27 +4,13 @@ # additionally tested packages from testing to the respective stable # repository (if possible [1]) -# 1] Condition for moving a package A from staging (testing) to -# testing (stable) is that: -# a) nothing on the build-list run-depends on A and -# b) no done package B (in a not-more stable repository) which is -# not being moved run-depends on A +# The condition [1] is explained in the stored function +# calculate_maximal_moveable_set which is created in bin/bootsrap-mysql # TODO: separate locks for staging, testing (and stable) -# TODO: handle deletion of parts of a split package - -# TODO: handle dependencies of parts of a split package separately - -# TODO: improve tracking of packages: -# track packages in stable, too -# know _exactly_ in which repository the packages are - -# TODO: fully integrate database (in other scripts too) - e.g. rely on -# info in db instead of state-files - -# TODO: add possibility to manually clone a (binary) package (e.g. -# for build-support) - this should also be possible via email +# TODO: we should delete more packages than just the ones in repositories +# where we move to (think of [extra] -> [community]) # shellcheck disable=SC2039 # shellcheck source=conf/default.conf @@ -34,369 +20,40 @@ usage() { >&2 echo '' >&2 echo 'db-update [options] [packages]:' - >&2 echo ' move possible packages from staging to testing.' >&2 echo ' move tested packages from testing to stable.' + >&2 echo ' move possible packages from staging to testing.' >&2 echo '' >&2 echo 'possible options:' - >&2 echo ' -b|--block: If necessary, wait for lock blocking.' - >&2 echo ' -h|--help: Show this help and exit.' - >&2 echo ' -n|--no-action: Only print what would be moved.' - >&2 echo ' -s|--stabilize $package:' - >&2 echo ' Assume, package $package can be stabilized.' - >&2 echo ' -u|--unstage $package:' - >&2 echo ' Assume, package $package can be unstaged.' + >&2 echo ' -b|--block:' + >&2 echo ' If necessary, wait for lock blocking.' + >&2 echo ' -f|--force $package-id:' + >&2 echo ' Force movement of Package with given id and move nothing else.' + >&2 echo ' -h|--help:' + >&2 echo ' Show this help and exit.' + >&2 echo ' -n|--no-action:' + >&2 echo ' Only print what would be moved.' + >&2 echo ' -p|--progressive:' + >&2 echo ' Move forward any package which replaces no package whose' + >&2 echo ' dependencies are all available somewhere.' + >&2 echo ' Note, that this _may_ move _less_ packages.' [ -z "$1" ] && exit 1 || exit "$1" } -# move_packages file with one "$package $from_repository $to_repository" per line -# the existence of a directory $tmp_dir is assumed - -move_packages() { - - if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then - >&2 echo 'move_packages: No tmp_dir provided.' - exit 2 - fi - - local package - local from_repo - local to_repo - local from_ending - local rm_ending - local to_ending - local repo - local part - local dummynator - local file - - if [ -e "${tmp_dir:?}/tmp" ]; then - rm -rf --one-file-system "${tmp_dir:?}/tmp" - fi - mkdir "${tmp_dir}/tmp" - - touch "${tmp_dir}/tmp/repos" - touch "${tmp_dir}/tmp/packages" - touch "${tmp_dir}/tmp/master-mirror-listing" - mkdir "${tmp_dir}/tmp/transit" - - if ${no_action}; then - dummynator='echo' - else - dummynator='' - fi - - ls_master_mirror 'i686' | \ - while read -r repo; do - ls_master_mirror "i686/${repo}" | \ - sed "s|^|i686/${repo}/|" >> \ - "${tmp_dir}/tmp/master-mirror-listing" - done - - while read -r package from_repo to_repo; do - if [ -z "${package}" ]; then - continue - fi - - if ${no_action}; then - printf \ - 'move "%s" from "%s" to "%s"\n' \ - "${package}" \ - "${from_repo}" \ - "${to_repo}" - fi - - echo "${package}" >> \ - "${tmp_dir}/tmp/packages" - - if echo "${from_repo}" | \ - grep -q 'staging$' && \ - echo "${to_repo}" | \ - grep -q 'testing$'; then - from_ending='done' - rm_ending='test\(ing\|ed\)' - to_ending='testing' - elif echo "${from_repo}" | \ - grep -q 'testing$' && \ - ! echo "${to_repo}" | \ - grep -q 'testing$\|staging$'; then - from_ending='tested' - rm_ending='' - to_ending='' - else - >&2 printf 'move_packages: Cannot move package "%s" from "%s" to "%s".\n' "${package}" "${from_repo}" "${to_repo}" - exit 2 - fi - - echo "${from_repo}" > \ - "${tmp_dir}/tmp/${package}.from_repo" - echo "${to_repo}" > \ - "${tmp_dir}/tmp/${package}.to_repo" - echo "${from_ending}" > \ - "${tmp_dir}/tmp/${package}.from_ending" - echo "${rm_ending}" > \ - "${tmp_dir}/tmp/${package}.rm_ending" - echo "${to_ending}" > \ - "${tmp_dir}/tmp/${package}.to_ending" - - if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then - >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}" - exit 2 - fi - - cp \ - "${work_dir}/package-states/${package}.${from_ending}" \ - "${tmp_dir}/tmp/${package}.parts" - - sed \ - 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \ - "${tmp_dir}/tmp/${package}.parts" > \ - "${tmp_dir}/tmp/${package}.parts_names" - - sed \ - 'p;s|$|.sig|' \ - "${tmp_dir}/tmp/${package}.parts" > \ - "${tmp_dir}/tmp/${package}.parts_and_signatures" - - while read -r part; do - if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then - >&2 printf \ - 'move_packages: Cannot find file "%s", part of package "%s".\n' \ - "i686/${from_repo}/${part}" \ - "${package}" - exit 2 - fi - done < \ - "${tmp_dir}/tmp/${package}.parts_and_signatures" - - mkdir -p "${tmp_dir}/tmp/${from_repo}" - mkdir -p "${tmp_dir}/tmp/${to_repo}" - - repos=$( - # shellcheck disable=SC2046 - printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \ - sort -u - ) - echo "${repos}" > \ - "${tmp_dir}/tmp/repos" - - done < \ - "$1" - - if ${no_action}; then - find "${tmp_dir}/tmp" -type f | \ - while read -r file; do - if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] || - [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then - echo "'${file}'" - else - echo "${file}:" - sed 's|^|<<|;s|$|>>|' "${file}" - fi - echo - done - fi - - # receive the *.db.tar.gz's and *.files.tar.gz's - - while read -r repo; do - - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \ - "${tmp_dir}/tmp/${repo}/" - - # add and remove the packages locally - - if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then - - # shellcheck disable=SC2046 - repo-remove -q \ - "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \ - $( - grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \ - sed ' - s|\.from_repo$|.parts_names| - ' | \ - xargs -rn1 cat - ) - fi - - if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then - grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \ - sed ' - s|\.to_repo$|| - ' | \ - while read -r package; do - while read -r part; do - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \ - "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \ - "${tmp_dir}/tmp/transit/" - repo-add -q \ - "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \ - "${tmp_dir}/tmp/transit/${part}" - rm \ - "${tmp_dir}/tmp/transit/${part}" \ - "${tmp_dir}/tmp/transit/${part}.sig" - done < \ - "${package}.parts" - done - fi - - done < "${tmp_dir}/tmp/repos" - - if ${no_action}; then - find "${tmp_dir}/tmp" -type f - fi - - # move the packages remotely via sftp - - { - while read -r package; do - - if [ -z "${package}" ]; then - continue - fi - - while read -r part; do - if [ -z "${part}" ]; then - continue - fi - printf \ - 'rename "%s" "%s"\n' \ - "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \ - "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}" - done < \ - "${tmp_dir}/tmp/${package}.parts_and_signatures" - - done < \ - "${tmp_dir}/tmp/packages" - echo 'quit' - } | \ - if ${no_action}; then - sed 's|^|sftp: |' - else - ${master_mirror_sftp_command} - fi - - # and push our local *.db.tar.gz via rsync - - while read -r repo; do - - # shellcheck disable=SC2086 - ${dummynator} ${master_mirror_rsync_command} \ - "${tmp_dir}/tmp/${repo}/${repo}.db."* \ - "${tmp_dir}/tmp/${repo}/${repo}.files."* \ - "${master_mirror_rsync_directory}/i686/${repo}/" - - done < \ - "${tmp_dir}/tmp/repos" - - while read -r package; do - - # then we can safely remove old versions - - while read -r part; do - ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}" - done < \ - "${tmp_dir}/tmp/${package}.parts" - - # and update the state files - - from_ending=$( - cat "${tmp_dir}/tmp/${package}.from_ending" - ) - rm_ending=$( - cat "${tmp_dir}/tmp/${package}.rm_ending" - ) - to_ending=$( - cat "${tmp_dir}/tmp/${package}.to_ending" - ) - - if [ -z "${to_ending}" ]; then - ${dummynator} rm \ - "${work_dir}/package-states/${package}.${from_ending}" - else - # remove old state files of $package with ending $rm_ending - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${package%.*.*.*}")"'\(\.[^.]\+\)\{3\}\.'"${rm_ending}" \ - -execdir ${dummynator} rm {} \; - ${dummynator} mv \ - "${work_dir}/package-states/${package}.${from_ending}" \ - "${work_dir}/package-states/${package}.${to_ending}" - fi - - done < \ - "${tmp_dir}/tmp/packages" - - # shellcheck disable=SC2016 - while read -r package; do - while read -r part; do - printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s"))' \ - "$( - base64_encode_each < \ - "${tmp_dir}/tmp/${package}.to_repo" - )" - printf ' WHERE' - printf ' `repositories`.`name`=from_base64("%s")' \ - "$( - base64_encode_each < \ - "${tmp_dir}/tmp/${package}.from_repo" - )" - printf '%s\n' "${part}" | \ - sed ' - s/\.pkg\.tar\.xz$// - s/-\([^-.]\+\)\(-[^-]\+\)$/-\1.0\2/ - s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/ - s/^\(.\+\)-\([^-:]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/\1\n\2\n\3\n\4\n\5\n\6/ - ' | \ - base64_encode_each | \ - tr '\n' ' ' | \ - sed ' - s,\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) $,'"$( - printf ' AND `binary_packages`.`%s`=from_base64("%s")' \ - 'pkgname' '\1' \ - 'epoch' '\2' \ - 'pkgver' '\3' \ - 'pkgrel' '\4' \ - 'sub_pkgrel' '\5' - printf ' AND `architectures`.`name`=from_base64("\\6")' - )"', - ' - printf ';\n' - done < \ - "${tmp_dir}/tmp/${package}.parts" - done < \ - "${tmp_dir}/tmp/packages" | \ - if ${no_action}; then - sed 's|^|mysql: |' - else - ${mysql_command} - fi - - ${dummynator} trigger_mirror_refreshs - - rm -rf --one-file-system "${tmp_dir:?}/tmp" - -} - eval set -- "$( - getopt -o bf:hns:u: \ + getopt -o bf:hnp \ --long block \ - --long from: \ + --long force \ --long help \ --long no-action \ - --long stabilize: \ - --long unstage: \ + --long progressive \ -n "$(basename "$0")" -- "$@" || \ echo usage )" block_flag='-n' no_action=false +progressive=false +force_ids='' while true do @@ -404,19 +61,22 @@ do -b|--block) block_flag='' ;; + -f|--force) + shift + force_ids=$( + printf '%s' "$1" | \ + base64 -w0 + printf '\n%s' "${force_ids}" + ) + ;; -h|--help) usage 0 ;; -n|--no-action) no_action=true ;; - -s|--stabilze) - shift - packages_to_force_stabilize="${packages_to_force_stabilize} $1" - ;; - -u|--unstage) - shift - packages_to_force_unstage="${packages_to_force_unstage} $1" + -p|--progressive) + progressive=true ;; --) shift @@ -435,237 +95,306 @@ if [ $# -ne 0 ]; then usage fi +if ${progressive} && \ + [ -n "${force_ids}" ]; then + >&2 echo 'db-update: conflicting arguments' + usage +fi + if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit fi -tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX") -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - -for package in ${packages_to_force_stabilize}; do - # some sanity checks - if [ ! -f "${work_dir}/package-states/${package}.tested" ] && \ - [ ! -f "${work_dir}/package-states/${package}.testing" ] && \ - [ ! -f "${work_dir}/package-states/${package}.done" ] && \ - ! tr ' ' '.' < \ - "${work_dir}/build-list" | \ - grep -qxF "${package}"; then - >&2 printf 'Package "%s" is not in testing, staging or on the build list!\n' "${package}" - exit 2 - fi -done +# Create tmp_dir, lock and trap. -for package in ${packages_to_force_unstage}; do - # some sanity checks - if [ ! -f "${work_dir}/package-states/${package}.done" ] && \ - ! tr ' ' '.' < \ - "${work_dir}/build-list" | \ - grep -qxF "${package}"; then - >&2 printf 'Package "%s" is not in staging or on the build list!\n' "${package}" - exit 2 - fi -done - -# Create a lock file and a trap. - -exec 9> "${build_list_lock_file}" +exec 9> "${package_database_lock_file}" if ! flock ${block_flag} 9; then - >&2 echo 'come back (shortly) later - I cannot lock build list.' - exit 0 -fi - -exec 8> "${package_database_lock_file}" -if ! flock ${block_flag} 8; then >&2 echo 'come back (shortly) later - I cannot lock package database.' exit 0 fi -exec 7> "${sanity_check_lock_file}" -if ! flock -s ${block_flag} 7; then +exec 8> "${sanity_check_lock_file}" +if ! flock -s ${block_flag} 8; then >&2 echo 'come back (shortly) later - sanity-check currently running.' exit 0 fi -clean_up_lock_file() { - rm -f "${package_database_lock_file}" "${build_list_lock_file}" - rm -rf --one-file-system "${tmp_dir}" -} - -trap clean_up_lock_file EXIT - -# sanity check - -for ending in 'done' 'tested'; do - errors=$( - find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - sort | \ - uniq -d - ) - if [ -n "${errors}" ]; then - >&2 echo 'Removing duplicates not yet implemented:' - >&2 echo "${errors}" - exit 42 - fi -done - -# packages which are done +tmp_dir=$(mktemp -d "${work_dir}/tmp.db-update.XXXXXXXXXX") +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \ - sed ' - s|\.done$|| - ' | \ - sort -u > \ - "${tmp_dir}/done-packages" +export TMPDIR="${tmp_dir}" -# packages still on the build-list +for source_stability in 'testing' 'staging'; do + find "${tmp_dir}" -mindepth 1 -delete -grep -vxF 'break_loops' "${work_dir}/build-list" | \ - tr ' ' '.' | \ - sort -u > \ - "${tmp_dir}/build-list-packages" + # shellcheck disable=SC2016 + { + if [ -n "${force_ids}" ]; then + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \ + 'moveable' 'replaced' + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' VALUES' + # shellcheck disable=SC2086 + printf '(from_base64("%s"),NULL),' \ + ${force_ids} | \ + sed 's/,$/;\n/' + printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' WHERE `repository_stabilities`.`name`!="%s";\n' \ + "${source_stability}" + printf 'UPDATE `moveable_binary_packages`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' + printf ' SET `moveable_binary_packages`.`to_repository`=`repository_moves`.`to_repository`;\n' + printf 'DELETE FROM `moveable_binary_packages` WHERE `moveable_binary_packages`.`to_repository` IS NULL;\n' + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' + printf ' SELECT `binary_packages`.`id`,`moveable_binary_packages`.`id`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` AS `subst_bp` ON `moveable_binary_packages`.`id`=`subst_bp`.`id`' + printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`' + printf ' AND `binary_packages`.`repository`=`moveable_binary_packages`.`to_repository`;\n' + elif ${progressive}; then + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages`;\n' \ + 'moveable' 'replaced' + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moveable_binary_packages` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' + printf ' SELECT `binary_packages`.`id`,`subst_bp`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`to_repository`=`binary_packages`.`repository`' + printf ' JOIN `binary_packages` AS `subst_bp`' + printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`' + printf ' AND `repository_moves`.`from_repository`=`subst_bp`.`repository`' + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + mysql_join_repositories_repository_stabilities 'subst_r' 'subst_rs' + printf ' AND `subst_rs`.`name`="%s"' \ + "${source_stability}" + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' + + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' SELECT `replaced_binary_packages`.`replaced_by`,`binary_packages`.`repository`' + printf ' FROM `replaced_binary_packages`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`' + printf ';\n' -find "${work_dir}/package-infos" -name '*.groups' \ - -exec grep -qxF 'base' {} \; \ - -printf '%f\n' | \ - sed ' - s|\.groups$|| - ' | \ - sort -u > \ - "${tmp_dir}/base-packages" + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + mysql_join_repositories_repository_stabilities + printf ' AND `repository_stabilities`.`name`="%s"' \ + "${source_stability}" + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `binary_packages` AS `repl_bp`' + printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`' + printf ' AND `repl_bp`.`repository`=`repository_moves`.`to_repository`' + printf ');\n' + else + printf 'CALL calculate_maximal_moveable_set("%s");\n' \ + "${source_stability}" + fi -{ - # shellcheck disable=SC2086 - printf '%s\n' ${packages_to_force_unstage} - print_list_of_archaic_packages 'build-list' -} > \ - "${tmp_dir}/force-unstage-packages" - ->&2 printf 'calculate what packages should be unstaged ...' -find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-unstage-packages" > \ - "${tmp_dir}/unstage-packages" ->&2 printf ' ok.\n' - -# no base packages on the build list anymore? -if [ -z "$( - join -j 1 \ - "${tmp_dir}/base-packages" \ - "${tmp_dir}/build-list-packages" - )" ]; then - - >&2 echo 'db-update unstage: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages' - for s in "${tmp_dir}/all-builds" "${tmp_dir}/all-depends"; do - sed '/ base$/d' "${s}" > \ - "${s}.no-base" - done + printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `moveable_binary_packages`.`to_repository`' + printf ' FROM `moveable_binary_packages`;\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `binary_packages`.`repository`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`;\n' + printf 'INSERT IGNORE INTO `rps` (`id`)' + printf ' SELECT `binary_packages`.`repository`' + printf ' FROM `replaced_binary_packages`' + printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`;\n' + + printf 'SELECT "repositories",`repositories`.`name`' + printf ' FROM `repositories`' + printf ' JOIN `rps` ON `rps`.`id`=`repositories`.`id`;\n' + + printf 'SELECT "mv.id",`moveable_binary_packages`.`id`,`moveable_binary_packages`.`to_repository`' + printf ' FROM `moveable_binary_packages`;\n' + + printf 'SELECT "mv",' + mysql_package_name_query + printf ',`repositories`.`name`,`new_repo`.`name`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` ON `moveable_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ' JOIN `repositories` AS `new_repo` ON `new_repo`.`id`=`moveable_binary_packages`.`to_repository`' + printf ';\n' + + printf 'SELECT "rm.id",`replaced_binary_packages`.`id`' + printf ' FROM `replaced_binary_packages`;\n' + + printf 'SELECT "rm",' + mysql_package_name_query + printf ',`repositories`.`name`' + printf ' FROM `replaced_binary_packages`' + printf ' JOIN `binary_packages` ON `replaced_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ';\n' + } | \ + mysql_run_query | \ + tr '\t' ' ' | \ + grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \ + while read -r what content; do + printf '%s\n' "${content}" >> \ + "${tmp_dir}/${what}" + done - >&2 printf 'calculate what packages should be unstaged ...' - find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-unstage-packages" > \ - "${tmp_dir}/unstage-packages" - >&2 printf ' ok.\n' + if [ ! -s "${tmp_dir}/repositories" ]; then + >&2 printf 'Nothing to move from %s.\n' "${source_stability}" + continue + fi -fi + touch \ + "${tmp_dir}/mv" \ + "${tmp_dir}/mv.id" \ + "${tmp_dir}/rm" \ + "${tmp_dir}/rm.id" -{ # shellcheck disable=SC2086 - printf '%s\n' ${packages_to_force_stabilize} - print_list_of_archaic_packages 'build-list' 'staging' 'testing' -} > \ - "${tmp_dir}/force-stabilize-packages" - -# calculate what packages should be stabilized - -{ - cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" - find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -printf '%f\n' | \ - sed 's|\.testing$||' -} | \ - sort -u > \ - "${tmp_dir}/keep-packages" - -find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.tested' -printf '%f\n' | \ - sed 's|\.tested$||' > \ - "${tmp_dir}/stabilize-packages" - -# no base packages on the build list or in staging anymore? -if [ -z "$( - cat \ - "${tmp_dir}/build-list-packages" \ - "${tmp_dir}/done-packages" | \ - sort -u | \ - join -j 1 \ - "${tmp_dir}/base-packages" \ - - - )" ]; then - - >&2 echo 'db-update stabilize: we pretend, the group "base" does not exist, so we only fetch "direct" dependencies on base-packages' - >&2 printf 'calculate what packages should be stabilized ...' - find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds.no-base" "${tmp_dir}/all-depends.no-base" "${tmp_dir}/force-stabilize-packages" | \ - sponge "${tmp_dir}/stabilize-packages" - >&2 printf ' ok.\n' - -else - - >&2 printf 'calculate what packages should be stabilized ...' - find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-builds" "${tmp_dir}/all-depends" "${tmp_dir}/force-stabilize-packages" | \ - sponge "${tmp_dir}/stabilize-packages" - >&2 printf ' ok.\n' - -fi + for s in "${tmp_dir}/"*; do + sort -u "${s}" | \ + sponge "${s}" + done -# unlock build list + # receive the repository databases from the master mirror + mkdir "${tmp_dir}/dbs" + while read -r repo; do + mkdir "${tmp_dir}/dbs/${repo}" + # shellcheck disable=SC2086 + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \ + "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \ + "${tmp_dir}/dbs/${repo}/" + tar -Oxzf "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" --wildcards '*/desc' | \ + sed -n ' + /^%FILENAME%$/{ + N + s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/ + T + p + } + ' + done < \ + "${tmp_dir}/repositories" | \ + while read -r old new; do + for file in 'rm' 'mv'; do + sed -i ' + s/\(\s\|^\)'"$(str_to_regex "${old}")"'\(\s\|$\)/\1'"${new}"'\2/ + ' "${tmp_dir}/${file}" + done + done -rm -f "${build_list_lock_file}" -flock -u 9 + # remove to-be-deleted packages + # shellcheck disable=SC2094 + cut -d' ' -f2 < \ + "${tmp_dir}/rm" | \ + sort -u | \ + while read -r repo; do + grep " $(str_to_regex "${repo}")\$" "${tmp_dir}/rm" | \ + sed ' + s/\(-[^-]\+\)\{3\} \S\+$// + ' | \ + xargs -r repo-remove -q "${tmp_dir}/dbs/${repo}/${repo}.db.tar.gz" + done -clean_up_lock_file() { - rm -rf --one-file-system "${tmp_dir}" - rm -f "${package_database_lock_file}" -} + # copy and delete moved packages + # shellcheck disable=SC2094 + cut -d' ' -f2,3 < \ + "${tmp_dir}/mv" | \ + sort -u | \ + while read -r from_repo to_repo; do + grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \ + sed ' + s/-[^-]\+ \S\+ \S\+$// + ' | \ + xargs -r "${base_dir}/bin/repo-copy" \ + "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" \ + "${tmp_dir}/dbs/${to_repo}/${to_repo}.db.tar.gz" + grep " $(str_to_regex "${from_repo}") $(str_to_regex "${to_repo}")\$" "${tmp_dir}/mv" | \ + sed ' + s/\(-[^-]\+\)\{3\} \S\+ \S\+$// + ' | \ + xargs -r repo-remove -q \ + "${tmp_dir}/dbs/${from_repo}/${from_repo}.db.tar.gz" + done -# testing -> stable + # move the packages remotely via sftp + { + sed ' + s,^\(\S\+\) \(\S\+\)$,rm "i686/\2/\1"\nrm "i686/\2/\1.sig", + ' "${tmp_dir}/rm" + sed ' + s,^\(\S\+\) \(\S\+\) \(\S\+\)$,rename "i686/\2/\1" "i686/\3/\1"\nrename "i686/\2/\1.sig" "i686/\3/\1.sig", + ' "${tmp_dir}/mv" + echo 'quit' + } | \ + if ${no_action}; then + sed 's|^|sftp: |' + else + ${master_mirror_sftp_command} + fi -while read -r package; do - if [ -z "${package}" ]; then - continue - fi - printf '%s %s %s\n' \ - "${package}" \ - "$(official_or_community "${package}" 'testing')" \ - "$(repository_of_package "${package}")" -done < \ - "${tmp_dir}/stabilize-packages" | \ - sponge "${tmp_dir}/stabilize-packages" - -# staging -> testing - -while read -r package; do - if [ -z "${package}" ]; then + if ${no_action}; then continue fi - printf '%s %s %s\n' \ - "${package}" \ - "$(official_or_community "${package}" 'staging')" \ - "$(official_or_community "${package}" 'testing')" -done < \ - "${tmp_dir}/unstage-packages" | \ - sponge "${tmp_dir}/unstage-packages" - -if [ -s "${tmp_dir}/stabilize-packages" ]; then - >&2 printf 'move packages from *testing/ to the stable repos ...' - move_packages "${tmp_dir}/stabilize-packages" - >&2 printf ' ok.\n' -else - >&2 printf 'Nothing to move from *testing to the stable repos.\n' -fi -if [ -s "${tmp_dir}/unstage-packages" ]; then - >&2 printf 'move packages from *staging to *testing ...' - move_packages "${tmp_dir}/unstage-packages" - >&2 printf ' ok.\n' -else - >&2 printf 'Nothing to move from *staging to *testing.\n' -fi + # and push our local *.db.tar.gz via rsync + while read -r repo; do + # shellcheck disable=SC2086 + ${master_mirror_rsync_command} \ + "${tmp_dir}/dbs/${repo}/${repo}.db."* \ + "${tmp_dir}/dbs/${repo}/${repo}.files."* \ + "${master_mirror_rsync_directory}/i686/${repo}/" + done < \ + "${tmp_dir}/repositories" + + # shellcheck disable=SC2016 + { + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `moved_binary_packages` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \ + "${tmp_dir}/mv.id" 'moved_binary_packages' \ + "${tmp_dir}/rm.id" 'replaced_binary_packages' + printf 'DELETE `binary_packages` FROM `binary_packages`' + printf ' JOIN `replaced_binary_packages` ON `binary_packages`.`id`=`replaced_binary_packages`.`id`;\n' + printf 'UPDATE `binary_packages`' + printf ' JOIN `moved_binary_packages` ON `binary_packages`.`id`=`moved_binary_packages`.`id`' + printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' + } | \ + mysql_run_query +done -clean_up_lock_file +trigger_mirror_refreshs diff --git a/bin/delete-packages b/bin/delete-packages index 26d323a..4343bc7 100755 --- a/bin/delete-packages +++ b/bin/delete-packages @@ -13,7 +13,7 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -# TODO: finish this +# TODO: delete other to-be-deleted packages if asked to do so # shellcheck disable=SC2016 usage() { @@ -69,18 +69,15 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX") -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - # Create a lock file and a trap. if ! ${no_action}; then - exec 9> "${build_list_lock_file}" - if ! flock ${block_flag} 9; then - >&2 echo 'come back (shortly) later - I cannot lock build list.' - exit 0 - fi +# exec 9> "${build_list_lock_file}" +# if ! flock ${block_flag} 9; then +# >&2 echo 'come back (shortly) later - I cannot lock build list.' +# exit 0 +# fi exec 8> "${package_database_lock_file}" if ! flock ${block_flag} 8; then @@ -96,260 +93,158 @@ if ! ${no_action}; then fi -clean_up_lock_file() { - if ! ${no_action}; then - rm -f "${package_database_lock_file}" "${build_list_lock_file}" - fi - rm -rf --one-file-system "${tmp_dir}" -} - -trap clean_up_lock_file EXIT - -cp \ - "${work_dir}/deletion-list" \ - "${work_dir}/build-list" \ - "${tmp_dir}/" - -all_repos="${standalone_package_repositories} ${stable_package_repositories} ${testing_package_repositories} ${staging_package_repositories}" -all_repos="core" - -for repo in ${all_repos}; do - mkdir "${tmp_dir}/${repo}" - ${master_mirror_rsync_command} \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \ - "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \ - "${tmp_dir}/${repo}/" -done - -for repo in ${all_repos}; do - tar -C "${tmp_dir}/${repo}" -xzf "${tmp_dir}/${repo}/${repo}.db.tar.gz" -done +tmp_dir=$(mktemp -d "${work_dir}/tmp.delete-packages.XXXXXXXXXX") +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -find "${tmp_dir}" -mindepth 3 -maxdepth 3 -name 'desc' -exec \ +# shellcheck disable=SC2016 +{ + printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'INSERT IGNORE INTO `to_deletes`' + printf ' SELECT `binary_packages`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + printf ' WHERE `binary_packages`.`is_to_be_deleted`' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + mysql_join_repositories_repository_stabilities 'prov_r' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND `repository_stabilities`.`name` NOT IN ("forbidden","virtual")' + printf ');\n' + + printf ' SELECT DISTINCT "repo",`repositories`.`name`' + printf ' FROM `to_deletes`' + printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + printf ';\n' + + printf ' SELECT "package",`repositories`.`name`,`binary_packages`.`pkgname`' + printf ' FROM `to_deletes`' + printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + printf ';\n' + + printf ' SELECT "package-file",`repositories`.`name`,' + mysql_package_name_query + printf ' FROM `to_deletes`' + printf ' JOIN `binary_packages` ON `to_deletes`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ';\n' + + printf ' SELECT "package-id",`to_deletes`.`id`' + printf ' FROM `to_deletes`' + printf ';\n' +} | \ + mysql_run_query | \ sed ' - /^%DEPENDS%$/{ - s/.*// - :dep_loop - N - /\n$/{ - s@\(^\|\n\)\(.\)@\1depends \2@g - bend - } - bdep_loop + y/\t/ / + /^repo /{ + s/^\S\+ // + w '"${tmp_dir}"'/repositories + d } - /^%PROVIDES%$/{ - s/.*// - :pro_loop - N - /\n$/{ - s@\(^\|\n\)\(.\)@\1provides \2@g - bend - } - bpro_loop + /^package /{ + s/^\S\+ // + w '"${tmp_dir}"'/packages + d } - d - :end - s/[<>=]\S*\($\|\n\)/\1/g - s#\(^\|\n\)\(.\)#\1{} \2#g - ' {} \; \ - -printf '%p provides %p\n' | \ - sed ' - /^$/d - s|^\S\+/\([^/ ]\+\)/\([^/ ]\+\)\(-[^-/ ]\+\)\{2\}/desc |\1 \2 | - s| \S\+/\([^/]\+\)\(-[^/-]\+\)\{2\}/desc$| \1| - ' | \ - awk '{print $3 " " $1 " " $2 " " $4}' | \ - sed -n ' - /^provides /{ - s/^provides // - w '"${tmp_dir}"'/db.provides - b + /^package-file /{ + s/^\S\+ // + s, ,/, + w '"${tmp_dir}"'/package-files + s/$/.sig/ + w '"${tmp_dir}"'/package-files + d } - /^depends /{ - s/^depends // - w '"${tmp_dir}"'/db.depends - b + /^package-id /{ + s/^\S\+ // + w '"${tmp_dir}"'/package-ids + d } ' +if [ ! -s "${tmp_dir}/packages" ]; then + printf 'Nothing to delete.\n' + exit +fi + +mkdir "${tmp_dir}/repos" + +while read -r repo; do + ${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db.tar.gz" \ + "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files.tar.gz" \ + "${tmp_dir}/repos/" + + tar -Oxzf "${tmp_dir}/repos/${repo}.db.tar.gz" --wildcards '*/desc' | \ + sed -n ' + /^%FILENAME%$/{ + N + s/^\S\+\n\(\S\+-[^-.]\+\)\(-[^-]\+\)/\1.0\2 \1\2/ + T + p + } + ' | \ + while read -r old new; do + printf 's,/%s\\(\\.sig\\)\\?$,/%s\\1,\n' \ + "$(str_to_regex "${old}")" \ + "$(str_to_regex "${new}")" + done >> \ + "${tmp_dir}/sub_pkgrel-removal.sed" + + # shellcheck disable=SC2046 + repo-remove "${tmp_dir}/repos/${repo}.db.tar.gz" \ + $( + grep "^$(str_to_regex "${repo}") " "${tmp_dir}/packages" | \ + cut -d' ' -f2 + ) + if ! ${no_action}; then + ${master_mirror_rsync_command} \ + "${tmp_dir}/repos/${repo}.db.tar.gz" \ + "${tmp_dir}/repos/${repo}.files.tar.gz" \ + "${master_mirror_rsync_directory}/i686/${repo}/" + fi +done < \ + "${tmp_dir}/repositories" + +if [ -s "${tmp_dir}/sub_pkgrel-removal.sed" ]; then + sed -i -f "${tmp_dir}/sub_pkgrel-removal.sed" "${tmp_dir}/package-files" +fi +if ${no_action}; then + printf 'Now, I would remove the packages from the database and delete the following files from the master mirror:\n' + sed ' + s,^, , + ' "${tmp_dir}/package-files" + exit +fi -echo 'OK' >&2 -exit 42 - -# # sanity check -# -# for ending in 'done' 'testing'; do -# if [ "${ending}" = 'testing' ] && \ -# [ -z "${packages_to_stabilize}" ]; then -# # if nothing is to be untested, we don't care about duplicate -# # testing packages (and maybe an unstaging fixes this anyway) -# continue -# fi -# if [ -n "$( -# find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \ -# sed 's|\(\.[^.]\+\)\{4\}$||' | \ -# sort | \ -# uniq -d -# )" ]; then -# >&2 echo 'Removing duplicates not yet implemented:' -# find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \ -# sed 's|\(\.[^.]\+\)\{4\}$||' | \ -# sort | \ -# uniq -d -# exit 42 -# fi -# done -# -# # packages which are done -# -# find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \ -# sed ' -# s|\.done$|| -# ' | \ -# sort -u > \ -# "${tmp_dir}/done-packages" -# -# # packages still on the build-list -# -# tr ' ' '.' < \ -# "${work_dir}/build-list" | \ -# sort -u > \ -# "${tmp_dir}/keep-packages" -# -# find "${work_dir}/package-infos" -name '*.groups' \ -# -exec grep -qx 'base\(-devel\)\?' {} \; \ -# -printf '%f\n' | \ -# sed ' -# s|\.groups$|| -# ' | \ -# sort -u > \ -# "${tmp_dir}/base-packages" -# -# # no base / base-devel packages on the build list? -# if [ -z "$( -# join -j 1 \ -# "${tmp_dir}/base-packages" \ -# "${tmp_dir}/keep-packages" -# )" ]; then -# # unstage all base / base-devel packages from staging -# cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages" "${tmp_dir}/keep-packages" | \ -# sort | \ -# uniq -u | \ -# sponge "${tmp_dir}/keep-packages" -# fi -# -# # find all dependencies of the unstageable packages -# mv \ -# "${tmp_dir}/keep-packages" \ -# "${tmp_dir}/new-keep-packages" -# touch "${tmp_dir}/keep-packages" -# -# while [ -s "${tmp_dir}/new-keep-packages" ]; do -# -# cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \ -# sort -u | \ -# sponge "${tmp_dir}/keep-packages" -# -# sed ' -# s|^|'"${work_dir}"'/package-infos/| -# s|$|.depends| -# ' "${tmp_dir}/keep-packages" | \ -# xargs -r grep -HF '' | \ -# sed ' -# s|^.*/|| -# s|\.depends:| | -# ' | \ -# sort -u | \ -# sort -k2,2 > \ -# "${tmp_dir}/keep-packages.depends" -# -# sed ' -# s|^|'"${work_dir}"'/package-infos/| -# s|$|.builds| -# ' "${tmp_dir}/done-packages" | \ -# xargs -r grep -HF '' | \ -# sed ' -# s|^.*/|| -# s|\.builds:| | -# ' | \ -# sort -u | \ -# sort -k2,2 > \ -# "${tmp_dir}/done-packages.builds" -# -# join -j 2 -o 1.1 \ -# "${tmp_dir}/done-packages.builds" \ -# "${tmp_dir}/keep-packages.depends" | \ -# sort -u > \ -# "${tmp_dir}/new-keep-packages" -# -# # "new" is only what has not been there before -# cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \ -# sort | \ -# uniq -u | \ -# sponge "${tmp_dir}/new-keep-packages" -# -# done -# -# # unlock build list -# -# rm -f "${build_list_lock_file}" -# flock -u 9 -# -# clean_up_lock_file() { -# rm -rf --one-file-system "${tmp_dir}" -# rm -f "${package_database_lock_file}" -# } -# -# # calculate unstageable packages from keep_packages and done_packages -# -# done_packages=$( -# cat \ -# "${tmp_dir}/keep-packages" \ -# "${tmp_dir}/keep-packages" \ -# "${tmp_dir}/done-packages" | \ -# sort | \ -# uniq -u -# ) -# -# # move packages in packages_to_stabilize from *testing/ to the stable repos -# -# # shellcheck disable=SC2046 -# move_packages $( -# -# for package in ${packages_to_stabilize}; do -# -# if [ -z "${package}" ]; then -# continue -# fi -# -# printf '%s/%s/%s\n' \ -# "${package}" \ -# "$(official_or_community "${package}" 'testing')" \ -# "$(repository_of_package "${package}")" -# -# done -# -# ) -# -# # move packages from *staging to *testing -# -# # shellcheck disable=SC2046 -# move_packages $( -# -# for package in ${done_packages}; do -# -# if [ -z "${package}" ]; then -# continue -# fi -# -# printf '%s/%s/%s\n' \ -# "${package}" \ -# "$(official_or_community "${package}" 'staging')" \ -# "$(official_or_community "${package}" 'testing')" -# -# done -# -# ) -# -# clean_up_lock_file +# shellcheck disable=SC2016 +{ + printf 'CREATE TEMPORARY TABLE `to_deletes` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `to_deletes`;\n' \ + "${tmp_dir}/package-ids" + + printf 'UPDATE `binary_packages` ' + printf ' JOIN `to_deletes` ON `to_deletes`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + printf ' SET `repository`=(' + printf 'SELECT `repositories`.`id`' + printf ' FROM `repositories`' + printf ' WHERE `repositories`.`name`="deletion-list"' + printf ');\n' +} | \ + mysql_run_query + +sed ' + s,^,rm "i686/, + s,$,", +' "${tmp_dir}/package-files" | \ + ${master_mirror_sftp_command} diff --git a/bin/get-assignment b/bin/get-assignment index d256248..7ac749f 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -15,49 +15,37 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -mkdir -p "${work_dir}/package-states" +# TODO: honor manual build order of tool-chain: +# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc hand_out_assignment() { - # find out the sub_pkgrel - sub_pkgrel=$( - next_sub_pkgrel "$1" "$2" "$3" "$4" - ) - - # we don't care anymore if an older version of this package was - # "locked" or "broken" - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \ - -delete - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - - echo "$1 $2 $3 $4 ${sub_pkgrel}" - { - # shellcheck disable=SC2154 - echo "${slave}" - if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - cat "${work_dir}/package-states/$1.$2.$3.$4.locked" - fi - } | \ - sort -u | \ - sponge "${work_dir}/package-states/$1.$2.$3.$4.locked" # shellcheck disable=SC2016 { + printf 'SELECT ' + printf '`package_sources`.`%s`,' \ + 'pkgbase' 'git_revision' 'mod_git_revision' + printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`' + printf ' FROM `upstream_repositories`' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `package_sources`.`%s`=from_base64("%s")' \ + 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ + 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ + 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" + printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ + "$(printf '%s' "$4" | base64 -w0)" + printf ' LIMIT 1;\n' + printf 'UPDATE `build_slaves`' printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -66,16 +54,24 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ')' - printf ' WHERE `build_slaves`.`name`=from_base64("%s");' \ + # shellcheck disable=SC2154 + printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" - } | \ - ${mysql_command} - # lock every loop this package breaks - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -exec touch '{}.locked' \; + printf 'UPDATE `build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' SET `build_assignments`.`priority`=0' + printf ' WHERE' + printf ' `package_sources`.`%s` = from_base64("%s") AND' \ + 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ + 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ + 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" + printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ + "$(printf '%s' "$4" | base64 -w0)" + } | \ + mysql_run_query | \ + tr '\t' ' ' exit 0 @@ -108,140 +104,106 @@ clean_up() { tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir) trap clean_up EXIT -# Check if there are any pending packages at all and if the requester -# has already got an assignment. - -pending_packages=false - -while read -r package git_revision mod_git_revision repository; do +# if we're building something already, hand it out (again) +currently_building=$( + # shellcheck disable=SC2016 + { + printf 'SELECT ' + mysql_query_select_pkgbase_and_revision + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ + "$(printf '%s' "${slave}" | base64 -w0)" + printf ' AND `repositories`.`name`="build-list"' + printf ' LIMIT 1;\n' + } | \ + mysql_run_query +) - generate_package_metadata "${package}.${git_revision}.${mod_git_revision}.${repository}" +if [ -n "${currently_building}" ]; then + # shellcheck disable=SC2086 + hand_out_assignment ${currently_building} +fi - if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then - # has this slave already got or does he prefer this assignment? - # note, that the dependencies are met, because this package is already locked - if grep -qxF "${slave}" "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" || \ - printf ',%s,' "$1" | \ - grep -qF ",${package},"; then - hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - fi - elif [ ! -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then - pending_packages=true - fi +# a package with [all dependencies met or which is part of a loop] +# and which is currently not being built, ordered by: +# 1: we requested it +# 2: its priority +# 3: is not yet built +# 4: was built the longest time ago +next_building=$( + # shellcheck disable=SC2016 + { + printf 'SELECT ' + printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \ + "$( + printf '%s' "$1" | \ + base64 -w0 + )" + printf '`build_assignments`.`priority`,' + printf 'COALESCE(' + printf 'MAX(`failed_builds`.`date`),0' + printf ') AS `last_trial`,' + mysql_query_is_part_of_loop '`build_assignments`.`id`' + printf ' AS `part_of_loop`,' + mysql_query_select_pkgbase_and_revision + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' LEFT' + mysql_join_build_assignments_failed_builds + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND NOT EXISTS (' + printf ' SELECT *' + printf ' FROM `build_slaves`' + printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`' + printf ') AND (' + printf '`build_assignments`.`is_blocked` IS NULL' + printf ' OR' + printf ' `package_sources`.`pkgbase`=from_base64("%s")' \ + "$( + printf '%s' "$1" | \ + base64 -w0 + )" + printf ') AND (' + mysql_query_is_part_of_loop '`build_assignments`.`id`' + printf ' OR NOT ' + mysql_query_has_pending_dependencies '`build_assignments`.`id`' + printf ')' + printf ' GROUP BY `build_assignments`.`id`' + printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' + printf ' LIMIT 1;\n' + } | \ + mysql_run_query | \ + sed ' + y/\t/ / + s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ + ' +) +if [ -n "${next_building}" ]; then + # shellcheck disable=SC2086 + hand_out_assignment ${next_building} +fi -done < "${work_dir}/build-list" +# Check if there are any pending packages at all +count_pending=$( + # shellcheck disable=SC2016 + { + printf 'SELECT count(*)' + printf ' FROM `build_assignments`' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_assignments`.`is_blocked` IS NULL' + printf ';\n' + } | \ + mysql_run_query +) -if ! ${pending_packages}; then +if [ "${count_pending}" -eq 0 ]; then >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages' exit 3 +else + >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready' + exit 2 fi - -# Find first package of build-list whose "dependencies" are all met -# 1st: prefered packages on the build list which have all dependencies met -# 2nd: unbroken packages on the build list which have all dependencies met -# 3rd: unbroken packages breaking a loop -# 4th: broken packages which have all dependencies met or break a loop - -for iteration in 'prefered' 'fresh' 'loops' 'broken'; do - - case "${iteration}" in - 'prefered') - hand_out_blocked=true - hand_out_broken=true - hand_out_loop=false - echo "$1" | \ - tr ',' '\n' | \ - sort -u > \ - "${tmp_dir}/hand-out-only-these-packages" - ;; - 'fresh') - hand_out_blocked=false - hand_out_broken=false - hand_out_loop=false - { - cat "${work_dir}/build-list" - find "${work_dir}/package-states" -name '*.broken' -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - p - ' - } | \ - sort -k1,1 -k2 | \ - uniq -u | \ - cut -d' ' -f1 | \ - uniq > \ - "${tmp_dir}/hand-out-only-these-packages" - ;; - 'loops') - hand_out_blocked=false - hand_out_broken=false - hand_out_loop=true - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat {} \; | \ - sort -u > \ - "${tmp_dir}/hand-out-only-these-packages" - ;; - 'broken') - hand_out_blocked=false - hand_out_broken=true - hand_out_loop=true - cut -d' ' -f1 < \ - "${work_dir}/build-list" | \ - sort -u > \ - "${tmp_dir}/hand-out-only-these-packages" - ;; - esac - - cat -n "${work_dir}/build-list" | \ - sort -k2,2 | \ - join -1 1 -2 2 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/hand-out-only-these-packages" - | \ - sort -k1,1 | \ - sed 's|^\s*\S\+\s\+||' > \ - "${tmp_dir}/try-to-hand-out-these-packages" - - while read -r package git_revision mod_git_revision repository; do - - # package locked? (we have taken care of those in the previous loop) - if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then - continue - fi - - # package blocked? - if ! ${hand_out_blocked} && - [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then - continue - fi - - # package broken? - if ! ${hand_out_broken} && \ - [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then - continue - fi - - # dependencies met? - if [ -n "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then - # do we hand out looped packages? - if ! ${hand_out_loop}; then - continue - fi - # is it a looped package? - if ! find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec cat {} \; | \ - grep -qxF "${package}"; then - continue - fi - fi - - hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - - done < \ - "${tmp_dir}/try-to-hand-out-these-packages" - -done - -# Remove the lock file - ->&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready' -exit 2 diff --git a/bin/get-package-updates b/bin/get-package-updates index 801aadd..cf2ea6c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -1,6 +1,6 @@ #!/bin/sh -# check for packages that need to be built, and build a list in the proper build order +# check for packages that need to be built # Details: # https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates @@ -9,9 +9,12 @@ # TODO: Find out, why sometimes package updates are missed. -# TODO: mark loops in mysql database +# TODO: read information from database -# TODO: test_exclusion does not yet cooperate with the database +# TODO: correctly handle if pkgbase of a split package is renamed, e.g.: +# $a -> ($a,$b) ==> $b -> ($a,$b) + +# TODO: keep database clean in case of abort # shellcheck disable=SC2016 usage() { @@ -86,57 +89,68 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi +# TODO: How should new deletion-list packages be handled? +# - packages deleted upstream should be marked as to-be-deleted if +# existent and otherwise be ignored +# - packages deleted due to black listing should be marked as black +# listed - and thus as to-be-deleted, but they should remain in the +# database after deletion! + # delete_package package # mark $package for deletion delete_package() { - echo "$1" >> \ - "${work_dir}/deletion-list.new" - sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - # shellcheck disable=SC2016 - { - printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' - printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' - printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \ - "$(printf '%s' "$1" | base64 -w0)" - } | \ - ${mysql_command} + # TODO: Once we want to rely on the database for test_exclusion, we + # need to run the command below unconditionally, but with some + # changes, so we can easily revert. + if [ -z "${test_exclusion}" ]; then + # shellcheck disable=SC2016 + { + # packages from the build-list/to-be-decided go straight to the deletion-list + printf 'UPDATE `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' + printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ + "$(printf '%s' "$1" | base64 -w0)" + # other packages are marked as `is_to_be_deleted` + printf 'UPDATE `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' SET `binary_packages`.`is_to_be_deleted`=1' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ + "$(printf '%s' "$1" | base64 -w0)" + } | \ + mysql_run_query + fi } -# create tmp_dir and trap -tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - -# Update git repositories (official packages, community packages and the repository of package customizations). +something_new=false for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' + # Update git repositories (official packages, community packages and the repository of package customizations). git -C "${repo_path}" fetch origin master:master -done - -# Read previous git revision numbers from files. - -something_new=false - -for repo in ${repo_names}; do + # read previous git revision numbers from database. + # shellcheck disable=SC2016 eval "old_repo_revisions__${repo}='$( - cat "${work_dir}/${repo}.revision" 2> /dev/null || \ - echo NONE + { + printf 'SELECT `git_repositories`.`head` FROM `git_repositories`' + printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${repo}" | base64 -w0)" + } | \ + mysql_run_query )'" - eval repo_path='"${repo_paths__'"${repo}"'}"' - eval "new_repo_revisions__${repo}='$( - if ${pull}; then + # determine new git revision + if ${pull}; then + eval "new_repo_revisions__${repo}='$( git -C "${repo_path}" rev-parse HEAD - else - cat "${work_dir}/${repo}.revision" - fi | \ - tee "${work_dir}/${repo}.revision.new" - )'" + )'" + else + eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"' + fi if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then something_new=true fi @@ -162,6 +176,11 @@ if ! flock -s ${block_flag} 8; then exit fi +trap mysql_cleanup EXIT + +# shellcheck disable=SC2119 +mysql_cleanup + echo 'Check modified packages from the last update, and put them to the build list.' # Check modified packages from the last update, and put them to the build list. @@ -169,13 +188,6 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -cp \ - "${work_dir}/build-list" \ - "${work_dir}/build-list.new" -cp \ - "${work_dir}/deletion-list" \ - "${work_dir}/deletion-list.new" - for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' @@ -191,26 +203,9 @@ for repo in ${repo_names}; do git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" fi } | \ - # Packages which are already on the build list should receive a git_revision bump if _any_ file changed. - # Thus, we rename any file "PKGBUILD" to trigger the successive logic. - if [ "${repo}" = 'archlinux32' ]; then - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t[^/]\\+/| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - else - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - fi | \ # only track changes in PKGBUILDs + # TODO: However, packages which are already on the build list should + # receive a git_revision bump if _any_ file changed. grep '/PKGBUILD$' | \ if [ "${repo}" = "archlinux32" ]; then # modify the directory structure from the modifiaction-repository @@ -245,24 +240,26 @@ done | \ while read -r mode package git_revision repository; do if [ "${mode}" = 'D' ]; then # deleted PKGBUILD - git_revision=$(cat "${work_dir}/archlinux32.revision.new") + # shellcheck disable=SC2154 + git_revision="${new_repo_revisions__archlinux32}" found_package=false for repository in ${repo_names}; do eval 'repo_path="${repo_paths__'"${repository}"'}"' if [ "${repository}" = "archlinux32" ]; then - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" 2> /dev/null | \ + if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \ tar -t 2> /dev/null | \ grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then found_package=true break; fi else - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" -- "${package}/repos" 2> /dev/null | \ - tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \ - cut -d/ -f3 | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -vq -- '-i686$'; then - git_revision=$(cat "${work_dir}/${repository}.revision.new") + # shellcheck disable=SC2154 + if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \ + 'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \ + 'cut -d/ -f3 | ' \ + 'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \ + 'grep -vq -- '"'"'-i686$'"'"; then + eval 'git_revision="${new_repo_revisions__'"${repository}"'}"' found_package=true break; fi @@ -280,31 +277,36 @@ done | \ fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - # new or modified PKGBUILD - sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - # shellcheck disable=SC2154 - echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ - "${work_dir}/build-list.new" - sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new" # shellcheck disable=SC2016 { # delete old build assignment and associated binary packages - # which are not yet build or on the deletion list + # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${package}" | \ base64 -w0 )" printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n' + # remove is-to-be-deleted marker from old binary packages + printf 'UPDATE `binary_packages`' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories + printf ' SET `is_to_be_deleted`=0' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ + "$( + printf '%s' "${package}" | \ + base64 -w0 + )" } | \ - ${mysql_command} - mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" + mysql_run_query + # shellcheck disable=SC2154 + mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi @@ -312,21 +314,6 @@ done | \ exit 1 done -sort -u "${work_dir}/deletion-list.new" | \ - sponge "${work_dir}/deletion-list.new" - -echo 'Extract dependencies of packages.' - -# First, we extract the dependencies of each package. - -mkdir -p "${work_dir}/package-infos" - -while read -r package git_revision mod_git_revision repository; do - - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - -done < "${work_dir}/build-list.new" - echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date @@ -334,249 +321,190 @@ echo 'apply blacklisting' black_listed='' black_listed_new=$( { - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \ - tar -Ox 'blacklist' | \ - sed ' - s/\s*#.*$// - /^\s*$/d - ' - if [ -n "${test_exclusion}" ]; then - echo "${test_exclusion}" - fi - find "${work_dir}/package-infos" -maxdepth 1 -name 'lib32-*' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - uniq + { + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ + tar -Ox 'blacklist' | \ + sed ' + s/\s*#.*$// + /^\s*$/d + ' + if [ -n "${test_exclusion}" ]; then + echo "${test_exclusion}" + fi + } | \ + base64_encode_each + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")' + printf ' FROM `package_sources`' + printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' + } | \ + mysql_run_query } | \ sort -u ) -find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \ - grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1| - ' | \ - sort -u | \ - sort -k4,4 > \ - "${tmp_dir}/packages" - -{ - uniq -f3 -u "${tmp_dir}/packages" | \ - awk '{print $4 " " $4 "." $1 "." $2 "." $3}' - uniq -f3 -D "${tmp_dir}/packages" | \ - uniq -f3 --group=append | \ - while read -r rev mod_rev repo pkg; do - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - # shellcheck disable=SC2031 - printf '%s %s.%s.%s.%s\n' \ - "${opkg}" \ - "${opkg}" \ - "$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} | \ - sort -u | \ - find_newest_of_git_revisions - )" \ - "$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} | \ - sort -u | \ - find_newest_of_git_revisions - )" \ - "${orepo}" | \ - grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$' - revs='' - mod_revs='' - continue - fi - revs=$( - printf '%s\n' \ - ${revs} \ - "${rev}" - ) - mod_revs=$( - printf '%s\n' \ - ${mod_revs} \ - "${mod_rev}" - ) - opkg="${pkg}" - orepo="${repo}" - done -} | \ - sort -k1,1 > \ - "${tmp_dir}/newest-revisions" - -sed ' - s|^\S\+ |'"${work_dir}"'/package-infos/| - s|$|.builds| -' "${tmp_dir}/newest-revisions" | \ - xargs -r cat | \ - sort | \ - uniq -c > \ - "${work_dir}/built-packages" - -sed ' - s|^\S\+ |'"${work_dir}"'/package-infos/| - s|$|.build-depends| -' "${tmp_dir}/newest-revisions" | \ - xargs -r grep -HF '' | \ - sed ' - s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2| - s|^.*/|| - ' | \ - sort -k2,2 > \ - "${work_dir}/newest-dependencies" - while [ -n "${black_listed_new}" ]; do black_listed=$( - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} ${black_listed_new} | \ + printf '%s\n' "${black_listed}" "${black_listed_new}" | \ + grep -vxF '' | \ sort -u ) black_listed_new=$( { - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} | \ - sort -k1,1 | \ - join -j1 -o 2.2 - "${tmp_dir}/newest-revisions" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' | \ - xargs -r cat | \ - sort | \ - uniq -c - cat "${work_dir}/built-packages" + printf '%s\n' "${black_listed}" "${black_listed}" + + # shellcheck disable=SC2016 + { + printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' + printf 'INSERT INTO `bl` (`pkgbase`) VALUES ' + printf '%s\n' "${black_listed}" | \ + sort -u | \ + sed ' + s/^/(from_base64("/ + s/$/")),/ + $ s/,$/;/ + ' + printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' + printf ' FROM `package_sources` AS `a_ps`' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_binary_packages_dependencies 'a_bp' + printf ' WHERE NOT EXISTS (' + printf ' SELECT *' + printf ' FROM `install_target_providers`' + printf ' WHERE NOT EXISTS (' + printf ' SELECT *' + printf ' FROM `bl`' + printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' + printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' + printf ')' + printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ') AND EXISTS (' + # TODO: This should be corrected at the root: automatic install targets, which are bogus should + # not be added in the first place - but how do we detect that? + printf ' SELECT *' + printf ' FROM `install_target_providers`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' + } | \ + mysql_run_query | \ + sort -u } | \ - sort | \ - uniq -d | \ - awk '{print $2}' | \ - sort -k1,1 | \ - join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \ - sort -u - ) - black_listed_new=$( - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \ + grep -vxF '' | \ sort | \ uniq -u ) done -echo "${black_listed}" | \ - while read -r package; do - [ -n "${package}" ] && \ - delete_package "${package}" - done -sort -u "${work_dir}/deletion-list.new" | \ - sponge "${work_dir}/deletion-list.new" - - -if [ -n "${test_exclusion}" ]; then - # we should not actually update the build-list et. al, but solely print this difference: - if diff --color -u "${work_dir}/deletion-list" "${work_dir}/deletion-list.new"; then - printf 'If you put "%s" on the blacklist, no additional packages will end up on the deletion list.\n' "${test_exclusion}" - fi - exit 0 -fi - -# Now we create the partial order. - -while read -r package git_revision mod_git_revision repository; do - # add "$pkgname -> $build-target" to build-order list - sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds" - # add "$dependency -> $pkgname" to build-order list - sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - # add "base/base-devel -> $pkgname" to build-order list - printf '%s '"${package}"'\n' 'base' 'base-devel' -done \ - < "${work_dir}/build-list.new" \ - > "${work_dir}/build-order" - -if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then - >&2 echo 'Created partial order file "build-order" is invalid.' - exit 2 -fi - -echo 'Now actually sort it.' - -{ - # this part will have the correct build order, but all the infos are missing - tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \ - nl -ba | \ - awk '{print $1 " not-git also-not-git whatever " $2}' - # this part has all the infos, but possibly the wrong order - awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new" -} | \ - sort -k5,5 -k1nr | \ - # now, we have the correct order and the infos, but in adjacent lines - uniq -f4 -D | \ - sed '/^0 /d;N;s|\n| |' | \ - # now in one line, each - sort -k1n,1 | \ - awk '{print $5 " " $7 " " $8 " " $9}' > \ - "${work_dir}/build-list.new.new" - -rm --one-file-system -rf "${work_dir}/build-list.loops.new" -mkdir "${work_dir}/build-list.loops.new" - -if [ -s "${work_dir}/tsort.error" ]; then - >&2 echo 'WARNING: There is a dependency cycle!' - >&2 cat "${work_dir}/tsort.error" - >&2 echo - >&2 echo 'I will continue anyway.' - # save loops in separate files each, so breaking them is easier - awk ' - /^tsort: \S+: input contains a loop:$/{ - n++; - getline - } - { - print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n - } - ' "${work_dir}/tsort.error" - - # remove lines from loop files which are no packages - find "${work_dir}/build-list.loops.new" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -printf '%p\n' | \ - while read -r loop; do - { - sort -u "${loop}" - cut -d' ' -f1 "${work_dir}/build-list.new.new" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - sponge "${loop}" - done -else - rm "${work_dir}/tsort.error" -fi +black_listed=$( + printf '%s\n' "${black_listed}" | \ + while read -r line; do + printf '%s' \ + "${line}" | \ + base64 -d + printf '\n' + done +) deletion_list_count=$( - wc -l < \ - "${work_dir}/deletion-list.new" + # shellcheck disable=SC2086 + printf '%s\n' ${black_listed} | \ + wc -l ) if [ "${deletion_list_count}" -gt 1000 ]; then >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ "${deletion_list_count}" + if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then + { + printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" + for repo_name in ${repo_names}; do + eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"' + eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"' + # shellcheck disable=SC2154 + if [ "${old_revision}" != "${new_revision}" ]; then + printf '%s: %s -> %s\n' \ + "${repo_name}" \ + "${old_revision}" \ + "${new_revision}" + fi + done + } | \ + tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ + irc_say + fi exit 3 fi +rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" -# Move the .new-files to the actual files +echo "${black_listed}" | \ + while read -r package; do + if [ -z "${package}" ]; then + continue + fi + delete_package "${package}" + done + + +if [ -n "${test_exclusion}" ]; then + # TODO: reimplement test_exclusion with information from the database + >&2 echo 'sry, not yet done.' + exit 0 +fi + +echo 'Done - mark decisions as final.' -rm -rf --one-file-system "${work_dir}/build-list.loops" +# shellcheck disable=SC2016 { - printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list" - # shellcheck disable=SC2086 - printf '%s.revision\n' ${repo_names} -} | \ - while read -r file; do - mv "${work_dir}/${file}.new" "${work_dir}/${file}" + # save blacklist into database + printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64), `reason` TEXT);\n' + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ + tar -Ox 'blacklist' | \ + sed -n ' + s/^\(\S\+\)\s*#\s*/\1 / + T + p + ' | \ + while read -r pkgbase reason; do + printf '(from_base64("%s"),from_base64("%s")),\n' \ + "$(printf '%s' "${pkgbase}" | base64 -w0)" \ + "$(printf '%s' "${reason}" | base64 -w0)" + done | \ + sed ' + 1 i INSERT INTO `bl` (`pkgbase`,`reason`) VALUES + $ s/,$/;/ + ' + printf 'UPDATE `build_assignments`' + mysql_join_build_assignments_package_sources + printf ' LEFT JOIN `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`' + printf ' SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n' + printf 'DROP TEMPORARY TABLE `bl`;\n' + + # update hashes of repositories in mysql database + for repo in ${repo_names}; do + printf 'UPDATE `git_repositories`' + printf ' SET `git_repositories`.`head`=from_base64("%s")' \ + "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')" + printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${repo}" | base64 -w0)" done + # move binary_packages from "to-be-decided" to "build-list" + printf 'UPDATE `binary_packages`' + mysql_join_binary_packages_repositories '' 'from_repo' + printf ' SET `repository`=(' + printf 'SELECT `to_repo`.`id`' + printf ' FROM `repositories` AS `to_repo`' + printf ' WHERE `to_repo`.`name`="build-list"' + printf ')' + printf ' WHERE `from_repo`.`name`="to-be-decided";\n' +} | \ + mysql_run_query + +# update loop list in database (beware, the packages are expected to be in "build-list", +# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". +mysql_find_build_assignment_loops # Remove the lock file diff --git a/bin/ii-connect b/bin/ii-connect index 54e1eb0..cf332c0 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -8,7 +8,7 @@ # start ii if it is not running if ! pgrep -x ii > /dev/null; then rm -rf --one-file-system "${irc_dir}" - screen -S ii -d -m ii -f buildmaster -n buildmaster + screen -S ii -d -m ii -s irc.freenode.net -f buildmaster -n buildmaster sleep 10 fi @@ -19,12 +19,12 @@ if tail -n1 "${irc_dir}/nickserv/out" 2> /dev/null | \ sponge "${irc_dir}/nickserv/in" fi -# join #archlinux-ports if not yet done +# join #archlinux32 if not yet done if ! grep ' buildmaster\.archlinux32\.org .* buildmaster$' "${irc_dir}/out" | \ tail -n1 | \ - grep -q ' #archlinux-ports '; then + grep -q ' #archlinux32 '; then { - echo '/j #archlinux-ports' + echo '/j #archlinux32' echo '/WHO buildmaster' } | \ sponge "${irc_dir}/in" @@ -39,7 +39,6 @@ fi if [ "$1" = 'watch' ]; then while pgrep -x 'ii' > /dev/null; do find "${irc_dir}" \ - -regextype sed \ -type f \ -name 'out' \ -printf '%p\n' | \ @@ -51,10 +50,10 @@ if [ "$1" = 'watch' ]; then else prefix='' fi - regex='^\S\+ \S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' + regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' if grep -q "${regex}" "${out_file}"; then sed -n ' - s/'"${regex}"'/\1 / + s/'"${regex}"'/\2 / T p ' "${out_file}" | \ diff --git a/bin/interpret-mail b/bin/interpret-mail index d16c07f..83c8999 100755 --- a/bin/interpret-mail +++ b/bin/interpret-mail @@ -5,6 +5,8 @@ # TODO: enable email interface to delete packages +# TODO: read information from database (?) + if [ $# -ne 0 ]; then >&2 echo '' >&2 echo 'usage: interpret-mail' @@ -21,15 +23,18 @@ if [ $# -ne 0 ]; then >&2 echo ' - "block: <state-file> <reason>":' >&2 echo ' Block the given packge for the given reason.' >&2 echo '' + >&2 echo ' - "copy-to-build-support: <pkgname>":' + >&2 echo ' Copy the given binary package into [build-support].' + >&2 echo '' + >&2 echo ' - "schedule: <pkgbase>":' + >&2 echo ' Put the given package on the build list (again).' + >&2 echo '' >&2 echo ' - "stabilize: <package-file>":' >&2 echo ' Mark the given package as tested.' >&2 echo '' >&2 echo ' - "unblock: <state-file>":' >&2 echo ' Unblock the given packge.' >&2 echo '' - >&2 echo ' - "schedule: <pkgname>":' - >&2 echo ' Put the given package on the build list (again).' - >&2 echo '' >&2 echo ' - ALL: all of the above (only valid in' >&2 echo ' "conf/admin-gpg-keys")' >&2 echo '' @@ -156,14 +161,14 @@ sed -n ' /\n$/!ba s/\n$// p -' "${tmp_dir}/plain-content" | +' "${tmp_dir}/plain-content" | \ sed ' :start_loop $!{ N bstart_loop } - s/=\s*\n//g + s/[=\]\s*\n//g s/:\s*\n/: /g s/\n\(\S\+[^: ]\(\s\|\n\|$\)\)/ \1/g ' > \ @@ -185,7 +190,7 @@ sed -n "$( )" "${tmp_dir}/raw-content" if [ -s "${tmp_dir}/block" ]; then - if run_and_log_on_error "${base_dir}/bin/modify-package-state" --block "${tmp_dir}/block"; then + if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --block "${tmp_dir}/block"; then log 'Successfully blocked %s packages.\n' "$(wc -l < "${tmp_dir}/block")" else log 'There was an error while blocking the packages - ignoring this message.\n' @@ -196,7 +201,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then sed -i ' /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/ ' "${tmp_dir}/stabilize" - if run_and_log_on_error "${base_dir}/bin/modify-package-state" --tested "${tmp_dir}/stabilize"; then + if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --tested "${tmp_dir}/stabilize"; then log 'Successfully marked %s packages as tested.\n' "$(wc -l < "${tmp_dir}/stabilize")" else log 'There was an error while marking the packages as tested - ignoring this message.\n' @@ -204,7 +209,7 @@ if [ -s "${tmp_dir}/stabilize" ]; then fi if [ -s "${tmp_dir}/unblock" ]; then - if run_and_log_on_error "${base_dir}/bin/modify-package-state" --unblock "${tmp_dir}/unblock"; then + if run_and_log_on_error "${base_dir}/bin/modify-package-state" --wait --unblock "${tmp_dir}/unblock"; then log 'Successfully unblocked %s packages.\n' "$(wc -l < "${tmp_dir}/unblock")" else log 'There was an error while unblocking the packages - ignoring this message.\n' @@ -213,7 +218,7 @@ fi if [ -s "${tmp_dir}/schedule" ]; then # shellcheck disable=SC2046 - "${base_dir}/bin/seed-build-list" $( + "${base_dir}/bin/seed-build-list" --wait $( tr '[:space:]' '\n' < \ "${tmp_dir}/schedule" | \ grep -vxF '' | \ @@ -224,3 +229,14 @@ if [ -s "${tmp_dir}/schedule" ]; then sponge "${tmp_dir}/schedule" log 'Successfully (re)scheduled %s packages.\n' "$(wc -l < "${tmp_dir}/schedule")" fi + +if [ -s "${tmp_dir}/copy-to-build-support" ]; then + sed -i ' + /\.pkg\.tar\.xz$/!s/$/.pkg.tar.xz/ + ' "${tmp_dir}/copy-to-build-support" + if run_and_log_on_error "${base_dir}/bin/copy-to-build-support" --wait "${tmp_dir}/copy-to-build-support"; then + log 'Successfully copied %s packages to [build-support].\n' "$(wc -l < "${tmp_dir}/copy-to-build-support")" + else + log 'There was an error while copying the packages to [build-support] - ignoring this message.\n' + fi +fi diff --git a/bin/modify-package-state b/bin/modify-package-state index c740d3d..4a2f5ac 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -1,5 +1,7 @@ #!/bin/sh +# shellcheck disable=SC2039 + # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" @@ -16,25 +18,28 @@ usage() { >&2 echo ' -n|--no-report: Do not report what packages were modified.' >&2 echo ' -t|--tested: Mark package(s) as tested.' >&2 echo ' -u|--unblock: Unblock package(s).' + >&2 echo ' -w|--wait: Wait for lock if necessary.' >&2 echo '' >&2 echo 'Exactly one of -b|-t|-u is needed for actual operation.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o bfhntu \ + getopt -o bfhntuw \ --long block \ --long faulty \ --long help \ --long no-report \ --long tested \ --long unblock \ + --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" action='' report=true +wait_for_lock='-n' while true do @@ -73,6 +78,9 @@ do fi action='unblock' ;; + -w|--wait) + wait_for_lock='' + ;; --) shift break @@ -95,26 +103,19 @@ if [ $# -ne 1 ]; then usage fi -clean_up() { - rm -rf --one-file-system "${tmp_dir}" -} -tmp_dir=$(mktemp -d 'tmp.modify-package-state.XXXXXXXXXX' --tmpdir) -trap clean_up EXIT - input_file="$1" if ${report}; then if ! [ -w "${input_file}" ]; then >&2 printf \ - 'Cannot open file "%s" for writing.' \ + 'Cannot open file "%s" for writing.\n' \ "${input_file}" exit 2 fi move_output() { cat "${output_file}" > "${input_file}" - clean_up + rm -f "${output_file}" } - output_file="${tmp_dir}/output-file" - touch "${output_file}" + output_file=$(mktemp 'tmp.modify-package-state.XXXXXXXXXX') trap 'move_output' EXIT else output_file='/dev/null' @@ -122,91 +123,130 @@ fi if ! [ -r "${input_file}" ]; then >&2 printf \ - 'Cannot open file "%s" for reading.' \ + 'Cannot open file "%s" for reading.\n' \ "${input_file}" exit 2 fi exec 9> "${sanity_check_lock_file}" -if ! flock -s -n 9; then +if ! flock -s ${wait_for_lock} 9; then >&2 echo 'Cannot get sanity-check lock.' exit fi -if [ "${action}" = 'tested' ]; then - receive_buglist 'Testing' > \ - "${tmp_dir}/package-bug-titles" +exec 8> "${package_database_lock_file}" +if ! flock ${wait_for_lock} 8; then + >&2 echo 'Cannot get package-database lock.' + exit fi -{ - err=0 - while read -r package reason; do - if echo "${package}" | \ - grep -q '\.pkg\.tar\.xz$'; then - package=$( - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.tested' -o -name '*.testing' \) \ - -exec grep -qxF "${package}" '{}' \; \ - -printf '%f\n' | \ - sed 's|\.[^.]\+$||' | \ - sort -u +while read -r package reason; do + case "${action}" in + 'faulty'|'tested') + # we expect a binary package identifier (pkgname-epoch:pkgver-pkgrel.sub_pkgrel-arch[".pkg.tar.xz"]) + # and we will only operate on packages in "testing" repositories + # shellcheck disable=SC2016 + combiner=$( + printf '`binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_architectures ) - if [ -z "${package}" ]; then - continue - fi - fi - if [ "${action}" = 'block' ] || \ - [ "${action}" = 'unblock' ]; then - # these packages need to be on the build list - if ! tr ' ' '.' < \ - "${work_dir}/build-list" | \ - grep -qxF "${package}"; then - >&2 printf 'Package "%s" is not on the build-list.\n' "${package}" - err=2 - continue + # shellcheck disable=SC2016,SC2031 + selector=$( + extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name "${package}" + printf 'WHERE `repository_stabilities`.`name`="testing"' + printf ' AND `binary_packages`.`%s`=from_base64("%s")' \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" + printf ' AND `architectures`.`name`=from_base64("%s")' \ + "$(printf '%s' "${arch}" | base64 -w0)" + ) + ;; + 'block'|'unblock') + # we expect a package source identifier (pkgbase.git_revision.mod_git_revision.repository) + # and we will only operate on packages in "unbuilt" repositories + # shellcheck disable=SC2016 + combiner=$( + printf '`build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + ) + # shellcheck disable=SC2016 + selector=$( + repository="${package##*.}" + pkgbase="${package%.*}" + mod_git_revision="${pkgbase##*.}" + pkgbase="${pkgbase%.*}" + git_revision="${pkgbase##*.}" + pkgbase="${pkgbase%.*}" + printf 'WHERE `repository_stabilities`.`name`="unbuilt"' + printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ + "$(printf '%s' "${repository}" | base64 -w0)" + printf ' AND `package_sources`.`%s`=from_base64("%s")' \ + 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ + 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ + 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" + ) + ;; + *) + >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" + exit 42 + ;; + esac + case "${action}" in + 'block') + if [ -z "${reason}" ]; then + >&2 printf 'No reason is given for blocking package "%s".\n' "${package}" + exit 2 fi - fi - case "${action}" in - 'block') - if [ -z "${reason}" ]; then - >&2 printf 'No reason is given for blocking package "%s".\n' "${package}" - err=2 - else - echo "${reason}" > \ - "${work_dir}/package-states/${package}.blocked" - printf '%s %s\n' "${package}" "${reason}" - fi - ;; - 'faulty') - if [ -f "${work_dir}/package-states/${package}.tested" ]; then - mv \ - "${work_dir}/package-states/${package}.tested" \ - "${work_dir}/package-states/${package}.testing" - printf '%s\n' "${package}" - fi - ;; - 'tested') - if [ -f "${work_dir}/package-states/${package}.testing" ] && \ - ! grep -qF "[${package%.*.*.*}]" "${tmp_dir}/package-bug-titles"; then - mv \ - "${work_dir}/package-states/${package}.testing" \ - "${work_dir}/package-states/${package}.tested" - printf '%s\n' "${package}" - fi - ;; - 'unblock') - if [ -f "${work_dir}/package-states/${package}.blocked" ]; then - rm "${work_dir}/package-states/${package}.blocked" - printf '%s\n' "${package}" - fi - ;; - *) - >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" - exit 42 - ;; - esac - done > \ - "${output_file}" - - exit ${err} -} < \ - "${input_file}" + tester='1' + # shellcheck disable=SC2016 + modifier=$( + printf '`build_assignments`.`is_blocked`=from_base64("%s")' \ + "$(printf '%s' "${reason}" | base64 -w0)" + ) + ;; + 'unblock') + # shellcheck disable=SC2016 + tester='NOT `build_assignments`.`is_blocked` IS NULL' + # shellcheck disable=SC2016 + modifier='`build_assignments`.`is_blocked`=NULL' + ;; + 'faulty') + # shellcheck disable=SC2016 + tester='`binary_packages`.`has_issues`=0' + # shellcheck disable=SC2016 + modifier='`binary_packages`.`is_tested`=0,`binary_packages`.`has_issues`=1' + ;; + 'tested') + # shellcheck disable=SC2016 + tester='`binary_packages`.`is_tested`=0' + # shellcheck disable=SC2016 + modifier='`binary_packages`.`is_tested`=1' + ;; + *) + >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" + exit 42 + ;; + esac + if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ + mysql_run_query | \ + grep -qxF '1'; then + # shellcheck disable=SC2016 + { + printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" + printf ';\n' + } | \ + mysql_run_query + printf '%s %s\n' "${action}" "${package}" + fi +done \ + < "${input_file}" \ + > "${output_file}" diff --git a/bin/mysql-functions b/bin/mysql-functions deleted file mode 100755 index ec2bf4c..0000000 --- a/bin/mysql-functions +++ /dev/null @@ -1,647 +0,0 @@ -#!/bin/sh - -# contains functions used to access mysql db - -# shellcheck disable=SC2039 - -if [ -z "${base_dir}" ]; then - # just to make shellcheck happy - . 'conf/default.conf' -fi - -# base64_encode_each encode each line of stdin with base64 - -base64_encode_each() { - local line - - while read -r line; do - printf '%s' \ - "${line}" | \ - base64 -w0 - printf '\n' - done -} - -# mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository - -# shellcheck disable=SC2016,SC2086 -mysql_add_package_source() { - local names='pkgbase git_revision mod_git_revision upstream_package_repository' - local values; - for _ in ${names}; do - values="${values}$( - printf '%s' "$1" | \ - base64 -w0 - ) " - shift - done - values="${values% }" - - { - printf 'INSERT IGNORE INTO package_sources' - printf ' (' - printf '`%s`, ' ${names} - printf ') SELECT' - printf ' from_base64("%s"), ' ${values% *} - printf ' `upstream_repositories`.`id`' - printf ' FROM `upstream_repositories`' - printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ - "${values##* }" - } | \ - sed 's|, )|)|g' | \ - ${mysql_command} -} - -# mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository - -# shellcheck disable=SC2016,SC2031,SC2086,SC2154 -mysql_add_binary_package() { - local names='pkgbase git_revision mod_git_revision upstream_package_repository pkgname sub_pkgrel architecture repository' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'INSERT IGNORE INTO binary_packages' - printf ' (' - printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' - printf ') SELECT' - printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" - printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' - printf ' FROM' - printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' - printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' - printf ' WHERE' - printf ' `%s`.`name` = from_base64("%s") AND' \ - 'repositories' "${repository}" \ - 'architectures' "${architecture}" - printf ' `package_sources`.`%s` = from_base64("%s") AND' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ' `upstream_repositories`.`name` = from_base64("%s")' \ - "${upstream_package_repository}" - } | \ - sed ' - s|, )|)|g - s|, FROM| FROM|g - ' | \ - ${mysql_command} -} - -# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel - -# shellcheck disable=SC2016,SC2031,SC2086,SC2154 -mysql_show_binary_package() { - local names='pkgname pkgver pkgrel sub_pkgrel' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - local joint - { - printf 'SELECT' - printf ' `%s`.`%s`,' \ - 'repositories' 'name' \ - 'binary_packages' 'pkgname' \ - 'package_sources' 'pkgver' \ - 'package_sources' 'pkgrel' \ - 'binary_packages' 'sub_pkgrel' \ - 'architectures' 'name' \ - 'package_sources' 'pkgbase' \ - 'package_sources' 'git_revision' \ - 'package_sources' 'mod_git_revision' \ - 'upstream_repositories' 'name' - printf ' FROM `binary_packages`' - for joint in \ - 'architectures:binary_packages:architecture' \ - 'package_sources:binary_packages:package_source' \ - 'repositories:binary_packages:repository' \ - 'upstream_repositories:package_sources:upstream_package_repository'; do - printf ' JOIN `%s` ON `%s`.`id` =' \ - "${joint%%:*}" "${joint%%:*}" - joint="${joint#*:}" - printf ' `%s`.`%s`' \ - "${joint%:*}" "${joint#*:}" - done - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'binary_packages' 'pkgname' "${pkgname}" \ - 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ - 'package_sources' 'pkgver' "${pkgver}" \ - 'package_sources' 'pkgrel' "${pkgrel}" - printf ';' - } | \ - sed ' - s|, FROM| FROM|g - s|AND;|;|g - ' | \ - ${mysql_command} --html -} - -# mysql_add_install_target $install_target - -# shellcheck disable=SC2016,2086 -mysql_add_install_target() { - local install_target - install_target=$( - printf "%s" "$1" | \ - base64 -w0 - ) - - { - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' VALUES (from_base64("%s"))' \ - "${install_target}" - } | \ - ${mysql_command} -} - -# mysql_generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# if sub_pkgrel should be determined automatically and the package is on the build-list -# and -# mysql_generate_package_metadata $sub_pkgrel $package $current_repository $git_revision $mod_git_revision $repository -# or -# mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository -# if $sub_pkgrel should be forced and the package is currently in $current_repository -# generate the meta data of a package (dependencies, built packages, ...) in the database - -# shellcheck disable=SC2016 -mysql_generate_package_metadata() { - - ( # new shell is intentional - case "$1" in - ''|*[!0-9]*) - unset forced_sub_pkgrel - current_repository=$( - printf 'build-list' | \ - base64 -w0 - ) - ;; - *) - forced_sub_pkgrel=$( - printf '%s' "$1" | \ - base64 -w0 - ) - shift - current_repository=$( - printf '%s' "$1" | \ - base64 -w0 - ) - shift - ;; - esac - package="$1" - git_revision="$2" - mod_git_revision="$3" - repository="$4" - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - printf '.' >&2 - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" - exit 2 - fi - # remove empty lines and unsupported architectures - sed -i ' - /^[^=]*=\s*$/d - /^\s*arch = /{ - / \(i686\|any\)$/!d - } - ' "${temp_dir}/SRCINFO" - - if [ ! -s "${temp_dir}/SRCINFO" ]; then - >&2 printf '"make_source_info" had empty output - eh, what?\n' - exit 2 - fi - printf '\n\n' >> "${temp_dir}/SRCINFO" - - printf '.' >&2 - pkgbase=$( - grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ - cut -d' ' -f3 - ) - if [ -z "${pkgbase}" ]; then - >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' - exit 2 - fi - - # add the package source - mysql_add_package_source "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" - printf '.' >&2 - - # now we encode everything in base64 - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - git_revision=$( - printf '%s' "${git_revision}" | \ - base64 -w0 - ) - mod_git_revision=$( - printf '%s' "${mod_git_revision}" | \ - base64 -w0 - ) - repository=$( - printf '%s' "${repository}" | \ - base64 -w0 - ) - - # add the build assignment(s) - { - archs=$( - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - grep -vxF 'any' | \ - sort -u - ) - if [ -z "${archs}" ]; then - echo 'any' - else - printf '%s\n' "${archs}" - fi - } | \ - while read -r arch; do - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL' - printf ' FROM `architectures` JOIN `package_sources`' - printf ' WHERE `architectures`.`name` = from_base64("%s")' \ - "$( - printf '%s' "${arch}" | \ - base64 -w0 - )" - printf ' AND `package_sources`.`%s` = from_base64("%s")' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ';\n' - done > \ - "${temp_dir}/add-build-assignments-command" - - # TODO: correctly link between binary_packages and build_assignments using any_arch - - # shellcheck disable=SC2034 - # select any specific arch (which will be building the 'any' part of a split package) - any_arch=$( - { - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - sort -r | \ - grep -vxFm 1 'any' || \ - echo 'any' - } | \ - base64_encode_each - ) - - grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ - cut -d' ' -f3 | \ - while read -r pkgname; do - pkgname64=$( - printf '%s' "${pkgname}" | \ - base64 -w0 - ) - sed -n ' - /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ - :a - N - /\n$/{ - p - T - } - ba - } - ' "${temp_dir}/SRCINFO" | \ - sed ' - /^\S/d - s/^\s*// - ' > \ - "${temp_dir}/BINARYINFO.${pkgname64}" - - grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ - cut -d' ' -f3 | \ - while read -r arch; do - arch64=$( - printf '%s' "${arch}" | \ - base64 -w0 - ) - sed ' - s/^\(\S\+\)_'"${arch}"' = /\1 = / - ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ - "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" - done - done - find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ - while read -r _ pkgname arch; do - pkgver=$( - grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - pkgrel=$( - grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - epoch=$( - { - grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \ - echo 'epoch = 0' - } | \ - cut -d' ' -f3 | \ - base64_encode_each - ) - provides=$( - grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - builddepends=$( - grep '^\(checkdepends\|makedepends\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - rundepends=$( - grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ - cut -d' ' -f3 | \ - sed 's/[<>=].*$//' | \ - base64_encode_each - ) - if [ -n "${forced_sub_pkgrel}" ]; then - sub_pkgrel='from_base64("'"${forced_sub_pkgrel}"'")' - else - sub_pkgrel=$( - printf '(SELECT COALESCE(' - # do not add binary packages which are currently on the build-list - printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - printf ' `architectures`.`name`=from_base64("%s")' \ - "${arch}" - printf ' AND `repositories`.`name`="build-list"),' - # max(sub_pkgrel)+1 - printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then - # 'any' gets higher sub_pkgrel than any architecture - printf ' 1' - else - # not-'any' gets higher sub_pkgrel than same or 'any' architecture - printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \ - "${arch}" - fi - printf ')' - printf ',0))' - ) - fi - { - printf 'INSERT IGNORE INTO `binary_packages` (' - printf '`%s`,' \ - 'build_assignment' \ - 'repository' \ - 'architecture' \ - 'epoch' \ - 'pkgver' \ - 'pkgrel' \ - 'pkgname' \ - 'sub_pkgrel' \ - 'has_issues' \ - 'is_tested' - printf ') SELECT ' - printf '`%s`.`id`,' \ - 'build_assignments' \ - 'repositories' \ - 'architectures' - printf 'from_base64("%s"),' \ - "${epoch}" \ - "${pkgver}" \ - "${pkgrel}" \ - "${pkgname}" - printf '%s,0,0 FROM' \ - "${sub_pkgrel}" - printf ' `%s` JOIN' \ - 'repositories' \ - 'architectures' \ - 'build_assignments' \ - 'package_sources' - printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`' - printf ' JOIN `upstream_repositories`' - printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'repositories' 'name' "${current_repository}" \ - 'architectures' 'name' "${arch}" \ - 'package_sources' 'pkgbase' "${pkgbase}" \ - 'package_sources' 'git_revision' "${git_revision}" \ - 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ - 'upstream_repositories' 'name' "${repository}" - printf ';\n' - } | \ - sed ' - s|,)|)|g - s|JOIN ON|ON|g - s| AND;$|;| - ' >> \ - "${temp_dir}/add-binary-packages-command" - { - printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ - 'provides' \ - 'builddepends' \ - 'rundepends' - - printf 'INSERT INTO `provides` VALUES\n' - echo "${provides}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' (from_base64("%s"));\n' \ - "${pkgname}" - - printf 'INSERT INTO `rundepends` VALUES\n' - echo "${rundepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' ("base");\n' - - printf 'INSERT INTO `builddepends` VALUES\n' - echo "${builddepends}" | \ - sort -u | \ - grep -vxF '' | \ - sed ' - s|^| (from_base64("| - s|$|")),| - ' - printf ' ("base-devel");\n' - - printf 'INSERT IGNORE INTO `install_targets` (`name`)' - printf ' SELECT (`name`) FROM `%s` UNION' \ - 'provides' \ - 'builddepends' \ - 'rundepends' | \ - sed 's| UNION$|;\n|' - - for link in 'provides' 'builddepends' 'rundepends'; do - case "${link}" in - 'provides') - printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' - ;; - 'builddepends'|'rundepends') - printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' - printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' - printf ' `dependency_types` JOIN' - ;; - esac - printf ' `binary_packages`' - printf ' JOIN `architectures`' - printf ' ON `binary_packages`.`architecture` = `architectures`.`id`' - printf ' JOIN `repositories`' - printf ' ON `binary_packages`.`repository` = `repositories`.`id`' - printf ' JOIN `install_targets`' - printf ' JOIN `%s`' "${link}" - printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" - printf ' WHERE' - if [ "${link}" = 'builddepends' ] || \ - [ "${link}" = 'rundepends' ]; then - printf ' `dependency_types`.`name` = "%s" AND' \ - "${link%depends}" - fi - printf ' `binary_packages`.`%s` = from_base64("%s") AND' \ - 'epoch' "${epoch}" \ - 'pkgver' "${pkgver}" \ - 'pkgrel' "${pkgrel}" \ - 'pkgname' "${pkgname}" - printf ' `binary_packages`.`sub_pkgrel` = %s AND' \ - "${sub_pkgrel}" - printf ' `architectures`.`name` = from_base64("%s") AND' \ - "${arch}" - printf ' `repositories`.`name` = from_base64("%s");\n' \ - "${current_repository}" - done - - printf 'DROP TABLE `%s`;\n' \ - 'provides' \ - 'builddepends' \ - 'rundepends' - } >> \ - "${temp_dir}/add-install-targets-command" - done - printf '.' >&2 - - { - if [ -s "${temp_dir}/add-build-assignments-command" ]; then - cat "${temp_dir}/add-build-assignments-command" - fi - if [ -s "${temp_dir}/add-binary-packages-command" ]; then - cat "${temp_dir}/add-binary-packages-command" - fi - if [ -s "${temp_dir}/add-install-targets-command" ]; then - cat "${temp_dir}/add-install-targets-command" - fi - } | \ - ${mysql_command} - printf '.' >&2 - - ) -} - -# mysql_sanity_check -# do a sanity check on the mysql database - -mysql_sanity_check() { - ( # new shell is intentional - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_sanity_check.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - - for dir in $(ls_master_mirror 'i686'); do - ls_master_mirror "i686/${dir}" | \ - sed -n ' - s/\.pkg\.tar\.xz$// - T - s/-\([0-9]\+\)-\([^-]\+\)$/-\1.0-\2/ - s/-\([^-:]\+-[^-]\+-[^-]\+\)$/-0:\1/ - s|^|'"${dir}"'/| - p - ' - done | \ - sort > \ - "${temp_dir}/master-mirror-listing" - - # shellcheck disable=SC2016 - { - printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' - printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' WHERE `repositories`.`is_on_master_mirror`' - } | \ - ${mysql_command} --batch | \ - sed ' - 1d - s,\t,/, - s,\t,-, - s,\t,:, - s,\t,-, - s,\t,., - s,\t,-, - ' | \ - sort > \ - "${temp_dir}/mysql-packages" - - diff -u \ - "${temp_dir}/master-mirror-listing" \ - "${temp_dir}/mysql-packages" - ) -} diff --git a/bin/ping-from-slave b/bin/ping-from-slave new file mode 100755 index 0000000..afaf018 --- /dev/null +++ b/bin/ping-from-slave @@ -0,0 +1,25 @@ +#!/bin/sh + +# should be called periodically on the build-master from the slaves to +# - report any update on the build process +# - show that the build is still running +# - get notified by the build master if the build is not necessary anymore + +# shellcheck source=conf/default.conf +. "${0%/*}/../conf/default.conf" + +# TODO: receive/save some statistics about current build + +if [ "$( + # shellcheck disable=SC2016,SC2154 + { + printf 'SELECT count(*) FROM `build_slaves`' + mysql_join_build_slaves_build_assignments + printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${slave}" | base64 -w0)" + } | \ + mysql_run_query + )" -ne 1 ]; then + >&2 echo 'You do not build anything currently - abort whatever you are doing.' + exit 2 +fi diff --git a/bin/ping-to-master b/bin/ping-to-master new file mode 100755 index 0000000..15c9120 --- /dev/null +++ b/bin/ping-to-master @@ -0,0 +1,39 @@ +#!/bin/sh + +# periodically connects to the buildmaster to call ping-from slave to: +# - report any update on the build process +# - show that the build is still running +# - get notified by the build master if the build is not necessary anymore + +# shellcheck source=conf/default.conf +. "${0%/*}/../conf/default.conf" + +# TODO: abort build if requested to + +parent_pid="$1" +parent_tmp_dir="$2" + +exec 9> "${work_dir}/ping-build-master.lock" + +while kill -0 "${parent_pid}" && \ + [ -f "${parent_tmp_dir}/.ping-build-master" ]; do + + # shellcheck disable=SC2029 + find "${parent_tmp_dir}" \ + -xdev \ + -type f \ + -name '*.build-log' \ + -exec wc -l {} \; | \ + sed 's, .*/, ,' | \ + ssh \ + -i "${master_build_server_identity}" \ + -p "${master_build_server_port}" \ + "${master_build_server_user}@${master_build_server}" \ + 'ping-from-slave' "$(cat "${parent_tmp_dir}/.ping-build-master")" + + # we wait upto 60 seconds to get the lock - if we get it, the parent + # must have released it and we're finished + if flock -w 60 9; then + break + fi +done diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index 06d4c06..db828ae 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -23,11 +23,24 @@ if ! flock -s -n 8; then exit 1 fi +# shellcheck disable=SC2016 { - sed -n "/^$1/p" "${work_dir}/build-list" - sed "/^$1/d" "${work_dir}/build-list" + printf 'UPDATE `build_assignments`' + mysql_join_build_assignments_package_sources + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' SET `build_assignments`.`priority`=(' + printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1' + printf ' FROM (' + printf 'SELECT `others`.`priority`' + printf ' FROM `build_assignments` AS `others`' + printf ') AS `all_priorities`' + printf ')' + printf ' WHERE `package_sources`.`pkgbase` REGEXP from_base64("%s")' \ + "$(printf '%s' "$1" | base64 -w0)" + printf ' AND `repositories`.`name`="build-list";\n' } | \ - sponge "${work_dir}/build-list" + mysql_run_query # Remove the lock file diff --git a/bin/repo-copy b/bin/repo-copy new file mode 100755 index 0000000..c3450fd --- /dev/null +++ b/bin/repo-copy @@ -0,0 +1,92 @@ +#!/bin/sh + +# Rudimentary copy a package from one repository to another. +# Note, that we do _not_ need to have the package itself, since all +# relevant information is already in the original package database. + +# "Rudimentary" means the following restrictions: +# - no arguments are accepted +# - no database signatures are handled +# - only *.db.tar.gz and *.files.tar.gz are recognized as database + +usage() { + >&2 echo 'usage:' + >&2 echo ' repo-copy from-repo.db.tar.gz to-repo.db.tar.gz package1 package2 ...' + >&2 echo + >&2 echo 'Note, that the packages must be given with version, e.g. "linux-4.15.7-1.0".' + exit 2 +} + +if [ $# -le 2 ]; then + usage +fi + +from_repo="$1" +to_repo="$2" +shift +shift + +tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX") +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + +# extract the databases +for repo in 'from' 'to'; do + for archive in 'db' 'files'; do + eval 'repo_db="${'"${repo}"'_repo}"' + if [ "${repo_db}" = "${repo_db%.db.tar.gz}" ]; then + >&2 printf '"%s" has an invalid suffix.\n' "${repo_db}" + usage + fi + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + if [ ! -f "${repo_db}" ]; then + >&2 printf 'Cannot open file "%s".\n' "${repo_db}" + usage + fi + mkdir "${tmp_dir}/${repo}.${archive}" + bsdtar -C "${tmp_dir}/${repo}.${archive}" -xf "${repo_db}" + done +done + +# move the packages +for package in "$@"; do + errors=$( + find "${tmp_dir}/to.db" "${tmp_dir}/to.files" -mindepth 1 -maxdepth 1 \ + -printf '%f\n' | \ + sed 's/-[^-]\+-[^-]\+$//' | \ + grep -xF "${package%-*-*}" + ) + if [ -n "${errors}" ]; then + >&2 printf 'The target repository "%s" already contains the following packages - "repo-remove" them first:\n' \ + "${to_repo}" + >&2 printf '%s\n' "${errors}" + exit 2 + fi + for archive in 'db' 'files'; do + if [ ! -d "${tmp_dir}/from.${archive}/${package}" ]; then + >&2 printf 'Repository "%s" does not contain package "%s"\n' \ + "${from_repo}" "${package}" + exit 2 + fi + mv "${tmp_dir}/from.${archive}/${package}" "${tmp_dir}/to.${archive}/" + done +done + +# pack the database +for archive in 'db' 'files'; do + repo_db="${to_repo}" + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + bsdtar -C "${tmp_dir}" -czf "${tmp_dir}/${repo_db##*/}" --strip-components=1 "to.${archive}" +done + +# move the database in place +for archive in 'db' 'files'; do + repo_db="${to_repo}" + if [ "${archive}" = 'files' ]; then + repo_db="${repo_db%.db.tar.gz}.files.tar.gz" + fi + mv "${tmp_dir}/${repo_db##*/}" "${repo_db}" +done diff --git a/bin/return-assignment b/bin/return-assignment index cd6d30e..89a1f85 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -47,17 +47,37 @@ trap clean_up_lock_file EXIT if [ "$5" = 'ERROR' ]; then # the build failed on the build slave - if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" || - ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - >&2 echo 'Too late, package already outdated, I ignore this error report.' - exit 2 - fi - - # shellcheck disable=SC2154 - if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then - >&2 echo 'You do not build this package - move on.' + # shellcheck disable=SC2016 + infos=$( + { + printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ + "$( + # shellcheck disable=SC2154 + printf '%s' "${slave}" | \ + base64 -w0 + )" + printf ' AND `package_sources`.`%s`=from_base64("%s")' \ + 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ + 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ + 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" + printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ + "$(printf '%s' "$4" | base64 -w0)" + printf ' AND `repositories`.`name`="build-list"' + } | \ + mysql_run_query | \ + tr '\t' ' ' + ) + if [ -z "${infos}" ]; then + >&2 echo 'You do not build this package (anymore) - move on.' exit 2 fi + was_broken_before="${infos##* }" # save sent build logs saved_build_logs=$( @@ -69,15 +89,6 @@ if [ "$5" = 'ERROR' ]; then '*.build-log.gz' ) - if [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ]; then - was_broken_before=true - else - was_broken_before=false - fi - - # shellcheck disable=SC2154 - echo "${slave}" >> \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then @@ -94,8 +105,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ - sed '1d' + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -139,7 +149,7 @@ if [ "$5" = 'ERROR' ]; then printf 'DROP TABLE `failures`;\n' fi printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' + mysql_join_build_assignments_build_slaves printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( @@ -147,36 +157,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} - - # shellcheck disable=SC2154 - sed -i ' - /^'"$(str_to_regex "${slave}")"'$/d - ' "${work_dir}/package-states/$1.$2.$3.$4.locked" - if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - - # unlock every loop this package would have broken and which is not - # broken by another locked package - locked_packages=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' -printf '%f\n' | \ - sed 's@^\(.\+\)\.\([0-9a-f]\{40\}\.\)\{2\}[^.]\+\.locked$@\1@' - ) - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -not -exec grep -qxF "${locked_packages}" '{}' \; \ - -exec rm '{}.locked' \; - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - fi + mysql_run_query # release lock on build-list - otherwise seed-build-list won't run flock -u 9 @@ -195,7 +176,7 @@ if [ "$5" = 'ERROR' ]; then ' | \ tr ' ' '\n' | \ sed ' - s/^/-p ^haskell-/ + s/^/-p ^(haskell-)?/ s/-[0-9.]\+$/\$/ ' | \ sort -u @@ -209,21 +190,19 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux-ports/in" ]; then - { - printf '%s is broken (says %s).' \ - "$1" \ - "${slave}" - if [ -n "${rescheduled_packages}" ]; then - printf -- ' - I rescheduled:' - # shellcheck disable=SC2086 - printf ' %s,' ${rescheduled_packages} | \ - sed 's/,$/./' - fi - printf '\n' - } | \ - sponge "${irc_dir}/#archlinux-ports/in" - fi + { + printf '%s is broken (says %s).' \ + "$1" \ + "${slave}" + if [ -n "${rescheduled_packages}" ]; then + printf -- ' - I rescheduled:' + # shellcheck disable=SC2086 + printf ' %s,' ${rescheduled_packages} | \ + sed 's/,$/./' + fi + printf '\n' + } | \ + irc_say fi exit 0 @@ -245,19 +224,33 @@ clean_up_lock_file() { rm -f "${package_database_lock_file}" } -if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" || - ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] || - ! [ "$5" = "$(next_sub_pkgrel "$1" "$2" "$3" "$4")" ]; then +# shellcheck disable=SC2016 +if ! { + printf 'SELECT count(*)' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_build_slaves + mysql_join_binary_packages_repositories + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `build_slaves`.`name`=from_base64("%s")' \ + "$(printf '%s' "${slave}" | base64 -w0)" + printf ' AND `package_sources`.`%s`=from_base64("%s")' \ + 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ + 'git_revision' "$(printf '%s' "$2" | base64 -w0)" \ + 'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)" + printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ + "$(printf '%s' "$4" | base64 -w0)" + printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ + "$(printf '%s' "$5" | base64 -w0)" + } | \ + mysql_run_query | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi -# shellcheck disable=SC2154 -if ! grep -qxF "${slave}" "${work_dir}/package-states/$1.$2.$3.$4.locked"; then - >&2 echo 'Whoops, this package is not built by this slave.' - exit 2 -fi - clean_up_tmp_dir() { cd "${base_dir}" rm -rf --one-file-system "${tmp_dir}" @@ -274,17 +267,41 @@ tar -x \ --no-wildcards-match-slash \ '*.pkg.tar.xz' \ '*.pkg.tar.xz.sig' \ - '*.pkg.tar.xz-namcap.log.gz' - -# check if all packages are signed and all signatures belong to a package + '*.pkg.tar.xz-namcap.log.gz' \ + '*.pkg.tar.xz.so.needs.gz' \ + '*.pkg.tar.xz.so.provides.gz' + +# check if all packages come with: +# - a package file +# - a signature +# - a namcap log +# - a list of needed libraries +# - a list of provided libraries missing_files=$( - find . -maxdepth 1 -name '*.pkg.tar.xz' -o -name '*.pkg.tar.xz.sig' -o -name '*.pkg.tar.xz-namcap.log.gz' | \ + find . -maxdepth 1 \( \ + \( \ + -name '*.pkg.tar.xz' \ + -printf '%f package\n' \ + \) -o \ + \( \ + -name '*.pkg.tar.xz.sig' \ + -printf '%f signature\n' \ + \) -o \ + \( \ + -name '*.pkg.tar.xz-namcap.log.gz' \ + -printf '%f namcap\n' \ + \) -o \ + \( \ + -name '*.pkg.tar.xz.so.needs.gz' \ + -printf '%f needed-libraries\n' \ + \) -o \ + \( \ + -name '*.pkg.tar.xz.so.provides.gz' \ + -printf '%f provided-libraries\n' \ + \) \ + \) | \ sed ' - s@\.sig$@ signature@ - t - s@-namcap\.log\.gz$@ namcap@ - t - s@$@ package@ + s/\(\.pkg\.tar\.xz\)\(\.sig\|-namcap\.log\.gz\|\.so\.\(provides\|needs\)\.gz\) /\1 / ' | \ sort -k1,1 -k2,2 | \ sed ' @@ -315,11 +332,23 @@ missing_files=$( p g } + / needed-libraries /!{ + h + s/^\(\S\+\) .*$/List of libraries needed by "\1" is missing./ + p + g + } + / provided-libraries /!{ + h + s/^\(\S\+\) .*$/List of libraries provided by "\1" is missing./ + p + g + } ' ) if [ -n "${missing_files}" ]; then - >&2 echo 'The following packages lack a signature, namcap log or package file:' + >&2 echo 'The following packages lack a signature, namcap log, package file or list of needed/provided libraries:' >&2 echo "${missing_files}" exit 3 fi @@ -353,16 +382,27 @@ package_errors=$( # shellcheck disable=SC2086 printf '%s\n' ${packages} | \ sed ' - s@\(-[^-]\+\)\{2\}-\([^-]\+\)\.pkg\.tar\.xz$@ \2@ - / any$/{ - s|any$|i686| - } s|^|was_built: | ' - sed ' - s|$| i686| - s|^|expected: | - ' "${work_dir}/package-infos/$1.$2.$3.$4.packages" + # shellcheck disable=SC2016 + { + printf 'SELECT CONCAT(' + printf '"expected: ",' + mysql_package_name_query + printf ')' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_build_slaves + mysql_join_binary_packages_repositories + printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ + "$( + printf '%s' "${slave}" | \ + base64 -w0 + )" + printf ' AND `repositories`.`name`="build-list"' + printf ';\n' + } | \ + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 @@ -377,6 +417,62 @@ fi # move namcap.logs find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_log_directory}/success/" \; +# insert provided libraries into database +# shellcheck disable=SC2016 +{ + for lib_link in 'pl:provides' 'nl:needs'; do + printf 'CREATE TEMPORARY TABLE `%s` (`pkgfile` VARCHAR(64), `lib` VARCHAR(128));\n' \ + "${lib_link%:*}" + find . -maxdepth 1 -name '*.pkg.tar.xz.so.'"${lib_link#*:}"'.gz' -execdir zgrep -HF '' '{}' \; | \ + sed -n ' + s,^\./\(.\+\.pkg\.tar\.xz\)\.so\.'"${lib_link#*:}"'\.gz:\([^:]\+\)$,\1\n\2, + T + p + ' | \ + base64_encode_each | \ + sed ' + N + s/^\(\S\+\)\n\(\S\+\)$/(from_base64("\1"),from_base64("\2")),/ + $s/,$/;/ + 2 s/^/INSERT INTO `'"${lib_link%:*}"'` (`pkgfile`,`lib`) VALUES / + ' + printf 'INSERT IGNORE INTO `install_targets` (`name`)' + printf ' SELECT DISTINCT `%s`.`lib` FROM `%s`;\n' \ + "${lib_link%:*}" "${lib_link%:*}" + if [ "${lib_link%:*}" = 'pl' ]; then + printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`)' + else + printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`)' + fi + printf ' SELECT `binary_packages`.`id`,`install_targets`.`id`' + if [ "${lib_link%:*}" = 'nl' ]; then + printf ',`dependency_types`.`id`' + fi + printf ' FROM `install_targets`' + if [ "${lib_link%:*}" = 'nl' ]; then + printf ' JOIN `dependency_types` ON `dependency_types`.`name`="link"' + fi + printf ' JOIN `%s` ON `%s`.`lib`=`install_targets`.`name`' \ + "${lib_link%:*}" "${lib_link%:*}" + printf ' JOIN `binary_packages`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_build_slaves + mysql_join_binary_packages_repositories + printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ + "$( + printf '%s' "${slave}" | \ + base64 -w0 + )" + printf ' AND `repositories`.`name`="build-list"' + printf ' AND ' + mysql_package_name_query + printf '=`%s`.`pkgfile`' \ + "${lib_link%:*}" + printf ';\n' + done +} | \ + mysql_run_query + # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -401,58 +497,46 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove old state files (these should be only "done" markers, but -# actually we don't care what it is) - as long as it's not "testing" or "tested" -find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -not -name '*.testing' \ - -not -name '*.tested' \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ - -exec rm '{}' \; - -# remove all loops which are broken by this package -find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -exec rm '{}.locked' \; - -# remove package from build list -sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" - -# remove package lock file -if ! [ "${destination}" = 'build-support' ]; then - # shellcheck disable=SC2086 - printf '%s\n' ${packages} > \ - "${work_dir}/package-states/$1.$2.$3.$4.done" -fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories '' 'old_repo' printf ' SET' printf ' `build_assignments`.`is_broken`=0,' - printf ' `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`=from_base64("%s")),' \ + printf ' `build_assignments`.`priority`=0,' + printf ' `binary_packages`.`repository`=(SELECT `new_repo`.`id` FROM `repositories` AS `new_repo` WHERE `new_repo`.`name`=from_base64("%s")),' \ "$( printf '%s' "${destination}" | \ base64 -w0 )" printf ' `binary_packages`.`has_issues`=0,' printf ' `binary_packages`.`is_tested`=0' - printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ + printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" - printf 'UPDATE `build_slaves` SET' - printf ' `build_slaves`.`currently_building`=NULL' - printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ + printf ' AND `old_repo`.`name`="build-list";\n' + printf 'UPDATE `build_slaves` AS `to_update`' + printf ' JOIN `build_slaves` AS `current_slave`' + printf ' ON `to_update`.`currently_building`=`current_slave`.`currently_building`' + printf ' SET `to_update`.`currently_building`=NULL' + printf ' WHERE `current_slave`.`name`=from_base64("%s");\n' \ "$( printf '%s' "${slave}" | \ base64 -w0 )" + printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n' + printf 'INSERT INTO `loops_to_delete`' + printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`' + mysql_join_build_dependency_loops_binary_packages + mysql_join_binary_packages_repositories + printf ' WHERE NOT `repositories`.`name` = "build-list";\n' + printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS (' + printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`' + printf ');\n' + printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} -rm -f \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" + mysql_run_query diff --git a/bin/sanity-check b/bin/sanity-check index 8736d47..7d1b2f0 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -20,7 +20,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + irc_say fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -103,7 +103,7 @@ touch "${tmp_dir}/messages" trap 'finish' EXIT if [ $# -eq 0 ]; then - set -- git-repositories build-list repos package-database state-files + set -- git-repositories build-list mysql repos package-database track-state fi while [ $# -gt 0 ]; do @@ -119,16 +119,28 @@ while [ $# -gt 0 ]; do for repo in ${repo_names}; do eval 'repo_path="${repo_paths__'"${repo}"'}"' repo_revision=$( - cat "${work_dir}/${repo}.revision" + # shellcheck disable=SC2016 + { + printf 'SELECT `git_repositories`.`head` FROM `git_repositories`' + printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${repo}" | base64 -w0)" + } | \ + mysql_run_query ) - if ! git -C "${repo_path}" archive "${repo_revision}" -- | \ - tar -t > /dev/null; then + if ! obj_type=$(git -C "${repo_path}" cat-file -t "${repo_revision}" 2>/dev/null); then if [ ${silence} -le 1 ]; then - printf '\nThe repository %s (%s) cannot archive the current revision %s.\n' \ + printf '\nThe repository %s (%s) does not know the current revision %s.\n' \ "${repo}" "${repo_path}" "${repo_revision}" >> \ "${tmp_dir}/messages" fi i_am_insane + elif [ "${obj_type}" != 'commit' ]; then + if [ ${silence} -le 1 ]; then + printf '\nThe repository %s (%s) knows the current revision %s, but it is not a commit, but a %s.\n' \ + "${repo}" "${repo_path}" "${repo_revision}" "${obj_type}" >> \ + "${tmp_dir}/messages" + fi + i_am_insane fi done @@ -145,20 +157,19 @@ while [ $# -gt 0 ]; do "${tmp_dir}/messages" errors=$( - grep -vn '^\S\+ [0-9a-f]\{40\} [0-9a-f]\{40\} \S\+$' "${work_dir}/build-list" - ) || true - if [ -n "${errors}" ]; then - if [ ${silence} -le 1 ]; then - printf '\nThe following build orders are wrongly formatted:\n%s\n' \ - "${errors}" >> \ - "${tmp_dir}/messages" - fi - i_am_insane - fi - - errors=$( - cut -d' ' -f1 < \ - "${work_dir}/build-list" | \ + # shellcheck disable=SC2016 + { + printf 'SELECT `architectures`.`name`,`package_sources`.`pkgbase` FROM `package_sources`' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_architectures + printf ' WHERE EXISTS(' + printf 'SELECT * FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' WHERE `repositories`.`name`="build-list"' + printf ' AND `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ');\n' + } | \ + mysql_run_query | \ sort | \ uniq -d ) @@ -172,13 +183,16 @@ while [ $# -gt 0 ]; do fi errors=$( + # shellcheck disable=SC2016 { - cut -d' ' -f1 < \ - "${work_dir}/build-list" - cat "${work_dir}/deletion-list" + printf 'SELECT `a`.`pkgname` FROM `binary_packages` AS `a`' + mysql_join_binary_packages_repositories 'a' 'a_r' + printf ' AND `a_r`.`name`="build-list"' + printf ' JOIN `binary_packages` AS `b` ON `a`.`pkgname`=`b`.`pkgname`' + mysql_join_binary_packages_repositories 'b' 'b_r' + printf ' AND `b_r`.`name`="deletion-list";\n' } | \ - sort | \ - uniq -d + mysql_run_query ) if [ -n "${errors}" ]; then if [ ${silence} -le 1 ]; then @@ -268,17 +282,18 @@ while [ $# -gt 0 ]; do errors=$( { - tar -tzf "${tmp_dir}/${repo}.db.tar.gz" | \ - grep '/$' | \ - sed ' - s|/$|| - s|^|in_database | + tar -Oxzf "${tmp_dir}/${repo}.db.tar.gz" --wildcards '*/desc' 2>/dev/null | \ + sed -n ' + /^%FILENAME%$/ { + N + s/^.*\n/in_database / + p + } ' echo "${packages}" | \ - grep '\S' | \ sed ' - s|-[^-]\+$|| - s|^|in_repository | + /\.pkg\.tar\.xz$/ !d + s/^/in_repository / ' | \ sort -u } | \ @@ -334,51 +349,80 @@ while [ $# -gt 0 ]; do ;; - state-files) - - for status in 'staging' 'testing'; do + track-state) - [ ${silence} -gt 0 ] || \ - printf 'checking state-files of "%s" ...' "${status}" >> \ - "${tmp_dir}/messages" + [ ${silence} -gt 0 ] || \ + printf 'checking if all packages are tracked correctly ...' >> \ + "${tmp_dir}/messages" - errors=$( + errors=$( + { + # shellcheck disable=SC2016 { - if [ "${status}" = 'staging' ]; then - find "${work_dir}/package-states" -name '*.done' \ - -exec sed 's|^|package-state-file |' '{}' \; - else - find "${work_dir}/package-states" \( -name '*.testing' -o -name '*.tested' \) \ - -exec sed 's|^|package-state-file |' '{}' \; - fi - ls_master_mirror 'i686' | \ - grep "${status}\$" | \ - while read -r repo; do - ls_master_mirror "i686/${repo}" - done | \ - grep '\.pkg\.tar\.xz$' | \ - sed 's|^|package-file |' + printf 'SELECT "mysql",CONCAT(`repositories`.`name`,"/",' + mysql_package_name_query + printf ') FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + mysql_join_binary_packages_architectures } | \ - sort -k2 | \ - uniq -cf1 | \ - grep -v '^\s*2\s' | \ - awk '{print $2 " " $3}' - ) - if [ -n "${errors}" ]; then - if [ ${silence} -le 1 ]; then - printf '\nThe following %s packages do not have state files or vice versa:\n%s\n' \ - "${status}" \ - "${errors}" >> \ - "${tmp_dir}/messages" - fi - i_am_insane + mysql_run_query | \ + tr '\t' ' ' + ls_master_mirror 'i686' | \ + while read -r repo; do + ls_master_mirror "i686/${repo}" | \ + sed ' + /\.pkg\.tar\.xz$/!d + s,^,package-file '"${repo}"'/, + ' + done + } | \ + sed 's/\(-[0-9]\+\)\.0\(-[^- ]\+$\)/\1\2/' | \ + sort -k2 | \ + uniq -uf1 + ) + if [ -n "${errors}" ]; then + if [ ${silence} -le 1 ]; then + printf '\nThe following packages from the master mirror are not tracked in the database or vice versa:\n%s\n' \ + "${errors}" >> \ + "${tmp_dir}/messages" fi + i_am_insane + fi - [ ${silence} -gt 0 ] || \ - echo ' passed.' >> \ + [ ${silence} -gt 0 ] || \ + echo ' passed.' >> \ + "${tmp_dir}/messages" + + ;; + + mysql) + + [ ${silence} -gt 0 ] || \ + printf 'checking mysql-sanity-check-file ...' >> \ + "${tmp_dir}/messages" + + if [ -s "${webserver_directory}/mysql-sanity.html" ]; then + if [ ${silence} -le 1 ]; then + printf '\nThere is something wrong with the database:\n' + cat "${webserver_directory}/mysql-sanity.html" + fi >> \ "${tmp_dir}/messages" + i_am_insane + fi - done + # hopefully, this gets rid of false positives :-) + sleep 1 + + if find "${work_dir}" -mindepth 1 -maxdepth 1 -name 'tmp.mysql-functions.query.*' | \ + grep '\S' >> \ + "${tmp_dir}/messages"; then + i_am_insane + fi + + [ ${silence} -gt 0 ] || \ + echo ' passed.' >> \ + "${tmp_dir}/messages" ;; @@ -399,5 +443,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + irc_say fi diff --git a/bin/seed-build-list b/bin/seed-build-list index acf25c3..ae08d57 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -5,8 +5,6 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -# TODO: put new packages also in mysql database - # shellcheck disable=SC2016 usage() { >&2 echo '' @@ -16,6 +14,9 @@ usage() { >&2 echo ' - a list of packages which need to be rebuilt' >&2 echo '' >&2 echo 'possible options:' + >&2 echo ' -a|--auto:' + >&2 echo ' Automatically reschedule packages which have run-time' + >&2 echo ' dependencies that are not available anywhere.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' >&2 echo ' -i|--ignore $package:' @@ -26,6 +27,8 @@ usage() { >&2 echo ' Do not actually update build-list, just print it.' >&2 echo ' -p|--package $pkg_regex:' >&2 echo ' Update packages matching $pkg_regex.' + >&2 echo ' -w|--wait:' + >&2 echo ' Wait for lock if necessary.' [ -z "$1" ] && exit 1 || exit "$1" } @@ -34,26 +37,32 @@ tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir) trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT eval set -- "$( - getopt -o hi:m:np: \ + getopt -o ahi:m:np:w \ + --long auto \ --long help \ --long ignore: \ --long mirror: \ --long no-action \ --long package: \ + --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" touch "${tmp_dir}/mirrors" -touch "${tmp_dir}/delta-packages" touch "${tmp_dir}/package-regexes" touch "${tmp_dir}/ignore-packages" +auto=false update=true +wait_for_lock='-n' while true do case "$1" in + -a|--auto) + auto=true + ;; -h|--help) usage 0 ;; @@ -75,6 +84,9 @@ do echo "$1" >> \ "${tmp_dir}/package-regexes" ;; + -w|--wait) + wait_for_lock='' + ;; --) shift break @@ -92,207 +104,280 @@ if [ $# -ne 0 ]; then fi if [ ! -s "${tmp_dir}/mirrors" ] && \ - [ ! -s "${tmp_dir}/package-regexes" ]; then + [ ! -s "${tmp_dir}/package-regexes" ] && \ + ! ${auto}; then # nothing to do exit 0 fi -repos="${stable_package_repositories}" - -find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \ - sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \ - sort -k1,1 > \ - "${tmp_dir}/known-packages" - -mod_repo_rev=$(cat "${work_dir}/archlinux32.revision") -{ - for repo in ${repo_names}; do - if [ "${repo_names}" = 'archlinux32' ]; then - continue - fi - eval 'repo_path="${repo_paths__'"${repo}"'}"' - repo_rev=$(cat "${work_dir}/${repo}.revision") - git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sed ' - s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2| - ' - done | \ - sort -u | \ - sort -k1,1 - awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \ - sort -u | \ - sed 'p' -} | \ - sort -k2,3 | \ - uniq -uf1 | \ - while read -r repo_rev pkg prepo; do - generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}" - done - -find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \ - sed ' - s|^.*/|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 | - ' | \ - sort -k5,5 > \ - "${tmp_dir}/known-packages" +# get locks +if ${update}; then + exec 9> "${sanity_check_lock_file}" + if ! flock -s ${wait_for_lock} 9; then + >&2 echo 'Cannot get sanity-check lock.' + exit 1 + fi -# generate delta-packages from package-regexes -while read -r pkg_regex; do - if [ -z "${pkg_regex}" ]; then - continue + exec 8> "${build_list_lock_file}" + if ! flock ${wait_for_lock} 8; then + >&2 echo 'Cannot get build-list lock.' + exit 1 fi - awk '{print $5}' "${tmp_dir}/known-packages" | \ - grep "${pkg_regex}" || \ - true -done < \ - "${tmp_dir}/package-regexes" >> \ - "${tmp_dir}/delta-packages" +fi -# genereate delta_packages from mirror delta +repos="${stable_package_repositories}" -while read -r mirror; do - if [ -z "${mirror}" ]; then - continue - fi +# genereate must-haves query from mirror delta +if [ -s "${tmp_dir}/mirrors" ]; then { # theirs - for repo in ${repos}; do - curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ - tar -tz - done | \ - grep '/$' | \ - sed ' - s|/$|| - s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1| - ' - + while read -r mirror; do + if [ -z "${mirror}" ]; then + continue + fi + for repo in ${repos}; do + curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ + tar -Oxz --wildcards '*/desc' | \ + sed ' + /^%FILENAME%$/!d + N + s/^.*\n// + s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/ + s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/theirs \2 \3 \1/ + ' + done + done < \ + "${tmp_dir}/mirrors" # ours - for repo in $(ls_master_mirror 'i686'); do - ls_master_mirror "i686/${repo}" | \ - grep '\.pkg\.tar\.xz$' | \ - sed 's|-[^-]\+$||' - done | \ + # shellcheck disable=SC2016 + { + printf 'SELECT ' + mysql_package_name_query + printf ' FROM `binary_packages`' + mysql_join_binary_packages_architectures + } | \ + mysql_run_query | \ sed ' - s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1| + s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ - expand_version 3 | \ - sort -k4,4 -k3Vr,3 -k1,1 | \ - shrink_version 3 | \ - uniq -f3 | \ - grep '^theirs ' | \ - awk '{print $4}' | \ - sort -k1,1 >> \ - "${tmp_dir}/delta-packages" -done < \ - "${tmp_dir}/mirrors" - -sort -k1,1 -u "${tmp_dir}/delta-packages" | \ - sponge "${tmp_dir}/delta-packages" - -cat \ - "${tmp_dir}/delta-packages" \ - "${tmp_dir}/ignore-packages" \ - "${tmp_dir}/ignore-packages" | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/delta-packages" - -join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \ - "${tmp_dir}/append-packages" + expand_version 2 | \ + sort -k3,4 -k2Vr,2 -k1,1 | \ + shrink_version 2 | \ + uniq -f2 | \ + sed -n ' + s/^theirs \(\S\+ \)\{2\}// + T + p + ' | \ + sort -u > \ + "${tmp_dir}/must-haves" +fi +# shellcheck disable=SC2016 { - awk '{print $5}' "${tmp_dir}/append-packages" - cat "${tmp_dir}/delta-packages" + printf 'CREATE TEMPORARY TABLE `ignore_packages` (`pkgname` VARCHAR(64));\n' + if [ -s "${tmp_dir}/ignore-packages" ]; then + grep -vxF '' "${tmp_dir}/ignore-packages" | \ + base64_encode_each | \ + sed ' + s/^/(from_base64("/ + s/$/")),/ + $s/,$/;/ + 1 s/^/INSERT INTO `ignore_packages` (`pkgname`) VALUES \n/ + ' + fi + # packages on the build-list or deletion-list should be ignored + printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)' + printf ' SELECT DISTINCT `ignore_bin`.`pkgname`' + printf ' FROM `binary_packages` AS `ignore_bin`' + mysql_join_binary_packages_repositories 'ignore_bin' + printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list")' + # packages with no not-to-be-deleted and at least on to-be-deleted version should be ignored + printf ' OR (' + printf '`ignore_bin`.`is_to_be_deleted`' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `binary_packages` AS `other_bin`' + printf ' WHERE NOT `other_bin`.`is_to_be_deleted`' + printf ' AND `other_bin`.`pkgname`=`ignore_bin`.`pkgname`' + printf '));\n' } | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/delta-packages" + sponge "${tmp_dir}/ignore-packages" -if [ -s "${tmp_dir}/delta-packages" ]; then - echo 'There are still packages newer for x86_64 which I cannot identify :-/' - cat "${tmp_dir}/delta-packages" - exit 2 -fi - -rm "${tmp_dir}/delta-packages" - -# convert append-packages to build-list.new (add git hashes) +# shellcheck disable=SC2016 +{ + printf 'CREATE TEMPORARY TABLE `must_haves` (`pkgname` VARCHAR(64));\n' + if [ -s "${tmp_dir}/must-haves" ]; then + grep -vxF '' "${tmp_dir}/must-haves" | \ + base64_encode_each | \ + sed ' + s/^/(from_base64("/ + s/$/")),/ + $s/,$/;/ + 1 s/^/INSERT INTO `must_haves` (`pkgname`) VALUES \n/ + ' + fi +} | \ + sponge "${tmp_dir}/must-haves" -cut -d' ' -f1,4 < \ - "${tmp_dir}/append-packages" | \ - sort -u | \ - while read -r pkg repo; do - git_repo=$(find_git_repository_to_package_repository "${repo}") - printf '%s %s %s %s\n' \ - "${pkg}" \ - "$(cat "${work_dir}/${git_repo}.revision")" \ - "$(cat "${work_dir}/archlinux32.revision")" \ - "${repo}" +# fetch unknown must-haves from upstream +# shellcheck disable=SC2016 +printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` VARCHAR(64));\n' > \ + "${tmp_dir}/pkgbases" +# shellcheck disable=SC2016 +{ + cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" + printf 'SELECT `must_haves`.`pkgname` FROM `must_haves`' + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `binary_packages`' + printf ' WHERE `binary_packages`.`pkgname`=`must_haves`.`pkgname`' + printf ') AND NOT EXISTS (' + printf 'SELECT * FROM `ignore_packages`' + printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' + printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' +} | \ + mysql_run_query | \ + while read -r pkgname; do + content=$( + curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ + tr ',' '\n' + ) + repo=$( + printf '%s\n' "${content}" | \ + sed -n ' + s/^\s*"repo"\s*:\s*"// + T + s/".*$// + T + p + ' + ) + pkgbase=$( + printf '%s\n' "${content}" | \ + sed -n ' + s/^\s*"pkgbase"\s*:\s*"// + T + s/".*$// + T + p + ' + ) + if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then + >&2 printf 'Could not find "%s" which is newer on x86_64!?\n' "${pkgname}" + exit 2 + fi + printf '(from_base64("%s"),from_base64("%s")),\n' \ + "$(printf '%s' "${pkgbase}" | base64 -w0)" \ + "$(printf '%s' "${repo}" | base64 -w0)" done | \ - sort -u > \ - "${tmp_dir}/build-list.append" - -# Create a lock file for build list. - -if ${update}; then - # always block if locked - exec 9> "${build_list_lock_file}" - flock 9 - - exec 8> "${sanity_check_lock_file}" - flock -s 8 -fi + sort -u | \ + sed ' + 1 s/^/INSERT IGNORE INTO `pkgbases` (`pkgbase`,`repository`) VALUES \n/ + $s/,$/;/ + ' >> \ + "${tmp_dir}/pkgbases" +# shellcheck disable=SC2016 { - awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append" - - # ignore packages on the build-list - awk '{print $2 " " $3 " " $4 " " $1}' < \ - "${work_dir}/build-list" | \ - sed 'p' - - # ignore packages on the deletion-list - awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \ - sed 'p' - - # ignore packages on the blacklist - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \ - tar -Ox blacklist | \ - sed ' - s/\s*#.*$// - /^\s*$/d - ' | \ - awk '{print "X X X " $1}' | \ - sed 'p' - - # ignore explicitely ignored packages - awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \ - sed 'p' + cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" "${tmp_dir}/pkgbases" + if ${auto}; then + printf 'CREATE TEMPORARY TABLE `bin_ids` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'INSERT IGNORE INTO `bin_ids`' + printf ' SELECT `binary_packages`.`id` FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`is_on_master_mirror`' + printf ' AND `repositories`.`name`!="build-support"' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + # nothing "less stable" has the same name + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `binary_packages` AS `subst_bp`' + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + printf ' AND `subst_r`.`name`!="build-support"' + printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`stability`' + printf ' WHERE `repository_stability_relations`.`more_stable`=`repositories`.`stability`' + printf ' AND `subst_bp`.`id`!=`binary_packages`.`id`' + printf ' AND `subst_bp`.`pkgname`=`binary_packages`.`pkgname`' + printf ') AND NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'subst_bp' + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ')' + printf ';\n' + fi + printf 'SELECT ' + printf '`pkgbases`.`pkgbase`,' + printf '`git_repositories`.`head`,' + printf '(' + printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`' + printf ' WHERE `al32`.`name`="archlinux32"' + printf '),' + printf '`pkgbases`.`repository`' + printf ' FROM `pkgbases`' + printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`' + mysql_join_upstream_repositories_git_repositories + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `ignore_packages`' + printf ' JOIN `binary_packages` AS `i_bp` ON `ignore_packages`.`pkgname`=`i_bp`.`pkgname`' + mysql_join_binary_packages_build_assignments 'i_bp' 'i_ba' + mysql_join_build_assignments_package_sources 'i_ba' 'i_ps' + printf ' WHERE `i_ps`.`pkgbase`=`pkgbases`.`pkgbase`' + printf ');\n' + printf ';\n' + printf 'SELECT ' + printf '`package_sources`.`pkgbase`,' + printf '`git_repositories`.`head`,' + printf '(' + printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`' + printf ' WHERE `al32`.`name`="archlinux32"' + printf '),' + printf '`upstream_repositories`.`name`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_git_repositories + printf ' WHERE (' + if [ -s "${tmp_dir}/package-regexes" ]; then + grep -vxF '' "${tmp_dir}/package-regexes" | \ + base64_encode_each | \ + sed ' + s/^/`binary_packages`.`pkgname` REGEXP from_base64("/ + s/$/") OR / + ' | \ + tr -d '\n' + fi + if ${auto}; then + printf 'EXISTS (' + printf 'SELECT * FROM `bin_ids`' + printf ' WHERE `bin_ids`.`id`=`binary_packages`.`id`' + printf ')' + printf ' OR ' + fi + printf 'EXISTS (' + printf 'SELECT * FROM `must_haves`' + printf ' WHERE `must_haves`.`pkgname`=`binary_packages`.`pkgname`' + printf ')) AND NOT EXISTS (' + printf 'SELECT * FROM `ignore_packages`' + printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' + printf ');\n' } | \ - sort -k4,4 | \ - uniq -uf3 > \ - "${tmp_dir}/build-list.new" - -if ${update}; then - awk '{print $4 " " $1 " " $2 " " $3}' \ - "${tmp_dir}/build-list.new" | \ - tee -a "${work_dir}/build-list" - - while read -r git_revision mod_git_revision repository pkgbase; do - mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" - done < \ - "${tmp_dir}/build-list.new" - - # Remove the lock file - - rm -f "${build_list_lock_file}" -else - awk '{print $4 " " $1 " " $2 " " $3}' \ - "${tmp_dir}/build-list.new" -fi + mysql_run_query | \ + sort -u | \ + tr '\t' ' ' | \ + if ${update}; then + while read -r pkgbase git_rev mod_git_rev repo; do + printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2 + printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ + sed 's/ $/\n/' + mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" + printf '\n' >&2 + done + + rm -f "${build_list_lock_file}" + else + cat + fi diff --git a/bin/show-dependencies b/bin/show-dependencies index efdf745..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -3,85 +3,19 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: can this be faster? + # Create a lock file. if [ $# -eq 0 ]; then broken=$( - find "${work_dir}/package-states" -maxdepth 1 -name '*.broken' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - sort -u - ) - new_sum='x' - sum='' - tmp_dir=$(mktemp -d 'tmp.show-dependencies.0.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - find "${work_dir}/package-infos" -maxdepth 1 -name '*.builds' -exec \ - grep -HF '' {} \; | \ - sed ' - s|^.*/|| - s|\(\.[^.]\+\)\{4\}:| | - ' | \ - sort -k2,2 > \ - "${tmp_dir}/builds" - while ! [ "${new_sum}" = "${sum}" ]; do - sum="${new_sum}" - # shellcheck disable=SC2086 - printf '%s\n' ${broken} | \ - sort > \ - "${tmp_dir}/broken" - broken=$( - { - find "${work_dir}/package-infos" -maxdepth 1 -name '*.build-depends' | \ - sed 's|^.*/\(.*\)\(\.[^.]\+\)\{4\}$|\1 \0|' | \ - sort -k1,1 | \ - join -1 1 -2 1 -o 1.2 - "${tmp_dir}/broken" | \ - xargs -r cat - } | \ - sort -u | \ - join -1 1 -2 2 -o 2.1 - "${tmp_dir}/builds" - ) - broken=$( - { - { - # shellcheck disable=SC2086 - printf '%s\n' ${broken} - cat "${tmp_dir}/broken" - } | \ - sort -u - cut -d' ' -f1 < \ - "${work_dir}/build-list" | \ - sort -u - } | \ - sort | \ - uniq -d - ) - new_sum=$( - # shellcheck disable=SC2086 - printf '%s\n' ${broken} | \ - sha512sum - ) - done - - rm -rf --one-file-system "${tmp_dir}" - trap - EXIT - broken=$( - { - # shellcheck disable=SC2086 - printf '%s\n' ${broken} 'ALL' - { - find "${webserver_directory}/graphs" -maxdepth 1 -name '*.png' -printf '%f\n' | \ - sed 's|\.png$||' - { - cut -d' ' -f1 < \ - "${work_dir}/build-list" - cat "${work_dir}/deletion-list" - } | \ - sort -u - } | \ - sort | \ - uniq -d - } | \ + printf 'CALL show_broken_packages_and_dependencies;\n' | \ + mysql_run_query | \ + sed ' + s/\s.*$// + ' | \ sort -u + printf 'ALL\n' ) # shellcheck disable=SC2086 "$0" ${broken} @@ -119,360 +53,248 @@ fi tmp_dir=$(mktemp -d 'tmp.show-dependencies.1.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -sort -u "${work_dir}/build-order" | \ - grep -v '^\(\S\+\) \1$' | \ - grep '^\S\+ \S\+$' > \ - "${tmp_dir}/original-build-order" - -sort -k1,1 "${work_dir}/build-list" > \ - "${tmp_dir}/build-list.sorted-by-package" - -{ - sort -k1,1 "${tmp_dir}/original-build-order" | \ - join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package" - sort -k2,2 "${tmp_dir}/original-build-order" | \ - join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/build-list.sorted-by-package" -} | \ - sort -u | \ - sponge "${tmp_dir}/original-build-order" - -find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - sort -u > \ - "${tmp_dir}/known-packages" - -find "${work_dir}/package-infos" -maxdepth 1 -name '*.groups' -printf '%f %p\n' | \ - while read -r name pf; do - xargs -r printf "${name%.*.*.*.*}"' %s\n' < \ - "${pf}" - done > \ - "${tmp_dir}/known-groups" - -find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f %p\n' | \ - while read -r name pf; do - grep -vxF "${name%.*.*.*.*}" "${pf}" | \ - xargs -r printf "${name%.*.*.*.*}"' %s\n' - done > \ - "${tmp_dir}/known-split-packages" - -find "${work_dir}/package-infos" -maxdepth 1 -name '*.run-depends' -printf '%f %p\n' | \ - while read -r name pf; do - xargs -r printf '%s '"${name%.*.*.*.*}"'\n' < \ - "${pf}" - done > \ - "${tmp_dir}/known-run-depends" - for target_package in "$@"; do output="${webserver_directory}/graphs/${target_package}.png" - if [ "${target_package}" = 'ALL' ]; then - - # groups and split packages built by jobs on the build list - cat "${tmp_dir}/known-groups" "${tmp_dir}/known-split-packages" | \ - sort -k1,1 | \ - join -1 1 -2 1 -o 1.2 - "${tmp_dir}/build-list.sorted-by-package" | \ - sort -u > \ - "${tmp_dir}/relevant-stuff" - - { - # groups and split packages built by jobs on the build list - which are also dependencies of packages on the build-list - sort -k1,1 "${tmp_dir}/original-build-order" | \ - join -1 1 -2 1 -o 2.1 - "${tmp_dir}/relevant-stuff" - # build list jobs themself - cut -d' ' -f1 < "${work_dir}/build-list" - } | \ - sort -u | \ - sponge "${tmp_dir}/relevant-stuff" - - { - sort -k1,1 "${tmp_dir}/original-build-order" | \ - join -1 1 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff" | \ - sort -k2,2 | \ - join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/relevant-stuff" - sed 's/.*/\0 \0/' "${tmp_dir}/relevant-stuff" - } > \ - "${tmp_dir}/build-order" - - rm "${tmp_dir}/relevant-stuff" - - find "${work_dir}/package-states" -name '*.done' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - sort -u | \ - tee "${tmp_dir}/done-packages" | \ - sed 's|^|0 staging-package |' > \ - "${tmp_dir}/knots" - - { - cat \ - "${tmp_dir}/known-groups" \ - "${tmp_dir}/known-run-depends" \ - "${tmp_dir}/known-split-packages" - awk '{print $1 " " $1}' < \ - "${tmp_dir}/known-packages" - } | \ - sort -k1,1 > \ - "${tmp_dir}/known-connections.sorted.1" - - sort -k2,2 \ - "${tmp_dir}/known-connections.sorted.1" > \ - "${tmp_dir}/known-connections.sorted.2" - - { - # staging -> something -> build-list - join -1 1 -2 1 -o 1.1,1.2 \ - "${tmp_dir}/known-connections.sorted.1" \ - "${tmp_dir}/done-packages" | \ - sort -k2,2 | \ - join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \ - - \ - "${tmp_dir}/known-connections.sorted.1" | \ - sort -k4,4 | \ - join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \ - - \ - "${tmp_dir}/build-list.sorted-by-package" | \ - sed ' - s|^\(\S\+ \S\+\) |\1\n| - ' - # staging -> something -> staging - join -1 1 -2 1 -o 1.1,1.2 \ - "${tmp_dir}/known-connections.sorted.1" \ - "${tmp_dir}/done-packages" | \ - sort -k2,2 | \ - join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \ - - \ - "${tmp_dir}/known-connections.sorted.1" | \ - sort -k4,4 | \ - join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \ - - \ - "${tmp_dir}/done-packages" | \ - sed ' - s|^\(\S\+ \S\+\) |\1\n| - ' - # build-list -> something -> staging - join -1 1 -2 1 -o 1.1,1.2 \ - "${tmp_dir}/known-connections.sorted.1" \ - "${tmp_dir}/build-list.sorted-by-package" | \ - sort -k2,2 | \ - join -1 2 -2 1 -o 1.1,1.2,2.1,2.2 \ - - \ - "${tmp_dir}/known-connections.sorted.1" | \ - sort -k4,4 | \ - join -1 4 -2 1 -o 1.1,1.2,1.3,1.4 \ - - \ - "${tmp_dir}/done-packages" | \ - sed ' - s|^\(\S\+ \S\+\) |\1\n| - ' - cat "${tmp_dir}/build-order" - } | \ - sort -u | \ - sponge "${tmp_dir}/build-order" - - rm \ - "${tmp_dir}/done-packages" \ - "${tmp_dir}/known-connections.sorted.1" \ - "${tmp_dir}/known-connections.sorted.2" - - else - - grep " $(str_to_regex "${target_package}")\$" "${tmp_dir}/original-build-order" | \ - sort -u > \ - "${tmp_dir}/build-order" - printf '0 target-package %s\n' "${target_package}" > \ - "${tmp_dir}/knots" - - last_sum='' - current_sum=$(sha512sum "${tmp_dir}/build-order") - while ! [ "${last_sum}" = "${current_sum}" ]; do - - last_sum="${current_sum}" - - awk '{print $1}' "${tmp_dir}/build-order" | \ - sort -u > \ - "${tmp_dir}/new" - { - cat \ - "${tmp_dir}/original-build-order" \ - "${tmp_dir}/known-split-packages" - sed '/ base\(-devel\)\?$/d' "${tmp_dir}/known-groups" - } | \ - sort -k2,2 | \ - join -1 2 -2 1 -o 1.1,1.2 - "${tmp_dir}/new" | \ - sponge -a "${tmp_dir}/build-order" - - rm "${tmp_dir}/new" - - sort -u "${tmp_dir}/build-order" | \ - sponge "${tmp_dir}/build-order" - - current_sum=$(sha512sum "${tmp_dir}/build-order") - - done - - fi + # shellcheck disable=SC2016 + { + printf 'CREATE TEMPORARY TABLE `relevant_binary_packages` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `relevant_binary_packages_copy` (`id` BIGINT, UNIQUE KEY (`id`));\n' + printf 'CREATE TEMPORARY TABLE `relevant_install_targets` (`id` BIGINT, UNIQUE KEY (`id`));\n' -# grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \ -# sponge "${tmp_dir}/build-order" + if [ "${target_package}" = 'ALL' ]; then - # shellcheck disable=SC2129 - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u | \ - join -j 1 - "${tmp_dir}/build-list.sorted-by-package" | \ - while read -r pkg rev mod_rev repo; do - if [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.broken" ]; then - printf '1 broken-build-list-package %s\n' "${pkg}" - elif [ -f "${work_dir}/package-states/${pkg}.${rev}.${mod_rev}.${repo}.blocked" ]; then - printf '2 blocked-build-list-package %s\n' "${pkg}" - else - printf '3 build-list-package %s\n' "${pkg}" - fi - done >> \ - "${tmp_dir}/knots" + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT DISTINCT `binary_packages`.`id`' + printf ' FROM `repositories`' + mysql_join_repositories_repository_stabilities + printf ' AND `repository_stabilities`.`name` IN ("unbuilt","staging")' + mysql_join_repositories_binary_packages + printf ';\n' - { - sort -u "${work_dir}/deletion-list" - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - xargs -r printf '4 deletion-list-package %s\n' >> \ - "${tmp_dir}/knots" + else - { - awk '{print $2}' "${tmp_dir}/known-split-packages" | \ - sort -u - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - xargs -r printf '5 split-package %s\n' >> \ - "${tmp_dir}/knots" + printf 'CALL calculate_dependencies_of_package_upto_first_built_one(from_base64("%s"));\n' \ + "$( + printf '%s' "${target_package}" | \ + base64 -w0 + )" - { - cat "${tmp_dir}/known-packages" - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - xargs -r printf '6 package %s\n' >> \ - "${tmp_dir}/knots" + fi - { - awk '{print $2}' "${tmp_dir}/known-groups" | \ - sort -u - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u + printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)' + printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`' + printf ';\n' + printf 'INSERT IGNORE INTO `relevant_install_targets` (`id`)' + printf ' SELECT DISTINCT `install_target_providers`.`install_target`' + printf ' FROM `relevant_binary_packages`' + mysql_join_binary_packages_install_target_providers 'relevant_binary_packages' + mysql_join_install_target_providers_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + printf ' JOIN `relevant_binary_packages_copy` ON `relevant_binary_packages_copy`.`id`=`dependencies`.`dependent`' + printf ';\n' + + # we return lines with either: + # "knot" $type $identifier $label + # or + # "edge" $type $from_knot $to_knot + printf 'SELECT DISTINCT' + printf ' "knot",' + printf 'IF(`build_assignments`.`is_%s`,"%s-build-list-pkgbase",' \ + 'blocked' 'broken' \ + 'broken' 'broken' + printf '"build-list-pkgbase"' + printf ')),' + printf 'CONCAT("pkgbase:",`package_sources`.`id`),' + printf '`package_sources`.`pkgbase`' + printf ' FROM `relevant_binary_packages`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' AND `repository_stabilities`.`name`="unbuilt"' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ';\n' + + printf 'SELECT DISTINCT' + printf ' "knot",' + printf 'CONCAT("pkgname-",`repository_stabilities`.`name`),' + printf 'CONCAT("pkgname:",`binary_packages`.`id`),' + printf '`binary_packages`.`pkgname`' + printf ' FROM `relevant_binary_packages`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ';\n' + + printf 'SELECT DISTINCT' + printf ' "knot","install-target",' + printf 'CONCAT("install-target:",`install_targets`.`id`),' + printf '`install_targets`.`name`' + printf ' FROM `relevant_install_targets`' + printf ' JOIN `install_targets` ON `install_targets`.`id`=`relevant_install_targets`.`id`' + printf ';\n' + + printf 'SELECT DISTINCT' + printf ' "edge","builds"' + printf ',CONCAT("%s:",`%s`.`id`)' \ + 'pkgbase' 'package_sources' \ + 'pkgname' 'binary_packages' + printf ' FROM `relevant_binary_packages`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + printf ' AND `repository_stabilities`.`name`="unbuilt"' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ';\n' + + printf 'SELECT DISTINCT' + printf ' "edge","provides"' + printf ',CONCAT("%s:",`%s`.`id`)' \ + 'pkgname' 'binary_packages' \ + 'install-target' 'install_targets' + printf ' FROM `relevant_binary_packages`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_install_target_providers + printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`install_target_providers`.`install_target`' + mysql_join_install_target_providers_install_targets + printf ';\n' + + printf 'SELECT DISTINCT' + printf ' "edge",CONCAT("depends:",`dependency_types`.`name`)' + printf ',CONCAT("%s:",`%s`.`id`)' \ + 'install-target' 'install_targets' \ + 'pkgname' 'binary_packages' + printf ' FROM `relevant_binary_packages`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_dependencies + printf ' JOIN `relevant_install_targets` ON `relevant_install_targets`.`id`=`dependencies`.`depending_on`' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_targets + printf ';\n' } | \ - sort | \ - uniq -d | \ - xargs -r printf '7 group %s\n' >> \ - "${tmp_dir}/knots" - - tr ' ' '\n' < \ - "${tmp_dir}/build-order" | \ - sort -u | \ - xargs -r printf '100 unknown %s\n' >> \ + mysql_run_query | \ + sed ' + y/\t/ / + /^knot /{ + s/^\S\+ // + w '"${tmp_dir}"'/knots + d + } + /^edge /{ + s/^\S\+ // + w '"${tmp_dir}"'/edges + d + } + ' >&2 + + mkdir "${tmp_dir}/find-identical" + + awk '{ + print $1 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2 + }' < \ "${tmp_dir}/knots" + awk '{ + print $1 " X " $3 >> "'"${tmp_dir}"'/find-identical/characteristics-"$2 + print $1 " " $2 " X" >> "'"${tmp_dir}"'/find-identical/characteristics-"$3 + }' < \ + "${tmp_dir}/edges" + + modifier=$( + find "${tmp_dir}/find-identical" -maxdepth 1 \ + -type f \ + -name 'characteristics-*' | \ + while read -r file; do + printf '%s ' "${file##*/}" + sort -u "${file}" | \ + sha512sum | \ + awk '{print $1}' + done | \ + sort -k2,2 | \ + uniq -Df1 | \ + sed 's,^characteristics-,,' | \ + sed ' + :a + $!N + s/^\(\S\+\) \(\S\+\)\n\(\S\+ \)\2$/\1+\3\2/ + ta + P + D + ' | \ + cut -d' ' -f1 | \ + sed ' + s/^/s,\\(/ + s/+\([^+[:space:]]\+\)$/\\)\\(\\s\\|$\\),\1\\2,g;/ + s/+/\\|/g + ' + ) + rm -rf --one-file-system "${tmp_dir}/find-identical" - sort -k3,3 -k1n,1 "${tmp_dir}/knots" | \ - uniq -f2 | \ - awk '{print $3 " " $1 " " $2}' | \ - sort -k2,3 | \ + sed "${modifier}" "${tmp_dir}/knots" | \ + sort -k2,2 -k1,1 | \ + sed ' + :a + $!N + s/^\(\S\+ \S\+ \)\(\S\+\)\n\1\(\S\+\)$/\1\2<nl>\3/ + ta + P + D + ' | \ sponge "${tmp_dir}/knots" - mkdir "${tmp_dir}/neighbours" - cat "${tmp_dir}/knots" "${tmp_dir}/build-order" | \ - awk '{ - print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$1; - print $1 " " $2 > "'"${tmp_dir}/neighbours/"'"$2; - }' - - find "${tmp_dir}/neighbours" -maxdepth 1 -type f | \ - while read -r file; do - sed ' - s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@ - s@\(^\| \)'"$(str_to_regex "${file##*/}")"'\( \|$\)@\1<knot>\2@ - ' "${file}" | \ - sort -u | \ - sponge "${file}" - done - - find "${tmp_dir}/neighbours" -maxdepth 1 -type f -exec \ - sha512sum {} \; | \ - sed 's|^\(\S\+\) .*/\([^/]\+\)$|\2 \1|' | \ - sort -k2,2 | \ - uniq -f1 -D | \ - awk '{print $2 " " $1}' > \ - "${tmp_dir}/sums" - - rm -rf --one-file-system "${tmp_dir}/neighbours" - - sed ' - :a - $!N - s|^\(\S\+\) \([^\n]\+\)\n\1 |\1 \2<nl>| - ta - P - D - ' "${tmp_dir}/sums" | \ - join -1 1 -2 1 -o 2.2,1.2 - "${tmp_dir}/sums" | \ - while read -r original replacement; do - sed -i ' - s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@ - s@\( \|^\)'"$(str_to_regex "${original}")"'\( \|$\)@\1'"${replacement}"'\2@ - ' "${tmp_dir}/build-order" "${tmp_dir}/knots" - done - - grep -v '^\(\S\+\) \1$' "${tmp_dir}/build-order" | \ + sed "${modifier}" "${tmp_dir}/edges" | \ sort -u | \ - sponge "${tmp_dir}/build-order" - sort -u "${tmp_dir}/knots" | \ - sponge "${tmp_dir}/knots" + sponge "${tmp_dir}/edges" { printf '%s\n' \ 'digraph dependencies {' \ ' fontname=dejavu;' - cut -d' ' -f1,3 < \ - "${tmp_dir}/knots" | \ - while read -r who what; do - if [ "${what}" = 'broken-build-list-package' ]; then + # knots: $type $identifier $label + # edges: $type $from_knot $to_knot + + while read -r type id label; do + case "${type}" in + 'broken-build-list-pkgbase') color='#ff0000' - elif [ "${what}" = 'blocked-build-list-package' ]; then + ;; + 'blocked-build-list-pkgbase') color='#d00000' - elif [ "${what}" = 'build-list-package' ]; then + ;; + 'build-list-pkgbase') color='#800000' - elif [ "${what}" = 'deletion-list-package' ]; then - color='#808080' - elif [ "${what}" = 'group' ]; then - color='#0000ff' - elif [ "${what}" = 'package' ]; then - color='#000000' - elif [ "${what}" = 'split-package' ]; then - color='#8080ff' - elif [ "${what}" = 'staging-package' ]; then + ;; + 'pkgname-unbuilt') + color='#800000' + ;; + 'pkgname-staging') color='#008000' - elif [ "${what}" = 'target-package' ]; then - color='#00ff00' - else + ;; + 'pkgname-stable'|'pkgname-testing'|'pkgname-standalone') + color='#000000' + ;; + 'install-target') + color='#000080' + ;; + *) color='#ff80ff' - fi - printf ' "%s" [fontcolor="%s"];\n' "${who}" "${color}" - done + label="${label} (${type})" + ;; + esac + printf ' "%s" [label="%s", fontcolor="%s"];\n' "${id}" "${label}" "${color}" + done < \ + "${tmp_dir}/knots" - sed 's|\\|\\\\|g' "${tmp_dir}/build-order" | \ - xargs -rn2 printf ' "%s" -> "%s";\n' | \ - sort -u + while read -r type from_knot to_knot; do + printf '"%s" -> "%s";\n' \ + "${from_knot}" "${to_knot}" + done < \ + "${tmp_dir}/edges" printf '%s\n' \ '}' @@ -486,16 +308,12 @@ for target_package in "$@"; do ' "${tmp_dir}/input" line_count=$(wc -l < "${tmp_dir}/input") - if [ "${line_count}" -gt 500 ] && [ "${target_package}" != 'ALL' ]; then - sed -i '/"base\(-devel\)\?"/d' "${tmp_dir}/input" - line_count=$(wc -l < "${tmp_dir}/input") - if [ "${line_count}" -gt 700 ]; then - rm -f "${output}" - >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \ - "${target_package}" \ - "${line_count}" - continue - fi + if [ "${target_package}" != 'ALL' ] && [ "${line_count}" -gt 1000 ]; then + rm -f "${output}" + >&2 printf 'Skipping graph for "%s" - would be too big (%d).\n' \ + "${target_package}" \ + "${line_count}" + continue fi printf 'small enough (%s): %d\n' \ "${target_package}" \ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 2ca72a3..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -3,11 +3,12 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" -if [ "${SSH_ORIGINAL_COMMAND%% *}" = "get-assignment" ] || \ - [ "${SSH_ORIGINAL_COMMAND%% *}" = "return-assignment" ]; then +if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ + [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xreturn-assignment' ] || \ + [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xping-from-slave' ]; then # small check to prevent some shell-injections - if echo "${SSH_ORIGINAL_COMMAND}" | \ + if printf '%s\n' "${SSH_ORIGINAL_COMMAND}" | \ grep -q '[^-a-zA-Z0-9.+_ ]'; then >&2 printf 'Invalid command: "%s".\n' "${SSH_ORIGINAL_COMMAND}" @@ -28,7 +29,7 @@ if [ "${SSH_ORIGINAL_COMMAND%% *}" = "get-assignment" ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/strict-bashism-check b/bin/strict-bashism-check index 3bb3884..cbf7e5b 100755 --- a/bin/strict-bashism-check +++ b/bin/strict-bashism-check @@ -49,37 +49,54 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT git archive "${tree}" | \ tar -C "${tmp_dir}" -x +if ! cd "${tmp_dir}"; then + echo 'Cannot cd.' + exit 1 +fi + errors=$( - cd "${tmp_dir}" || \ - echo 'Cannot cd.' - find "bin" "conf" -type f -not -executable -not -name '.gitignore' + find bin conf lib -type f -not -executable -not -name '.gitignore' ) if [ -n "${errors}" ]; then - >&2 echo 'Non-executable files found in bin/ or conf/:' + >&2 echo 'Non-executable files found in bin/, conf/ or lib/:' >&2 echo "${errors}" exit 1 fi errors=$( -# shellcheck disable=SC2016 - find bin conf -type f -executable -exec grep -H '="\$(' {} \; + grep -r '\(\s\|^\)mysql buildmaster\($\|\s\)' bin lib conf ) if [ -n "${errors}" ]; then - >&2 echo 'Unnecessary quotes found:' + >&2 echo 'Style error: call "mysql_run_query" instead of "mysql buildmaster":' >&2 echo "${errors}" exit 1 fi errors=$( - cd "${tmp_dir}" || \ - echo 'Cannot cd.' - shellcheck -x bin/* conf/* 2>&1 +# shellcheck disable=SC2016 + find bin conf lib -type f -executable -exec grep -H '="\$(' {} \; ) if [ -n "${errors}" ]; then - >&2 echo 'shellcheck complains about the following:' + >&2 echo 'Unnecessary quotes found:' >&2 echo "${errors}" exit 1 fi + +if which shellcheck >/dev/null 2>&1; then + errors=$( + find bin conf lib \ + -type f \ + -not -name 'opcode_list' \ + -not -name '.*' \ + -exec shellcheck -x '{}' \; 2>&1 + ) + + if [ -n "${errors}" ]; then + >&2 echo 'shellcheck complains about the following:' + >&2 echo "${errors}" + exit 1 + fi +fi diff --git a/bin/why-dont-you b/bin/why-dont-you index 2e3ea86..e873c71 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -7,6 +7,9 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: reintrocude "keep", "stubbornly_keep", "stabilize" and "unstage" +# using information from the database. + action="$1" shift @@ -17,342 +20,164 @@ case "${action}" in 'build') - for pkg in "$@"; do - { - grep "^$(str_to_regex "${pkg}") " "${work_dir}/build-list" || \ - >&2 printf '"%s" is not on the build list.\n' "${pkg}" - } | \ - while read -r package git_revision mod_git_revision repository; do - - if [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - printf '"%s" is locked by ' "$1" - sort -u < \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - sed ' - :a - $!{ - N - s/\n/, / - ba - } - s/$/./ - ' - continue - fi - if [ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ]; then - printf '"%s" is blocked: "' "${pkg}" - tr '[:space:]' ' ' < \ - "${work_dir}/package-states/$1.$2.$3.$4.blocked" | \ - sed ' - s| \+| | - s|^ || - s| $|| - ' - printf '"\n' - continue - fi - - unmet_dependencies=$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}") - if [ -n "${unmet_dependencies}" ]; then - printf '"%s" has unmet dependencies:\n' "${package}" - echo "${unmet_dependencies}" | \ - while read -r dep; do - grep -lxF "${dep}" "${work_dir}/package-infos/"*".builds" | \ - sed ' - s|^.*/|| - s|\(\.[^.]\+\)\{4\}|| - ' - done | \ - sort -u - printf '\n' - - continue - fi - - if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then - printf '"%s" is broken (%sx built), but would be built\n' \ - "${pkg}" \ - "$(wc -l < "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken")" - else - printf '"%s" would be built\n' "${pkg}" - fi - done - - done - - ;; - - 'stabilize'|'unstage') - - if [ "${action}" = 'stabilize' ]; then - suffix='tested' - else - suffix='done' - fi - + # shellcheck disable=SC2016 { - awk '{print $1 "." $2 "." $3 "." $4}' < \ - "${work_dir}/build-list" - if [ "${action}" = 'stabilize' ]; then - find "${work_dir}/package-states" -maxdepth 1 \( \ - -name '*.done' -o \ - -name '*.testing' \ - \) -printf '%f\n' | \ - sed 's|\.[^.]\+$||' - fi + printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64));\n' + printf 'INSERT INTO `pkgbases` VALUES ' + # shellcheck disable=SC2046 + printf '(from_base64("%s")),' \ + $( + printf '%s\n' "$@" | \ + base64_encode_each + ) | \ + sed 's/,$/;\n/' + # we select everything which is possibly of any interest: + # - id (to see if it actually is on the build-list + # - to_build.is_broken + # - failed_builds_count + # - to_build.is_blocked + # - deps.pkgbase (any dependency pending?) + # - build_slaves.name (is anyone building this?) + # - pkgbase + printf 'SELECT DISTINCT `to_build`.`ba_id`,' + printf 'If(`to_build`.`is_broken`,1,0),' + printf '(' + printf 'SELECT count(*) FROM `failed_builds`' + printf 'WHERE `failed_builds`.`build_assignment`=`to_build`.`ba_id`' + printf ')' + printf ',replace(to_base64(`%s`.`%s`),"\\n","")' \ + 'to_build' 'is_blocked' \ + 'deps' 'pkgbase' \ + 'build_slaves' 'name' \ + 'pkgbases' 'pkgbase' + # at least one row for each given `pkgbase` + printf ' FROM `pkgbases`' + printf ' LEFT JOIN ' + printf '(' + # join the tables for the to-be-built packages: + # package_source, build_assignment, binary_package, repostory + printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`' + printf ' FROM `package_sources` AS `tb_ps`' + mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba' + mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin' + mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep' + printf ' WHERE `tb_rep`.`name`="build-list"' + printf ') AS `to_build`' + printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`' + printf ' LEFT JOIN ' + printf '(' + # same join as above, but with different names - for the + # potential dependencies + printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`' + printf ' FROM `package_sources` AS `dep_ps`' + mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba' + mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin' + mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep' + # now we have some (=3) additional joins, + # because we are interested in dependency relations to `to_build` + mysql_join_binary_packages_install_target_providers 'dep_bin' + mysql_join_install_target_providers_dependencies + mysql_join_dependencies_dependency_types + printf ' WHERE `dep_rep`.`name`="build-list"' + printf ' AND `dependency_types`.`relevant_for_building`' + printf ') AS `deps`' + printf ' ON `deps`.`dependent`=`to_build`.`bin_id`' + # now we join with build slaves to see if someone builds this + printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' + printf ';\n' } | \ - sort -u > \ - "${tmp_dir}/unmoveable-list" - - find "${work_dir}/package-states" -maxdepth 1 -name "*.${suffix}" -printf '%f\n' | \ - sed 's|\.[^.]\+$||' | \ - sort -u > \ - "${tmp_dir}/moveable-list" - - cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \ + mysql_run_query | \ + tr '\t' ' ' | \ + sort -k7,7 -k6,6 -k5,5 | \ sed ' - s|^|'"${work_dir}/package-infos/"'| - s|$|.run-depends| + / NULL \S\+$/ b multi-dep + :multi-slave + $!N + s/^\(\(\S\+ \)\{5\}\)\(\S\+\)\( \S\+\)\n\(\S\+ \)\{5\}\(\S\+\)\4/\1\3,\6\4/ + t multi-slave + P + D + :multi-dep + / NULL\( \S\+\)\{3\}$/! b + $!N + s/^\(\(\S\+ \)\{4\}\)\(\S\+\)\(\( \S\+\)\{2\}\)\n\(\S\+ \)\{4\}\(\S\+\)\4/\1\3,\7\4/ + t multi-dep + P + D ' | \ - # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged! - xargs -r grep -vHxF 'base' | \ sed ' - s|^[^:]*/|| - s|\.run-depends:| | + s/NULL,//g ' | \ - sort -k2,2 > \ - "${tmp_dir}/all-run-depends" - - cat "${tmp_dir}/moveable-list" "${tmp_dir}/unmoveable-list" | \ - sed ' - s|^|'"${work_dir}/package-infos/"'| - s|$|.builds| - ' | \ - # base is boring, so we ignore it (might give result "... can be unstaged" although it _will_not_ be unstaged! - xargs -r grep -vHxF 'base' | \ - sed ' - s|^[^:]*/|| - s|\.builds:| | - ' | \ - sort -k2,2 > \ - "${tmp_dir}/all-builds" - - for pkg in "$@"; do - - if ! state_file=$( - grep '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}$' "${tmp_dir}/moveable-list" - ) || \ - [ ! -f "${work_dir}/package-states/${state_file}.${suffix}" ]; then - printf '"%s" is not %s yet!\n' "${pkg}" "${suffix}" - continue - fi - - echo "${state_file}" > "${tmp_dir}/dependent.new" - touch "${tmp_dir}/dependent" - - while [ -s "${tmp_dir}/dependent.new" ]; do - cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" | \ - sort -u | \ - sponge "${tmp_dir}/dependent" - - { - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${tmp_dir}/dependent.new" | \ - xargs -r cat | \ - sort -u | \ - join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-run-depends" - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${tmp_dir}/dependent.new" | \ - xargs -r cat | \ - sort -u | \ - join -1 1 -2 2 -o 2.1 - "${tmp_dir}/all-builds" - } | \ - sort -u | \ - sponge "${tmp_dir}/dependent.new" - - cat "${tmp_dir}/dependent.new" "${tmp_dir}/dependent" "${tmp_dir}/dependent" | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/dependent.new" - done - - dependent_unmoveable=$( - join -1 1 -2 1 -o 2.1 "${tmp_dir}/unmoveable-list" "${tmp_dir}/dependent" - ) - - if [ -n "${dependent_unmoveable}" ]; then - printf 'The following packages are dependent on "%s", but cannot be %sd:\n' "${pkg}" "${action}" - echo "${dependent_unmoveable}" | \ - while read -r sf; do - printf '%s' "${sf}" - if [ -f "${work_dir}/package-states/${sf}.testing" ]; then - printf ' (not tested yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.testing")) / 3600 / 24 ))" - elif [ -f "${work_dir}/package-states/${sf}.done" ]; then - printf ' (not unstaged yet for %s days)' "$(( ($(date '+%s') - $(stat -c '%Y' "${work_dir}/package-states/${sf}.done")) / 3600 / 24 ))" - elif tr ' ' '.' < \ - "${work_dir}/build-list" | \ - grep -qxF "${sf}"; then - printf ' (not built yet)' - fi - printf '\n' - done - printf '\n' - continue - fi - - printf 'Package "%s" can be %sd.\n' "${pkg}" "${action}" - - done - - ;; - - 'keep'|'stubbornly_keep') - - find '/var/lib/pacman/sync' -name '*.db' -execdir bsdtar -tf '{}' \; | \ - sed -n ' - s|-[^-]\+-[^-]\+/$|| - T - p - ' | \ - sort -u > \ - "${tmp_dir}/upstream-packages" - - while read -r pkg; do - - if builds_file=$( - find "${work_dir}/package-infos" -maxdepth 1 -printf '%f\n' | \ - grep -m1 '^'"$(str_to_regex "${pkg}")"'\(\.[^.]\+\)\{3\}\.builds$' - ); then - - builds_file="${builds_file%.*}" - prepo="${builds_file##*.}" - builds_file="${builds_file%.*}" - mod_rev="${builds_file##*.}" - builds_file="${builds_file%.*}" - rev="${builds_file##*.}" - - else - - found_PKGBUILD=false - mod_rev=$(cat "${work_dir}/archlinux32.revision") - for repo in ${repo_names}; do - eval 'repo_path="${repo_paths__'"${repo}"'}"' - rev=$(cat "${work_dir}/${repo}.revision") - if [ "${repo}" = 'archlinux32' ]; then - if git -C "${repo_path}" archive "${mod_rev}" | \ - grep -q '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$'; then - prepo=$( - git -C "${repo_path}" archive "${mod_rev}" | \ - grep '^[^/]\+/'"$(str_to_regex "${pkg}")"'/PKGBUILD$' | \ - cut -d/ -f1 - ) - found_PKGBUILD=true - break - fi - else - prepo=$( - git -C "${repo_path}" archive "${rev}" -- "${pkg}/repos" 2>/dev/null | \ - tar -t 2> /dev/null | \ - grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sed ' - s|^[^/]\+/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1| - ' | \ - head -n1 - ) - if [ -n "${prepo}" ]; then - found_PKGBUILD=true - break - fi - fi - done - if ! ${found_PKGBUILD}; then + while read -r id is_broken trials is_blocked dependency slave pkgbase; do + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -d + ) + if [ "${id}" = 'NULL' ]; then + >&2 printf '"%s" is not on the build list.\n' \ + "${pkgbase}" continue fi - - generate_package_metadata "${pkg}" "${rev}" "${mod_rev}" "${prepo}" - - fi - - sed "s|^|${pkg} builds |" "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${prepo}.builds" - - done < \ - "${work_dir}/deletion-list" > \ - "${tmp_dir}/deleted.builds" - - sort -k3,3 "${tmp_dir}/deleted.builds" | \ - sponge "${tmp_dir}/deleted.builds" - - for pkg in "$@"; do - - if ! grep -qxF "${pkg}" "${work_dir}/deletion-list"; then - printf 'Package "%s" is not on the deletion list.\n' "${pkg}" - continue - fi - - if ! grep -qxF "${pkg}" "${tmp_dir}/upstream-packages"; then - printf 'Package "%s" is not available upstream.\n' "${pkg}" - if [ "${action}" = 'keep' ]; then + if [ "${slave}" != 'NULL' ]; then + # beware: A slave named "5BË" will look exactly like this! + printf '"%s" is locked by %s.\n' \ + "${pkgbase}" \ + "$( + printf '%s\n' "${slave}" | \ + tr ',' '\n' | \ + while read -r line; do + printf '%s\n' "${line}" | \ + base64 -d + printf ',' + done | \ + sed 's/,$//' + )" continue fi - fi - - if git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \ - tar -Ox 'blacklist' | \ - sed ' - s/\s*#.*$// - /^\s*$/d - ' | \ - grep -qxF "${pkg}"; then - printf 'Package "%s" is explicitely blacklisted.\n' "${pkg}" - continue - fi + if [ "${is_blocked}" != 'NULL' ]; then + # beware: A block-reason "5BË" will look exactly like this! + printf '"%s" is blocked: "%s".\n' \ + "${pkgbase}" \ + "$( + printf '%s' "${is_blocked}" | \ + base64 -d + )" + continue + fi + if [ "${dependency}" != 'NULL' ]; then + printf '"%s" has unmet dependencies:\n' \ + "${pkgbase}" + printf '%s\n' "${dependency}" | \ + tr ',' '\n' | \ + while read -r line; do + printf ' ' + printf '%s\n' "${line}" | \ + base64 -d + printf '\n' + done + continue + fi + if [ "${is_broken}" = '1' ]; then + printf '"%s" is broken (%sx built), but would be built.\n' \ + "${pkgbase}" \ + "${trials}" + continue + fi + printf '"%s" would be built.\n' \ + "${pkgbase}" + done - if [ "lib32-${pkg#lib32-}" = "${pkg}" ]; then - printf 'Package "%s" is a library from multilib.\n' "${pkg}" - continue - fi + ;; - build_depends=$( - find "${work_dir}/package-infos" -maxdepth 1 -name "${pkg}.*.build-depends" -exec cat {} \; - ) - if [ -z "${build_depends}" ]; then - printf 'Package "%s" was deleted in the git repositories.\n' "${pkg}" - continue - fi + 'stabilize'|'unstage') - build_depends=$( - echo "${build_depends}" | \ - sort -u - ) + printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}" - errors=$( - { - # shellcheck disable=SC2086 - printf '%s\n' ${build_depends} - awk '{print $3}' "${tmp_dir}/deleted.builds" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - join -1 1 -2 3 -o 2.1,2.2,2.3 - "${tmp_dir}/deleted.builds" - ) - if [ -n "${errors}" ]; then - printf 'Package "%s" has dependencies on the deletion list:\n' "${pkg}" - # shellcheck disable=SC2086,SC2183 - printf '%s %s %s\n' ${errors} - printf '\n' - continue - fi + ;; - printf 'It seems, package "%s" should not be deleted.\n' "${pkg}" + 'keep'|'stubbornly_keep') - done + printf 'Sry, "why-dont-you %s" is unavailable, until someone recodes it to look into the database.\n' "${action}" ;; |