diff options
| -rwxr-xr-x | bin/bootstrap-mysql | 1250 | ||||
| -rwxr-xr-x | bin/build-master-status | 1 | ||||
| -rwxr-xr-x | bin/build-packages | 5 | ||||
| -rwxr-xr-x | bin/get-package-updates | 1 | ||||
| -rwxr-xr-x | bin/ii-answer | 116 | ||||
| -rwxr-xr-x | bin/ii-connect | 106 | ||||
| -rwxr-xr-x | bin/return-assignment | 1 | ||||
| -rwxr-xr-x | bin/sanity-check | 2 | ||||
| -rwxr-xr-x | lib/common-functions | 75 | ||||
| -rwxr-xr-x | lib/mysql-functions | 350 |
10 files changed, 582 insertions, 1325 deletions
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 35f5398..265bbe5 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,13 +3,6 @@ # shellcheck source=../conf/default.conf . "${0%/*}/../conf/default.conf" -# TODO: this should become some sort of mysql dump to bootstrap the -# tables and stored functions only - when the database is gone, we're -# pretty much screwed anyway :-D - -tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - exec 9> "${build_list_lock_file}" if ! flock -n 9; then >&2 echo 'Cannot get build-list lock.' @@ -26,914 +19,361 @@ if ! flock -s -n 7; then exit 1 fi -for dir in $(ls_master_mirror 'i686'); do - ls_master_mirror "i686/${dir}" | \ - sed ' - /\.pkg\.tar\.xz$/!d - s|^|'"${dir}"' | - ' -done | \ - sort -k2,2 > \ - "${tmp_dir}/master-mirror-listing" - -if [ ! "$1" = 'slim' ]; then - tables=$( - printf '%s\n' \ - 'dependency_types SMALLINT' \ - ' name VARCHAR(32)' \ - ' relevant_for_building BIT' \ - ' relevant_for_binary_packages BIT' \ - ' UNIQUE name' \ - 'dependencies BIGINT' \ - ' dependent BIGINT :binary_packages' \ - ' depending_on BIGINT :install_targets' \ - ' dependency_type SMALLINT :dependency_types' \ - ' UNIQUE content dependent depending_on dependency_type' \ - 'install_targets BIGINT' \ - ' name VARCHAR(64)' \ - ' UNIQUE name' \ - 'install_target_providers BIGINT' \ - ' package BIGINT :binary_packages' \ - ' install_target BIGINT :install_targets' \ - ' UNIQUE content package install_target' \ - 'binary_packages BIGINT' \ - ' build_assignment BIGINT :build_assignments' \ - ' repository MEDIUMINT :repositories' \ - ' epoch MEDIUMINT' \ - ' pkgver VARCHAR(64)' \ - ' pkgrel MEDIUMINT' \ - ' sub_pkgrel MEDIUMINT' \ - ' has_issues BIT' \ - ' is_tested BIT' \ - ' is_to_be_deleted BIT' \ - ' pkgname VARCHAR(64)' \ - ' architecture SMALLINT :architectures' \ - ' sha512sum NULL' \ - ' UNIQUE content build_assignment sub_pkgrel pkgname architecture repository' \ - ' UNIQUE file_name pkgname epoch pkgver pkgrel sub_pkgrel architecture repository' \ - 'repositories MEDIUMINT' \ - ' name VARCHAR(64)' \ - ' stability MEDIUMINT :repository_stabilities' \ - ' is_on_master_mirror BIT' \ - ' UNIQUE name' \ - 'repository_stabilities MEDIUMINT' \ - ' name VARCHAR(32)' \ - ' bugtracker_category VARCHAR(64) NULL' \ - ' UNIQUE name' \ - 'architectures SMALLINT' \ - ' name VARCHAR(16)' \ - ' UNIQUE name' \ - 'build_assignments BIGINT' \ - ' package_source BIGINT :package_sources' \ - ' architecture SMALLINT :architectures' \ - ' is_blocked VARCHAR(128) NULL' \ - ' is_black_listed TEXT NULL' \ - ' is_broken BIT' \ - ' priority SMALLINT' \ - ' UNIQUE content package_source architecture' \ - 'build_dependency_loops BIGINT' \ - ' loop MEDIUMINT' \ - ' build_assignment BIGINT :build_assignments' \ - ' UNIQUE content loop build_assignment' \ - 'build_slaves MEDIUMINT' \ - ' name VARCHAR(32)' \ - ' ssh_key VARCHAR(1024)' \ - ' operator VARCHAR(32)' \ - ' currently_building BIGINT NULL :build_assignments' \ - ' last_connection DATETIME' \ - ' logged_lines BIGINT NULL' \ - ' last_action VARCHAR(32) NULL' \ - ' UNIQUE name' \ - 'package_sources BIGINT' \ - ' pkgbase VARCHAR(64)' \ - ' git_revision VARCHAR(40)' \ - ' mod_git_revision VARCHAR(40)' \ - ' upstream_package_repository SMALLINT :upstream_repositories' \ - ' uses_upstream BIT' \ - ' uses_modification BIT' \ - ' commit_time DATETIME' \ - ' UNIQUE content pkgbase git_revision mod_git_revision' \ - 'upstream_repositories SMALLINT' \ - ' name VARCHAR(64)' \ - ' git_repository SMALLINT :git_repositories' \ - ' UNIQUE content name git_repository' \ - 'git_repositories SMALLINT' \ - ' name VARCHAR(64)' \ - ' url VARCHAR(128)' \ - ' directory VARCHAR(128)' \ - ' head VARCHAR(40)' \ - ' UNIQUE name' \ - ' UNIQUE url' \ - ' UNIQUE directory' \ - 'fail_reasons SMALLINT' \ - ' name VARCHAR(32)' \ - ' identifier VARCHAR(64)' \ - ' severity SMALLINT' \ - ' UNIQUE name' \ - 'failed_builds MEDIUMINT' \ - ' build_slave MEDIUMINT :build_slaves' \ - ' build_assignment BIGINT :build_assignments' \ - ' date DATETIME' \ - ' reason SMALLINT :fail_reasons' \ - ' log_file VARCHAR(512)' \ - 'todos MEDIUMINT' \ - ' file VARCHAR(64)' \ - ' line MEDIUMINT' \ - ' description VARCHAR(512)' \ - 'todo_links MEDIUMINT' \ - ' dependent MEDIUMINT' \ - ' depending_on MEDIUMINT' \ - 'repository_stability_relations MEDIUMINT' \ - ' more_stable MEDIUMINT :repository_stabilities' \ - ' less_stable MEDIUMINT :repository_stabilities' \ - 'repository_moves MEDIUMINT' \ - ' from_repository MEDIUMINT :repositories' \ - ' to_repository MEDIUMINT :repositories' \ - ' upstream_package_repository SMALLINT :upstream_repositories' \ - ' UNIQUE source from_repository upstream_package_repository' \ - 'statistics BIGINT' \ - ' date DATETIME' \ - ' stable_packages_count MEDIUMINT' \ - ' pending_tasks_count MEDIUMINT' \ - ' pending_packages_count MEDIUMINT' \ - ' staging_packages_count MEDIUMINT' \ - ' testing_packages_count MEDIUMINT' \ - ' tested_packages_count MEDIUMINT' \ - ' broken_tasks_count MEDIUMINT' \ - ' dependency_loops_count MEDIUMINT' \ - ' dependency_looped_tasks_count MEDIUMINT' \ - ' locked_tasks_count MEDIUMINT' \ - ' blocked_tasks_count MEDIUMINT' \ - ' next_tasks_count MEDIUMINT' \ - ' UNIQUE date' - ) - - # shellcheck disable=SC2016 - { - printf '%s\n' \ - '/*!40014 SET UNIQUE_CHECKS=0 */;' \ - '/*!40014 SET FOREIGN_KEY_CHECKS=0 */;' - for turn in 'drop' 'create' 'link'; do - echo "${tables}" | \ - sed -n ' - /^\S/p - ' | \ - while read -r table size; do - case "${turn}" in - 'drop') - printf 'DROP TABLE IF EXISTS `%s`;\n' "${table}" - ;; - 'create'|'link') - if [ "${turn}" = 'create' ]; then - printf 'CREATE TABLE `%s` (\n' "${table}" - printf ' `id` %s NOT NULL AUTO_INCREMENT,\n' "${size}" - fi - echo "${tables}" | \ - sed -n ' - s/^'"$(str_to_regex "${table} ${size}")"'$// - T - :a - $!N - s/\n\S.*$// - $!Ta - s/^\n// - p - ' | \ - while read -r column type rest; do - case "${column}" in - 'UNIQUE') - if [ ! "${turn}" = 'create' ]; then - continue - fi - if [ -z "${rest}" ]; then - rest="${type}" - fi - printf ' UNIQUE KEY `%s` (' "${type}" - # shellcheck disable=SC2086 - printf '`%s`,' ${rest} | \ - sed 's|,$||' - printf ')' - ;; - *) - if [ "${turn}" = 'create' ]; then - printf ' `%s` %s' "${column}" "${type}" - if ! echo "${rest}" | \ - grep -qwF 'NULL'; then - printf ' NOT NULL' - fi - else - echo "${rest}" | \ - tr ' ' '\n' | \ - sed -n 's/^://;T;p' | \ - while read -r link; do - printf 'ALTER TABLE `%s` ADD FOREIGN KEY (`%s`) REFERENCES `%s` (`id`) ON UPDATE CASCADE ON DELETE ' \ - "${table}" \ - "${column}" \ - "${link}" - if echo "${rest}" | \ - grep -qwF 'NULL'; then - printf 'SET NULL' - else - printf 'CASCADE' - fi - printf ';\n' - done - fi - ;; - esac - if [ "${turn}" = 'create' ]; then - printf ',\n' - fi - done - if [ "${turn}" = 'create' ]; then - printf ' PRIMARY KEY (`id`));\n' - fi - ;; - *) - >&2 printf 'unknown turn "%s"\n' "${turn}" - exit 2 - esac - done - done - printf '%s\n' \ - '/*!40014 SET UNIQUE_CHECKS=1 */;' \ - '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;' - - # show_broken_packages_and_dependencies - # gives a list of broken packages and their dependencies if they're - # still on the build- or deletion-list - printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n' - printf 'DELIMITER //\n' - printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n' - printf 'BEGIN\n' - printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ - 'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old' - printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' - printf ' SELECT `build_assignments`.`id`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_build_assignments - printf ' AND `build_assignments`.`is_broken`' - mysql_join_binary_packages_repositories - printf ' AND `repositories`.`name`="build-list";\n' - printf 'REPEAT\n' - printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)' - printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n' - printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' - printf ' SELECT `new_bp`.`build_assignment`' - printf ' FROM `broken_packages_and_dependencies_old`' - mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp' - mysql_join_binary_packages_dependencies 'old_bp' - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_building`' - mysql_join_dependencies_install_target_providers - mysql_join_install_target_providers_binary_packages '' 'new_bp' - mysql_join_binary_packages_repositories 'new_bp' 'new_repo' - printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n' - printf 'UNTIL ROW_COUNT()=0\n' - printf 'END REPEAT;\n' - printf 'SELECT ' - mysql_query_select_pkgbase_and_revision - printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n' - printf 'END\n' - printf '//\n' - printf 'DELIMITER ;\n' - - # calculate_dependencies_of_package_upto_first_built_one - # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy` - printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n' - printf 'DELIMITER //\n' - printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n' - printf 'BEGIN\n' - printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' - printf ' SELECT `binary_packages`.`id`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_repositories - printf ' AND `repositories`.`name`="build-list"' - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n' - printf 'REPEAT\n' - printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)' - printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n' - printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' - printf ' SELECT `install_target_providers`.`package`' - printf ' FROM `relevant_binary_packages_copy`' - printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`' - mysql_join_binary_packages_repositories - printf ' AND `repositories`.`name`="build-list"' - mysql_join_binary_packages_dependencies - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_building`' - mysql_join_dependencies_install_target_providers - printf ';\n' - printf 'UNTIL ROW_COUNT()=0\n' - printf 'END REPEAT;\n' - printf 'END\n' - printf '//\n' - printf 'DELIMITER ;\n' - - # calculate_maximal_moveable_set - # stores results in `moveable_binary_packages` and `replaced_binary_packages` - # Give a maximal list of packages to be moved, while implementing the - # condition from db-update: - - # Every package which is replaced[1], must have its provided install_targets: - # a) provided by another moved or not-replaced package or - # b) not required by any not-replaced package. - - # Every package being moved needs to have all dependencies - # installable in the target repository. - - # TODO: [1] A "replaced" package may also be in a different repository - # e.g. if a-2 is moved from [staging] to [testing] and there is only - # a-1 in [core], then this will be "replaced" by a-2 on a system - # running on [testing] repositories. - - printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' - printf 'DELIMITER //\n' - printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' - printf 'BEGIN\n' - # variables to store count of changed rows - printf 'DECLARE row_count_saved INT DEFAULT 0;\n' - for copy in '' '_copy' '_copy2'; do - printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ - 'moveable' "${copy}" \ - 'replaced' "${copy}" - printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \ - "${copy}" - if [ "${copy}" = '_copy2' ]; then +#shellcheck disable=SC2016 +{ + # show_broken_packages_and_dependencies + # gives a list of broken packages and their dependencies if they're + # still on the build- or deletion-list + printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n' + printf 'BEGIN\n' + printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ + 'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `build_assignments`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + printf ' AND `build_assignments`.`is_broken`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list";\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)' + printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `new_bp`.`build_assignment`' + printf ' FROM `broken_packages_and_dependencies_old`' + mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp' + mysql_join_binary_packages_dependencies 'old_bp' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'new_bp' + mysql_join_binary_packages_repositories 'new_bp' 'new_repo' + printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'SELECT ' + mysql_query_select_pkgbase_and_revision + printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + + # calculate_dependencies_of_package_upto_first_built_one + # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy` + printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n' + printf 'BEGIN\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `binary_packages`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)' + printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `install_target_providers`.`package`' + printf ' FROM `relevant_binary_packages_copy`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + printf ';\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + + # calculate_maximal_moveable_set + # stores results in `moveable_binary_packages` and `replaced_binary_packages` + # Give a maximal list of packages to be moved, while implementing the + # condition from db-update: + + # Every package which is replaced[1], must have its provided install_targets: + # a) provided by another moved or not-replaced package or + # b) not required by any not-replaced package. + + # Every package being moved needs to have all dependencies + # installable in the target repository. + + # TODO: [1] A "replaced" package may also be in a different repository + # e.g. if a-2 is moved from [staging] to [testing] and there is only + # a-1 in [core], then this will be "replaced" by a-2 on a system + # running on [testing] repositories. + + printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' + printf 'BEGIN\n' + # variables to store count of changed rows + printf 'DECLARE row_count_saved INT DEFAULT 0;\n' + for copy in '' '_copy' '_copy2'; do + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ + 'moveable' "${copy}" \ + 'replaced' "${copy}" + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \ + "${copy}" + if [ "${copy}" = '_copy2' ]; then + continue + fi + printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \ + "${copy}" + done + printf 'DROP TEMPORARY TABLE IF EXISTS `package_blobs`;\n' + printf 'CREATE TEMPORARY TABLE `package_blobs` (`ps_a` BIGINT, `ps_b` BIGINT, UNIQUE KEY `content` (`ps_a`,`ps_b`));\n' + # all packages being vaguely relevant enter `package_blobs`, because + # they might block any other package from moving + printf 'INSERT IGNORE INTO `package_blobs` (`ps_a`,`ps_b`)' + printf ' SELECT `a_ps`.`id`,`b_ps`.`id`' + printf ' FROM `package_sources` AS `a_ps`' + printf ' JOIN `package_sources` AS `b_ps`' + printf ' ON UNIX_TIMESTAMP(`a_ps`.`commit_time`) - UNIX_TIMESTAMP(`b_ps`.`commit_time`) BETWEEN -10 AND 10' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' + mysql_join_binary_packages_repositories 'a_bp' 'a_r' + mysql_join_binary_packages_repositories 'b_bp' 'b_r' + mysql_join_repositories_repository_stabilities 'a_r' 'a_rs' + mysql_join_repositories_repository_stabilities 'b_r' 'b_rs' + printf ' WHERE `a_rs`.`name` = `from_stability`' + printf ' AND `b_rs`.`name` = `from_stability`;\n' + # these packages are considered for moving: + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' + # correct stability: "testing"/"staging" - as chosen + printf ' WHERE `repository_stabilities`.`name` = `from_stability`' + printf ' AND (' + printf '`from_stability`="staging"' + # "testing" packages must be tested + printf ' OR `binary_packages`.`is_tested`' + printf ')' + # no open issues + printf ' AND NOT `binary_packages`.`has_issues`' + printf ';\n' + # these packages are considered for being replaced: + # for each moved package + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' + printf ' SELECT `r_bp`.`id`,`m_bp`.`id`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories 'm_bp' 'm_r' + mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' + mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' + mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' + mysql_join_upstream_repositories_repository_moves 'm_ur' + printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' + # in its target repository + printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' + mysql_join_repositories_binary_packages 'r_r' 'r_bp' + # all packages with identical names + printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n' + # now we delete all unreplaceable and unmoveable packages from the respective + # list until no further changes are required + printf 'REPEAT\n' + printf 'SET row_count_saved = 0;\n' + + # create copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then continue fi - printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \ - "${copy}" + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" done - printf 'DROP TEMPORARY TABLE IF EXISTS `package_blobs`;\n' - printf 'CREATE TEMPORARY TABLE `package_blobs` (`ps_a` BIGINT, `ps_b` BIGINT, UNIQUE KEY `content` (`ps_a`,`ps_b`));\n' - # all packages being vaguely relevant enter `package_blobs`, because - # they might block any other package from moving - printf 'INSERT IGNORE INTO `package_blobs` (`ps_a`,`ps_b`)' - printf ' SELECT `a_ps`.`id`,`b_ps`.`id`' - printf ' FROM `package_sources` AS `a_ps`' - printf ' JOIN `package_sources` AS `b_ps`' - printf ' ON UNIX_TIMESTAMP(`a_ps`.`commit_time`) - UNIX_TIMESTAMP(`b_ps`.`commit_time`) BETWEEN -10 AND 10' - mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' - mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' - mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' - mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' - mysql_join_binary_packages_repositories 'a_bp' 'a_r' - mysql_join_binary_packages_repositories 'b_bp' 'b_r' - mysql_join_repositories_repository_stabilities 'a_r' 'a_rs' - mysql_join_repositories_repository_stabilities 'b_r' 'b_rs' - printf ' WHERE `a_rs`.`name` = `from_stability`' - printf ' AND `b_rs`.`name` = `from_stability`;\n' - # these packages are considered for moving: - printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' - printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_repositories - mysql_join_repositories_repository_stabilities - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - mysql_join_upstream_repositories_repository_moves - printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' - # correct stability: "testing"/"staging" - as chosen - printf ' WHERE `repository_stabilities`.`name` = `from_stability`' - printf ' AND (' - printf '`from_stability`="staging"' - # "testing" packages must be tested - printf ' OR `binary_packages`.`is_tested`' - printf ')' - # no open issues - printf ' AND NOT `binary_packages`.`has_issues`' - printf ';\n' - # these packages are considered for being replaced: - # for each moved package - printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' - printf ' SELECT `r_bp`.`id`,`m_bp`.`id`' - printf ' FROM `moveable_binary_packages`' - printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' - mysql_join_binary_packages_repositories 'm_bp' 'm_r' - mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' - mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' - mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' - mysql_join_upstream_repositories_repository_moves 'm_ur' - printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' - # in its target repository - printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' - mysql_join_repositories_binary_packages 'r_r' 'r_bp' - # all packages with identical names - printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n' - # now we delete all unreplaceable and unmoveable packages from the respective - # list until no further changes are required - printf 'REPEAT\n' - printf 'SET row_count_saved = 0;\n' - - # create copies of our temporary tables *yuck* - for table in 'replaced' 'moveable'; do - for copy in '' '2'; do - if [ "${table}" = 'moveable' ] && \ - [ "${copy}" = '2' ]; then - continue - fi - printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" - printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" - printf ' SELECT `%s_binary_packages`.*' "${table}" - printf ' FROM `%s_binary_packages`;\n' "${table}" - done - done - # a package is not moveable if its dependencies are not provided ... - printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`' - printf ' FROM `replaced_binary_packages`' - printf ' RIGHT JOIN `moveable_binary_packages`' - printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' - printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' - printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`' - mysql_join_binary_packages_dependencies - mysql_join_dependencies_dependency_types - printf ' AND `dependency_types`.`relevant_for_binary_packages`' - # ... by a not-deleted, "more stable" package already in place or ... - printf ' WHERE NOT EXISTS (' - printf 'SELECT 1 FROM `install_target_providers`' - mysql_join_install_target_providers_binary_packages '' 'prov_bp' - mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' - printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`' - printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`' - printf ' AND NOT EXISTS (' - printf 'SELECT 1 FROM `replaced_binary_packages_copy`' - printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' - printf ')' - # ... by a moved package - printf ') AND NOT EXISTS (' - printf 'SELECT 1 FROM `install_target_providers`' - printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`' - printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ');\n' - printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' - - # refresh copies of our temporary tables *yuck* - for table in 'replaced' 'moveable'; do - for copy in '' '2'; do - if [ "${table}" = 'moveable' ] && \ - [ "${copy}" = '2' ]; then - continue - fi - printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" - printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" - printf ' SELECT `%s_binary_packages`.*' "${table}" - printf ' FROM `%s_binary_packages`;\n' "${table}" - done - done - - # packages which should not be replaced: ones providing something that is: - # a) still needed by a not-replaced package x "less stable" than the target repository and - # b) not provided by: - # 1) a moved package or - # 2) any current, not-replaced package in a repository more stable than x's repository - # - # Note, that this is not 100% clean from an academic point of view: - # It might require _downgrading_ of a package to keep fulfilling a dependency - # and it might require to do so _without_ any chance for the user to notice, - # because there may be more "dependencies" in the database than in the package files. - # - # However, in practice both should not happen. - # - printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' - printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' - printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`' - mysql_join_binary_packages_install_target_providers 'repl_bp' - mysql_join_install_target_providers_dependencies + done + # a package is not moveable if its dependencies are not provided ... + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`' + printf ' FROM `replaced_binary_packages`' + printf ' RIGHT JOIN `moveable_binary_packages`' + printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + # ... by a not-deleted, "more stable" package already in place or ... + printf ' WHERE NOT EXISTS (' + printf 'SELECT 1 FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`' printf ' AND NOT EXISTS (' - # dependencies of replaced packages don't matter printf 'SELECT 1 FROM `replaced_binary_packages_copy`' - printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' printf ')' - mysql_join_dependencies_dependency_types - # consider only runtime dependencies - printf ' AND `dependency_types`.`relevant_for_binary_packages`' - mysql_join_dependencies_binary_packages '' 'req_bp' - # we need to check wether req_bp's dependency is (un)critical - mysql_join_binary_packages_repositories 'repl_bp' 'repl_r' - mysql_join_binary_packages_repositories 'req_bp' 'req_r' - # dependent package is "less stable" than dependency - printf ' JOIN `repository_stability_relations` AS `repl_rr`' - printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`' - printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`' - # a) ^ - printf ' WHERE NOT EXISTS (' - # no moved package ... - printf 'SELECT 1' - printf ' FROM `moveable_binary_packages_copy`' - mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp' - # ... provides the same - printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`' - # b) 1) ^ - printf ') AND NOT EXISTS (' - # no current package ... - printf 'SELECT 1' - printf ' FROM `binary_packages` AS `subst_bp`' - mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp' - # ... in a repository ... - mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' - # ... more stable ... - printf ' JOIN `repository_stability_relations` AS `subst_rr`' - printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`' - # ... than x's repository ... - printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`' - printf ' AND NOT EXISTS (' - # ... and which is not replaced ... - printf 'SELECT 1 FROM `replaced_binary_packages_copy2`' - printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`' - printf ')' - # ... and provides the same - printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' - # b) 2) ^ - printf ');\n' - printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' - - # refresh copies of our temporary tables *yuck* - for table in 'replaced' 'moveable'; do - for copy in '' '2'; do - if [ "${table}" = 'moveable' ] && \ - [ "${copy}" = '2' ]; then - continue - fi - printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" - printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" - printf ' SELECT `%s_binary_packages`.*' "${table}" - printf ' FROM `%s_binary_packages`;\n' "${table}" - done - done - - printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' - printf ' RIGHT JOIN `moveable_binary_packages`' - printf ' ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' - printf ' JOIN `binary_packages`' - printf ' ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' - mysql_join_binary_packages_build_assignments - printf ' JOIN `package_blobs`' - printf ' ON `build_assignments`.`package_source`=`package_blobs`.`ps_a`' - printf ' JOIN `build_assignments` AS `bl_ba`' - printf ' ON `bl_ba`.`package_source`=`package_blobs`.`ps_b`' - mysql_join_build_assignments_binary_packages 'bl_ba' 'bl_bp' - printf ' WHERE NOT EXISTS (' - printf 'SELECT 1 FROM `moveable_binary_packages_copy`' - printf ' WHERE `moveable_binary_packages_copy`.`id`=`bl_bp`.`id`' - printf ');\n' - printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' - - printf 'UNTIL row_count_saved=0\n' - printf 'END REPEAT;\n' - for table in 'moveable' 'replaced'; do - for copy in '' '2'; do - if [ "${table}" = 'moveable' ] && \ - [ "${copy}" = '2' ]; then - continue - fi - printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \ - "${table}" "${copy}" - done + # ... by a moved package + printf ') AND NOT EXISTS (' + printf 'SELECT 1 FROM `install_target_providers`' + printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' + + # refresh copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" done - printf 'END\n' - printf '//\n' - printf 'DELIMITER ;\n' - - printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ - 'CREATE ROUTINE' 'buildmaster.*' \ - 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ - 'EXECUTE' 'buildmaster.*' \ - 'RELOAD' '*.*' \ - 'SELECT' 'buildmaster.*' \ - 'SELECT' 'mysql.proc' \ - 'SHOW VIEW' 'buildmaster.*' \ - 'UPDATE' 'buildmaster.*' - printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \ - 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ - 'SELECT' 'buildmaster.*' \ - 'SHOW VIEW' 'buildmaster.*' - printf 'FLUSH PRIVILEGES;\n' - } | \ - mysql_run_query -u root -p -fi + done -# shellcheck disable=SC2016 -{ - printf 'INSERT IGNORE INTO `architectures` (`name`) VALUES ' - printf '("%s"),' \ - 'any' 'i686' | \ - sed 's|,$||' - printf ';\n' - printf 'INSERT IGNORE INTO `fail_reasons` (`identifier`,`name`,`severity`) VALUES ' - printf '(from_base64("%s"),"%s",%s),' \ - "$( - printf '%s' '==> ERROR: A failure occurred in build()\.' | \ - base64 -w0 - )" 'build()' 0 \ - "$( - printf '%s' '==> ERROR: Could not download sources\.' | \ - base64 -w0 - )" 'source' 1 \ - "$( - printf '%s' '.*error: failed to commit transaction (invalid or corrupted package)' | \ - base64 -w0 - )" 'package-cache' 2 \ - "$( - printf '%s' '==> ERROR: A failure occurred in prepare()\.' | \ - base64 -w0 - )" 'prepare()' 2 \ - "$( - printf '%s' '==> ERROR: A failure occurred in package\(_\S\+\)\?()\.' | \ - base64 -w0 - )" 'package()' 3 \ - "$( - printf '%s' '==> ERROR: A failure occurred in check()\.' | \ - base64 -w0 - )" 'check()' 4 \ - "$( - printf '%s' '==> ERROR: '"'"'pacman'"'"' failed to install missing dependencies\.' | \ - base64 -w0 - )" 'dependencies' 1 \ - "$( - printf '%s' '==> ERROR: Running makepkg as root is not allowed as it can cause permanent,.*' | \ - base64 -w0 - )" 'run-as-root' 1 \ - "$( - printf '.*' | \ - base64 -w0 - )" 'unknown' 100 | \ - sed 's|,$||' - printf ';\n' - printf 'INSERT IGNORE INTO `git_repositories` (`name`,`url`,`directory`,`head`) VALUES' - { - for repo in ${repo_names}; do - eval 'repo_path="${repo_paths__'"${repo}"'}"' - printf '\n (' - printf 'from_base64("%s"),' \ - "$( - printf '%s' "${repo}" | \ - base64 -w0 - )" \ - "$( - git -C "${repo_path}" remote -v | \ - awk '{print $2}' | \ - tail -n1 | \ - base64_encode_each - )" \ - "$( - printf '%s' "${repo_path}" | \ - base64 -w0 - )" \ - "$( - git -C "${repo_path}" rev-parse HEAD | \ - base64_encode_each - )" | \ - sed 's|,$|),|' + # packages which should not be replaced: ones providing something that is: + # a) still needed by a not-replaced package x "less stable" than the target repository and + # b) not provided by: + # 1) a moved package or + # 2) any current, not-replaced package in a repository more stable than x's repository + # + # Note, that this is not 100% clean from an academic point of view: + # It might require _downgrading_ of a package to keep fulfilling a dependency + # and it might require to do so _without_ any chance for the user to notice, + # because there may be more "dependencies" in the database than in the package files. + # + # However, in practice both should not happen. + # + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' + printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' + printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`' + mysql_join_binary_packages_install_target_providers 'repl_bp' + mysql_join_install_target_providers_dependencies + printf ' AND NOT EXISTS (' + # dependencies of replaced packages don't matter + printf 'SELECT 1 FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`' + printf ')' + mysql_join_dependencies_dependency_types + # consider only runtime dependencies + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + mysql_join_dependencies_binary_packages '' 'req_bp' + # we need to check wether req_bp's dependency is (un)critical + mysql_join_binary_packages_repositories 'repl_bp' 'repl_r' + mysql_join_binary_packages_repositories 'req_bp' 'req_r' + # dependent package is "less stable" than dependency + printf ' JOIN `repository_stability_relations` AS `repl_rr`' + printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`' + printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`' + # a) ^ + printf ' WHERE NOT EXISTS (' + # no moved package ... + printf 'SELECT 1' + printf ' FROM `moveable_binary_packages_copy`' + mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp' + # ... provides the same + printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 1) ^ + printf ') AND NOT EXISTS (' + # no current package ... + printf 'SELECT 1' + printf ' FROM `binary_packages` AS `subst_bp`' + mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp' + # ... in a repository ... + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + # ... more stable ... + printf ' JOIN `repository_stability_relations` AS `subst_rr`' + printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`' + # ... than x's repository ... + printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`' + printf ' AND NOT EXISTS (' + # ... and which is not replaced ... + printf 'SELECT 1 FROM `replaced_binary_packages_copy2`' + printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`' + printf ')' + # ... and provides the same + printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 2) ^ + printf ');\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' + + # refresh copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" done - printf ';\n' - } | \ - sed ' - s|,;|;| - ' - - for repo in \ - 'core:packages' \ - 'extra:packages' \ - 'multilib:packages' \ - 'community:community'; do - printf 'INSERT IGNORE INTO `upstream_repositories` (`name`,`git_repository`) SELECT\n' - printf ' from_base64("%s"),`id` FROM `git_repositories` WHERE `name` = from_base64("%s");\n' \ - "$( - printf '%s' "${repo%:*}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${repo#*:}" | \ - base64 -w0 - )" done - printf 'INSERT IGNORE INTO `build_slaves` (`name`,`ssh_key`,`operator`,`last_connection`) VALUES' - { - sed -n ' - s/^command="\S\+ \(\S\+\)" \S\+ \(\S\+\) \S\+$/\1 \2/ - T - p - ' ~/.ssh/authorized_keys | \ - while read -r name key; do - case "${name}" in - 'nlopc'*|'rechenknecht') - operator='deep42thought' - ;; - 'buildknecht'*) - operator='deep42thought/vollzornbrot' - ;; - 'eurobuild3') - operator='abaumann' - ;; - *) - operator="${name}" - ;; - esac - printf '\n (' - printf 'from_base64("%s"),' \ - "$( - printf '%s' "${name}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${key}" | \ - base64 -w0 - )" \ - "$( - printf '%s' "${operator}" | \ - base64 -w0 - )" - printf 'NOW()),' - done - printf ';\n' - } | \ - sed 's|,;|;|' - - printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES' - { - printf '\n ("%s",%s),' \ - 'stable' '"Packages: Stable"' \ - 'testing' '"Packages: Testing"' \ - 'staging' 'NULL' \ - 'standalone' 'NULL' \ - 'unbuilt' '"Packages: Build-list"' \ - 'forbidden' 'NULL' - printf ';\n' - } | \ - sed 's|,;|;|' - - { - printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)' - printf ' SELECT `ms`.`id`,`ls`.`id`' - printf ' FROM `repository_stabilities` AS `ms` JOIN `repository_stabilities` AS `ls`' - printf ' WHERE ' - printf '(`ms`.`name`="%s" AND `ls`.`name`="%s") OR ' \ - 'stable' 'stable' \ - 'stable' 'testing' \ - 'stable' 'staging' \ - 'stable' 'standalone' \ - 'stable' 'unbuilt' \ - 'stable' 'forbidden' \ - 'testing' 'testing' \ - 'testing' 'staging' \ - 'testing' 'standalone' \ - 'testing' 'unbuilt' \ - 'testing' 'forbidden' \ - 'staging' 'staging' \ - 'staging' 'standalone' \ - 'staging' 'unbuilt' \ - 'staging' 'forbidden' \ - 'unbuilt' 'forbidden' \ - 'standalone' 'standalone' - printf ';\n' - } | \ - sed 's| OR ;|;|' - - { - printf 'INSERT IGNORE INTO `repository_moves` (`from_repository`,`to_repository`,`upstream_package_repository`)' - printf ' SELECT `f`.`id`,`t`.`id`,`u`.`id`' - printf ' FROM' - printf ' `repositories` AS `%s` JOIN' \ - 'f' 't' - printf ' `upstream_repositories` AS `u`' - printf ' WHERE ' - printf '(`f`.`name`="%s" AND `t`.`name`="%s" AND `u`.`name`="%s") OR ' \ - 'staging' 'testing' 'core' \ - 'staging' 'testing' 'extra' \ - 'staging' 'testing' 'multilib' \ - 'community-staging' 'community-testing' 'community' \ - 'community-staging' 'community-testing' 'multilib' \ - 'testing' 'core' 'core' \ - 'testing' 'extra' 'extra' \ - 'testing' 'extra' 'multilib' \ - 'community-testing' 'community' 'community' \ - 'community-testing' 'community' 'multilib' - printf ';\n' - } | \ - sed 's| OR ;|;|' - - for repo in \ - 'core:stable:AQ==' \ - 'extra:stable:AQ==' \ - 'community:stable:AQ==' \ - 'build-support:standalone:AQ==' \ - 'testing:testing:AQ==' \ - 'community-testing:testing:AQ==' \ - 'staging:staging:AQ==' \ - 'community-staging:staging:AQ==' \ - 'build-list:unbuilt:AA==' \ - 'deletion-list:forbidden:AA=='; do - printf 'INSERT IGNORE INTO `repositories` (`name`,`stability`,`is_on_master_mirror`) SELECT' - printf ' from_base64("%s"),`id`,from_base64("%s") FROM `repository_stabilities` WHERE `name`=from_base64("%s");\n' \ - "$( - printf '%s' "${repo}" | \ - cut -d: -f1 | \ - base64_encode_each - )" \ - "$( - printf '%s' "${repo}" | \ - cut -d: -f3 - )" \ - "$( - printf '%s' "${repo}" | \ - cut -d: -f2 | \ - base64_encode_each - )" + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' + printf ' RIGHT JOIN `moveable_binary_packages`' + printf ' ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' + printf ' JOIN `binary_packages`' + printf ' ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_build_assignments + printf ' JOIN `package_blobs`' + printf ' ON `build_assignments`.`package_source`=`package_blobs`.`ps_a`' + printf ' JOIN `build_assignments` AS `bl_ba`' + printf ' ON `bl_ba`.`package_source`=`package_blobs`.`ps_b`' + mysql_join_build_assignments_binary_packages 'bl_ba' 'bl_bp' + printf ' WHERE NOT EXISTS (' + printf 'SELECT 1 FROM `moveable_binary_packages_copy`' + printf ' WHERE `moveable_binary_packages_copy`.`id`=`bl_bp`.`id`' + printf ');\n' + printf 'SET row_count_saved = row_count_saved + ROW_COUNT();\n' + + printf 'UNTIL row_count_saved=0\n' + printf 'END REPEAT;\n' + for table in 'moveable' 'replaced'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \ + "${table}" "${copy}" + done done - - printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES' - { - printf '\n ("%s",%s,%s),' \ - 'make' '1' '0' \ - 'check' '0' '0' \ - 'link' '0' '1' \ - 'run' '1' '1' - printf ';\n' - } | \ - sed 's|,;|;|' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + + printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE ROUTINE' 'buildmaster.*' \ + 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ + 'EXECUTE' 'buildmaster.*' \ + 'RELOAD' '*.*' \ + 'SELECT' 'buildmaster.*' \ + 'SELECT' 'mysql.proc' \ + 'SHOW VIEW' 'buildmaster.*' \ + 'UPDATE' 'buildmaster.*' + printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ + 'SELECT' 'buildmaster.*' \ + 'SHOW VIEW' 'buildmaster.*' + printf 'FLUSH PRIVILEGES;\n' } | \ - mysql_run_query - -grep '^\('"$( - # shellcheck disable=SC2086 - printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} -)"'\) ' "${tmp_dir}/master-mirror-listing" | \ - sed ' - s/\.pkg\.tar\.xz$// - s/-\([^-:]\+\)\(\(-[^-]\+\)\{2\}\)$/-0:\1\2/ - s/\(-[0-9]\+\)\(-[^-]\+\)$/\1.0\2/ - s/-\([^-:]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/ - ' | \ - while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do - # we don't care too much about those - they won't be moved anyways - printf '(' - # shellcheck disable=SC2046 - printf 'from_base64("%s"),' \ - $( - printf '%s\n' \ - "${pkgname}" \ - "${epoch}" \ - "${pkgver}" \ - "${pkgrel}" \ - "${sub_pkgrel}" \ - "${arch}" \ - "${repo}" | \ - base64_encode_each - ) | \ - sed 's/,$/),/' - printf '\n' - done > \ - "${tmp_dir}/new-stable-packages" - -if [ -s "${tmp_dir}/new-stable-packages" ]; then - # shellcheck disable=SC2016 - { - printf 'CREATE TEMPORARY TABLE `stable_packages` (' - printf '`%s` %s,' \ - 'pkgname' 'VARCHAR(64)' \ - 'epoch' 'MEDIUMINT' \ - 'pkgver' 'VARCHAR(64)' \ - 'pkgrel' 'MEDIUMINT' \ - 'sub_pkgrel' 'MEDIUMINT' \ - 'architecture' 'VARCHAR(16)' \ - 'repository' 'VARCHAR(64)' \ - 'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT' - printf 'PRIMARY KEY (`build_assignment`));\n' - sed ' - 1~10 ! b not_start - s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/ - :not_start - $ b end - 0~10 b end - b - :end - s/,$/;/ - ' "${tmp_dir}/new-stable-packages" - printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' - printf ' SELECT ' - printf '`stable_packages`.`%s`,' \ - 'pkgname' \ - 'epoch' \ - 'pkgver' \ - 'pkgrel' \ - 'sub_pkgrel' - printf '`%s`.`id`,' \ - 'architectures' \ - 'repositories' - printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' - printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ - 'repositories' 'repository' 'repositories' \ - 'architectures' 'architecture' 'architectures' - printf ';\n' - printf 'DROP TABLE `stable_packages`;\n' - } | \ - mysql_run_query -fi - -mysql_repair_binary_packages_without_build_assignment + mysql_run_query -u root -p diff --git a/bin/build-master-status b/bin/build-master-status index 7ab38d4..f46d668 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -111,6 +111,7 @@ rm -f "${tmp_dir}/todos" if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then + # shellcheck disable=SC2119 printf 'girls, my database is dirty again ...\n' | \ irc_say echo 'build master is insane' > \ diff --git a/bin/build-packages b/bin/build-packages index fc80ab1..12fe9b7 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -9,6 +9,11 @@ # TODO: build other 'architectures', too (pentium4, i486) +# TODO: The any_arch logic linking build_assignments and binary_packages +# needs to be implemented here, too, so "any" binary_packages are only +# built (or at least only uploaded) if they're expected from the build +# master (e.g. when they're linked in the database). + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/get-package-updates b/bin/get-package-updates index 73766c8..8239097 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -503,6 +503,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ "${deletion_list_count}" if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then + # shellcheck disable=SC2119 { printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" for repo_name in ${repo_names}; do diff --git a/bin/ii-answer b/bin/ii-answer new file mode 100755 index 0000000..9b25abb --- /dev/null +++ b/bin/ii-answer @@ -0,0 +1,116 @@ +#!/bin/sh + +# answer to stuff on irc + +# shellcheck source=../conf/default.conf +. "${0%/*}/../conf/default.conf" + +# welcome devs (in #archlinux32 only) +welcome_user_regex='^\S\+ -!- \(abaumann\|deep42thought\|tyzoid\)(.* has joined \S\+$' +new_users=$( + sed -n ' + s/'"${welcome_user_regex}"'/\1/ + T + p + ' "${irc_dir}/#archlinux32/out" | \ + sort -u +) +if [ -n "${new_users}" ]; then + sed -i ' + /'"${welcome_user_regex}"'/ d + ' "${irc_dir}/#archlinux32/out" + printf '%s\n' "${new_users}" | \ + sed ' + s/^/Hi / + s/$/!/ + ' | \ + irc_say '' 'copy' +fi + +find "${irc_dir}" \ + -type f \ + -name 'out' \ + -printf '%p\n' | \ + while read -r out_file; do + channel="${out_file%/out}" + channel="${channel##*/}" + if [ -z "${channel%%#*}" ]; then + prefix='buildmaster: ' + sloppy_salutation='buildmaster' + else + prefix='' + sloppy_salutation='' + fi + # answer "why don't you?" + regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' + if grep -q "${regex}" "${out_file}"; then + sed -n ' + s/'"${regex}"'/\2 / + T + p + ' "${out_file}" | \ + while read -r line; do + reason=$( + echo "${line}" | \ + xargs "${base_dir}/bin/why-dont-you" 2>&1 + ) + if [ "$(echo "${reason}" | wc -l)" -le 5 ]; then + echo "${reason}" + else + echo "${reason}" | \ + head -n5 + printf '... (%s lines total)\n' "$(echo "${reason}" | wc -l)" + fi | \ + irc_say "${channel}" 'copy' + done + sed -i "/${regex}/d" "${out_file}" + fi + # answer "wtf?" + regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'wtf ' + if grep -q "${regex}" "${out_file}"; then + sed -n ' + s/'"${regex}"'// + T + p + ' "${out_file}" | \ + while read -r line; do + reason=$("${base_dir}/bin/wtf" "${line}"); + if [ -z "${reason}" ]; then + reason="Huh, I don't know that one." + fi + if [ "$(echo "${reason}" | wc -l)" -le 5 ]; then + echo "${reason}" + else + echo "${reason}" | \ + head -n5 + printf '... (%s lines total)\n' "$(echo "${reason}" | wc -l)" + fi | \ + irc_say "${channel}" 'copy' + done + sed -i "/${regex}/d" "${out_file}" + fi + # answer "What's up?" + regex='^\(\S\+ \)\?\S\+ <\S\+> .*[Ww]hat'"'"'\?s *[Uu]p' + if grep "${regex}" "${out_file}" | \ + grep -q "${sloppy_salutation}"; then + sed -i "/${regex}/d" "${out_file}" + { + printf 'up? I'"'"'m up for %s, %s' \ + "$(uptime -p | sed 's/^up //')" \ + "$(uptime | sed 's/.*, //')" + if [ -s "${work_dir}/build-master-sanity" ]; then + printf ' ... and I'"'"'m insane :-D' + fi + printf '\n' + } | \ + irc_say "${channel}" 'copy' + fi + # answer "Thanks!" + regex='^\(\S\+ \)\?\S\+ <\S\+> \(.* \)\?[Tt]h\(anks\|x\)\([ ,.!]\|$\)' + if grep "${regex}" "${out_file}" | \ + grep -q "${sloppy_salutation}"; then + sed -i "/${regex}/d" "${out_file}" + printf 'np\n' | \ + irc_say "${channel}" 'copy' + fi + done diff --git a/bin/ii-connect b/bin/ii-connect index 5fc5f37..6e543c5 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -37,10 +37,9 @@ fi # watch if asked to if [ "$1" = 'watch' ]; then - done_something=false while pgrep -x 'ii' > /dev/null; do # this avoids missing modifications during our last execution - if ! ${done_something}; then + if [ -z "${said}" ]; then # shellcheck disable=SC2046 inotifywait -e 'CLOSE_WRITE,CLOSE' $( find "${irc_dir}" \ @@ -48,108 +47,9 @@ if [ "$1" = 'watch' ]; then -name 'out' ) fi - done_something=false - welcome_user_regex='^\S\+ -!- \(abaumann\|deep42thought\|tyzoid\)(.* has joined \S\+$' - new_users=$( - sed -n ' - s/'"${welcome_user_regex}"'/\1/ - T - p - ' "${irc_dir}/#archlinux32/out" | \ - sort -u + said=$( + "${base_dir}/bin/ii-answer" ) - if [ -n "${new_users}" ]; then - sed -i ' - /'"${welcome_user_regex}"'/ d - ' "${irc_dir}/#archlinux32/out" - done_something=true - printf '%s\n' "${new_users}" | \ - sed ' - s/^/Hi / - s/$/!/ - ' | \ - irc_say - fi - find "${irc_dir}" \ - -type f \ - -name 'out' \ - -printf '%p\n' | \ - while read -r out_file; do - channel="${out_file%/out}" - channel="${channel##*/}" - if [ -z "${channel%%#*}" ]; then - prefix='buildmaster: ' - sloppy_salutation='buildmaster' - else - prefix='' - sloppy_salutation='' - fi - regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'why[- ]don'"'"'\?t[- ]you \(build\|stabilize\|unstage\|keep\|stubbornly_keep\) ' - if grep -q "${regex}" "${out_file}"; then - done_something=true - sed -n ' - s/'"${regex}"'/\2 / - T - p - ' "${out_file}" | \ - while read -r line; do - reason=$( - echo "${line}" | \ - xargs "${base_dir}/bin/why-dont-you" 2>&1 - ) - if [ "$(echo "${reason}" | wc -l)" -le 5 ]; then - echo "${reason}" - else - echo "${reason}" | \ - head -n5 - printf '... (%s lines total)\n' "$(echo "${reason}" | wc -l)" - fi | \ - sponge "${out_file%/out}/in" - done - sed -i "/${regex}/d" "${out_file}" - fi - regex='^\(\S\+ \)\?\S\+ <\S\+> '"${prefix}"'wtf ' - if grep -q "${regex}" "${out_file}"; then - done_something=true - sed -n ' - s/'"${regex}"'// - T - p - ' "${out_file}" | \ - while read -r line; do - reason=$("${base_dir}/bin/wtf" "${line}"); - if [ -z "${reason}" ]; then - reason="Huh, I don't know that one." - fi - if [ "$(echo "${reason}" | wc -l)" -le 5 ]; then - echo "${reason}" - else - echo "${reason}" | \ - head -n5 - printf '... (%s lines total)\n' "$(echo "${reason}" | wc -l)" - fi | \ - sponge "${out_file%/out}/in" - done - sed -i "/${regex}/d" "${out_file}" - fi - regex='^\(\S\+ \)\?\S\+ <\S\+> .*[Ww]hat'"'"'\?s *[Uu]p' - if grep "${regex}" "${out_file}" | \ - grep -q "${sloppy_salutation}"; then - done_something=true - sed -i "/${regex}/d" "${out_file}" - printf 'up? I'"'"'m up for %s, %s\n' "$(uptime -p | sed 's/^up //')" "$(uptime | sed 's/.*, //')" | \ - sponge "${out_file%/out}/in" - fi - regex='^\(\S\+ \)\?\S\+ <\S\+> \(.* \)\?[Tt]h\(anks\|x\)\([ ,.!]\|$\)' - if grep "${regex}" "${out_file}" | \ - grep -q "${sloppy_salutation}"; then - done_something=true - sed -i "/${regex}/d" "${out_file}" - printf 'np\n' | \ - sponge "${out_file%/out}/in" - fi - done - sleep 1 done exit fi diff --git a/bin/return-assignment b/bin/return-assignment index bcdb717..b4fdf1c 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -193,6 +193,7 @@ if [ "$5" = 'ERROR' ]; then fi ) + # shellcheck disable=SC2119 { printf '%s is broken (says %s).' \ "$1" \ diff --git a/bin/sanity-check b/bin/sanity-check index 96f028f..fdcc25a 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -19,6 +19,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then + # shellcheck disable=SC2119 printf '\001ACTION goes insane.\001\n' | \ irc_say fi @@ -461,6 +462,7 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" + # shellcheck disable=SC2119 printf '\001ACTION resumes sanity.\001\n' | \ irc_say fi diff --git a/lib/common-functions b/lib/common-functions index ef10367..7093c12 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -549,8 +549,7 @@ recursively_umount_and_rm() { # mangle_pkgbuild $PKGBUILD [$sub_pkgrel] # mangle $arch in PKBUILDs to contain i486, i586, i686 # append $sub_pkgrel to the pkgrel - -# TODO: actually include the other architectures +# remove "lib32-" from {make,check,opt,}depends mangle_pkgbuild() { local PKGBUILD="$1" @@ -568,53 +567,20 @@ mangle_pkgbuild() { sed -i ' /^arch=[^#]*any/!{ - /^arch=(/s/(/(i686 / + /^arch=(/s/(/(i686 i486 pentium3 / } s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ ' "${PKGBUILD}" -} -# find_newest_of_git_revisions -# find newest git revision of the ones provided at stdin -# (assuming linear history) - -find_newest_of_git_revisions() { - local revisions - local repo - revisions=$(cat) - - if [ "$( - echo "${revisions}" | \ - wc -l - )" -eq 1 ]; then - - echo "${revisions}" - return - - fi - - repo=$( - find_repository_with_commit \ - "$( - echo "${revisions}" | \ - grep -xm1 '[0-9a-f]\{40\}' - )" - ) - - eval 'repo="${repo_paths__'"${repo}"'}"' - - echo "${revisions}" | \ - xargs -rn1 git -C "${repo}" rev-parse | \ - { - newest='' - while read -r current; do - if [ -z "${newest}" ] || \ - git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then - newest="${current}" - fi - done - echo "${newest}" + sed -i ' + /^\s*\(\(make\|check\|opt\|\)depends\)=(/ { + :a + s,^\s*\(\S[^=]*\)=(\([^()]*\(#[^\n]*\n\)\?\)*),\0\n\1=(${\1[\@]/lib32-/})\n, + t + $!N + ba } + ' "${PKGBUILD}" } # find_package_repository_to_package $package $git_repository $git_commit @@ -921,10 +887,23 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { fi } -# irc_say -# say content of stdin in irc channel +# irc_say $channel [copy] +# say content of stdin in irc channel $channel (default: #archlinux32) +# and print copy to stdout if 'copy' is given + +# shellcheck disable=SC2120 irc_say() { - if [ -p "${irc_dir}/#archlinux32/in" ]; then - sponge "${irc_dir}/#archlinux32/in" + local channel + if [ -z "$1" ]; then + channel='#archlinux32' + else + channel="$1" + fi + if [ -p "${irc_dir}/${channel}/in" ]; then + if [ "$2" = 'copy' ]; then + pee cat 'sponge '"${irc_dir}/${channel}/in" + else + sponge "${irc_dir}/${channel}/in" + fi fi } diff --git a/lib/mysql-functions b/lib/mysql-functions index 83b9296..71097cb 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -2,6 +2,9 @@ # contains functions used to access mysql db +# TODO: return ids on INSERT/UPDATE queries and use those in subsequent +# queries + # shellcheck disable=SC2016,SC2039 if [ -z "${base_dir}" ]; then @@ -64,6 +67,7 @@ mysql_run_query() { >&2 printf 'I could not complete a mysql query!\n' if [ ! -s "${work_dir}/build-master-sanity" ] && \ [ -z "${file_name_extra}" ]; then + # shellcheck disable=SC2119 printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_stdin##*/}" \ | irc_say @@ -122,10 +126,8 @@ mysql_add_package_source() { else uses_modification=0 fi - printf 'A' >&2 if grep -qx 'PKGBUILD = \S\+' "${srcinfo_file}"; then uses_upstream=1 - printf 'B' >&2 eval 'repo_path="${repo_paths__'"$(sed -n 's/^upstream_git_repository = //;T;p' "${srcinfo_file}")"'}"' pkgbuild=$( sed -n ' @@ -134,26 +136,19 @@ mysql_add_package_source() { p ' "${srcinfo_file}" ) - printf 'C' >&2 commit_time=$( git -C "${repo_path}" log -n 1 --pretty=format:%ct "${old_git_revision}${git_revision}" -- "${pkgbuild}" ) - printf 'D' >&2 if [ -z "${commit_time}" ]; then # We may have modified git_mod_revision! - printf 'E' >&2 commit_time=$( git -C "${repo_path}" log -n 1 --pretty=format:%ct "${git_revision}" -- "${pkgbuild}" ) - printf 'F' >&2 fi else - printf 'G' >&2 uses_upstream=0 commit_time=$(date '+%s') - printf 'H' >&2 fi - printf 'I' >&2 upstream_package_repository_id=$( { @@ -176,7 +171,7 @@ mysql_add_package_source() { fi { - printf 'INSERT IGNORE INTO package_sources' + printf 'INSERT INTO `package_sources`' printf ' (`pkgbase`,`git_revision`,`mod_git_revision`,`upstream_package_repository`,`uses_upstream`,`uses_modification`,`commit_time`)' printf ' VALUES (' printf 'from_base64("%s"),' \ @@ -196,60 +191,12 @@ mysql_add_package_source() { "${upstream_package_repository_id}" \ "${uses_upstream}" \ "${uses_modification}" - printf 'from_unixtime(%s));\n' \ + printf 'from_unixtime(%s))' \ "${commit_time}" + printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`package_sources`.`id`);\n' + printf 'SELECT LAST_INSERT_ID();\n' } | \ mysql_run_query - printf 'J' >&2 -} - -# mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel - -# shellcheck disable=SC2031,SC2086,SC2154 -mysql_show_binary_package() { - local names='pkgname pkgver pkgrel sub_pkgrel' - local name - for name in ${names}; do - eval 'local '"${name}" - eval "${name}"'=$( - printf "%s" "$1" | - base64 -w0 - )' - shift - done - - { - printf 'SELECT' - printf ' `%s`.`%s`,' \ - 'repositories' 'name' \ - 'binary_packages' 'pkgname' \ - 'package_sources' 'pkgver' \ - 'package_sources' 'pkgrel' \ - 'binary_packages' 'sub_pkgrel' \ - 'architectures' 'name' \ - 'package_sources' 'pkgbase' \ - 'package_sources' 'git_revision' \ - 'package_sources' 'mod_git_revision' \ - 'upstream_repositories' 'name' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - mysql_join_binary_packages_repositories - mysql_join_binary_packages_build_assignments - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' WHERE' - printf ' `%s`.`%s` = from_base64("%s") AND' \ - 'binary_packages' 'pkgname' "${pkgname}" \ - 'binary_packages' 'sub_pkgrel' "${sub_pkgrel}" \ - 'package_sources' 'pkgver' "${pkgver}" \ - 'package_sources' 'pkgrel' "${pkgrel}" - printf ';' - } | \ - sed ' - s|, FROM| FROM|g - s|AND;|;|g - ' | \ - mysql_run_query --html --column-names } # mysql_generate_package_metadata $current_repository $package $git_revision $old_git_revision $mod_git_revision $repository @@ -286,7 +233,6 @@ mysql_generate_package_metadata() { temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${temp_dir}"' EXIT - printf '.a' >&2 if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" exit 2 @@ -306,7 +252,6 @@ mysql_generate_package_metadata() { fi printf '\n\n' >> "${temp_dir}/SRCINFO" - printf '.b' >&2 pkgbase=$( grep '^pkgbase = ' "${temp_dir}/SRCINFO" | \ cut -d' ' -f3 @@ -315,86 +260,55 @@ mysql_generate_package_metadata() { >&2 printf '"make_source_info" did not return a "pkgbase" - eh, what?\n' exit 2 fi - printf '.c' >&2 # add the package source - mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO" - printf '.d' >&2 + package_source_id=$( + mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO" + ) # now we encode everything in base64 current_repository=$( printf '%s' "${current_repository}" | \ base64 -w0 ) - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - git_revision=$( - printf '%s' "${git_revision}" | \ - base64 -w0 - ) - mod_git_revision=$( - printf '%s' "${mod_git_revision}" | \ - base64 -w0 - ) - repository=$( - printf '%s' "${repository}" | \ - base64 -w0 - ) # add the build assignment(s) { - archs=$( - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - grep -vxF 'any' | \ - sort -u - ) - if [ -z "${archs}" ]; then + sed -n ' + s/^\tarch = // + T + p + ' "${temp_dir}/SRCINFO" | \ + sort -u | \ + grep -vxF 'any' || \ echo 'any' - else - printf '%s\n' "${archs}" - fi } | \ + base64_encode_each | \ while read -r arch; do - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' - printf ' FROM `architectures` JOIN `package_sources`' + printf 'INSERT INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT %s,`architectures`.`id`,NULL,0,0' \ + "${package_source_id}" + printf ' FROM `architectures`' printf ' WHERE `architectures`.`name` = from_base64("%s")' \ - "$( - printf '%s' "${arch}" | \ - base64 -w0 - )" - printf ' AND `package_sources`.`%s` = from_base64("%s")' \ - 'pkgbase' "${pkgbase}" \ - 'git_revision' "${git_revision}" \ - 'mod_git_revision' "${mod_git_revision}" - printf ';\n' - done > \ - "${temp_dir}/add-build-assignments-command" - - # TODO: correctly link between binary_packages and build_assignments using any_arch + "${arch}" + printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`build_assignments`.`id`);\n' + printf 'SELECT "%s",LAST_INSERT_ID();\n' \ + "${arch}" + done | \ + mysql_run_query > \ + "${temp_dir}/build-assignments.id" - # shellcheck disable=SC2034 # select any specific arch (which will be building the 'any' part of a split package) any_arch=$( - { - sed -n ' - s/^\tarch = // - T - p - ' "${temp_dir}/SRCINFO" | \ - sort -r | \ - grep -vxFm 1 'any' || \ - echo 'any' - } | \ - base64_encode_each + cut -f1 < \ + "${temp_dir}/build-assignments.id" | \ + sort | \ + head -n1 | \ + tr -d '\n' | \ + base64 -w0 ) + # iterate over all pkgnames grep '^pkgname = ' "${temp_dir}/SRCINFO" | \ cut -d' ' -f3 | \ while read -r pkgname; do @@ -403,22 +317,15 @@ mysql_generate_package_metadata() { base64 -w0 ) sed -n ' - /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/{ - :a - N - /\n$/{ - p - T - } - ba - } - ' "${temp_dir}/SRCINFO" | \ - sed ' + /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/,/^$/ { /^\S/d s/^\s*// - ' > \ + p + } + ' "${temp_dir}/SRCINFO" > \ "${temp_dir}/BINARYINFO.${pkgname64}" + # iterate over all archs grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \ cut -d' ' -f3 | \ while read -r arch; do @@ -426,24 +333,51 @@ mysql_generate_package_metadata() { printf '%s' "${arch}" | \ base64 -w0 ) - sed ' - s/^\(\S\+\)_'"${arch}"' = /\1 = / - ' "${temp_dir}/BINARYINFO.${pkgname64}" > \ + { + { + # this binary package is either built by the build_assignment + # with the identical arch (if existent) or by the one + # with arch=$any_arch + grep "^${arch64}"'\s' "${temp_dir}/build-assignments.id" || \ + grep "^${any_arch}"'\s' "${temp_dir}/build-assignments.id" + } | \ + sed ' + s/^.*\s/build-assignment-id = / + ' + sed ' + s/^\(\S\+\)_'"${arch}"' = /\1 = / + ' "${temp_dir}/BINARYINFO.${pkgname64}" + } > \ "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}" done done + + # TODO: This should become multiple mysql queries: + # 1st inserting install_targets + # 2nd inserting binary_packages (and binary_packages_in_repositories) + # 3rd,4th inserting dependencies and install_target_providers + # The links between the above should _solely_ be made by the ids of + # the respective rows. + + # now each "ARCHINFO $pkgname $arch" file represents one binary package find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \ while read -r _ pkgname arch; do + build_assignment_id=$( + grep '^build-assignment-id = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 + ) pkgver=$( grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) pkgrel=$( grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ sed 's/\.[0-9]\+$//' | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) epoch=$( { @@ -451,8 +385,10 @@ mysql_generate_package_metadata() { echo 'epoch = 0' } | \ cut -d' ' -f3 | \ - base64_encode_each + tr -d '\n' | \ + base64 -w0 ) + # TODO: allow versioned install_targets provides=$( grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ @@ -534,8 +470,8 @@ mysql_generate_package_metadata() { 'is_tested' \ 'is_to_be_deleted' printf ') SELECT ' + printf '%s,' "${build_assignment_id}" printf '`%s`.`id`,' \ - 'build_assignments' \ 'repositories' \ 'architectures' printf 'from_base64("%s"),' \ @@ -547,23 +483,16 @@ mysql_generate_package_metadata() { "${sub_pkgrel}" printf ' `%s` JOIN' \ 'repositories' \ - 'architectures' \ - 'build_assignments' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories + 'architectures' printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'repositories' 'name' "${current_repository}" \ - 'architectures' 'name' "${arch}" \ - 'package_sources' 'pkgbase' "${pkgbase}" \ - 'package_sources' 'git_revision' "${git_revision}" \ - 'package_sources' 'mod_git_revision' "${mod_git_revision}" \ - 'upstream_repositories' 'name' "${repository}" + 'architectures' 'name' "${arch}" printf ';\n' } | \ sed ' s|,)|)|g - s| JOIN JOIN | JOIN | + s| JOIN WHERE | WHERE | s| AND;$|;| ' >> \ "${temp_dir}/add-binary-packages-command" @@ -689,12 +618,8 @@ mysql_generate_package_metadata() { } >> \ "${temp_dir}/add-install-targets-command" done - printf '.e' >&2 { - if [ -s "${temp_dir}/add-build-assignments-command" ]; then - cat "${temp_dir}/add-build-assignments-command" - fi if [ -s "${temp_dir}/add-binary-packages-command" ]; then cat "${temp_dir}/add-binary-packages-command" fi @@ -703,7 +628,6 @@ mysql_generate_package_metadata() { fi } | \ mysql_run_query - printf '.f' >&2 ) } @@ -999,118 +923,6 @@ mysql_query_select_pkgbase_and_revision() { mysql_join_package_sources_upstream_repositories } -# mysql_repair_binary_packages_without_build_assignment -# try to generate valid build assignments to binary packages without -# a valid one yet -mysql_repair_binary_packages_without_build_assignment() { - { - printf 'SELECT ' - printf '`binary_packages`.`id`' - printf ',replace(to_base64(%s),"\\n","")' \ - '`binary_packages`.`pkgname`' \ - '`architectures`.`name`' - printf ' FROM `binary_packages`' - mysql_join_binary_packages_architectures - printf ' WHERE `binary_packages`.`build_assignment`<0' - } | \ - mysql_run_query | \ - while read -r id pkgname arch; do - pkgname=$( - printf '%s' "${pkgname}" | \ - base64 -d - ) - pkgbase=$( - curl -Ss "$( - printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ - "${pkgname}" - )" | \ - sed ' - s/^.*"results":\s*\[// - s/}\s*,\s*{/\n/g - ' | \ - grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ - tr ',' '\n' | \ - grep '"pkgbase":' | \ - cut -d'"' -f4 | \ - sort -u | \ - head -n1 - ) - if [ -z "${pkgbase}" ] && \ - { - printf 'SELECT count(*) FROM `package_sources`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "${pkgname}" | base64 -w0)" - } | \ - mysql_run_query | \ - grep -qvxF '0'; then - pkgbase="${pkgname}" - fi - if [ -z "${pkgbase}" ]; then - >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" - continue - fi - pkgbase=$( - printf '%s' "${pkgbase}" | \ - base64 -w0 - ) - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' - printf ' FROM `package_sources`' - printf ' JOIN `architectures`' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" - printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" - printf ' LIMIT 1;\n' - printf 'UPDATE `binary_packages`' - printf ' JOIN `build_assignments`' - mysql_join_binary_packages_build_assignments - printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' WHERE `binary_packages`.`id`=%s' "${id}" - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" - done | \ - mysql_run_query -} - -# mysql_remove_duplicate_binary_packages -# remove duplicate binary_packages, matching pkgname, epoch, pkgver, pkgrel, -# having difference of 1 in sub_pkgrel - -mysql_remove_duplicate_build_order() { - { - printf 'CREATE TEMPORARY TABLE `ren`' - printf ' (`old` BIGINT, `new` BIGINT, `repo` BIGINT, `sub_pkgrel` BIGINT);\n' - printf 'INSERT INTO `ren` (`old`,`new`,`repo`,`sub_pkgrel`)' - printf ' SELECT `old`.`id`,`new`.`id`,`old`.`repository`,`old`.`sub_pkgrel`' - printf ' FROM `binary_packages` as `old`' - printf ' JOIN `binary_packages` as `new` ON' - printf ' `old`.`%s`=`new`.`%s` AND' \ - 'pkgname' 'pkgname' \ - 'epoch' 'epoch' \ - 'pkgver' 'pkgver' \ - 'pkgrel' 'pkgrel' - printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - mysql_join_binary_packages_repositories 'old' 'orep' - mysql_join_binary_packages_repositories 'new' 'nrep' - printf ' WHERE `orep`.`name`!="to-be-decided"' - printf ' AND `nrep`.`name`="to-be-decided";\n' - printf 'UPDATE IGNORE `dependencies`' - printf ' JOIN `ren` ON `ren`.`old`=`dependencies`.`dependent`' - printf ' SET `dependencies`.`dependent`=`ren`.`new`;\n' - printf 'UPDATE IGNORE `install_target_providers`' - printf ' JOIN `ren` ON `ren`.`old`=`install_target_providers`.`package`' - printf ' SET `install_target_providers`.`package`=`ren`.`new`;\n' - printf 'DELETE FROM `binary_packages`' - printf ' WHERE EXISTS (' - printf 'SELECT * FROM `ren`' - printf ' WHERE `ren`.`old`=`binary_packages`.`id`' - printf ');\n' - printf 'UPDATE IGNORE `binary_packages`' - printf ' JOIN `ren` ON `ren`.`new`=`binary_packages`.`id`' - printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' - printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' - } | \ - mysql_run_query -} - # mysql_package_name_query # print a mysql query of the full name of a package file |