#!/bin/sh # get-assignment $architecture $prefered-package-list # receive one package to be built from the build-list whose dependencies # are already satisfied or which breaks a dependency cycle # accepts a comma separated list of prefered packages as the first argument # exit code shows state of success: # 0: ok, I gave you an assignment # 1: come back (shortly) later - I was running already # 2: come back later - there are still packages to be built, # but currently none has all its dependencies ready # 3: come back after the next run of get-package-updates - currently # there are no pending packages # 4: wrong number of arguments # shellcheck disable=SC2119,SC2120 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" # hand_out_assignment $build_assignments.id hand_out_assignment() { # shellcheck disable=SC2016 { printf 'SELECT ' printf '`package_sources`.`%s`,' \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`,' printf '`binary_packages`.`sub_pkgrel`,' printf '`architectures`.`name`' printf ' FROM `upstream_repositories`' mysql_join_upstream_repositories_package_sources mysql_join_package_sources_build_assignments mysql_join_build_assignments_architectures mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' AND `build_assignments`.`id`=%s' \ "$1" printf ' LIMIT 1;\n' printf 'SELECT ' mysql_package_name_query printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures printf ' LEFT' mysql_join_binary_packages_compressions mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE `binary_packages`.`build_assignment`=%s' \ "$1" printf ' AND `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ';\n' printf 'UPDATE `build_slaves`' printf ' SET `build_slaves`.`currently_building`=%s,' \ "$1" printf ' `build_slaves`.`is_sane`=1' # shellcheck disable=SC2154 printf ' WHERE `build_slaves`.`id`=from_base64("%s");\n' \ "$(printf '%s' "${slave_id}" | base64 -w0)" printf 'UPDATE `build_assignments`' printf ' SET `build_assignments`.`priority`=0' printf ' WHERE `build_assignments`.`id`=%s;\n' \ "$1" } | \ mysql_run_query | \ tr '\t' ' ' exit 0 } if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit 1 fi # Create a lock file and a trap. exec 9> "${build_list_lock_file}" if ! verbose_flock -n 9; then >&2 echo 'come back (shortly) later - I cannot lock build list.' exit 1 fi exec 8> "${sanity_check_lock_file}" if ! verbose_flock -s -n 8; then >&2 echo 'come back (shortly) later - sanity-check running.' exit 1 fi clean_up() { rm -rf --one-file-system "${tmp_dir}" } tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir) trap clean_up EXIT arch=$( printf '%s' "$1" | \ base64 -w0 ) shift if [ $# -eq 1 ]; then requested=$( printf '%s' "$1" | \ base64 -w0 ) shift else requested='' fi # shellcheck disable=SC2016 if [ $# -ne 0 ]; then >&2 echo '"get-assignment" was called with wrong number of arguments.' >&2 echo 'call either:' >&2 echo ' get-assignment $arch' >&2 echo ' get-assignment $arch $favourite_package' exit 4 fi # if we're building something already, hand it out (again) currently_building=$( # shellcheck disable=SC2016 { printf 'SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' mysql_join_build_assignments_build_slaves mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' JOIN `architecture_compatibilities`' printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`' printf ' JOIN `architectures`' printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`' printf ' AND `architecture_compatibilities`.`build_slave_compatible`' printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \ "$(printf '%s' "${slave_id}" | base64 -w0)" printf ' AND `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' AND `architectures`.`name` = from_base64("%s")' \ "${arch}" printf ' LIMIT 1;\n' } | \ mysql_run_query ) if [ -n "${currently_building}" ]; then hand_out_assignment "${currently_building}" fi # shellcheck disable=SC2154 while [ -s "${work_dir}/forced-package-builds.${slave}" ]; do next_building=$( sed -i ' 1 { w /dev/stdout d } ' "${work_dir}/forced-package-builds.${slave}" ) if [ ! -s "${work_dir}/forced-package-builds.${slave}" ]; then rm "${work_dir}/forced-package-builds.${slave}" fi if [ -n "${next_building}" ]; then next_building=$( # shellcheck disable=SC2016 { printf 'SELECT' printf ' `build_assignments`.`id`' printf ' FROM `build_assignments`' mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' AND `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" mysql_join_build_assignments_package_sources mysql_join_build_assignments_architectures printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${next_building#* }" | \ base64 -w0 )" printf ' AND `architectures`.`name`=from_base64("%s")' \ "$( printf '%s' "${next_building%% *}" | \ base64 -w0 )" printf ' LIMIT 1;\n' } | \ mysql_run_query ) if [ -n "${next_building}" ]; then hand_out_assignment "${next_building}" fi fi done # a package with [all dependencies met (or unsuccessfully built) or which is part of a loop] # and which is currently not being built, ordered by: # 1: we requested it # 2: tool chain priority # 3: its priority # 4: was built the least times # 5: architecture matches (or the slave is from the majority architecture) # 6: is part of a loop # 7: has the oldes package source # 8: build_assignment id next_building=$( # shellcheck disable=SC2016 { for suffix in '' '_2' '_3' '_4'; do printf 'CREATE TEMPORARY TABLE `considered_build_assignments%s` (' \ "${suffix}" printf '`id` BIGINT,' printf ' UNIQUE KEY `id`(`id`)' printf ');\n' done printf 'INSERT IGNORE INTO `considered_build_assignments` (`id`)' printf ' SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' mysql_join_build_assignments_package_sources mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' JOIN `architecture_compatibilities`' printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`' printf ' AND `architecture_compatibilities`.`build_slave_compatible`' printf ' JOIN `architectures`' printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`' # "any" packages are built with the highest-level build command architecture printf ' JOIN `architectures` AS `build_command_architectures`' printf ' ON `build_command_architectures`.`id`=IF(' printf '`build_assignments`.`architecture`=%s,' \ "${architecture_ids__any}" printf '`architectures`.`id`,' printf '`build_assignments`.`architecture`' printf ')' printf ' LEFT' mysql_join_package_sources_toolchain_order printf ' WHERE (' printf '`build_assignments`.`is_blocked` IS NULL' if [ -n "${requested}" ]; then printf ' OR' printf ' `package_sources`.`pkgbase`=from_base64("%s")' \ "${requested}" fi printf ') AND (' mysql_query_is_part_of_loop '`build_assignments`.`id`' printf ' OR NOT ' mysql_query_has_pending_dependencies '`build_command_architectures`.`id`' '`build_assignments`.`id`' printf ' OR `toolchain_order`.`pkgbase` IS NOT NULL' printf ')' printf ' AND `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' AND `architectures`.`name` = from_base64("%s");\n' \ "${arch}" printf 'INSERT IGNORE INTO `considered_build_assignments_2` (`id`)' printf ' SELECT `considered_build_assignments`.`id`' printf ' FROM `considered_build_assignments`;\n' # delete all non-toolchain packages from considered list iff any # toolchain package is on that list printf 'CREATE TEMPORARY TABLE `toolchain_architectures` (' printf '`id` SMALLINT NOT NULL,' printf 'PRIMARY KEY (`id`)' printf ');\n' printf 'INSERT IGNORE INTO `toolchain_architectures` (`id`)' printf ' SELECT `build_assignments`.`architecture`' printf ' FROM `considered_build_assignments`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`' mysql_join_build_assignments_package_sources mysql_join_package_sources_toolchain_order printf ';\n' printf 'DELETE `considered_build_assignments`' printf ' FROM `toolchain_architectures`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`' printf ' JOIN `considered_build_assignments`' printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`' mysql_join_build_assignments_package_sources printf ' LEFT' mysql_join_package_sources_toolchain_order printf ' WHERE `toolchain_order`.`number` IS NULL;\n' printf 'DELETE `considered_build_assignments_2`' printf ' FROM `considered_build_assignments_2`;\n' for suffix in '_2' '_3' '_4'; do printf 'INSERT IGNORE INTO `considered_build_assignments%s` (`id`)' \ "${suffix}" printf ' SELECT `considered_build_assignments`.`id`' printf ' FROM `considered_build_assignments`;\n' done # delete all but one package from considered list iff any toolchain # package is on that list # the order is: # - does not exist blocked before exists blocked # - unbuilt before built # - exists unblocked before does not exist unblocked printf 'CREATE TEMPORARY TABLE `ordered_considered_build_assignments`(' printf '`order` BIGINT NOT NULL AUTO_INCREMENT,' printf '`id` BIGINT NOT NULL,' printf '`exists_blocked` BIT NOT NULL,' printf '`exists_built` BIT NOT NULL,' printf '`exists_unblocked` BIT NOT NULL,' printf 'PRIMARY KEY (`order`),' printf 'KEY (`id`)' printf ');\n' printf 'CREATE TEMPORARY TABLE `orders`(' printf '`order` BIGINT NOT NULL AUTO_INCREMENT,' printf 'PRIMARY KEY (`order`)' printf ');\n' # TODO: This is broken, it cannot work and probably never will do this way. printf 'INSERT IGNORE INTO `ordered_considered_build_assignments` (' printf '`id`,' printf '`exists_blocked`,' printf '`exists_built`,' printf '`exists_unblocked`' printf ')' printf 'SELECT' printf ' `build_assignments`.`id`,' printf 'EXISTS (' printf 'SELECT 1' printf ' FROM `toolchain_order` AS `late`' printf ' JOIN `toolchain_order` AS `early`' printf ' ON `late`.`pkgbase`!=`early`.`pkgbase`' printf ' AND `late`.`number`>`early`.`number`' mysql_join_toolchain_order_package_sources 'early' 'e_ps' mysql_join_package_sources_build_assignments 'e_ps' 'e_ba' printf ' JOIN `considered_build_assignments_3` AS `e_cba`' printf ' ON `e_cba`.`id`=`e_ba`.`id`' printf ' WHERE `late`.`pkgbase`=`package_sources`.`pkgbase`' # the toolchain_order-blocked build-assignment printf ' AND (' # must be marked as requires_all_dependencies_built (second half of list) printf '`late`.`requires_all_dependencies_built` OR' # or all blocking dependencies must not yet been built printf ' NOT EXISTS (' printf 'SELECT 1 FROM `binary_packages` AS `e_bp`' mysql_join_binary_packages_binary_packages_in_repositories 'e_bp' 'e_bpir' mysql_join_binary_packages_in_repositories_repositories 'e_bpir' 'e_r' printf ' WHERE `e_bp`.`build_assignment`=`e_ba`.`id`' printf ' AND `e_r`.`is_on_master_mirror`' printf ')' printf ')' printf ') AS `exists_blocked`,' printf 'EXISTS (' printf 'SELECT 1' printf ' FROM `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_repositories printf ' AND `repositories`.`is_on_master_mirror`' printf ' JOIN `architecture_compatibilities`' printf ' ON `repositories`.`architecture`=`architecture_compatibilities`.`built_for`' printf ' AND `architecture_compatibilities`.`build_slave_compatible`' printf ' JOIN `architectures`' printf ' ON `architectures`.`id`=`architecture_compatibilities`.`runs_on`' printf ' AND `architectures`.`name`=from_base64("%s")' \ "${arch}" mysql_join_binary_packages_in_repositories_binary_packages printf ' WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`' printf ') AS `exists_built`,' printf 'EXISTS (' printf 'SELECT 1' printf ' FROM `toolchain_order` AS `late`' printf ' WHERE NOT EXISTS (' printf 'SELECT 1' printf ' FROM `toolchain_order` AS `early`' mysql_join_toolchain_order_package_sources 'early' 'e_ps' mysql_join_package_sources_build_assignments 'e_ps' 'e_ba' printf ' JOIN `considered_build_assignments_4` AS `e_cba`' printf ' ON `e_ba`.`id`=`e_cba`.`id`' printf ' WHERE `late`.`pkgbase`!=`early`.`pkgbase`' printf ' AND `late`.`number`>`early`.`number`' printf ')' printf ' AND `late`.`pkgbase`=`package_sources`.`pkgbase`' printf ') AS `exists_unblocked`' printf ' FROM `considered_build_assignments`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`' printf ' JOIN `toolchain_architectures`' printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`' mysql_join_build_assignments_package_sources printf ' ORDER BY' # does it exist toolchain-blocked? printf ' `exists_blocked`,' # does it exist built? printf '`exists_built`,' # does it exist not toolchain-blocked? printf '`exists_unblocked` DESC' printf ';\n' printf 'INSERT IGNORE INTO `orders`(`order`)' printf ' SELECT' printf ' MIN(`ordered_considered_build_assignments`.`order`)' printf ' FROM `ordered_considered_build_assignments`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`id`=`ordered_considered_build_assignments`.`id`' printf ' JOIN `toolchain_architectures`' printf ' JOIN `architecture_compatibilities`' printf ' ON `architecture_compatibilities`.`fully_compatible`' printf ' AND (' printf '(' printf '`architecture_compatibilities`.`runs_on`=`build_assignments`.`architecture`' printf ' AND `architecture_compatibilities`.`built_for`=`toolchain_architectures`.`id`' printf ') OR (' printf '`architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`' printf ' AND `architecture_compatibilities`.`runs_on`=`toolchain_architectures`.`id`' printf ')' printf ')' printf ' GROUP BY `toolchain_architectures`.`id`' printf ';\n' printf 'DELETE `considered_build_assignments`' printf ' FROM `toolchain_architectures`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`' printf ' JOIN `considered_build_assignments`' printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`;\n' printf 'INSERT IGNORE INTO `considered_build_assignments` (`id`)' printf ' SELECT' printf ' `ordered_considered_build_assignments`.`id`' printf ' FROM `ordered_considered_build_assignments`' printf ' JOIN `orders`' printf ' ON `orders`.`order`=`ordered_considered_build_assignments`.`order`' printf ';\n' printf 'DROP TEMPORARY TABLE `ordered_considered_build_assignments`;\n' printf 'DROP TEMPORARY TABLE `orders`;\n' # delete all currently building packages from considered list printf 'DELETE `considered_build_assignments`' printf ' FROM `considered_build_assignments`' mysql_join_build_assignments_build_slaves 'considered_build_assignments' printf ';\n' printf 'SELECT `considered_build_assignments`.`id` FROM' printf ' `considered_build_assignments`' printf ' JOIN `build_assignments`' printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`' mysql_join_build_assignments_package_sources mysql_join_build_assignments_binary_packages printf ' LEFT JOIN (' printf 'SELECT' printf ' `failed_builds`.`build_assignment`,' printf 'COUNT(DISTINCT `failed_builds`.`id`) AS `count`' printf ' FROM `failed_builds`' printf ' GROUP BY `failed_builds`.`build_assignment`' printf ') AS `latest_failed_builds`' printf ' ON `latest_failed_builds`.`build_assignment`=`build_assignments`.`id`' printf ' LEFT JOIN (' # Does this job wait for something else in `toolchain_order`? printf 'SELECT ' printf '`tc_q`.`pkgbase`,' printf '`tc_q`.`architecture`,' printf 'IF(' printf '`tc_q`.`is_blocked`,' printf 'IF(' printf '`tc_q`.`is_built`,' printf '2,' printf '1' printf '),' printf '0' printf ') AS `priority`' printf ' FROM (' printf 'SELECT ' printf '`late`.`pkgbase`,' printf '`binary_packages`.`architecture`,' printf 'MAX(' printf 'IF(' printf 'EXISTS (' printf 'SELECT 1' printf ' FROM `binary_packages` AS `built_bp`' mysql_join_binary_packages_binary_packages_in_repositories 'built_bp' 'built_bpir' mysql_join_binary_packages_in_repositories_repositories 'built_bpir' 'built_r' printf ' AND `built_r`.`is_on_master_mirror`' printf ' WHERE `built_bp`.`build_assignment`=`build_assignments`.`id`' printf '),' printf '1,' printf '0' printf ')' printf ') AS `is_built`,' printf 'MAX(' printf 'IF(' printf 'EXISTS (' printf 'SELECT 1' printf ' FROM `package_sources` AS `block_ps`' mysql_join_package_sources_build_assignments 'block_ps' 'block_ba' mysql_join_build_assignments_binary_packages 'block_ba' 'block_bp' mysql_join_binary_packages_binary_packages_in_repositories 'block_bp' 'block_bpir' printf ' AND `block_bpir`.`repository`=%s' \ "${repository_ids__any_build_list}" mysql_join_package_sources_toolchain_order 'block_ps' 'early' printf ' WHERE `early`.`number`<`late`.`number`' printf ' AND `early`.`pkgbase`!=`late`.`pkgbase`' printf ' AND (' printf '`block_bp`.`architecture`=`binary_packages`.`architecture`' # shellcheck disable=SC2154 printf ' OR `%s`.`architecture`=%s' \ 'block_bp' "${architecture_ids__any}" \ 'binary_packages' "${architecture_ids__any}" printf ')' printf '),' printf '1,' printf '0' printf ')' printf ') AS `is_blocked`' printf ' FROM `toolchain_order` AS `late`' mysql_join_toolchain_order_package_sources 'late' mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' AND `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' GROUP BY CONCAT(`late`.`pkgbase`,"-",`binary_packages`.`architecture`)' printf ') AS `tc_q`' printf ') AS `toolchain_query`' printf ' ON `toolchain_query`.`pkgbase`=`package_sources`.`pkgbase`' printf ' AND `toolchain_query`.`architecture`=`binary_packages`.`architecture`' printf ' JOIN `architectures`' printf ' ON `architectures`.`name`=from_base64("%s")' \ "${arch}" printf ' ORDER BY ' if [ -n "${requested}" ]; then printf 'IF(`package_sources`.`pkgbase`=from_base64("%s"),1,0) DESC,' \ "${requested}" fi # 0: in toolchain, should be built now # 1: in toolchain, should be built later # 2: not in toolchain printf 'IFNULL(`toolchain_query`.`priority`,2),' printf '`build_assignments`.`priority` DESC,' printf 'IFNULL(`latest_failed_builds`.`count`,0),' printf 'IF(' printf '`architectures`.`id`=%s' \ "$(mysql_determine_majority_build_slave_architecture_id)" printf ' OR `architectures`.`id`=`build_assignments`.`architecture`,' printf '1,0) DESC,' printf 'IF(' mysql_query_is_part_of_loop '`build_assignments`.`id`' printf ',1,0),' printf '`package_sources`.`commit_time`,' printf '`build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ mysql_run_query ) if [ -n "${next_building}" ]; then hand_out_assignment "${next_building}" fi # Check if there are any pending packages at all count_pending=$( # shellcheck disable=SC2016 { printf 'SELECT COUNT(1)' printf ' FROM `build_assignments`' mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories mysql_join_binary_packages_in_repositories_repositories printf ' JOIN `architecture_compatibilities`' printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`id`' printf ' AND `architecture_compatibilities`.`build_slave_compatible`' printf ' JOIN `architectures`' printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`' printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ' AND `architectures`.`name`=from_base64("%s")' \ "${arch}" printf ';\n' } | \ mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages' exit 3 else >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready' exit 2 fi