path: root/bin/get-assignment

#!/bin/sh

# get-assignment $architecture $prefered-package-list

# receive one package to be built from the build-list whose dependencies
# are already satisfied or which breaks a dependency cycle
# accepts a comma separated list of prefered packages as the first argument

# exit code shows state of success:
#  0: ok, I gave you an assignment
#  1: come back (shortly) later - I was running already
#  2: come back later - there are still packages to be built,
#     but currently none has all its dependencies ready
#  3: come back after the next run of get-package-updates - currently
#     there are no pending packages
#  4: wrong number of arguments

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# hand_out_assignment $build_assignments.id
hand_out_assignment() {

  # shellcheck disable=SC2016
  {
    printf 'SELECT '
    printf '`package_sources`.`%s`,' \
      'pkgbase' 'git_revision' 'mod_git_revision'
    printf '`upstream_repositories`.`name`,'
    printf '`binary_packages`.`sub_pkgrel`,'
    printf '`architectures`.`name`'
    printf ' FROM `upstream_repositories`'
    mysql_join_upstream_repositories_package_sources
    mysql_join_package_sources_build_assignments
    mysql_join_build_assignments_architectures
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' WHERE `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ' AND `build_assignments`.`id`=%s' \
      "$1"
    printf ' LIMIT 1;\n'

    printf 'SELECT '
    mysql_package_name_query
    printf ' FROM `binary_packages`'
    mysql_join_binary_packages_architectures
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' WHERE `binary_packages`.`build_assignment`=%s' \
      "$1"
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ';\n'

    printf 'UPDATE `build_slaves`'
    printf ' SET `build_slaves`.`currently_building`=%s,' \
      "$1"
    printf ' `build_slaves`.`is_sane`=1'
    # shellcheck disable=SC2154
    printf ' WHERE `build_slaves`.`id`=from_base64("%s");\n' \
      "$(printf '%s' "${slave_id}" | base64 -w0)"

    printf 'UPDATE `build_assignments`'
    printf ' SET `build_assignments`.`priority`=0'
    printf ' WHERE `build_assignments`.`id`=%s;\n' \
      "$1"
  } | \
    mysql_run_query | \
    tr '\t' ' '

  exit 0

}

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! verbose_flock -n 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${sanity_check_lock_file}"
if ! verbose_flock -s -n 8; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit 1
fi

clean_up() {
  rm -rf --one-file-system "${tmp_dir}"
}

tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
trap clean_up EXIT

arch=$(
  printf '%s' "$1" | \
    base64 -w0
)
shift
if [ $# -eq 1 ]; then
  requested=$(
    printf '%s' "$1" | \
      base64 -w0
  )
  shift
else
  requested=''
fi
# shellcheck disable=SC2016
if [ $# -ne 0 ]; then
  >&2 echo '"get-assignment" was called with wrong number of arguments.'
  >&2 echo 'call either:'
  >&2 echo '  get-assignment $arch'
  >&2 echo '  get-assignment $arch $favourite_package'
  exit 4
fi

# if we're building something already, hand it out (again)
currently_building=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT `build_assignments`.`id`'
    printf ' FROM `build_assignments`'
    mysql_join_build_assignments_build_slaves
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' JOIN `architecture_compatibilities`'
    printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
    printf ' JOIN `architectures`'
    printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
    printf ' AND `architecture_compatibilities`.`build_slave_compatible`'
    printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \
      "$(printf '%s' "${slave_id}" | base64 -w0)"
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ' AND `architectures`.`name` = from_base64("%s")' \
      "${arch}"
    printf ' LIMIT 1;\n'
  } | \
    mysql_run_query
)

if [ -n "${currently_building}" ]; then
  hand_out_assignment "${currently_building}"
fi

# shellcheck disable=SC2154
while [ -s "${work_dir}/forced-package-builds.${slave}" ]; do
  next_building=$(
    sed -i '
      1 {
        w /dev/stdout
        d
      }
    ' "${work_dir}/forced-package-builds.${slave}"
  )
  if [ ! -s "${work_dir}/forced-package-builds.${slave}" ]; then
    rm "${work_dir}/forced-package-builds.${slave}"
  fi
  if [ -n "${next_building}" ]; then
    next_building=$(
      # shellcheck disable=SC2016
      {
        printf 'SELECT'
        printf ' `build_assignments`.`id`'
        printf ' FROM `build_assignments`'
        mysql_join_build_assignments_binary_packages
        mysql_join_binary_packages_binary_packages_in_repositories
        printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
          "${repository_ids__any_build_list}"
        mysql_join_build_assignments_package_sources
        mysql_join_build_assignments_architectures
        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
          "$(
            printf '%s' "${next_building#* }" | \
              base64 -w0
          )"
        printf ' AND `architectures`.`name`=from_base64("%s")' \
          "$(
            printf '%s' "${next_building%% *}" | \
              base64 -w0
          )"
        printf ' LIMIT 1;\n'
      } | \
        mysql_run_query
    )
    if [ -n "${next_building}" ]; then
      hand_out_assignment "${next_building}"
    fi
  fi
done

# a package with [all dependencies met or which is part of a loop]
# and which is currently not being built, ordered by:
# 1: we requested it
# 2: tool chain priority
# 3: its priority
# 4: was built the least times
# 5: architecture matches (or the slave is from the majority architecture)
# 6: is part of a loop
# 7: has the oldes package source
# 8: build_assignment id
next_building=$(
  # shellcheck disable=SC2016
  {
    for suffix in '' '_2' '_3' '_4'; do
      printf 'CREATE TEMPORARY TABLE `considered_build_assignments%s` (' \
        "${suffix}"
        printf '`id` BIGINT,'
        printf ' UNIQUE KEY `id`(`id`)'
      printf ');\n'
    done

    printf 'INSERT IGNORE INTO `considered_build_assignments` (`id`)'
    printf ' SELECT `build_assignments`.`id`'
    printf ' FROM `build_assignments`'
    mysql_join_build_assignments_package_sources
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' JOIN `architecture_compatibilities`'
    printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
    printf ' AND `architecture_compatibilities`.`build_slave_compatible`'
    printf ' JOIN `architectures`'
    printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
    # "any" packages are built with the highest-level build command architecture
    printf ' JOIN `architectures` AS `build_command_architectures`'
    printf ' ON `build_command_architectures`.`id`=IF('
      printf '`build_assignments`.`architecture`=%s,' \
        "${architecture_ids__any}"
      printf '`architectures`.`id`,'
      printf '`build_assignments`.`architecture`'
    printf ')'
    printf ' LEFT'
    mysql_join_package_sources_toolchain_order
    printf ' WHERE ('
      printf '`build_assignments`.`is_blocked` IS NULL'
      if [ -n "${requested}" ]; then
        printf ' OR'
        printf ' `package_sources`.`pkgbase`=from_base64("%s")' \
          "${requested}"
      fi
    printf ') AND ('
      mysql_query_is_part_of_loop '`build_assignments`.`id`'
      printf ' OR NOT '
      mysql_query_has_pending_dependencies '`build_command_architectures`.`id`' '`build_assignments`.`id`'
      printf ' OR `toolchain_order`.`pkgbase` IS NOT NULL'
    printf ')'
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ' AND `architectures`.`name` = from_base64("%s");\n' \
      "${arch}"

    printf 'INSERT IGNORE INTO `considered_build_assignments_2` (`id`)'
    printf ' SELECT `considered_build_assignments`.`id`'
    printf ' FROM `considered_build_assignments`;\n'

    # delete all non-toolchain packages from considered list iff any
    # toolchain package is on that list
    printf 'CREATE TEMPORARY TABLE `toolchain_architectures` ('
      printf '`id` SMALLINT NOT NULL,'
      printf 'PRIMARY KEY (`id`)'
    printf ');\n'
    printf 'INSERT IGNORE INTO `toolchain_architectures` (`id`)'
    printf ' SELECT `build_assignments`.`architecture`'
    printf ' FROM `considered_build_assignments`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`'
    mysql_join_build_assignments_package_sources
    mysql_join_package_sources_toolchain_order
    printf ';\n'

    printf 'DELETE `considered_build_assignments`'
    printf ' FROM `toolchain_architectures`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`'
    printf ' JOIN `considered_build_assignments`'
    printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`'
    mysql_join_build_assignments_package_sources
    printf ' LEFT'
    mysql_join_package_sources_toolchain_order
    printf ' WHERE `toolchain_order`.`number` IS NULL;\n'

    printf 'DELETE `considered_build_assignments_2`'
    printf ' FROM `considered_build_assignments_2`;\n'
    for suffix in '_2' '_3' '_4'; do
      printf 'INSERT IGNORE INTO `considered_build_assignments%s` (`id`)' \
        "${suffix}"
      printf ' SELECT `considered_build_assignments`.`id`'
      printf ' FROM `considered_build_assignments`;\n'
    done

    # delete all but one package from considered list iff any toolchain
    # package is on that list
    # the order is:
    #  - does not exist blocked before exists blocked
    #  - unbuilt before built
    #  - exists unblocked before does not exist unblocked

    printf 'CREATE TEMPORARY TABLE `ordered_considered_build_assignments`('
      printf '`order` BIGINT NOT NULL AUTO_INCREMENT,'
      printf '`id` BIGINT NOT NULL,'
      printf '`exists_blocked` BIT NOT NULL,'
      printf '`exists_built` BIT NOT NULL,'
      printf '`exists_unblocked` BIT NOT NULL,'
      printf 'PRIMARY KEY (`order`),'
      printf 'KEY (`id`)'
    printf ');\n'
    printf 'CREATE TEMPORARY TABLE `orders`('
      printf '`order` BIGINT NOT NULL AUTO_INCREMENT,'
      printf 'PRIMARY KEY (`order`)'
    printf ');\n'

    printf 'INSERT IGNORE INTO `ordered_considered_build_assignments` ('
      printf '`id`,'
      printf '`exists_blocked`,'
      printf '`exists_built`,'
      printf '`exists_unblocked`'
    printf ')'
    printf 'SELECT'
    printf ' `build_assignments`.`id`,'
    printf 'EXISTS ('
      printf 'SELECT 1'
      printf ' FROM `toolchain_order` AS `late`'
      printf ' JOIN `toolchain_order` AS `early`'
      printf ' ON `late`.`pkgbase`!=`early`.`pkgbase`'
      printf ' AND `late`.`number`>`early`.`number`'
      mysql_join_toolchain_order_package_sources 'early' 'e_ps'
      mysql_join_package_sources_build_assignments 'e_ps' 'e_ba'
      printf ' JOIN `considered_build_assignments_3` AS `e_cba`'
      printf ' ON `e_cba`.`id`=`e_ba`.`id`'
      printf ' WHERE `late`.`pkgbase`=`package_sources`.`pkgbase`'
    printf ') AS `exists_blocked`,'
    printf 'EXISTS ('
      printf 'SELECT 1'
      printf ' FROM `binary_packages_in_repositories`'
      mysql_join_binary_packages_in_repositories_repositories
      printf ' AND `repositories`.`is_on_master_mirror`'
      printf ' JOIN `architecture_compatibilities`'
      printf ' ON `repositories`.`architecture`=`architecture_compatibilities`.`built_for`'
      printf ' AND `architecture_compatibilities`.`build_slave_compatible`'
      printf ' JOIN `architectures`'
      printf ' ON `architectures`.`id`=`architecture_compatibilities`.`runs_on`'
      printf ' AND `architectures`.`name`=from_base64("%s")' \
        "${arch}"
      mysql_join_binary_packages_in_repositories_binary_packages
      printf ' WHERE `binary_packages`.`build_assignment`=`build_assignments`.`id`'
    printf ') AS `exists_built`,'
    printf 'EXISTS ('
      printf 'SELECT 1'
      printf ' FROM `toolchain_order` AS `late`'
      printf ' WHERE NOT EXISTS ('
        printf 'SELECT 1'
        printf ' FROM `toolchain_order` AS `early`'
        mysql_join_toolchain_order_package_sources 'early' 'e_ps'
        mysql_join_package_sources_build_assignments 'e_ps' 'e_ba'
        printf ' JOIN `considered_build_assignments_4` AS `e_cba`'
        printf ' ON `e_ba`.`id`=`e_cba`.`id`'
        printf ' WHERE `late`.`pkgbase`!=`early`.`pkgbase`'
        printf ' AND `late`.`number`>`early`.`number`'
      printf ')'
      printf ' AND `late`.`pkgbase`=`package_sources`.`pkgbase`'
    printf ') AS `exists_unblocked`'
    printf ' FROM `considered_build_assignments`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`'
    printf ' JOIN `toolchain_architectures`'
    printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`'
    mysql_join_build_assignments_package_sources
    printf ' ORDER BY'
    # does it exist toolchain-blocked?
    printf ' `exists_blocked`,'
    # does it exist built?
    printf '`exists_built`,'
    # does it exist not toolchain-blocked?
    printf '`exists_unblocked` DESC'
    printf ';\n'

    printf 'INSERT INTO `orders`(`order`)'
    printf ' SELECT'
    printf ' MIN(`ordered_considered_build_assignments`.`order`)'
    printf ' FROM `ordered_considered_build_assignments`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`id`=`ordered_considered_build_assignments`.`id`'
    printf ' JOIN `toolchain_architectures`'
    printf ' JOIN `architecture_compatibilities`'
    printf ' ON `architecture_compatibilities`.`fully_compatible`'
    printf ' AND ('
      printf '('
        printf '`architecture_compatibilities`.`runs_on`=`build_assignments`.`architecture`'
        printf ' AND `architecture_compatibilities`.`built_for`=`toolchain_architectures`.`id`'
      printf ') OR ('
        printf '`architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
        printf ' AND `architecture_compatibilities`.`runs_on`=`toolchain_architectures`.`id`'
      printf ')'
    printf ')'
    printf ' GROUP BY `toolchain_architectures`.`id`'
    printf ';\n'

    printf 'DELETE `considered_build_assignments`'
    printf ' FROM `toolchain_architectures`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`architecture`=`toolchain_architectures`.`id`'
    printf ' JOIN `considered_build_assignments`'
    printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`;\n'

    printf 'INSERT IGNORE INTO `considered_build_assignments` (`id`)'
    printf ' SELECT'
    printf ' `ordered_considered_build_assignments`.`id`'
    printf ' FROM `ordered_considered_build_assignments`'
    printf ' JOIN `orders`'
    printf ' ON `orders`.`order`=`ordered_considered_build_assignments`.`order`'
    printf ';\n'

    printf 'DROP TEMPORARY TABLE `ordered_considered_build_assignments`;\n'
    printf 'DROP TEMPORARY TABLE `orders`;\n'

    # delete all currently building packages from considered list
    printf 'DELETE `considered_build_assignments`'
    printf ' FROM `considered_build_assignments`'
    mysql_join_build_assignments_build_slaves 'considered_build_assignments'
    printf ';\n'

    printf 'SELECT `considered_build_assignments`.`id` FROM'
    printf ' `considered_build_assignments`'
    printf ' JOIN `build_assignments`'
    printf ' ON `build_assignments`.`id`=`considered_build_assignments`.`id`'
    mysql_join_build_assignments_package_sources
    mysql_join_build_assignments_binary_packages
    printf ' LEFT JOIN ('
      printf 'SELECT'
      printf ' `failed_builds`.`build_assignment`,'
      printf 'COUNT(DISTINCT `failed_builds`.`id`) AS `count`'
      printf ' FROM `failed_builds`'
      printf ' GROUP BY `failed_builds`.`build_assignment`'
    printf ') AS `latest_failed_builds`'
    printf ' ON `latest_failed_builds`.`build_assignment`=`build_assignments`.`id`'
    printf ' LEFT JOIN ('
      # Does this job wait for something else in `toolchain_order`?
      printf 'SELECT '
      printf '`tc_q`.`pkgbase`,'
      printf '`tc_q`.`architecture`,'
      printf 'IF('
        printf '`tc_q`.`is_blocked`,'
        printf 'IF('
          printf '`tc_q`.`is_built`,'
          printf '2,'
          printf '1'
        printf '),'
        printf '0'
      printf ') AS `priority`'
      printf ' FROM ('
        printf 'SELECT '
        printf '`late`.`pkgbase`,'
        printf '`binary_packages`.`architecture`,'
        printf 'MAX('
          printf 'IF('
            printf 'EXISTS ('
              printf 'SELECT 1'
              printf ' FROM `binary_packages` AS `built_bp`'
              mysql_join_binary_packages_binary_packages_in_repositories 'built_bp' 'built_bpir'
              mysql_join_binary_packages_in_repositories_repositories 'built_bpir' 'built_r'
              printf ' AND `built_r`.`is_on_master_mirror`'
              printf ' WHERE `built_bp`.`build_assignment`=`build_assignments`.`id`'
            printf '),'
            printf '1,'
            printf '0'
          printf ')'
        printf ') AS `is_built`,'
        printf 'MAX('
          printf 'IF('
            printf 'EXISTS ('
              printf 'SELECT 1'
              printf ' FROM `package_sources` AS `block_ps`'
              mysql_join_package_sources_build_assignments 'block_ps' 'block_ba'
              mysql_join_build_assignments_binary_packages 'block_ba' 'block_bp'
              mysql_join_binary_packages_binary_packages_in_repositories 'block_bp' 'block_bpir'
              printf ' AND `block_bpir`.`repository`=%s' \
                "${repository_ids__any_build_list}"
              mysql_join_package_sources_toolchain_order 'block_ps' 'early'
              printf ' WHERE `early`.`number`<`late`.`number`'
              printf ' AND `early`.`pkgbase`!=`late`.`pkgbase`'
              printf ' AND ('
                printf '`block_bp`.`architecture`=`binary_packages`.`architecture`'
                # shellcheck disable=SC2154
                printf ' OR `%s`.`architecture`=%s' \
                  'block_bp' "${architecture_ids__any}" \
                  'binary_packages' "${architecture_ids__any}"
              printf ')'
            printf '),'
            printf '1,'
            printf '0'
          printf ')'
        printf ') AS `is_blocked`'
        printf ' FROM `toolchain_order` AS `late`'
        mysql_join_toolchain_order_package_sources 'late'
        mysql_join_package_sources_build_assignments
        mysql_join_build_assignments_binary_packages
        mysql_join_binary_packages_binary_packages_in_repositories
        printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
          "${repository_ids__any_build_list}"
        printf ' GROUP BY CONCAT(`late`.`pkgbase`,"-",`binary_packages`.`architecture`)'
      printf ') AS `tc_q`'
    printf ') AS `toolchain_query`'
    printf ' ON `toolchain_query`.`pkgbase`=`package_sources`.`pkgbase`'
    printf ' AND `toolchain_query`.`architecture`=`binary_packages`.`architecture`'
    printf ' JOIN `architectures`'
    printf ' ON `architectures`.`name`=from_base64("%s")' \
      "${arch}"
    printf ' ORDER BY '
    if [ -n "${requested}" ]; then
      printf 'IF(`package_sources`.`pkgbase`=from_base64("%s"),1,0) DESC,' \
        "${requested}"
    fi
    # 0: in toolchain, should be built now
    # 1: in toolchain, should be built later
    # 2: not in toolchain
    printf 'IFNULL(`toolchain_query`.`priority`,2),'
    printf '`build_assignments`.`priority` DESC,'
    printf 'IFNULL(`latest_failed_builds`.`count`,0),'
    printf 'IF('
      printf '`architectures`.`id`=%s' \
        "$(mysql_determine_majority_build_slave_architecture_id)"
      printf ' OR `architectures`.`id`=`build_assignments`.`architecture`,'
      printf '1,0) DESC,'
    printf 'IF('
      mysql_query_is_part_of_loop '`build_assignments`.`id`'
    printf ',1,0),'
    printf '`package_sources`.`commit_time`,'
    printf '`build_assignments`.`id`'
    printf ' LIMIT 1;\n'
  } | \
    mysql_run_query
)
if [ -n "${next_building}" ]; then
  hand_out_assignment "${next_building}"
fi

# Check if there are any pending packages at all
count_pending=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT COUNT(1)'
    printf ' FROM `build_assignments`'
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    mysql_join_binary_packages_in_repositories_repositories
    printf ' JOIN `architecture_compatibilities`'
    printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`id`'
    printf ' AND `architecture_compatibilities`.`build_slave_compatible`'
    printf ' JOIN `architectures`'
    printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
    printf ' WHERE `repositories`.`name`="build-list"'
    printf ' AND `build_assignments`.`is_blocked` IS NULL'
    printf ' AND `architectures`.`name`=from_base64("%s")' \
      "${arch}"
    printf ';\n'
  } | \
    mysql_run_query
)

if [ "${count_pending}" -eq 0 ]; then
  >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages'
  exit 3
else
  >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
  exit 2
fi