path: root/bin/get-assignment

#!/bin/sh

# receive one package to be built from the build-list whose dependencies
# are already satisfied or which breaks a dependency cycle
# accepts a comma separated list of prefered packages as the first argument

# exit code shows state of success:
#  0: ok, I gave you an assignment
#  1: come back (shortly) later - I was running already
#  2: come back later - there are still packages to be built,
#     but currently none has all its dependencies ready
#  3: come back after the next run of get-package-updates - currently
#     there are no pending packages

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# hand_out_assignment $build_assignments.id
hand_out_assignment() {

  # shellcheck disable=SC2016
  {
    printf 'SELECT '
    printf '`package_sources`.`%s`,' \
      'pkgbase' 'git_revision' 'mod_git_revision'
    printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`'
    printf ' FROM `upstream_repositories`'
    mysql_join_upstream_repositories_package_sources
    mysql_join_package_sources_build_assignments
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' WHERE `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ' AND `build_assignments`.`id`=%s' \
      "$1"
    printf ' LIMIT 1;\n'

    printf 'SELECT '
    mysql_package_name_query
    printf ' FROM `binary_packages`'
    mysql_join_binary_packages_architectures
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' WHERE `binary_packages`.`build_assignment`=%s' \
      "$1"
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ';\n'

    printf 'UPDATE `build_slaves`'
    printf ' SET `build_slaves`.`currently_building`=%s,' \
      "$1"
    printf ' `build_slaves`.`is_sane`=1'
    # shellcheck disable=SC2154
    printf ' WHERE `build_slaves`.`id`=from_base64("%s");\n' \
      "$(printf '%s' "${slave_id}" | base64 -w0)"

    printf 'UPDATE `build_assignments`'
    printf ' SET `build_assignments`.`priority`=0'
    printf ' WHERE `build_assignments`.`id`=%s;\n' \
      "$1"
  } | \
    mysql_run_query | \
    tr '\t' ' '

  exit 0

}

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! verbose_flock -n 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${sanity_check_lock_file}"
if ! verbose_flock -s -n 8; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit 1
fi

clean_up() {
  rm -f "${build_list_lock_file}"
  rm -rf --one-file-system "${tmp_dir}"
}

tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
trap clean_up EXIT

arch=$(
  printf '%s' "$1" | \
    base64 -w0
)
shift

# if we're building something already, hand it out (again)
currently_building=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT `build_assignments`.`id`'
    printf ' FROM `build_assignments`'
    mysql_join_build_assignments_build_slaves
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' JOIN `architecture_compatibilities`'
    printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
    printf ' JOIN `architectures`'
    printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
    printf ' WHERE `build_slaves`.`id`=from_base64("%s")' \
      "$(printf '%s' "${slave_id}" | base64 -w0)"
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_build_list}"
    printf ' AND `architectures`.`name` = from_base64("%s")' \
      "${arch}"
    printf ' LIMIT 1;\n'
  } | \
    mysql_run_query
)

if [ -n "${currently_building}" ]; then
  hand_out_assignment "${currently_building}"
fi

# TODO: currently, toolchain packages ARE HANDED OUT IN PARALLEL - THIS SHOULD NOT BE!

# a package with [all dependencies met or which is part of a loop]
# and which is currently not being built, ordered by:
# 1: we requested it
# 2: architecture matches
# 3: its priority
# 4: is not yet built
# 5: was built the longest time ago
next_building=$(
  # shellcheck disable=SC2016
  {
    mysql_query_create_toolchain_order
    printf 'SELECT `q`.`ba_id` FROM'
    printf '('
      printf 'SELECT '
      printf '`package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
        "$(
          printf '%s' "$1" | \
            base64 -w0
        )"
      printf 'IFNULL(`toolchain_query`.`priority`,3) AS `toolchain_priority`,'
      # 0: in toolchain, never blocked
      # 1: in toolchain, unblocked exists, not-yet-built
      # 2: in toolchain, unblocked exists, already built
      # (should normally become "0", before being handed out)
      # 3: not in toolchain or only blocked in toolchain
      printf '`build_assignments`.`priority`,'
      printf 'COALESCE('
      printf 'MAX(`failed_builds`.`date`),0'
      printf ') AS `last_trial`,'
      mysql_query_is_part_of_loop '`build_assignments`.`id`'
      printf ' AS `part_of_loop`,'
      printf '`build_assignments`.`id` AS `ba_id`'
      printf ' FROM `build_assignments`'
      mysql_join_build_assignments_package_sources
      mysql_join_build_assignments_binary_packages
      mysql_join_binary_packages_binary_packages_in_repositories
      printf ' JOIN `architecture_compatibilities`'
      printf ' ON `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
      printf ' AND `architecture_compatibilities`.`fully_compatible`'
      printf ' JOIN `architectures`'
      printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
      printf ' LEFT'
      mysql_join_build_assignments_failed_builds
      printf ' LEFT JOIN ('
        # Does this job wait for something else in `toolchain_order`?
        printf 'SELECT '
        printf '`tc_q`.`pkgbase`,'
        printf '`tc_q`.`architecture`,'
        printf 'IF('
          printf '`tc_q`.`is_blocked`,'
          printf 'IF('
            printf '`tc_q`.`is_built`,'
            printf '2,'
            printf '1'
          printf '),'
          printf '0'
        printf ') AS `priority`'
        printf ' FROM ('
          printf 'SELECT '
          printf '`late`.`pkgbase`,'
          printf '`binary_packages`.`architecture`,'
          printf 'MAX('
            printf 'IF('
              printf 'EXISTS ('
                printf 'SELECT 1'
                printf ' FROM `binary_packages` AS `built_bp`'
                mysql_join_binary_packages_binary_packages_in_repositories 'built_bp' 'built_bpir'
                mysql_join_binary_packages_in_repositories_repositories 'built_bpir' 'built_r'
                printf ' AND `built_r`.`is_on_master_mirror`'
                printf ' WHERE `built_bp`.`build_assignment`=`build_assignments`.`id`'
              printf '),'
              printf '1,'
              printf '0'
            printf ')'
          printf ') AS `is_built`,'
          printf 'MAX('
            printf 'IF('
              printf 'EXISTS ('
                printf 'SELECT 1'
                printf ' FROM `package_sources` AS `block_ps`'
                mysql_join_package_sources_build_assignments 'block_ps' 'block_ba'
                mysql_join_build_assignments_binary_packages 'block_ba' 'block_bp'
                mysql_join_binary_packages_binary_packages_in_repositories 'block_bp' 'block_bpir'
                printf ' AND `block_bpir`.`repository`=%s' \
                  "${repository_ids__any_build_list}"
                printf ' JOIN `toolchain_order_copy` AS `early`'
                printf ' ON `early`.`pkgbase`=`block_ps`.`pkgbase`'
                printf ' WHERE `early`.`number`<`late`.`number`'
                printf ' AND `early`.`pkgbase`!=`late`.`pkgbase`'
                printf ' AND ('
                  printf '`block_bp`.`architecture`=`binary_packages`.`architecture`'
                  # shellcheck disable=SC2154
                  printf ' OR `%s`.`architecture`=%s' \
                    'block_bp' "${architecture_ids__any}" \
                    'binary_packages' "${architecture_ids__any}"
                printf ')'
              printf '),'
              printf '1,'
              printf '0'
            printf ')'
          printf ') AS `is_blocked`'
          printf ' FROM `toolchain_order` AS `late`'
          printf ' JOIN `package_sources`'
          printf ' ON `package_sources`.`pkgbase`=`late`.`pkgbase`'
          mysql_join_package_sources_build_assignments
          mysql_join_build_assignments_binary_packages
          mysql_join_binary_packages_binary_packages_in_repositories
          printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
            "${repository_ids__any_build_list}"
          printf ' GROUP BY CONCAT(`late`.`pkgbase`,"-",`binary_packages`.`architecture`)'
        printf ') AS `tc_q`'
      printf ') AS `toolchain_query`'
      printf ' ON `toolchain_query`.`pkgbase`=`package_sources`.`pkgbase`'
      printf ' AND `toolchain_query`.`architecture`=`binary_packages`.`architecture`'
      printf ' WHERE NOT EXISTS ('
        printf ' SELECT 1'
        printf ' FROM `build_slaves`'
        printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
      printf ') AND ('
        printf '`build_assignments`.`is_blocked` IS NULL'
        printf ' OR'
        printf ' `package_sources`.`pkgbase`=from_base64("%s")' \
          "$(
            printf '%s' "$1" | \
              base64 -w0
          )"
      printf ') AND ('
        mysql_query_is_part_of_loop '`build_assignments`.`id`'
        printf ' OR NOT '
        mysql_query_has_pending_dependencies '`architectures`.`name`' '`build_assignments`.`id`'
      printf ')'
      printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
        "${repository_ids__any_build_list}"
      printf ' AND `architectures`.`name` = from_base64("%s")' \
        "${arch}"
      printf ' GROUP BY `build_assignments`.`id`'
      printf ' ORDER BY'
      printf ' `requested` DESC,'
      printf ' `toolchain_priority`,'
      printf ' `priority` DESC,'
      printf ' `last_trial`,'
      printf ' `architectures`.`id`=`build_assignments`.`architecture` DESC,'
      printf ' `part_of_loop`,'
      printf ' `package_sources`.`commit_time`,'
      printf ' `build_assignments`.`id`'
      printf ' LIMIT 1'
    printf ') AS `q`;\n'
  } | \
    mysql_run_query
)
if [ -n "${next_building}" ]; then
  hand_out_assignment "${next_building}"
fi

# Check if there are any pending packages at all
count_pending=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT COUNT(1)'
    printf ' FROM `build_assignments`'
    mysql_join_build_assignments_binary_packages
    mysql_join_binary_packages_binary_packages_in_repositories
    mysql_join_binary_packages_in_repositories_repositories
    printf ' WHERE `repositories`.`name`="build-list"'
    printf ' AND `build_assignments`.`is_blocked` IS NULL'
    printf ';\n'
  } | \
    mysql_run_query
)

if [ "${count_pending}" -eq 0 ]; then
  >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages'
  exit 3
else
  >&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
  exit 2
fi