path: root/bin/get-assignment

#!/bin/sh

# receive one package to be built from the build-list whose dependencies
# are already satisfied or which breaks a dependency cycle
# accepts a comma separated list of prefered packages as the first argument

# exit code shows state of success:
#  0: ok, I gave you an assignment
#  1: come back (shortly) later - I was running already
#  2: come back later - there are still packages to be built,
#     but currently none has all its dependencies ready
#  3: come back after the next run of get-package-updates - currently
#     there are no pending packages

# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

# TODO: honor manual build order of tool-chain:
# toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc

mkdir -p "${work_dir}/package-states"

hand_out_assignment() {

  # find out the sub_pkgrel
  sub_pkgrel=$(
    next_sub_pkgrel "$1" "$2" "$3" "$4"
  )

  # we don't care anymore if an older version of this package was
  # "locked" or "broken"
  find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
    -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \
    -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \
    -delete

  # move that build order to the end of the build-list
  sed -i '
    /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
      $ b
      d
    }
    $ a '"$1 $2 $3 $4" \
    "${work_dir}/build-list"

  echo "$1 $2 $3 $4 ${sub_pkgrel}"
  {
    # shellcheck disable=SC2154
    echo "${slave}"
    if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
      cat "${work_dir}/package-states/$1.$2.$3.$4.locked"
    fi
  } | \
    sort -u | \
    sponge "${work_dir}/package-states/$1.$2.$3.$4.locked"
  # shellcheck disable=SC2016
  {
    printf 'UPDATE `build_slaves`'
    printf ' SET `currently_building` = ('
    printf ' SELECT `build_assignments`.`id`'
    printf ' FROM `build_assignments`'
    printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`'
    printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`'
    printf ' WHERE'
    printf ' `package_sources`.`%s` = from_base64("%s") AND' \
      'pkgbase'          "$(printf '%s' "$1" | base64 -w0)" \
      'git_revision'     "$(printf '%s' "$2" | base64 -w0)" \
      'mod_git_revision' "$(printf '%s' "$3" | base64 -w0)"
    printf ' `upstream_repositories`.`name` = from_base64("%s")' \
      "$(printf '%s' "$4" | base64 -w0)"
    printf ')'
    printf ' WHERE `build_slaves`.`name`=from_base64("%s");' \
      "$(printf '%s' "${slave}" | base64 -w0)"
  } | \
    ${mysql_command}

  # lock every loop this package breaks
  find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
    -regex '.*/loop_[0-9]\+' \
    -exec grep -qxF "$1" '{}' \; \
    -exec touch '{}.locked' \;

  exit 0

}

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! flock -n 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${sanity_check_lock_file}"
if ! flock -s -n 8; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit 1
fi

clean_up() {
  rm -f "${build_list_lock_file}"
  rm -rf --one-file-system "${tmp_dir}"
}

tmp_dir=$(mktemp -d 'tmp.get-assignment.XXXXXXXXXX' --tmpdir)
trap clean_up EXIT

# if we're building something already, hand it out (again)
currently_building=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT'
    printf ' `package_sources`.`%s`,' \
      'pkgbase' \
      'git_revision' \
      'mod_git_revision'
    printf '`upstream_repositories`.`name`'
    printf ' FROM `build_slaves`'
    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
      'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \
      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository'
    printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \
      "$(printf '%s' "${slave}" | base64 -w0)"
    printf ' LIMIT 1;\n'
  } | \
    ${mysql_command} --batch --raw | \
    sed '1d'
)

if [ -n "${currently_building}" ]; then
  # shellcheck disable=SC2086
  hand_out_assignment ${currently_building}
fi

# packages with all dependencies met, ordered by number of failed trials
next_building=$(
  # shellcheck disable=SC2016
  {
    printf 'SELECT'
    printf ' `package_sources`.`pkgbase`=from_base64("%s") AS `requested`,' \
      "$(
        printf '%s' "$1" | \
          base64 -w0
      )"
    printf '('
      printf 'SELECT COUNT(*)'
      printf ' FROM `failed_builds`'
      printf ' WHERE `failed_builds`.`build_assignment`=`build_assignments`.`id`'
    printf ') AS `trials`,'
    printf '`package_sources`.`%s`,' \
      'pkgbase' \
      'git_revision' \
      'mod_git_revision'
    printf '`upstream_repositories`.`name`'
    printf ' FROM `binary_packages`'
    printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
      'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \
      'package_sources' 'package_sources' 'build_assignments' 'package_source' \
      'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \
      'repositories' 'repositories' 'binary_packages' 'repository'
    printf ' WHERE `repositories`.`name`="build-list"'
    printf ' AND NOT EXISTS ('
      printf ' SELECT *'
      printf ' FROM `build_slaves`'
      printf ' WHERE `build_slaves`.`currently_building`=`build_assignments`.`id`'
    printf ') AND ('
      printf '`build_assignments`.`is_blocked` IS NULL'
      printf ' OR'
      printf ' `package_sources`.`pkgbase`=from_base64("%s")' \
        "$(
          printf '%s' "$1" | \
            base64 -w0
        )"
    printf ') AND ('
      printf 'EXISTS( '
        printf 'SELECT * FROM `build_dependency_loops`'
        printf ' WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`'
# TODO: ignore loops which contain a build_assignment currently being handed out (?)
      printf ') OR'
      printf ' NOT EXISTS ('
        printf 'SELECT * FROM `binary_packages` as `to_dos`'
        printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \
          'repositories` AS `to_do_repos' 'to_do_repos'  'id'           'to_dos'                   'repository' \
          'dependencies'                  'to_dos'       'id'           'dependencies'             'dependent' \
          'install_target_providers'      'dependencies' 'depending_on' 'install_target_providers' 'install_target' \
          'binary_packages` AS `bin_deps' 'bin_deps'     'id'           'install_target_providers' 'package' \
          'repositories` AS `dep_repos'   'dep_repos'    'id'           'bin_deps'                 'repository'
        printf ' WHERE'
        printf ' `%s`.`name`="build-list" AND' \
          'dep_repos' 'to_do_repos'
        printf ' `bin_deps`.`build_assignment`!=`to_dos`.`build_assignment` AND'
        printf ' `to_dos`.`build_assignment`=`build_assignments`.`id`'
      printf ')'
    printf ')'
    printf ' ORDER BY `requested` DESC, `trials`, `build_assignments`.`id`'
    printf ' LIMIT 1;\n'
  } | \
    ${mysql_command} --batch --raw | \
    sed '
      1d
      y/\t/ /
      s/^\S\+ \S\+ //
    '
)
# not ready yet!
if [ -n "${next_building}" ]; then
  # shellcheck disable=SC2086
  echo hand_out_assignment ${next_building} >&2
fi

# Check if there are any pending packages at all and if the requester
# has already got an assignment.

pending_packages=false

while read -r package git_revision mod_git_revision repository; do

  generate_package_metadata "${package}.${git_revision}.${mod_git_revision}.${repository}"

  if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
    # has this slave already got or does he prefer this assignment?
    # note, that the dependencies are met, because this package is already locked
    if grep -qxF "${slave}" "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" || \
      printf ',%s,' "$1" | \
        grep -qF ",${package},"; then
      hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
    fi
  elif [ ! -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
    pending_packages=true
  fi

done < "${work_dir}/build-list"

if ! ${pending_packages}; then
  >&2 echo 'come back after the next run of get-package-updates - currently there are no pending packages'
  exit 3
fi

# Find first package of build-list whose "dependencies" are all met
# 1st: prefered packages on the build list which have all dependencies met
# 2nd: unbroken packages on the build list which have all dependencies met
# 3rd: unbroken packages breaking a loop
# 4th: broken packages which have all dependencies met or break a loop

for iteration in 'prefered' 'fresh' 'loops' 'broken'; do

  case "${iteration}" in
    'prefered')
      hand_out_blocked=true
      hand_out_broken=true
      hand_out_loop=false
      echo "$1" | \
        tr ',' '\n' | \
        sort -u > \
        "${tmp_dir}/hand-out-only-these-packages"
    ;;
    'fresh')
      hand_out_blocked=false
      hand_out_broken=false
      hand_out_loop=false
      {
        cat "${work_dir}/build-list"
        find "${work_dir}/package-states" -name '*.broken' -printf '%f\n' | \
          sed '
            s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
            p
          '
      } | \
        sort -k1,1 -k2 | \
        uniq -u | \
        cut -d' ' -f1 | \
        uniq > \
        "${tmp_dir}/hand-out-only-these-packages"
    ;;
    'loops')
      hand_out_blocked=false
      hand_out_broken=false
      hand_out_loop=true
      find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
        -regex '.*/loop_[0-9]\+' \
        -exec cat {} \; | \
        sort -u > \
        "${tmp_dir}/hand-out-only-these-packages"
    ;;
    'broken')
      hand_out_blocked=false
      hand_out_broken=true
      hand_out_loop=true
      cut -d' ' -f1 < \
        "${work_dir}/build-list" | \
        sort -u > \
        "${tmp_dir}/hand-out-only-these-packages"
    ;;
  esac

  cat -n "${work_dir}/build-list" | \
    sort -k2,2 | \
    join -1 1 -2 2 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/hand-out-only-these-packages" - | \
    sort -k1,1 | \
    sed 's|^\s*\S\+\s\+||' > \
    "${tmp_dir}/try-to-hand-out-these-packages"

  while read -r package git_revision mod_git_revision repository; do

    # package locked? (we have taken care of those in the previous loop)
    if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.locked" ]; then
      continue
    fi

    # package blocked?
    if ! ${hand_out_blocked} &&
      [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
      continue
    fi

    # package broken?
    if ! ${hand_out_broken} && \
      [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ]; then
      continue
    fi

    # dependencies met?
    if [ -n "$(find_dependencies_on_build_list "${package}" "${git_revision}" "${mod_git_revision}" "${repository}")" ]; then
      # do we hand out looped packages?
      if ! ${hand_out_loop}; then
        continue
      fi
      # is it a looped package?
      if ! find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \
        -regex '.*/loop_[0-9]\+' \
        -exec cat {} \; | \
        grep -qxF "${package}"; then
        continue
      fi
    fi

    hand_out_assignment "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"

  done < \
    "${tmp_dir}/try-to-hand-out-these-packages"

done

# Remove the lock file

>&2 echo 'come back later - there are still packages to be built, but currently none has all its dependencies ready'
exit 2