path: root/bin/seed-build-list

#!/bin/sh

# seed the build list from differences between an x86_64 and our master mirror

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: conf/seed-ignore-packages should only make upstream packages being ignored,
# not all packages!

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'seed-build-list [options]:'
  >&2 echo '  seed the build list from'
  >&2 echo '    - differences between an x86_64 and our master mirror'
  >&2 echo '    - a list of packages which need to be rebuilt'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -a|--auto:'
  >&2 echo '    Automatically reschedule packages which have run-time'
  >&2 echo '    dependencies that are not available from staging and'
  >&2 echo '    which cannot be replaced by another less stable version'
  >&2 echo '    of the same package.'
  >&2 echo '  -c|--cron-exit:'
  >&2 echo '    Exit with 0 if not getting a lock - makes the script'
  >&2 echo '    suitable for a crontab.'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  >&2 echo '  -i|--ignore $package:'
  >&2 echo '    Do not update $package.'
  >&2 echo '  -m|--mirror $url:'
  >&2 echo '    Get x86_64 packages from mirror at $url.'
  >&2 echo '  -n|--no-action:'
  >&2 echo '    Do not actually update build-list, just print it.'
  >&2 echo '  -p|--package $pkg_regex:'
  >&2 echo '    Update packages matching $pkg_regex.'
  >&2 echo '  -u|--undelete $url: '
  >&2 echo '    Schedule all former deletion-list packages which do not'
  >&2 echo '    belong on the deletion-list anymore and which are'
  >&2 echo '    available on $url.'
  >&2 echo '  -w|--wait:'
  >&2 echo '    Wait for lock if necessary.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
# shellcheck disable=SC2064
trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT

eval set -- "$(
  getopt -o achi:m:np:u:w \
    --long auto \
    --long cron-exit \
    --long help \
    --long ignore: \
    --long mirror: \
    --long no-action \
    --long package: \
    --long undelete: \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
    echo usage
  )"

touch "${tmp_dir}/ignore-packages"
touch "${tmp_dir}/mirrors"
touch "${tmp_dir}/package-regexes"
touch "${tmp_dir}/undelete-mirrors"

auto=false
update=true
wait_for_lock='-n'
lock_error=1

if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  cp "${base_dir}/conf/seed-ignore-packages" "${tmp_dir}/ignore-packages"
fi

while true
do
  case "$1" in
    -a|--auto)
      auto=true
    ;;
    -c|--cron-exit)
      lock_error=0
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore)
      shift
      echo "$1" >> \
        "${tmp_dir}/ignore-packages"
    ;;
    -m|--mirror)
      shift
      echo "$1" >> \
        "${tmp_dir}/mirrors"
    ;;
    -n|--no-action)
      update=false
    ;;
    -p|--package)
      shift
      echo "$1" >> \
        "${tmp_dir}/package-regexes"
    ;;
    -u|--undelete)
      shift
      echo "$1" >> \
        "${tmp_dir}/undelete-mirrors"
    ;;
    -w|--wait)
      wait_for_lock=''
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  usage 1
fi

if [ ! -s "${tmp_dir}/mirrors" ] && \
  [ ! -s "${tmp_dir}/package-regexes" ] && \
  [ ! -s "${tmp_dir}/undelete-mirrors" ] && \
  ! ${auto}; then
  # nothing to do
  exit 0
fi

# get locks
if ${update}; then
  exec 9> "${sanity_check_lock_file}"
  if ! verbose_flock -s ${wait_for_lock} 9; then
    >&2 echo 'Cannot get sanity-check lock.'
    exit ${lock_error}
  fi

  exec 8> "${build_list_lock_file}"
  if ! verbose_flock ${wait_for_lock} 8; then
    >&2 echo 'Cannot get build-list lock.'
    exit ${lock_error}
  fi
fi

# shellcheck disable=SC2016
repos=$(
  {
    printf 'SELECT DISTINCT `repositories`.`name`'
    printf ' FROM `repositories`'
    mysql_join_repositories_repository_stabilities
    printf ' WHERE `repository_stabilities`.`name`="stable";\n'
  } | \
    mysql_run_query
  printf '%s\n' 'multilib'
)

# genereate must-haves query from mirror delta
if [ -s "${tmp_dir}/mirrors" ]; then
  {
    # theirs
    while read -r mirror; do
      if [ -z "${mirror}" ]; then
        continue
      fi
      for repo in ${repos}; do
        curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
          tar -Oxz --wildcards '*/desc' | \
          sed '
            /^%FILENAME%$/!d
            N
            s/^.*\n//
            s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
            s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
          '
      done
    done < \
      "${tmp_dir}/mirrors"
    # ours
    # shellcheck disable=SC2016
    {
      printf 'SELECT '
      mysql_package_name_query
      printf ' FROM `binary_packages`'
      mysql_join_binary_packages_architectures
      mysql_join_binary_packages_binary_packages_in_repositories
      printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
    } | \
      mysql_run_query | \
      sed '
        s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/ours \2 \3 \1/
      '
  } | \
    expand_version 2 | \
    sort -k3,4 -k2Vr,2 -k1,1 | \
    shrink_version 2 | \
    uniq -f2 | \
    sed -n '
      s/^theirs \(\S\+ \)\{2\}//
      T
      p
    ' | \
    sort -u > \
    "${tmp_dir}/must-haves"
fi

if [ -s "${tmp_dir}/undelete-mirrors" ]; then
  while read -r mirror; do
    if [ -z "${mirror}" ]; then
      continue
    fi
    for repo in ${repos}; do
      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
        tar -Oxz --wildcards '*/desc' | \
        sed '
          /^%FILENAME%$/!d
          N
          s/^.*\n//
          s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
          s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/
          /^lib32-/d
        '
    done
  done < \
    "${tmp_dir}/undelete-mirrors" | \
    sort -u > \
    "${tmp_dir}/available"
fi

# shellcheck disable=SC2016
{
  printf 'CREATE TEMPORARY TABLE `ignore_packages` (`pkgname` VARCHAR(64));\n'
  if [ -s "${tmp_dir}/ignore-packages" ]; then
    grep -vxF '' "${tmp_dir}/ignore-packages" | \
      base64_encode_each | \
      sed '
        s/^/(from_base64("/
        s/$/")),/
        $s/,$/;/
        1 s/^/INSERT INTO `ignore_packages` (`pkgname`) VALUES \n/
      '
  fi
  # packages on the build-list or deletion-list should be ignored
  printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)'
  printf ' SELECT DISTINCT `ignore_bin`.`pkgname`'
  printf ' FROM `binary_packages` AS `ignore_bin`'
  mysql_join_binary_packages_binary_packages_in_repositories 'ignore_bin'

  printf ' WHERE `binary_packages_in_repositories`.`repository` IN (%s,%s)' \
    "${repository_ids__any_build_list}" \
    "${repository_ids__any_deletion_list}"
  # packages with no not-to-be-deleted and at least on to-be-deleted version should be ignored
  printf ' OR ('
  printf '`binary_packages_in_repositories`.`is_to_be_deleted`'
  printf ' AND NOT EXISTS ('
    printf 'SELECT 1 FROM `binary_packages` AS `other_bin`'
    mysql_join_binary_packages_binary_packages_in_repositories 'other_bin' 'other_bpir'
    printf ' WHERE NOT `other_bpir`.`is_to_be_deleted`'
    printf ' AND `other_bin`.`pkgname`=`ignore_bin`.`pkgname`'
  printf '));\n'
} | \
  sponge "${tmp_dir}/ignore-packages"

# shellcheck disable=SC2016
{
  printf 'CREATE TEMPORARY TABLE `must_haves` (`pkgname` VARCHAR(64));\n'
  if [ -s "${tmp_dir}/must-haves" ]; then
    grep -vxF '' "${tmp_dir}/must-haves" | \
      base64_encode_each | \
      sed '
        s/^/(from_base64("/
        s/$/")),/
        $s/,$/;/
        1 s/^/INSERT INTO `must_haves` (`pkgname`) VALUES \n/
      '
  fi
} | \
  sponge "${tmp_dir}/must-haves"

# fetch unknown must-haves from upstream
# shellcheck disable=SC2016
printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` VARCHAR(64));\n' > \
  "${tmp_dir}/pkgbases"
# shellcheck disable=SC2016
{
  cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages"
  printf 'SELECT `must_haves`.`pkgname` FROM `must_haves`'
  printf ' WHERE NOT EXISTS ('
    printf 'SELECT 1 FROM `binary_packages`'
    printf ' WHERE `binary_packages`.`pkgname`=`must_haves`.`pkgname`'
  printf ') AND NOT EXISTS ('
    printf 'SELECT 1 FROM `ignore_packages`'
    printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`'
  printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n'
} | \
  mysql_run_query | \
  while read -r pkgname; do
    content=$(
      curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
        tr ',' '\n'
    )
    repo=$(
      printf '%s\n' "${content}" | \
        sed -n '
          s/^\s*"repo"\s*:\s*"//
          T
          s/".*$//
          T
          p
        '
    )
    pkgbase=$(
      printf '%s\n' "${content}" | \
        sed -n '
          s/^\s*"pkgbase"\s*:\s*"//
          T
          s/".*$//
          T
          p
        '
    )
    if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
      >&2 printf 'Could not find "%s" which is newer on x86_64!?\n' "${pkgname}"
      exit 2
    fi
    printf '(from_base64("%s"),from_base64("%s")),\n' \
      "$(printf '%s' "${pkgbase}" | base64 -w0)" \
      "$(printf '%s' "${repo}" | base64 -w0)"
  done | \
  sort -u | \
  sed '
    1 s/^/INSERT IGNORE INTO `pkgbases` (`pkgbase`,`repository`) VALUES \n/
    $s/,$/;/
  ' >> \
  "${tmp_dir}/pkgbases"

# shellcheck disable=SC2016
{
  cat "${tmp_dir}/must-haves" "${tmp_dir}/ignore-packages" "${tmp_dir}/pkgbases"
  if ${auto} || \
    [ -s "${tmp_dir}/available" ]; then
    printf 'CREATE TEMPORARY TABLE `bin_ids` (`id` BIGINT, UNIQUE KEY (`id`));\n'
  fi
  if ${auto}; then
    printf 'INSERT IGNORE INTO `bin_ids`'
    printf ' SELECT `binary_packages`.`id` FROM `binary_packages`'
    mysql_join_binary_packages_binary_packages_in_repositories
    mysql_join_binary_packages_in_repositories_repositories
    printf ' AND `repositories`.`is_on_master_mirror`'
    printf ' AND `repositories`.`name`!="build-support"'
    mysql_join_binary_packages_dependencies
    mysql_join_dependencies_dependency_types
    printf ' AND `dependency_types`.`relevant_for_binary_packages`'
    # nothing "less stable" has the same name
    printf ' WHERE NOT EXISTS ('
      printf 'SELECT 1 FROM `binary_packages` AS `subst_bp`'
      mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bir'
      mysql_join_binary_packages_in_repositories_repositories 'subst_bir' 'subst_r'
      printf ' AND `subst_r`.`name`!="build-support"'
      printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`stability`'
      printf ' WHERE `repository_stability_relations`.`more_stable`=`repositories`.`stability`'
      printf ' AND `subst_bp`.`id`!=`binary_packages`.`id`'
      printf ' AND `subst_bp`.`pkgname`=`binary_packages`.`pkgname`'
    printf ') AND NOT EXISTS ('
      printf 'SELECT 1 FROM `install_target_providers`'
      mysql_join_install_target_providers_binary_packages '' 'subst_bp'
      mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bir'
      mysql_join_binary_packages_in_repositories_repositories 'subst_bir' 'subst_r'
      printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")'
      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
      # this is the least stable, built install_target_provider with that name
      printf ' AND NOT EXISTS ('
        printf 'SELECT 1 FROM `binary_packages` AS `ss_bp`'
        mysql_join_binary_packages_binary_packages_in_repositories 'ss_bp' 'ss_bir'
        mysql_join_binary_packages_in_repositories_repositories 'ss_bir' 'ss_r'
        printf ' AND `ss_r`.`name` NOT IN ("build-list","build-support","deletion-list","to-be-decided")'
        printf ' JOIN `repository_stability_relations` AS `ss_rsr`'
        printf ' ON `ss_rsr`.`less_stable`=`ss_r`.`stability`'
        printf ' WHERE `ss_bp`.`pkgname`=`subst_bp`.`pkgname`'
        printf ' AND `ss_bp`.`id`!=`subst_bp`.`id`'
        printf ' AND `ss_rsr`.`more_stable`=`subst_r`.`stability`'
      printf ')'
    printf ')'
    printf ';\n'
  fi
  if [ -s "${tmp_dir}/available" ]; then
    printf 'CREATE TEMPORARY TABLE `available` (`pkgname` VARCHAR(64), UNIQUE KEY (`pkgname`));\n'
    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `available`;\n' \
      "${tmp_dir}/available"

    printf 'INSERT IGNORE INTO `bin_ids`'
    printf ' SELECT `binary_packages`.`id` FROM `binary_packages`'
    mysql_join_binary_packages_binary_packages_in_repositories
    printf ' AND `binary_packages_in_repositories`.`repository`=%s' \
      "${repository_ids__any_deletion_list}"
    printf ' JOIN `available` ON `available`.`pkgname`=`binary_packages`.`pkgname`'
    mysql_join_binary_packages_build_assignments
    printf ' WHERE `binary_packages`.`pkgname` NOT LIKE "lib32-%%"'
    printf ' AND `build_assignments`.`is_black_listed` IS NULL'
    printf ' AND NOT EXISTS ('
      printf 'SELECT * FROM `dependencies`'
      mysql_join_dependencies_dependency_types
      printf ' AND ('
        printf '`dependency_types`.`relevant_for_building`'
        printf ' OR `dependency_types`.`relevant_for_binary_packages`'
      printf ')'
      mysql_join_dependencies_install_target_providers '' 'it_dummy'
      printf ' WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
      printf ' AND NOT EXISTS ('
        printf 'SELECT 1 FROM `install_target_providers`'
        mysql_join_install_target_providers_binary_packages '' 'it_bp'
        mysql_join_binary_packages_binary_packages_in_repositories 'it_bp' 'it_bpir'
        printf ' WHERE `it_bp`.`build_assignment`!=`binary_packages`.`build_assignment`'
        printf ' AND NOT `it_bpir`.`is_to_be_deleted`'
        printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
      printf ')'
    printf ');\n'

    printf 'DELETE `ignore_packages` FROM `ignore_packages`'
    printf ' JOIN `available` ON `available`.`pkgname`=`ignore_packages`.`pkgname`;\n'
  fi
  printf 'SELECT '
  printf '`pkgbases`.`pkgbase`,'
  printf '`git_repositories`.`head`,'
  printf '('
    printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
    printf ' WHERE `al32`.`name`="archlinux32"'
  printf '),'
  printf '`pkgbases`.`repository`'
  printf ' FROM `pkgbases`'
  printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`'
  mysql_join_upstream_repositories_git_repositories
  printf ' WHERE NOT EXISTS ('
    printf 'SELECT 1 FROM `ignore_packages`'
    printf ' JOIN `binary_packages` AS `i_bp` ON `ignore_packages`.`pkgname`=`i_bp`.`pkgname`'
    mysql_join_binary_packages_build_assignments 'i_bp' 'i_ba'
    mysql_join_build_assignments_package_sources 'i_ba' 'i_ps'
    printf ' WHERE `i_ps`.`pkgbase`=`pkgbases`.`pkgbase`'
  printf ');\n'
  printf 'SELECT '
  printf '`package_sources`.`pkgbase`,'
  printf '`git_repositories`.`head`,'
  printf '('
    printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
    printf ' WHERE `al32`.`name`="archlinux32"'
  printf '),'
  printf '`upstream_repositories`.`name`'
  printf ' FROM `binary_packages`'
  mysql_join_binary_packages_build_assignments
  mysql_join_build_assignments_package_sources
  mysql_join_package_sources_upstream_repositories
  mysql_join_upstream_repositories_git_repositories
  printf ' WHERE ('
  if [ -s "${tmp_dir}/package-regexes" ]; then
    grep -vxF '' "${tmp_dir}/package-regexes" | \
      base64_encode_each | \
      sed '
        s/^/`binary_packages`.`pkgname` REGEXP from_base64("/
        s/$/") OR /
      ' | \
      tr -d '\n'
  fi
  if ${auto} || \
    [ -s "${tmp_dir}/available" ]; then
    printf 'EXISTS ('
      printf 'SELECT 1 FROM `bin_ids`'
      printf ' WHERE `bin_ids`.`id`=`binary_packages`.`id`'
    printf ')'
    printf ' OR '
  fi
  printf 'EXISTS ('
    printf 'SELECT 1 FROM `must_haves`'
    printf ' WHERE `must_haves`.`pkgname`=`binary_packages`.`pkgname`'
  printf ')) AND NOT EXISTS ('
    printf 'SELECT 1 FROM `ignore_packages`'
    printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`'
  printf ');\n'
} | \
  mysql_run_query | \
  sort -u | \
  tr '\t' ' ' | \
  if ${update}; then
    while read -r pkgbase git_rev mod_git_rev repo; do
      printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
        sed 's/ $/\n/'
      mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${git_rev}" '' "${mod_git_rev}" "${repo}"
    done

    rm -f "${build_list_lock_file}"
  else
    cat
  fi