path: root/bin/get-package-updates

#!/bin/sh

# check for packages that need to be built

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: Find out, why sometimes package updates are missed.

# TODO: correctly handle if pkgbase of a split package is renamed, e.g.:
# $a -> ($a,$b)  ==>  $b -> ($a,$b)

# TODO: keep database clean in case of abort

# TODO: Being on the deletion-list is not a reason not to be scheduled
# if a dependency formerly on the deletion-list is now being built.

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'get-package-updates: check for packages that need to be built,'
  >&2 echo ' and build a list in the proper build order'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -d|--date $datetime:'
  >&2 echo '                Pull latest commit before $datetime'
  >&2 echo '                (yyyy-mm-ddThh:mm:ss). Conflicts -n.'
  >&2 echo '  -h|--help:    Show this help and exit.'
  >&2 echo '  -i|--ignore-insanity:'
  >&2 echo '    Do not abort when insane.'
  >&2 echo '  -n|--no-pull: Do not pull git repos, merely reorder build list.'
  >&2 echo '                Conflicts -d.'
  >&2 echo '  -w|--wait:    If necessary, wait for lock blocking.'
  >&2 echo '  -x|--test-exclusion $package:'
  >&2 echo '                Print additionally deleted/excluded packages if'
  >&2 echo '                "$package" would be black listed.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o d:hinwx: \
    --long date: \
    --long help \
    --long ignore-insanity \
    --long no-pull \
    --long test-exclusion: \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
date_time=''
ignore_insanity=false
pull=true
test_exclusion=''

while true
do
  case "$1" in
    -d|--date)
      shift
      date_time="$1"
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore-insanity)
      ignore_insanity=true
    ;;
    -n|--no-pull)
      pull=false
    ;;
    -w|--wait)
      block_flag=''
    ;;
    -x|--test-exclusion)
      shift
      if [ -n "${test_exclusion}" ]; then
        >&2 printf 'I already have --test-exclusion=%s and you gave me another one.\n' "${test_exclusion}"
        >&2 printf 'But I can only handle one exclusion test at a time.\n'
        exit 2
      fi
      test_exclusion="$1"
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -n "${date_time}" ] && ! ${pull}; then
  >&2 printf -- '-d and -n are mutually exclusive.\n'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  if ! ${ignore_insanity}; then
    exit
  fi
fi

# delete_package arch package
# mark $arch/$package for deletion
delete_package() {
  # TODO: Once we want to rely on the database for test_exclusion, we
  # need to run the command below unconditionally, but with some
  # changes, so we can easily revert.
  if [ -z "${test_exclusion}" ]; then
    # shellcheck disable=SC2016
    query_delete_packages=$(
      printf '`architectures` AS `d_a`'
      printf ' JOIN `architecture_compatibilities` AS `a_c`'
      printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
      printf ' AND `d_a`.`name`=from_base64("%s")' \
        "$(printf '%s' "$1" | base64 -w0)"
      printf ' JOIN `build_assignments`'
      printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
      # "any" references all architectures, but this is not represented
      # in `architecture_compatibilities`: If a package is not buildable
      # for "any", this means literally, that it is not buildable for
      # the _least_ architecture (e.g. it is not generic)
      printf ' OR `d_a`.`name`="any")'
      mysql_join_build_assignments_package_sources
      printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
        "$(printf '%s' "$2" | base64 -w0)"
      mysql_join_build_assignments_binary_packages
      mysql_join_binary_packages_binary_packages_in_repositories
    )
    # shellcheck disable=SC2016
    {
      # packages from the build-list/to-be-decided go straight to the deletion-list
      # this happens in two steps, because we need to create one item per
      # target architecture
      printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
        printf '`package`,'
        printf '`repository`,'
        printf '`last_moved`,'
        printf '`is_to_be_deleted`'
      printf ') SELECT'
      printf ' `binary_packages`.`id`,'
      printf '`d_r`.`id`,'
      printf 'NOW(),'
      printf '1'
      printf ' FROM'
      printf ' %s' "${query_delete_packages}"
      mysql_join_binary_packages_in_repositories_repositories
      # shellcheck disable=SC2154
      printf ' AND `repositories`.`stability` in (%s,%s)' \
        "${repository_stability_ids__unbuilt}" \
        "${repository_stability_ids__virtual}"
      mysql_join_build_assignments_architectures '' 'ba_a'
      printf ' JOIN `repositories` AS `d_r`'
      printf ' ON ('
        # arch-specific build_assignments must match exactly
        printf '`d_r`.`architecture`=`ba_a`.`id`'
        # "any" build_assignments build for all architectures
        printf ' OR `ba_a`.`name`="any"'
      printf ')'
      # shellcheck disable=SC2154
      printf ' AND `d_r`.`stability`=%s;\n' \
        "${repository_stability_ids__forbidden}"

      printf 'COMMIT;\n'

      printf 'DELETE `binary_packages_in_repositories`'
      printf ' FROM %s' "${query_delete_packages}"
      mysql_join_binary_packages_in_repositories_repositories
      # shellcheck disable=SC2154
      printf ' AND `repositories`.`stability` in (%s,%s);\n' \
        "${repository_stability_ids__unbuilt}" \
        "${repository_stability_ids__virtual}"

      printf 'COMMIT;\n'

      # other packages are marked as `is_to_be_deleted`
      printf 'UPDATE %s' "${query_delete_packages}"
      printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
    } | \
      mysql_run_query
  fi
}

something_new=false

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  # Update git repositories (official packages, community packages and the repository of package customizations).
  if [ -d "${repo_path}/.git" ]; then
    git -C "${repo_path}" remote update
  else
    git -C "${repo_path}" fetch origin master:master
  fi || \
    true
  # read previous git revision numbers from database.
  # shellcheck disable=SC2016
  eval "old_repo_revisions__${repo}='$(
    {
      printf 'SELECT `git_repositories`.`head` FROM `git_repositories`'
      printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
        "$(printf '%s' "${repo}" | base64 -w0)"
    } | \
      mysql_run_query
  )'"
  # determine new git revision
  if ${pull}; then
    if [ -z "${date_time}" ]; then
      eval "new_repo_revisions__${repo}='$(
        git -C "${repo_path}" rev-parse HEAD
      )'"
    else
      eval "new_repo_revisions__${repo}='$(
        git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD
      )'"
    fi
  else
    eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"'
  fi
  if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
    something_new=true
  fi
done

if ${pull} && \
  ! ${something_new}; then
  >&2 echo 'Nothing changed.'
  exit
fi

# Create a lock file for build list.

exec 9> "${build_list_lock_file}"
if ! verbose_flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit
fi

exec 8> "${sanity_check_lock_file}"
if ! verbose_flock -s ${block_flag} 8; then
  >&2 echo 'come back (shortly) later - sanity-check running.'
  exit
fi

cleanup() {
  mysql_cleanup
  rm -rf --one-file-system "${tmp_dir:?}"
}
tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)

trap cleanup EXIT

# shellcheck disable=SC2119
mysql_cleanup

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

trigger_rebuild_on_any_file_sed_expression=$(
  # note: This only works for upstream sources!
  # shellcheck disable=SC2016
  {
    printf 'SELECT DISTINCT CONCAT('
    printf '`package_sources`.`pkgbase`,"/repos/",'
    printf '`upstream_repositories`.`name`,"-",'
    printf 'IF(`architectures`.`name`="any","any","x86_64"))'
    printf ' FROM `binary_packages`'
    mysql_join_binary_packages_binary_packages_in_repositories
    mysql_join_binary_packages_build_assignments
    mysql_join_build_assignments_architectures
    mysql_join_build_assignments_package_sources
    mysql_join_package_sources_upstream_repositories
    printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
      "${repository_ids__any_build_list}"
  } | \
    mysql_run_query | \
    sed '
      s@.*@s,^\\(.\\t\0/\\)[^/]\\+,\\1PKGBUILD,@
    '
)

{
  # trigger rebuild of packages removed from blacklist
  # shellcheck disable=SC2016
  {
    printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`,`git_repositories`.`name`'
    printf ' FROM `package_sources`'
    mysql_join_package_sources_upstream_repositories
    mysql_join_upstream_repositories_git_repositories
    printf ' WHERE `package_sources`.`pkgbase` IN ('
    # shellcheck disable=SC2154
    git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \
      sed -n '
        s/^-\([^-]\)/\1/
        T
        s/\s*#.*$//
        p
      ' | \
      base64_encode_each | \
      sed '
        s/^/from_base64("/
        s/$/")/
      ' | \
      tr '\n' ','
    printf '"");\n'
  } | \
    mysql_run_query | \
    while read -r pkgbase repository git_repository; do
      printf 'A %s ' \
        "${pkgbase}"
      eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${git_repository}"'}"'
      printf ' %s\n' \
        "${repository}"
    done
  # actual updates/removes
  for repo in ${repo_names}; do
    eval repo_path='"${repo_paths__'"${repo}"'}"'
    eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
    eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"'
    # if old revision unknown, mimic "git diff"-output
    # shellcheck disable=SC2154
    if [ "${old_repo_revision}" = "NONE" ]; then
      git -C "${repo_path}" archive --format=tar HEAD | \
        tar -t | \
        sed 's|^|A\t|'
    else
      git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
    fi | \
      # rename any file to "PKGBUILD" for packages on the build-list
      sed "${trigger_rebuild_on_any_file_sed_expression}" | \
      # only track changes in PKGBUILDs
      grep '/PKGBUILD$' | \
      if [ "${repo}" = "archlinux32" ]; then
      # modify the directory structure from the modifiaction-repository
      # to the one of an original source repository
        # shellcheck disable=SC2016
        sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
          while read -r pkg_repo rest; do
            repo=$(find_git_repository_to_package_repository "${pkg_repo}")
            eval 'printf '"'"'%s %s %s\n'"'" \
              "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \
              "'${rest}'"
          done
      else
        sed "s|^|${new_repo_revision} |"
      fi | \
      grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
      # ignore i686
      grep -v -- '-i686/PKGBUILD$' | \
      # ignore staging and testing
      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
      sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|'
  done | \
    sort -u | \
    sed '
      s|^D\s|0 \0|
      t
      s|^[AM]\s|1 \0|
      t
      s|^|2 |
    ' | \
    sort -k1,1 | \
    sed 's|^[012] ||'
} | \
  while read -r mode package git_revision repository; do
    if [ "${mode}" = 'D' ]; then
      # deleted PKGBUILD
      # shellcheck disable=SC2154
      git_revision="${new_repo_revisions__archlinux32}"
      found_package=false
      for repository in ${repo_names}; do
        eval 'repo_path="${repo_paths__'"${repository}"'}"'
        if [ "${repository}" = "archlinux32" ]; then
          if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \
            tar -t 2> /dev/null | \
            grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then
            found_package=true
          fi
        else
          # shellcheck disable=SC2154
          if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \
            'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \
            'cut -d/ -f3 | ' \
            'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \
            'grep -vq -- '"'"'-i686$'"'"; then
            eval 'git_revision="${new_repo_revisions__'"${repository}"'}"'
            found_package=true
            break
          fi
        fi
      done
      if ${found_package}; then
# TODO: moving packages between community64 and packages64 is currently broken
# - what exactly is broken? I guess, we're setting the is_to_be_deleted
# flag but should not do so - is that right? Maybe we shuold incorporate
# some detection like http://mirror.archlinux32.org/irc-logs/%23archlinux32/2018-06-13.html#21:14:53
# I guess, this works now - 2018-09-01 deep42thought
        mode='M'
        repository=$(
          find_package_repository_to_package "${package}" "${repository}" "${git_revision}"
        )
      else
        delete_package 'any' "${package}"
        continue
      fi
    fi
    if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
      # shellcheck disable=SC2016
      {
        # delete old build assignment and associated binary packages
        # which are not yet built or on the deletion list
        printf 'DELETE `build_assignments`,`binary_packages`,`binary_packages_in_repositories`'
        printf ' FROM `binary_packages`'
        mysql_join_binary_packages_build_assignments
        mysql_join_build_assignments_package_sources
        mysql_join_binary_packages_binary_packages_in_repositories
        mysql_join_binary_packages_in_repositories_repositories
        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
          "$(
            printf '%s' "${package}" | \
              base64 -w0
          )"
        # shellcheck disable=SC2154
        printf ' AND `repositories`.`stability` IN (%s,%s);\n' \
          "${repository_stability_ids__unbuilt}" \
          "${repository_stability_ids__forbidden}"
        # remove is-to-be-deleted marker from old binary packages
        printf 'UPDATE `binary_packages_in_repositories`'
        mysql_join_binary_packages_in_repositories_binary_packages
        mysql_join_binary_packages_build_assignments
        mysql_join_build_assignments_package_sources
        printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0'
        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
          "$(
            printf '%s' "${package}" | \
              base64 -w0
          )"
      } | \
        mysql_run_query
      # shellcheck disable=SC2154
      printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2
      mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
      printf '\n' >&2
      continue
    fi

    >&2 echo "unknown git diff mode '${mode}'"
    exit 1
  done

echo 'apply blacklisting'
# ignore blacklisted packages and dependent packages
# this is the first time when all the information is available and up to date

{
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
    tar -Ox 'blacklist' | \
    sed '
      s/\s*#.*$//
      /^\S\+\s\+\S\+$/!d
    '
  if [ -n "${test_exclusion}" ]; then
    echo "${test_exclusion}"
  fi
  # shellcheck disable=SC2016
  {
    printf 'SELECT DISTINCT "any",`package_sources`.`pkgbase`'
    printf ' FROM `package_sources`'
    printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
  } | \
    mysql_run_query
} | \
  tr ' ' '\t' | \
  sort -u > \
  "${tmp_dir}/black-listed"

# shellcheck disable=SC2016
{
  printf 'CREATE TEMPORARY TABLE `blacklist` ('
  printf '`arch` VARCHAR(16),'
  printf '`pkgbase` VARCHAR(64),'
  printf 'UNIQUE KEY `content`(`arch`,`pkgbase`)'
  printf ');\n'
  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `blacklist` (`arch`,`pkgbase`);\n' \
    "${tmp_dir}/black-listed"
  printf 'CALL `blacklist_packages`;\n'
} | \
  mysql_run_query | \
  sort -u | \
  sponge "${tmp_dir}/black-listed"

deletion_list_count=$(
  wc -l < \
    "${tmp_dir}/black-listed"
)
if [ "${deletion_list_count}" -gt 1000 ]; then
  >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
    "${deletion_list_count}"
  if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
    # shellcheck disable=SC2119
    {
      printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
      for repo_name in ${repo_names}; do
        eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"'
        eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"'
        # shellcheck disable=SC2154
        if [ "${old_revision}" != "${new_revision}" ]; then
          printf '%s: %s -> %s\n' \
            "${repo_name}" \
            "${old_revision}" \
            "${new_revision}"
        fi
      done
    } | \
      tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
      irc_say
  fi
  exit 3
fi
rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"

while read -r arch package; do
  delete_package "${arch}" "${package}"
done < \
  "${tmp_dir}/black-listed"

if [ -n "${test_exclusion}" ]; then
  # TODO: reimplement test_exclusion with information from the database
  >&2 echo 'sry, not yet done.'
  exit 0
fi

echo 'Done - mark decisions as final.'

# shellcheck disable=SC2016
{
  # save blacklist into database
  printf 'CREATE TEMPORARY TABLE `blacklist` (`arch` VARCHAR(16), `pkgbase` VARCHAR(64), `reason` TEXT);\n'
  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
    tar -Ox 'blacklist' | \
    sed -n '
      s/^\(\S\+\)\s\+\(\S\+\)\s*#\s*/\1 \2 /
      T
      p
    ' | \
    while read -r arch pkgbase reason; do
      printf '(from_base64("%s"),from_base64("%s"),from_base64("%s")),\n' \
        "$(printf '%s' "${arch}" | base64 -w0)" \
        "$(printf '%s' "${pkgbase}" | base64 -w0)" \
        "$(printf '%s' "${reason}" | base64 -w0)"
    done | \
    sed '
      1 i INSERT INTO `blacklist` (`arch`,`pkgbase`,`reason`) VALUES
      $ s/,$/;/
    '
  printf 'UPDATE `build_assignments`'
  mysql_join_build_assignments_package_sources
  printf ' JOIN `architecture_compatibilities`'
  printf ' ON `build_assignments`.`architecture`=`architecture_compatibilities`.`built_for`'
  printf ' LEFT JOIN ('
    printf 'SELECT '
    printf '`blacklist`.`pkgbase`,'
    printf '`blacklist`.`reason`,'
    printf '`architectures`.`id` AS `arch`'
    printf ' FROM `blacklist`'
    printf ' JOIN `architectures`'
    printf ' ON `architectures`.`name`=`blacklist`.`arch`'
  printf ') AS `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`'
  printf ' AND ('
    printf '`architecture_compatibilities`.`runs_on`=`bl`.`arch`'
    # shellcheck disable=SC2154
    printf ' OR `bl`.`arch`=%s' \
      "${architecture_ids__any}"
  printf ') SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n'
  printf 'DROP TEMPORARY TABLE `blacklist`;\n'
  printf 'COMMIT;\n'

  # update hashes of repositories in mysql database
  for repo in ${repo_names}; do
    printf 'UPDATE `git_repositories`'
    printf ' SET `git_repositories`.`head`=from_base64("%s")' \
      "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')"
    printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
      "$(printf '%s' "${repo}" | base64 -w0)"
  done
  # move binary_packages from "to-be-decided" to "build-list"
  printf 'UPDATE `binary_packages_in_repositories`'
  mysql_join_binary_packages_in_repositories_binary_packages
  printf ' SET `binary_packages_in_repositories`.`repository`=%s' \
    "${repository_ids__any_build_list}"
  printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
    "${repository_ids__any_to_be_decided}"
} | \
  mysql_run_query

mysql_sort_versions

# update loop list in database (beware, the packages are expected to be in "build-list",
# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list".
mysql_find_build_assignment_loops

# remove duplicate binary_packages from "build-list"
mysql_query_remove_old_binary_packages_from_build_list | \
  mysql_run_query