path: root/bin/get-package-updates

#!/bin/sh

# check for packages that need to be built, and build a list in the proper build order
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates

# TODO: be more secure in case of update while build(s) is/are still in
# progress -> (stale) lock files, moving (or changing content of) loop
# lock files

# TODO: (might be connected to above): find out who deletes loop-files which
# should not be deleted

# TODO: remove extra whitelist cases

# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'get-package-updates: check for packages that need to be built,'
  >&2 echo ' and build a list in the proper build order'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--block:   If necessary, wait for lock blocking.'
  >&2 echo '  -h|--help:    Show this help and exit.'
  >&2 echo '  -n|--no-pull: Do not pull git repos, merely reorder build list.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o bhn \
    --long block \
    --long help \
    --long no-pull \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
pull=true

while true
do
  case "$1" in
    -b|--block)
      block_flag=''
    ;;
    -h|--help)
      usage 0
    ;;
    -n|--no-pull)
      pull=false
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

# delete_package package
# mark $package for deletion
delete_package() {
  echo "$1" >> \
    "${work_dir}/deletion-list.new"
  sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
}

# Update git repositories (official packages, community packages and the repository of package customizations).

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  git -C "${repo_path}" fetch
done

# Read previous git revision numbers from files.

something_new=false

for repo in ${repo_names}; do
  eval "old_repo_revisions__${repo}='$(
    cat "${work_dir}/${repo}.revision" 2> /dev/null || \
      echo NONE
  )'"
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  eval "new_repo_revisions__${repo}='$(
    if ${pull}; then
      git -C "${repo_path}" rev-parse HEAD
    else
      cat "${work_dir}/${repo}.revision"
    fi | \
      tee "${work_dir}/${repo}.revision.new"
  )'"
  if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then
    something_new=true
  fi
done

if ${pull} && \
  ! ${something_new}; then
  >&2 echo 'Nothing changed.'
  exit
fi

# Create a lock file for build list.

exec 9> "${build_list_lock_file}"
if ! flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit
fi

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

grep -vxF 'break_loops' \
  "${work_dir}/build-list" > \
  "${work_dir}/build-list.new" || \
  true
cp \
  "${work_dir}/deletion-list" \
  "${work_dir}/deletion-list.new"

for repo in ${repo_names}; do
  eval repo_path='"${repo_paths__'"${repo}"'}"'
  eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
  eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"'
  (
    # if old revision unknown, mimic "git diff"-output
    # shellcheck disable=SC2154
    if [ "${old_repo_revision}" = "NONE" ]; then
      git -C "${repo_path}" archive --format=tar HEAD | \
        tar -t | \
        sed 's|^|A\t|'
    else
      git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
    fi
  ) | \
    # only track changes in PKGBUILDs
    grep '/PKGBUILD$' | \
    if [ "${repo}" = "archlinux32" ]; then
    # modify the directory structure from the modifiaction-repository
    # to the one of an original source repository
      # shellcheck disable=SC2016
      sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
        while read -r pkg_repo rest; do
          eval "printf '%s %s\n'" \
            "$(printf '"${new_repo_revisions__%s}"' "$(find_git_repository_to_package_repository "${pkg_repo}")")" \
            "'${rest}'"
        done
    else
      sed "s|^|${new_repo_revision} |"
    fi | \
    grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
    # ignore i686
    grep -v -- '-i686/PKGBUILD$' | \
    # ignore staging and testing
    grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
    sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|'
done | \
  sort -u | \
  sed '
    s|^D\s|0 \0|
    t
    s|^[AM]\s|1 \0|
    t
    s|^|2 |
  ' | \
  sort -k1,1 | \
  sed 's|^[012] ||' | \
  while read -r mode package git_revision repository; do
    case "${mode}" in

      # new or modified PKGBUILD
      "A"|"M")
        sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
        # shellcheck disable=SC2154
        echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
          "${work_dir}/build-list.new"
        sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
      ;;

      # deleted PKGBUILD
      "D")
        found_package=false
        for repository in ${repo_names}; do
          eval 'repo_path="${repo_paths__'"${repository}"'}"'
          if [ "${repository}" = "archlinux32" ]; then
            if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" 2> /dev/null | \
              tar -t 2> /dev/null | \
              grep -q "$(str_to_regex "${package}")/PKGBUILD$"; then
              found_package=true
              break;
            fi
          else
            if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- "${package}/repos" 2> /dev/null | \
              tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \
              cut -d/ -f3 | \
              grep -v 'staging\|testing\|-unstable' | \
              grep -vq -- '-i686$'; then
              found_package=true
              break;
            fi
          fi
        done
        if ! ${found_package}; then
          delete_package "${package}"
        fi
      ;;

      *)
        >&2 echo "unknown git diff mode '${mode}'"
        exit 1
      ;;

    esac
  done

sort -u "${work_dir}/deletion-list.new" | \
  sponge "${work_dir}/deletion-list.new"

echo 'Extract dependencies of packages.'

# First, we extract the dependencies of each package.

mkdir -p "${work_dir}/package-infos"

while read -r package git_revision mod_git_revision repository; do

  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"

done < "${work_dir}/build-list.new"

echo 'apply blacklisting'
# ignore blacklisted packages and dependent packages
# this is the first time when all the information is available and up to date

black_listed=''
black_listed_new=$(
  (
    git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
      tar -Ox 'blacklist' | \
      sed '
        s/\s*#.*$//
        /^\s*$/d
      '
    find "${work_dir}/package-infos" -maxdepth 1 -name 'lib32-*' -printf '%f\n' | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      uniq
  ) | \
    sort -u
)

find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \
  grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \
  sed '
    s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1|
  ' | \
  sort -k4,4 | \
  uniq -f3 --group=append | \
  while read -r rev mod_rev repo pkg; do
    if [ -z "${rev}" ] && \
      [ -z "${mod_rev}" ] && \
      [ -z "${repo}" ] && \
      [ -z "${pkg}" ]; then
      # shellcheck disable=SC2031
      printf '%s %s.%s.%s.%s\n' \
        "${opkg}" \
        "${opkg}" \
        "$(
          # shellcheck disable=SC2086
          printf '%s\n' ${revs} | \
            sort -u | \
            find_newest_of_git_revisions
        )" \
        "$(
          # shellcheck disable=SC2086
          printf '%s\n' ${mod_revs} | \
            sort -u | \
            find_newest_of_git_revisions
        )" \
        "${orepo}" | \
        grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$'
      revs=''
      mod_revs=''
      continue
    fi
    revs=$(
      printf '%s\n' \
        ${revs} \
        "${rev}"
    )
    mod_revs=$(
      printf '%s\n' \
        ${mod_revs} \
        "${mod_rev}"
    )
    opkg="${pkg}"
    orepo="${repo}"
  done | \
  sort -k1,1 > \
  "${work_dir}/newest-revisions"

sed "
  s|^\S\+ |${work_dir}/package-infos/|
  s|\$|.builds|
" "${work_dir}/newest-revisions" | \
  xargs -r cat | \
  sort | \
  uniq -c > \
  "${work_dir}/built-packages"

sed "
  s|^\S\+ |${work_dir}/package-infos/|
  s|\$|.build-depends|
" "${work_dir}/newest-revisions" | \
  xargs -r grep -HF '' | \
  sed '
    s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2|
    s|^.*/||
  ' | \
  sort -k2,2 > \
  "${work_dir}/newest-dependencies"

while [ -n "${black_listed_new}" ]; do
  black_listed=$(
    # shellcheck disable=SC2086
    printf '%s\n' ${black_listed} ${black_listed_new} | \
      sort -u
  )
  black_listed_new=$(
    (
      # shellcheck disable=SC2086
      printf '%s\n' ${black_listed} | \
        sort -k1,1 | \
        join -j1 -o 2.2 - "${work_dir}/newest-revisions" | \
        sed "
          s|^|${work_dir}/package-infos/|
          s|\$|.builds|
        " | \
        xargs -r cat | \
        sort | \
        uniq -c
      cat "${work_dir}/built-packages"
    ) | \
      sort | \
      uniq -d | \
      awk '{print $2}' | \
      sort -k1,1 | \
      join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \
      while read -r pkg; do
        if [ "${pkg}" = 'dev86' ] || \
          [ "${pkg}" = 'python-pysocks' ] || \
          [ "${pkg}" = 'wine' ] || \
          [ "${pkg}" = 'wine-staging' ] || \
          [ "${pkg}" = 'wine-staging-nine' ]; then
          >&2 printf 'blacklisting: %s would be black listed, but is ignored.\n' "${pkg}"
        else
          echo "${pkg}"
        fi
      done | \
      sort -u
  )
  black_listed_new=$(
    # shellcheck disable=SC2086
    printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \
      sort | \
      uniq -u
  )
done
echo "${black_listed}" | \
  while read -r package; do
    [ -n "${package}" ] && \
      delete_package "${package}"
  done

sort -u "${work_dir}/deletion-list.new" | \
  sponge "${work_dir}/deletion-list.new"

# Now we create the partial order.

while read -r package git_revision mod_git_revision repository; do
  # add "$pkgname -> $build-target" to build-order list
  sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds"
  # add "$dependency -> $pkgname" to build-order list
  sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
  # add "base/base-devel -> $pkgname" to build-order list
  printf "%s ${package}\n" 'base' 'base-devel'
done \
  < "${work_dir}/build-list.new" \
  > "${work_dir}/build-order"

if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then
  >&2 echo 'Created partial order file "build-order" is invalid.'
  exit 2
fi

echo 'Now actually sort it.'

(
  # this part will have the correct build order, but all the infos are missing
  tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
    nl -ba | \
    awk '{print $1 " not-git also-not-git whatever " $2}'
  # this part has all the infos, but possibly the wrong order
  awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new"
) | \
  sort -k5,5 -k1nr | \
  # now, we have the correct order and the infos, but in adjacent lines
  uniq -f4 -D | \
  sed '/^0 /d;N;s|\n| |' | \
  # now in one line, each
  sort -k1n,1 | \
  awk '{print $5 " " $7 " " $8 " " $9}' > \
  "${work_dir}/build-list.new.new"

rm --one-file-system -rf "${work_dir}/build-list.loops.new"
mkdir "${work_dir}/build-list.loops.new"

if [ -s "${work_dir}/tsort.error" ]; then
  >&2 echo 'WARNING: There is a dependency cycle!'
  >&2 cat "${work_dir}/tsort.error"
  >&2 echo
  >&2 echo 'I will continue anyway.'
  # save loops in separate files each, so breaking them is easier
  awk '
    /^tsort: \S+: input contains a loop:$/{
      n++;
      getline
    }
    {
      print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n
    }
    ' "${work_dir}/tsort.error"

  # remove lines from loop files which are no packages
  find "${work_dir}/build-list.loops.new" -maxdepth 1 -printf '%f\n' | \
    grep '^loop_[0-9]\+$' | \
    while read -r loop; do
      (
        sort -u "${work_dir}/build-list.loops.new/${loop}"
        cut -d' ' -f1 "${work_dir}/build-list.new.new" | \
          sort -u
      ) | \
        sort | \
        uniq -d > \
        "${work_dir}/build-list.loops.new/${loop}.new"
      mv \
        "${work_dir}/build-list.loops.new/${loop}.new" \
        "${work_dir}/build-list.loops.new/${loop}"
    done

  # add orders to break loops to build list
  insert_break_loops_orders "${work_dir}/build-list.new.new"

else
  rm "${work_dir}/tsort.error"
fi

# Move the .new-files to the actual files

rm -rf --one-file-system "${work_dir}/build-list.loops"
(
  printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list"
  # shellcheck disable=SC2086
  printf '%s.revision\n' ${repo_names}
) | \
  while read -r file; do
    mv "${work_dir}/${file}.new" "${work_dir}/${file}"
  done

# Remove the lock file

rm -f "${build_list_lock_file}"