path: root/bin/get-package-updates

#!/bin/bash

# check for packages that need to be built, and build a list in the proper build order
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates

# TODOs:
#  additional to blacklisted packages ignore any packages depending on them
#  be more secure in case of update while build(s) is/are still in progress
#   -> (stale) lock files, moving loop lock files

. "${0%/*}/../conf/default.conf"

# Create a lock file for build list.

exec 9> "${build_list_lock_file}"
if ! flock -n 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit
fi

# Update git repositories (official packages, community packages and the repository of package customizations).

for repo in "${repo_paths[@]}"; do
  git -C "${repo}" clean -df
  git -C "${repo}" reset --hard
  git -C "${repo}" checkout master
  git -C "${repo}" pull
done

# Read previous git revision numbers from files.

declare -A old_repo_revisions
declare -A new_repo_revisions

for repo in "${!repo_paths[@]}"; do
  old_repo_revisions["${repo}"]="$(
    cat "${work_dir}/${repo}.revision" 2> /dev/null || \
      echo NONE
  )"
  new_repo_revisions["${repo}"]="$(
    git -C "${repo_paths["${repo}"]}" rev-parse HEAD | \
      tee "${work_dir}/${repo}.revision.new"
  )"
done

echo 'Check modified packages from the last update, and put them to the build list.'

# Check modified packages from the last update, and put them to the build list.
# If a package is updated, but already on the rebuild list, then just update the git revision number.
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.

cp "${work_dir}/build-list"{,.new}
cp "${work_dir}/deletion-list"{,.new}

(
  for repo in "${!repo_paths[@]}"; do
    (
      # if old revision unknown, mimic "git diff"-output
      if [ "${old_repo_revisions["${repo}"]}" == "NONE" ]; then
        git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \
          tar -t | \
          sed 's|^|A\t|'
      else
        git -C "${repo_paths["${repo}"]}" diff --no-renames --name-status "${old_repo_revisions["${repo}"]}" HEAD
      fi
    ) | \
      # only track changes in PKGBUILDs
      grep '/PKGBUILD$' | \
      if [ "${repo}" == "archlinux32" ]; then
      # modify the directory structure from the modifiaction-repository
      # to the one of an original source repository
        sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
          while read -r pkg_repo rest; do
            echo "${new_repo_revisions["$(find_git_repository_to_package_repository "${pkg_repo}")"]} ${rest}"
          done
      else
        sed "s|^|${new_repo_revisions["${repo}"]} |"
      fi | \
      grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
      # ignore i686
      grep -v -- '-i686/PKGBUILD$' | \
      sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' | \
      # ignore staging and testing
      grep -v '\(staging\|testing\)$'
  done | \
    sort -u
  # ignore blacklisted packages by explicitely deleting them _at_the_end_
  sed "s|^\(.*\)\$|D \1 ${new_repo_revisions["archlinux32"]} archlinux32|" "${repo_paths["archlinux32"]}/blacklist"
) | \
  while read -r mode package git_revision repository; do
    case "${mode}" in

      # new or modified PKGBUILD
      "A"|"M")
        sed -i "/^${package} /d" "${work_dir}/build-list"
        echo "${package} ${git_revision} ${new_repo_revisions["archlinux32"]} ${repository}" >> \
          "${work_dir}/build-list.new"
        sed -i "/^${package}\$/d" "${work_dir}/deletion-list.new"
      ;;

      # deleted PKGBUILD
      "D")
        echo "${package}" >> \
          "${work_dir}/deletion-list.new"
        sed -i "/^${package} /d" "${work_dir}/build-list.new"
      ;;

      *)
        >&2 echo "unknown git diff mode '${mode}'"
        exit 1
      ;;

    esac
  done

echo 'Extract dependencies of packages.'

# Put the list in the proper build order.
# First, we extract the dependencies of each package.

mkdir -p "${work_dir}/package-infos"

rm -f "${work_dir}/build-order"
touch "${work_dir}/build-order"

while read -r package git_revision mod_git_revision repository; do
  # delete cached values of old versions of this PKGBUILD
  ls -1 "${work_dir}/package-infos/${package}."* 2> /dev/null | \
    sed 's|^.*/||' | \
    grep "^${package//./\\.}"'\.\([0-9a-f]\{40\}\.\)\{2\}\(builds\|needs\|packages\)$' | \
    grep -v "^${package//./\\.}\.${git_revision}\.${new_repo_revisions["archlinux32"]}"'\.\(builds\|needs\|packages\)$' | \
    while read file; do
      rm "${work_dir}/package-infos/${file}"
    done

  # extract infos from PKGBUILD if not existent yet
  if [ ! -e "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.builds" ] || \
    [ ! -e "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.needs" ] || \
    [ ! -e "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.packages" ]; then

    PKGBUILD="$(find_pkgbuild "${package}" "${repository}")"

    if [ ! -r "${PKGBUILD}" ]; then
      echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'"
      exit 1
    fi

    (
      cd "${PKGBUILD%/*}"
      apply_package_customizations
      mksrcinfo -o "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO"
    )

    # extract "builds" = provides \cup pkgname
    grep '^\('$'\t''provides\|pkgname\) = ' "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO" | \
      cut -d= -f2 | \
      sed 's|^\s\+||; s|[<>]$||' | \
      sort -u > \
      "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.builds"

    # extract "packages" = pkgname
    grep '^pkgname = ' "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO" | \
      cut -d= -f2 | \
      sed 's|^\s\+||; s|[<>]$||' | \
      sort -u > \
      "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.packages"

    # extract "needs" = ( makedepends \cup checkdepends ) \setminus "builds"
    (
      (
        # this would include runtime dependencies, too:
        # sed -n '/^pkgname = /q;/^'$'\t''depends = /p' "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO"
        grep '^'$'\t''\(makedepends\|checkdepends\) = ' "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO"
      ) | \
        cut -d= -f2 | \
        sed 's|^\s\+||; s|[<>]$||' | \
        sort -u
      sed 'p' "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.builds"
    ) | \
      sort | \
      uniq -u > \
      "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.needs"

    rm "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.SRCINFO"

  fi

  # add "$pkgname -> $build-target" to build-order list
  while read target; do
    echo "${package} ${target}" >> \
      "${work_dir}/build-order"
  done < "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.builds"

  # add "$dependency -> $pkgname" to build-order list
  while read dependency; do
    echo "${dependency} ${package}" >> \
      "${work_dir}/build-order"
  done < "${work_dir}/package-infos/${package}.${git_revision}.${new_repo_revisions["archlinux32"]}.needs"

done < "${work_dir}/build-list.new"

echo 'Now actually sort it.'

(
  # this part will have the correct build order, but all the infos are missing
  tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \
    nl -ba | \
    awk '{print $1 " not-git also-not-git whatever " $2}'
  # this part has all the infos, but possibly the wrong order
  awk '{print "0 " $2 " " $3 " " $4 " " $1}' < \
    "${work_dir}/build-list.new"
) | \
  sort -k5,5 -k1nr | \
  # now, we have the correct order and the infos, but in adjacent lines
  uniq -f4 -D | \
  sed '/^0 /d;N;s|\n| |' | \
  # now in one line, each
  sort -k1n,1 | \
  awk '{print $5 " " $7 " " $8 " " $9}' > \
  "${work_dir}/build-list.new.new"

rm --one-file-system -rf "${work_dir}/build-order.loops.new"
mkdir "${work_dir}/build-order.loops.new"

if [ -s "${work_dir}/tsort.error" ]; then
  >&2 echo 'WARNING: There is a dependency cycle!'
  >&2 cat "${work_dir}/tsort.error"
  >&2 echo
  >&2 echo 'I will continue anyway.'
  # save loops in separate files each, so breaking them is easier
  awk '
    /^tsort: \S+: input contains a loop:$/{
      n++;
      getline
    }
    {
      print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n
    }
    ' "${work_dir}/tsort.error"
else
  rm "${work_dir}/tsort.error"
fi

# Move the .new-files to the actual files

rm -rf --one-file-system "${work_dir}/build-order.loops"
(
  echo "build-order.loops" "build-list.new" "build-list" "deletion-list"
  echo "${!repo_paths[@]}" | \
    sed 's@\( \|$\)@.revision\1@g'
) | \
  tr ' ' '\n' | \
  while read -r file; do
    mv "${work_dir}/${file}.new" "${work_dir}/${file}"
  done

# Remove the lock file

rm -f "${build_list_lock_file}"