path: root/bin/seed-build-list

#!/bin/sh

# seed the build list from differences between an x86_64 and our master mirror

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'seed-build-list [options]:'
  >&2 echo '  seed the build list from'
  >&2 echo '    - differences between an x86_64 and our master mirror'
  >&2 echo '    - a list of packages which need to be rebuilt'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  >&2 echo '  -i|--ignore $package:'
  >&2 echo '    Do not update $package.'
  >&2 echo '  -m|--mirror $url:'
  >&2 echo '    Get x86_64 packages from mirror at $url.'
  >&2 echo '  -n|--no-action:'
  >&2 echo '    Do not actually update build-list, just print it.'
  >&2 echo '  -p|--package $pkg_regex:'
  >&2 echo '    Update packages matching $pkg_regex.'
  [ -z "$1" ] && exit 1 || exit $1
}

tmp_dir="$(mktemp -d)"
trap "rm -rf --one-file-system '${tmp_dir}'" EXIT

eval set -- "$(
  getopt -o hi:m:np: \
    --long help \
    --long ignore: \
    --long mirror: \
    --long no-action \
    --long package: \
    -n "$(basename "$0")" -- "$@" || \
    echo usage
  )"

touch "${tmp_dir}/mirrors"
touch "${tmp_dir}/delta_packages"
touch "${tmp_dir}/delta_pkgbases"
touch "${tmp_dir}/package_regexes"
touch "${tmp_dir}/ignore_packages"

update=true

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -i|--ignore)
      shift
      echo "$1" >> \
        "${tmp_dir}/ignore_packages"
    ;;
    -m|--mirror)
      shift
      echo "$1" >> \
        "${tmp_dir}/mirrors"
    ;;
    -n|--no-action)
      update=false
    ;;
    -p|--package)
      shift
      echo "$1" >> \
        "${tmp_dir}/package_regexes"
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  usage 1
fi

repos='core community extra gnome-unstable kde-unstable'

for repo in ${repo_names}; do
  if [ "${repo_names}" = 'archlinux32' ]; then
    continue
  fi
  eval 'repo_path="${repo_paths__'"${repo}"'}"'
  git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
    tar -t | \
    grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
    grep -v -- '-i686/PKGBUILD$' | \
    grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
    sed '
      s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2|
    '
done | \
  sort -k1,1 > \
  "${tmp_dir}/known_packages"

# generate delta_packages from package_regexes
while read -r pkg_regex; do
  if [ -z "${pkg_regex}" ]; then
    continue
  fi
  awk '{print $1}' "${tmp_dir}/known_packages" | \
    grep "${pkg_regex}" || \
    true
done < \
  "${tmp_dir}/package_regexes" >> \
  "${tmp_dir}/delta_packages"

# genereate delta_packages from mirror delta

while read -r mirror; do
  if [ -z "${mirror}" ]; then
    continue
  fi
  (
    # theirs
    for repo in ${repos}; do
      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
        tar -tz
    done | \
      grep '/$' | \
      sed '
        s|/$||
        s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1|
      '

    # ours
    for repo in $(ls_master_mirror 'i686'); do
      ls_master_mirror "i686/$repo" | \
        grep '\.pkg\.tar\.xz$' | \
        sed 's|-[^-]\+$||'
    done | \
      sed '
        s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
      '
    ) | \
    sort -k4,4 -k3Vr,3 -k1,1 | \
    uniq -f3 | \
    grep '^theirs ' | \
    awk '{print $4}' | \
    sort -k1,1 >> \
    "${tmp_dir}/delta_packages"
done < \
  "${tmp_dir}/mirrors"

sort -k1,1 -u "${tmp_dir}/delta_packages" | \
  sponge "${tmp_dir}/delta_packages"

join -j1 "${tmp_dir}/delta_packages" "${tmp_dir}/known_packages" >> \
  "${tmp_dir}/append_packages"

(
  awk '{print $1}' "${tmp_dir}/append_packages"
  cat "${tmp_dir}/delta_packages"
) | \
  sort | \
  uniq -u | \
  sponge "${tmp_dir}/delta_packages"

while read -r pkg; do
  if [ -z "${pkg}" ]; then
    continue
  fi
  success=false
  for arch in x86_64 any; do
    pkgbase="$(
      curl -sS "https://www.archlinux.org/packages/community/x86_64/${pkg}/" | \
        sed -n '
          /Base Package:/{
            :a
            N
            s|</tr>.*$||
            Ta
            p
          }
        ' | \
        tr '<>' '\n' | \
        grep '^a href="' | \
        sed '
          s|^a href="||
          s|".*||
          s|/$||
          s|^.*/||
        '
    )"
    if [ -n "${pkgbase}" ]; then
      echo "${pkgbase}" >> \
        "${tmp_dir}/delta_pkgbases"
      success=true
      break
    fi
  done
  if ! ${sucess}; then
    echo "${pkg}"
  fi
done < \
  "${tmp_dir}/delta_packages" | \
  sponge "${tmp_dir}/delta_packages"

sort -u "${tmp_dir}/delta_pkgbases" | \
  sponge "${tmp_dir}/delta_pkgbases"

join -j1 "${tmp_dir}/delta_pkgbases" "${tmp_dir}/known_packages" >> \
  "${tmp_dir}/append_packages"

sort -u "${tmp_dir}/append_packages" | \
  sponge "${tmp_dir}/append_packages"

rm "${tmp_dir}/delta_pkgbases"

if [ -s "${tmp_dir}/delta_packages" ]; then
  echo 'There are still packages newer for x86_64 which I cannot identify :-/'
  exit 2
fi

rm "${tmp_dir}/delta_packages"

# convert append_packages to build-list.new (add git hashes)

while read -r pkg repo; do
  git_repo="$(find_git_repository_to_package_repository "${repo}")"
  printf '%s %s %s %s\n' \
    "${pkg}" \
    "$(cat "${work_dir}/${git_repo}.revision")" \
    "$(cat "${work_dir}/archlinux32.revision")" \
    "${repo}"
done < \
  "${tmp_dir}/append_packages" | \
  awk '{print "append " $1 " " $2 " " $3 " " $4 " " $1}' > \
  "${tmp_dir}/build-list.append"

# Create a lock file for build list.

if ${update}; then
  exec 9> "${build_list_lock_file}"
  flock 9
fi

# ignore packages already on the build list
grep -vxF 'break_loops' "${work_dir}/build-list" | \
  awk '{print "old " $1 " " $2 " " $3 " " $4 " " $1}' > \
  "${tmp_dir}/build-list.old"

# ignore packages on the deletion-list
awk '{print "delete " $1 " X X X " $1}' "${work_dir}/deletion-list" >> \
  "${tmp_dir}/build-list.old"

# ignore packages on the blacklist
git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
  tar -Ox blacklist | \
  awk '{print "blacklist " $1 " X X X " $1}' >> \
  "${tmp_dir}/build-list.old"

# ignore explicitely ignored packages
awk '{print "ignore " $1 " X X X " $1}' "${tmp_dir}/ignore_packages" >> \
  "${tmp_dir}/build-list.old"

if ${update}; then
  cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
    sort -k6,6 -k1,1 | \
    uniq -uf5 | \
    grep '^append ' | \
    awk '{print $2 " " $3 " " $4 " " $5}' >> \
    "${work_dir}/build-list"

  # Remove the lock file

  rm -f "${build_list_lock_file}"
else
  cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \
    sort -k6,6 | \
    uniq -uf5 | \
    grep '^append ' | \
    awk '{print $2 " " $3 " " $4 " " $5}'
fi