#!/bin/sh # seed the build list from differences between an x86_64 and our master mirror # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" # shellcheck disable=SC2016 usage() { >&2 echo '' >&2 echo 'seed-build-list [options]:' >&2 echo ' seed the build list from' >&2 echo ' - differences between an x86_64 and our master mirror' >&2 echo ' - a list of packages which need to be rebuilt' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' >&2 echo ' -i|--ignore $package:' >&2 echo ' Do not update $package.' >&2 echo ' -m|--mirror $url:' >&2 echo ' Get x86_64 packages from mirror at $url.' >&2 echo ' -n|--no-action:' >&2 echo ' Do not actually update build-list, just print it.' >&2 echo ' -p|--package $pkg_regex:' >&2 echo ' Update packages matching $pkg_regex.' [ -z "$1" ] && exit 1 || exit "$1" } tmp_dir=$(mktemp -d) # shellcheck disable=SC2064 trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT eval set -- "$( getopt -o hi:m:np: \ --long help \ --long ignore: \ --long mirror: \ --long no-action \ --long package: \ -n "$(basename "$0")" -- "$@" || \ echo usage )" touch "${tmp_dir}/mirrors" touch "${tmp_dir}/delta_packages" touch "${tmp_dir}/delta_pkgbases" touch "${tmp_dir}/package_regexes" touch "${tmp_dir}/ignore_packages" update=true while true do case "$1" in -h|--help) usage 0 ;; -i|--ignore) shift echo "$1" >> \ "${tmp_dir}/ignore_packages" ;; -m|--mirror) shift echo "$1" >> \ "${tmp_dir}/mirrors" ;; -n|--no-action) update=false ;; -p|--package) shift echo "$1" >> \ "${tmp_dir}/package_regexes" ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then usage 1 fi repos='core community extra gnome-unstable kde-unstable' for repo in ${repo_names}; do if [ "${repo_names}" = 'archlinux32' ]; then continue fi eval 'repo_path="${repo_paths__'"${repo}"'}"' git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \ tar -t | \ grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \ grep -v -- '-i686/PKGBUILD$' | \ grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ sed ' s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2| ' done | \ sort -k1,1 > \ "${tmp_dir}/known_packages" # generate delta_packages from package_regexes while read -r pkg_regex; do if [ -z "${pkg_regex}" ]; then continue fi awk '{print $1}' "${tmp_dir}/known_packages" | \ grep "${pkg_regex}" || \ true done < \ "${tmp_dir}/package_regexes" >> \ "${tmp_dir}/delta_packages" # genereate delta_packages from mirror delta while read -r mirror; do if [ -z "${mirror}" ]; then continue fi ( # theirs for repo in ${repos}; do curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ tar -tz done | \ grep '/$' | \ sed ' s|/$|| s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1| ' # ours for repo in $(ls_master_mirror 'i686'); do ls_master_mirror "i686/$repo" | \ grep '\.pkg\.tar\.xz$' | \ sed 's|-[^-]\+$||' done | \ sed ' s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1| ' ) | \ sort -k4,4 -k3Vr,3 -k1,1 | \ uniq -f3 | \ grep '^theirs ' | \ awk '{print $4}' | \ sort -k1,1 >> \ "${tmp_dir}/delta_packages" done < \ "${tmp_dir}/mirrors" sort -k1,1 -u "${tmp_dir}/delta_packages" | \ sponge "${tmp_dir}/delta_packages" join -j1 "${tmp_dir}/delta_packages" "${tmp_dir}/known_packages" >> \ "${tmp_dir}/append_packages" ( awk '{print $1}' "${tmp_dir}/append_packages" cat "${tmp_dir}/delta_packages" ) | \ sort | \ uniq -u | \ sponge "${tmp_dir}/delta_packages" while read -r pkg; do if [ -z "${pkg}" ]; then continue fi pkgbase=$( curl -sS "https://www.archlinux.org/packages/community/x86_64/${pkg}/" | \ sed -n ' /Base Package:/{ :a N s|.*$|| Ta p } ' | \ tr '<>' '\n' | \ grep '^a href="' | \ sed ' s|^a href="|| s|".*|| s|/$|| s|^.*/|| ' ) if [ -n "${pkgbase}" ]; then echo "${pkgbase}" >> \ "${tmp_dir}/delta_pkgbases" else echo "${pkg}" fi done < \ "${tmp_dir}/delta_packages" | \ sponge "${tmp_dir}/delta_packages" sort -u "${tmp_dir}/delta_pkgbases" | \ sponge "${tmp_dir}/delta_pkgbases" join -j1 "${tmp_dir}/delta_pkgbases" "${tmp_dir}/known_packages" >> \ "${tmp_dir}/append_packages" sort -u "${tmp_dir}/append_packages" | \ sponge "${tmp_dir}/append_packages" rm "${tmp_dir}/delta_pkgbases" if [ -s "${tmp_dir}/delta_packages" ]; then echo 'There are still packages newer for x86_64 which I cannot identify :-/' exit 2 fi rm "${tmp_dir}/delta_packages" # convert append_packages to build-list.new (add git hashes) while read -r pkg repo; do git_repo=$(find_git_repository_to_package_repository "${repo}") printf '%s %s %s %s\n' \ "${pkg}" \ "$(cat "${work_dir}/${git_repo}.revision")" \ "$(cat "${work_dir}/archlinux32.revision")" \ "${repo}" done < \ "${tmp_dir}/append_packages" | \ awk '{print "append " $1 " " $2 " " $3 " " $4 " " $1}' > \ "${tmp_dir}/build-list.append" # Create a lock file for build list. if ${update}; then exec 9> "${build_list_lock_file}" flock 9 fi { # ignore packages already on the build list grep -vxF 'break_loops' "${work_dir}/build-list" | \ awk '{print "old " $1 " " $2 " " $3 " " $4 " " $1}' # ignore packages on the deletion-list awk '{print "delete " $1 " X X X " $1}' "${work_dir}/deletion-list" # ignore packages on the blacklist git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \ tar -Ox blacklist | \ awk '{print "blacklist " $1 " X X X " $1}' # ignore explicitely ignored packages awk '{print "ignore " $1 " X X X " $1}' "${tmp_dir}/ignore_packages" } > \ "${tmp_dir}/build-list.old" if ${update}; then cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \ sort -k6,6 -k1,1 | \ uniq -uf5 | \ grep '^append ' | \ awk '{print $2 " " $3 " " $4 " " $5}' >> \ "${work_dir}/build-list" # Remove the lock file rm -f "${build_list_lock_file}" else cat "${tmp_dir}/build-list.append" "${tmp_dir}/build-list.old" | \ sort -k6,6 | \ uniq -uf5 | \ grep '^append ' | \ awk '{print $2 " " $3 " " $4 " " $5}' fi