#!/bin/sh # check for packages that need to be built # shellcheck disable=SC2119,SC2120 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" # TODO: Find out, why sometimes package updates are missed. # TODO: correctly handle if pkgbase of a split package is renamed, e.g.: # $a -> ($a,$b) ==> $b -> ($a,$b) # TODO: keep database clean in case of abort # TODO: Being on the deletion-list is not a reason not to be scheduled # if a dependency formerly on the deletion-list is now being built. # shellcheck disable=SC2016 usage() { >&2 echo '' >&2 echo 'get-package-updates: check for packages that need to be built,' >&2 echo ' and build a list in the proper build order' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -d|--date $datetime:' >&2 echo ' Pull latest commit before $datetime' >&2 echo ' (yyyy-mm-ddThh:mm:ss). Conflicts -n.' >&2 echo ' -h|--help: Show this help and exit.' >&2 echo ' -n|--no-pull: Do not pull git repos, merely reorder build list.' >&2 echo ' Conflicts -d.' >&2 echo ' -w|--wait: If necessary, wait for lock blocking.' >&2 echo ' -x|--test-exclusion $package:' >&2 echo ' Print additionally deleted/excluded packages if' >&2 echo ' "$package" would be black listed.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( getopt -o d:hnwx: \ --long date: \ --long help \ --long no-pull \ --long test-exclusion: \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" block_flag='-n' date_time='' test_exclusion='' pull=true while true do case "$1" in -d|--date) shift date_time="$1" ;; -h|--help) usage 0 ;; -n|--no-pull) pull=false ;; -w|--wait) block_flag='' ;; -x|--test-exclusion) shift if [ -n "${test_exclusion}" ]; then >&2 printf 'I already have --test-exclusion=%s and you gave me another one.\n' "${test_exclusion}" >&2 printf 'But I can only handle one exclusion test at a time.\n' exit 2 fi test_exclusion="$1" ;; --) shift break ;; *) >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' exit 42 ;; esac shift done if [ $# -ne 0 ]; then >&2 echo 'Too many arguments.' usage fi if [ -n "${date_time}" ] && ! ${pull}; then >&2 printf -- '-d and -n are mutually exclusive.\n' usage fi if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit fi # TODO: How should new deletion-list packages be handled? # - packages deleted upstream should be marked as to-be-deleted if # existent and otherwise be ignored # - packages deleted due to black listing should be marked as black # listed - and thus as to-be-deleted, but they should remain in the # database after deletion! # delete_package package # mark $package for deletion delete_package() { # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. if [ -z "${test_exclusion}" ]; then # shellcheck disable=SC2016 { # packages from the build-list/to-be-decided go straight to the deletion-list printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources printf ' SET ' # shellcheck disable=SC2154 printf '`binary_packages_in_repositories`.`%s`=%s,' \ 'repository' "${repository_ids__any_deletion_list}" \ 'is_to_be_deleted' '1' \ 'last_moved' 'NOW()' | \ sed 's/,$//' # shellcheck disable=SC2154 printf ' WHERE `binary_packages_in_repositories`.`repository` in (%s,%s)' \ "${repository_ids__any_build_list}" \ "${repository_ids__any_to_be_decided}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$(printf '%s' "$1" | base64 -w0)" printf 'COMMIT;\n' # other packages are marked as `is_to_be_deleted` printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ mysql_run_query fi } something_new=false for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' # Update git repositories (official packages, community packages and the repository of package customizations). git -C "${repo_path}" remote update # read previous git revision numbers from database. # shellcheck disable=SC2016 eval "old_repo_revisions__${repo}='$( { printf 'SELECT `git_repositories`.`head` FROM `git_repositories`' printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" } | \ mysql_run_query )'" # determine new git revision if ${pull}; then if [ -z "${date_time}" ]; then eval "new_repo_revisions__${repo}='$( git -C "${repo_path}" rev-parse HEAD )'" else eval "new_repo_revisions__${repo}='$( git -C "${repo_path}" rev-list -n1 --until "${date_time}" HEAD )'" fi else eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"' fi if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then something_new=true fi done if ${pull} && \ ! ${something_new}; then >&2 echo 'Nothing changed.' exit fi # Create a lock file for build list. exec 9> "${build_list_lock_file}" if ! verbose_flock ${block_flag} 9; then >&2 echo 'come back (shortly) later - I cannot lock build list.' exit fi exec 8> "${sanity_check_lock_file}" if ! verbose_flock -s ${block_flag} 8; then >&2 echo 'come back (shortly) later - sanity-check running.' exit fi trap mysql_cleanup EXIT # shellcheck disable=SC2119 mysql_cleanup echo 'Check modified packages from the last update, and put them to the build list.' # Check modified packages from the last update, and put them to the build list. # If a package is updated, but already on the rebuild list, then just update the git revision number. # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. trigger_rebuild_on_any_file_sed_expression=$( # note: This only works for upstream sources! # shellcheck disable=SC2016 { printf 'SELECT DISTINCT CONCAT(' printf '`package_sources`.`pkgbase`,"/repos/",' printf '`upstream_repositories`.`name`,"-",' printf 'IF(`architectures`.`name`="any","any","x86_64"))' printf ' FROM `binary_packages`' mysql_join_binary_packages_binary_packages_in_repositories mysql_join_binary_packages_build_assignments mysql_join_build_assignments_architectures mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \ "${repository_ids__any_build_list}" } | \ mysql_run_query | \ sed ' s@.*@s,^\\(.\\t\0/\\)[^/]\\+,\\1PKGBUILD,@ ' ) { # trigger rebuild of packages removed from blacklist # shellcheck disable=SC2016 { printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`,`git_repositories`.`name`' printf ' FROM `package_sources`' mysql_join_package_sources_upstream_repositories mysql_join_upstream_repositories_git_repositories printf ' WHERE `package_sources`.`pkgbase` IN (' # shellcheck disable=SC2154 git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \ sed -n ' s/^-\([^-]\)/\1/ T s/\s*#.*$// p ' | \ base64_encode_each | \ sed ' s/^/from_base64("/ s/$/")/ ' | \ tr '\n' ',' printf '"");\n' } | \ mysql_run_query | \ while read -r pkgbase repository git_repository; do printf 'A %s ' \ "${pkgbase}" eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${git_repository}"'}"' printf ' empty %s\n' \ "${repository}" done # actual updates/removes for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"' # if old revision unknown, mimic "git diff"-output # shellcheck disable=SC2154 if [ "${old_repo_revision}" = "NONE" ]; then git -C "${repo_path}" archive --format=tar HEAD | \ tar -t | \ sed 's|^|A\t|' else git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" fi | \ # rename any file to "PKGBUILD" for packages on the build-list sed "${trigger_rebuild_on_any_file_sed_expression}" | \ # only track changes in PKGBUILDs grep '/PKGBUILD$' | \ if [ "${repo}" = "archlinux32" ]; then # modify the directory structure from the modifiaction-repository # to the one of an original source repository # shellcheck disable=SC2016 sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ while read -r pkg_repo rest; do repo=$(find_git_repository_to_package_repository "${pkg_repo}") eval 'printf '"'"'%s %s %s\n'"'" \ "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \ "$(printf '"${old_repo_revisions__%s}"' "${repo}")" \ "'${rest}'" done else sed "s|^|${new_repo_revision} ${old_repo_revision} |" fi | \ grep '^\S\+ \S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ # ignore i686 grep -v -- '-i686/PKGBUILD$' | \ # ignore staging and testing grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ sed 's|^\(\S\+ \S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|' done | \ sort -u | \ sed ' s|^D\s|0 \0| t s|^[AM]\s|1 \0| t s|^|2 | ' | \ sort -k1,1 | \ sed 's|^[012] ||' } | \ while read -r mode package git_revision old_git_revision repository; do if [ "${old_git_revision}" = 'empty' ]; then old_git_revision='' fi if [ "${mode}" = 'D' ]; then # deleted PKGBUILD # shellcheck disable=SC2154 git_revision="${new_repo_revisions__archlinux32}" old_git_revision='' found_package=false for repository in ${repo_names}; do eval 'repo_path="${repo_paths__'"${repository}"'}"' if [ "${repository}" = "archlinux32" ]; then if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \ tar -t 2> /dev/null | \ grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then found_package=true break; fi else # shellcheck disable=SC2154 if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \ 'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \ 'cut -d/ -f3 | ' \ 'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \ 'grep -vq -- '"'"'-i686$'"'"; then eval 'git_revision="${new_repo_revisions__'"${repository}"'}"' found_package=true break; fi fi done if ${found_package}; then # TODO: moving packages between community64 and packages64 is currently broken (?) mode='M' repository=$( find_package_repository_to_package "${package}" "${repository}" "${git_revision}" ) else delete_package "${package}" continue fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # shellcheck disable=SC2016 { # delete old build assignment and associated binary packages # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`,`binary_packages_in_repositories`' printf ' FROM `binary_packages`' mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${package}" | \ base64 -w0 )" printf ' AND `binary_packages_in_repositories`.`repository` IN (%s,%s);\n' \ "${repository_ids__any_build_list}" \ "${repository_ids__any_deletion_list}" # remove is-to-be-deleted marker from old binary packages printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$( printf '%s' "${package}" | \ base64 -w0 )" } | \ mysql_run_query # shellcheck disable=SC2154 printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2 mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${old_git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" printf '\n' >&2 continue fi >&2 echo "unknown git diff mode '${mode}'" exit 1 done echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date black_listed='' black_listed_new=$( { { git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ tar -Ox 'blacklist' | \ sed ' s/\s*#.*$// /^\s*$/d ' if [ -n "${test_exclusion}" ]; then echo "${test_exclusion}" fi } | \ base64_encode_each # shellcheck disable=SC2016 { printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")' printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ mysql_run_query } | \ sort -u ) while [ -n "${black_listed_new}" ]; do black_listed=$( printf '%s\n' "${black_listed}" "${black_listed_new}" | \ grep -vxF '' | \ sort -u ) black_listed_new=$( { printf '%s\n' "${black_listed}" "${black_listed}" # shellcheck disable=SC2016 { printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' printf 'INSERT INTO `bl` (`pkgbase`) VALUES ' printf '%s\n' "${black_listed}" | \ sort -u | \ sed ' s/^/(from_base64("/ s/$/")),/ $ s/,$/;/ ' printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' printf ' FROM `package_sources` AS `a_ps`' mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' mysql_join_binary_packages_dependencies 'a_bp' mysql_join_dependencies_dependency_types printf ' AND (`dependency_types`.`relevant_for_building`' printf ' OR `dependency_types`.`relevant_for_binary_packages`)' printf ' WHERE NOT EXISTS (' printf ' SELECT 1' printf ' FROM `install_target_providers`' printf ' WHERE NOT EXISTS (' printf ' SELECT 1' printf ' FROM `bl`' printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' printf ')' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ') AND EXISTS (' # TODO: This should be corrected at the root: automatic install targets, which are bogus should # not be added in the first place - but how do we detect that? printf ' SELECT 1' printf ' FROM `install_target_providers`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ mysql_run_query | \ sort -u } | \ grep -vxF '' | \ sort | \ uniq -u ) done black_listed=$( printf '%s\n' "${black_listed}" | \ while read -r line; do printf '%s' \ "${line}" | \ base64 -d printf '\n' done ) deletion_list_count=$( # shellcheck disable=SC2086 printf '%s\n' ${black_listed} | \ wc -l ) if [ "${deletion_list_count}" -gt 1000 ]; then >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ "${deletion_list_count}" if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then # shellcheck disable=SC2119 { printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" for repo_name in ${repo_names}; do eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"' eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"' # shellcheck disable=SC2154 if [ "${old_revision}" != "${new_revision}" ]; then printf '%s: %s -> %s\n' \ "${repo_name}" \ "${old_revision}" \ "${new_revision}" fi done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ irc_say fi exit 3 fi rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" echo "${black_listed}" | \ while read -r package; do if [ -z "${package}" ]; then continue fi delete_package "${package}" done if [ -n "${test_exclusion}" ]; then # TODO: reimplement test_exclusion with information from the database >&2 echo 'sry, not yet done.' exit 0 fi echo 'Done - mark decisions as final.' # shellcheck disable=SC2016 { # save blacklist into database printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64), `reason` TEXT);\n' git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ tar -Ox 'blacklist' | \ sed -n ' s/^\(\S\+\)\s*#\s*/\1 / T p ' | \ while read -r pkgbase reason; do printf '(from_base64("%s"),from_base64("%s")),\n' \ "$(printf '%s' "${pkgbase}" | base64 -w0)" \ "$(printf '%s' "${reason}" | base64 -w0)" done | \ sed ' 1 i INSERT INTO `bl` (`pkgbase`,`reason`) VALUES $ s/,$/;/ ' printf 'UPDATE `build_assignments`' mysql_join_build_assignments_package_sources printf ' LEFT JOIN `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`' printf ' SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n' printf 'DROP TEMPORARY TABLE `bl`;\n' printf 'COMMIT;\n' # update hashes of repositories in mysql database for repo in ${repo_names}; do printf 'UPDATE `git_repositories`' printf ' SET `git_repositories`.`head`=from_base64("%s")' \ "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')" printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" done # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages_in_repositories`' mysql_join_binary_packages_in_repositories_binary_packages printf ' SET `binary_packages_in_repositories`.`repository`=%s' \ "${repository_ids__any_build_list}" printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \ "${repository_ids__any_to_be_decided}" } | \ mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". mysql_find_build_assignment_loops # Remove the lock file rm -f "${build_list_lock_file}"