From 6e4da7ba406c99a2ada801fcf6f1cf2ff71478a9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 6 Aug 2019 09:54:49 +0200 Subject: bin/get-package-updates: change layout of blacklist --- bin/get-package-updates | 263 +++++++++++++++++------------------------------- 1 file changed, 90 insertions(+), 173 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index d159ace..3b7f440 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -40,20 +40,16 @@ usage() { >&2 echo ' git repository (e.g. ignore -d for it).' >&2 echo ' Requires -d.' >&2 echo ' -w|--wait: If necessary, wait for lock blocking.' - >&2 echo ' -x|--test-exclusion $package:' - >&2 echo ' Print additionally deleted/excluded packages if' - >&2 echo ' "$package" would be black listed.' [ -z "$1" ] && exit 1 || exit "$1" } eval set -- "$( - getopt -o d:hinrwx: \ + getopt -o d:hinrw \ --long date: \ --long help \ --long ignore-insanity \ --long no-pull \ --long recent-modifications \ - --long test-exclusion: \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage @@ -64,7 +60,6 @@ date_time='' ignore_insanity=false pull=true recent_modifications=false -test_exclusion='' while true do @@ -88,15 +83,6 @@ do -w|--wait) block_flag='' ;; - -x|--test-exclusion) - shift - if [ -n "${test_exclusion}" ]; then - >&2 printf 'I already have --test-exclusion=%s and you gave me another one.\n' "${test_exclusion}" - >&2 printf 'But I can only handle one exclusion test at a time.\n' - exit 2 - fi - test_exclusion="$1" - ;; --) shift break @@ -134,81 +120,77 @@ fi # delete_package arch package # mark $arch/$package for deletion delete_package() { - # TODO: Run the command below unconditionally, but with some changes, - # so we can easily revert. - if [ -z "${test_exclusion}" ]; then - # shellcheck disable=SC2016 - query_delete_packages=$( - printf '`architectures` AS `d_a`' - printf ' JOIN `architecture_compatibilities` AS `a_c`' - printf ' ON `a_c`.`runs_on`=`d_a`.`id`' - printf ' AND `d_a`.`name`=from_base64("%s")' \ - "$(printf '%s' "$1" | base64 -w0)" - printf ' JOIN `build_assignments`' - printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`' - # "any" references all architectures, but this is not represented - # in `architecture_compatibilities`: If a package is not buildable - # for "any", this means literally, that it is not buildable for - # the _least_ architecture (e.g. it is not generic) - printf ' OR `d_a`.`name`="any")' - mysql_join_build_assignments_package_sources - printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "$2" | base64 -w0)" - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_binary_packages_in_repositories - ) - # shellcheck disable=SC2016 - { - # packages from the build-list/to-be-decided go straight to the deletion-list - # this happens in two steps, because we need to create one item per - # target architecture - printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (' - printf '`package`,' - printf '`repository`,' - printf '`last_moved`,' - printf '`is_to_be_deleted`' - printf ') SELECT' - printf ' `binary_packages`.`id`,' - printf '`d_r`.`id`,' - printf 'NOW(),' - printf '1' - printf ' FROM' - printf ' %s' "${query_delete_packages}" - mysql_join_binary_packages_in_repositories_repositories - # shellcheck disable=SC2154 - printf ' AND `repositories`.`stability` in (%s,%s)' \ - "${repository_stability_ids__unbuilt}" \ - "${repository_stability_ids__virtual}" - mysql_join_build_assignments_architectures '' 'ba_a' - printf ' JOIN `repositories` AS `d_r`' - printf ' ON (' - # arch-specific build_assignments must match exactly - printf '`d_r`.`architecture`=`ba_a`.`id`' - # "any" build_assignments build for all architectures - printf ' OR `ba_a`.`name`="any"' - printf ')' - # shellcheck disable=SC2154 - printf ' AND `d_r`.`stability`=%s;\n' \ - "${repository_stability_ids__forbidden}" + # shellcheck disable=SC2016 + query_delete_packages=$( + printf '`architectures` AS `d_a`' + printf ' JOIN `architecture_compatibilities` AS `a_c`' + printf ' ON `a_c`.`runs_on`=`d_a`.`id`' + printf ' AND `d_a`.`name`=from_base64("%s")' \ + "$(printf '%s' "$1" | base64 -w0)" + printf ' JOIN `build_assignments`' + printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`' + # "any" references all architectures, but this is not represented + # in `architecture_compatibilities`: If a package is not buildable + # for "any", this means literally, that it is not buildable for + # the _least_ architecture (e.g. it is not generic) + printf ' OR `d_a`.`name`="any")' + mysql_join_build_assignments_package_sources + printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \ + "$(printf '%s' "$2" | base64 -w0)" + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_binary_packages_in_repositories + ) + # shellcheck disable=SC2016 + { + # packages from the build-list/to-be-decided go straight to the deletion-list + # this happens in two steps, because we need to create one item per + # target architecture + printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (' + printf '`package`,' + printf '`repository`,' + printf '`last_moved`,' + printf '`is_to_be_deleted`' + printf ') SELECT' + printf ' `binary_packages`.`id`,' + printf '`d_r`.`id`,' + printf 'NOW(),' + printf '1' + printf ' FROM' + printf ' %s' "${query_delete_packages}" + mysql_join_binary_packages_in_repositories_repositories + # shellcheck disable=SC2154 + printf ' AND `repositories`.`stability` in (%s,%s)' \ + "${repository_stability_ids__unbuilt}" \ + "${repository_stability_ids__virtual}" + mysql_join_build_assignments_architectures '' 'ba_a' + printf ' JOIN `repositories` AS `d_r`' + printf ' ON (' + # arch-specific build_assignments must match exactly + printf '`d_r`.`architecture`=`ba_a`.`id`' + # "any" build_assignments build for all architectures + printf ' OR `ba_a`.`name`="any"' + printf ')' + # shellcheck disable=SC2154 + printf ' AND `d_r`.`stability`=%s;\n' \ + "${repository_stability_ids__forbidden}" - printf 'COMMIT;\n' + printf 'COMMIT;\n' - printf 'DELETE `binary_packages_in_repositories`' - printf ' FROM %s' "${query_delete_packages}" - mysql_join_binary_packages_in_repositories_repositories - # shellcheck disable=SC2154 - printf ' AND `repositories`.`stability` in (%s,%s);\n' \ - "${repository_stability_ids__unbuilt}" \ - "${repository_stability_ids__virtual}" + printf 'DELETE `binary_packages_in_repositories`' + printf ' FROM %s' "${query_delete_packages}" + mysql_join_binary_packages_in_repositories_repositories + # shellcheck disable=SC2154 + printf ' AND `repositories`.`stability` in (%s,%s);\n' \ + "${repository_stability_ids__unbuilt}" \ + "${repository_stability_ids__virtual}" - printf 'COMMIT;\n' + printf 'COMMIT;\n' - # other packages are marked as `is_to_be_deleted` - printf 'UPDATE %s' "${query_delete_packages}" - printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n' - } | \ - mysql_run_query - fi + # other packages are marked as `is_to_be_deleted` + printf 'UPDATE %s' "${query_delete_packages}" + printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n' + } | \ + mysql_run_query } something_new=false @@ -330,12 +312,11 @@ echo 'Check modified packages from the last update, and put them to the build li mysql_join_upstream_repositories_git_repositories printf ' WHERE `package_sources`.`pkgbase` IN (' # shellcheck disable=SC2154 - git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \ - sed -n ' - s/^-\([^-]\)/\1/ - T - s/\s*#.*$// - p + git -C "${repo_paths__archlinux32}" diff --no-renames --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ + sed ' + s@^D\tblacklist/[^/]\+/[^/]\+/@@ + t + d ' | \ base64_encode_each | \ sed ' @@ -444,6 +425,7 @@ echo 'Check modified packages from the last update, and put them to the build li fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then + # shellcheck disable=SC2016 { # delete old build assignment and associated binary packages @@ -476,6 +458,7 @@ echo 'Check modified packages from the last update, and put them to the build li )" } | \ mysql_run_query + printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages" # shellcheck disable=SC2154 printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2 mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" @@ -487,89 +470,23 @@ echo 'Check modified packages from the last update, and put them to the build li exit 1 done -echo 'apply blacklisting' -echo ' - generate list of packages' -# ignore blacklisted packages and dependent packages -# this is the first time when all the information is available and up to date - -{ +if [ -s "${tmp_dir}/modified-packages" ]; then + echo 'Delete black-listed packages.' + sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages" + # extract black-listed packages git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - tar -Ox 'blacklist' | \ + tar -t 'blacklist' | \ sed ' - s/\s*#.*$// - /^\S\+\s\+\S\+$/!d - ' - if [ -n "${test_exclusion}" ]; then - echo "${test_exclusion}" - fi - # shellcheck disable=SC2016 - { - printf 'SELECT DISTINCT "any",`package_sources`.`pkgbase`' - printf ' FROM `package_sources`' - printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' - } | \ - mysql_run_query -} | \ - tr ' ' '\t' | \ - sort -u > \ - "${tmp_dir}/black-listed" - -# shellcheck disable=SC2016 -{ - printf 'CREATE TEMPORARY TABLE `blacklist` (' - printf '`arch` VARCHAR(16),' - printf '`pkgbase` VARCHAR(64),' - printf 'UNIQUE KEY `content`(`arch`,`pkgbase`)' - printf ');\n' - printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `blacklist` (`arch`,`pkgbase`);\n' \ - "${tmp_dir}/black-listed" - printf 'CALL `blacklist_packages`;\n' -} | \ - mysql_run_query | \ - sort -u | \ - sponge "${tmp_dir}/black-listed" - -deletion_list_count=$( - grep -cv '^i486\s' < \ - "${tmp_dir}/black-listed" -) -if [ "${deletion_list_count}" -gt 2000 ]; then - >&2 printf 'There are %s > 2000 packages on the deletion-list. This does not seem right.\n' \ - "${deletion_list_count}" - if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then - # shellcheck disable=SC2119 - { - printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" - for repo_name in ${repo_names}; do - eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"' - eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"' - # shellcheck disable=SC2154 - if [ "${old_revision}" != "${new_revision}" ]; then - printf '%s: %s -> %s\n' \ - "${repo_name}" \ - "${old_revision}" \ - "${new_revision}" - fi - done - } | \ - tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - irc_say - fi - exit 3 -fi -rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" - -echo ' - delete packages' - -while read -r arch package; do - delete_package "${arch}" "${package}" -done < \ - "${tmp_dir}/black-listed" - -if [ -n "${test_exclusion}" ]; then - # TODO: reimplement test_exclusion with information from the database - >&2 echo 'sry, not yet done.' - exit 0 + s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@ + t + d + ' | \ + sort -u | \ + sort -k2,2 | \ + join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - | \ + while read -r arch pkgbase; do + delete_package "${arch}" "${pkgbase}" + done fi echo 'Done - mark decisions as final.' -- cgit v1.2.3