summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2019-08-06 09:54:49 +0200
committerErich Eckner <git@eckner.net>2019-08-06 09:54:49 +0200
commit6e4da7ba406c99a2ada801fcf6f1cf2ff71478a9 (patch)
tree474455e213308b911f14aa38c58a752f9483686b
parent79c45fa9281bb48f2904290717264f56676341b8 (diff)
downloadbuilder-6e4da7ba406c99a2ada801fcf6f1cf2ff71478a9.tar.xz
bin/get-package-updates: change layout of blacklist
-rwxr-xr-xbin/get-package-updates263
1 files changed, 90 insertions, 173 deletions
diff --git a/bin/get-package-updates b/bin/get-package-updates
index d159ace..3b7f440 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -40,20 +40,16 @@ usage() {
>&2 echo ' git repository (e.g. ignore -d for it).'
>&2 echo ' Requires -d.'
>&2 echo ' -w|--wait: If necessary, wait for lock blocking.'
- >&2 echo ' -x|--test-exclusion $package:'
- >&2 echo ' Print additionally deleted/excluded packages if'
- >&2 echo ' "$package" would be black listed.'
[ -z "$1" ] && exit 1 || exit "$1"
}
eval set -- "$(
- getopt -o d:hinrwx: \
+ getopt -o d:hinrw \
--long date: \
--long help \
--long ignore-insanity \
--long no-pull \
--long recent-modifications \
- --long test-exclusion: \
--long wait \
-n "$(basename "$0")" -- "$@" || \
echo usage
@@ -64,7 +60,6 @@ date_time=''
ignore_insanity=false
pull=true
recent_modifications=false
-test_exclusion=''
while true
do
@@ -88,15 +83,6 @@ do
-w|--wait)
block_flag=''
;;
- -x|--test-exclusion)
- shift
- if [ -n "${test_exclusion}" ]; then
- >&2 printf 'I already have --test-exclusion=%s and you gave me another one.\n' "${test_exclusion}"
- >&2 printf 'But I can only handle one exclusion test at a time.\n'
- exit 2
- fi
- test_exclusion="$1"
- ;;
--)
shift
break
@@ -134,81 +120,77 @@ fi
# delete_package arch package
# mark $arch/$package for deletion
delete_package() {
- # TODO: Run the command below unconditionally, but with some changes,
- # so we can easily revert.
- if [ -z "${test_exclusion}" ]; then
- # shellcheck disable=SC2016
- query_delete_packages=$(
- printf '`architectures` AS `d_a`'
- printf ' JOIN `architecture_compatibilities` AS `a_c`'
- printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
- printf ' AND `d_a`.`name`=from_base64("%s")' \
- "$(printf '%s' "$1" | base64 -w0)"
- printf ' JOIN `build_assignments`'
- printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
- # "any" references all architectures, but this is not represented
- # in `architecture_compatibilities`: If a package is not buildable
- # for "any", this means literally, that it is not buildable for
- # the _least_ architecture (e.g. it is not generic)
- printf ' OR `d_a`.`name`="any")'
- mysql_join_build_assignments_package_sources
- printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
- "$(printf '%s' "$2" | base64 -w0)"
- mysql_join_build_assignments_binary_packages
- mysql_join_binary_packages_binary_packages_in_repositories
- )
- # shellcheck disable=SC2016
- {
- # packages from the build-list/to-be-decided go straight to the deletion-list
- # this happens in two steps, because we need to create one item per
- # target architecture
- printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
- printf '`package`,'
- printf '`repository`,'
- printf '`last_moved`,'
- printf '`is_to_be_deleted`'
- printf ') SELECT'
- printf ' `binary_packages`.`id`,'
- printf '`d_r`.`id`,'
- printf 'NOW(),'
- printf '1'
- printf ' FROM'
- printf ' %s' "${query_delete_packages}"
- mysql_join_binary_packages_in_repositories_repositories
- # shellcheck disable=SC2154
- printf ' AND `repositories`.`stability` in (%s,%s)' \
- "${repository_stability_ids__unbuilt}" \
- "${repository_stability_ids__virtual}"
- mysql_join_build_assignments_architectures '' 'ba_a'
- printf ' JOIN `repositories` AS `d_r`'
- printf ' ON ('
- # arch-specific build_assignments must match exactly
- printf '`d_r`.`architecture`=`ba_a`.`id`'
- # "any" build_assignments build for all architectures
- printf ' OR `ba_a`.`name`="any"'
- printf ')'
- # shellcheck disable=SC2154
- printf ' AND `d_r`.`stability`=%s;\n' \
- "${repository_stability_ids__forbidden}"
+ # shellcheck disable=SC2016
+ query_delete_packages=$(
+ printf '`architectures` AS `d_a`'
+ printf ' JOIN `architecture_compatibilities` AS `a_c`'
+ printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
+ printf ' AND `d_a`.`name`=from_base64("%s")' \
+ "$(printf '%s' "$1" | base64 -w0)"
+ printf ' JOIN `build_assignments`'
+ printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
+ # "any" references all architectures, but this is not represented
+ # in `architecture_compatibilities`: If a package is not buildable
+ # for "any", this means literally, that it is not buildable for
+ # the _least_ architecture (e.g. it is not generic)
+ printf ' OR `d_a`.`name`="any")'
+ mysql_join_build_assignments_package_sources
+ printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
+ "$(printf '%s' "$2" | base64 -w0)"
+ mysql_join_build_assignments_binary_packages
+ mysql_join_binary_packages_binary_packages_in_repositories
+ )
+ # shellcheck disable=SC2016
+ {
+ # packages from the build-list/to-be-decided go straight to the deletion-list
+ # this happens in two steps, because we need to create one item per
+ # target architecture
+ printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
+ printf '`package`,'
+ printf '`repository`,'
+ printf '`last_moved`,'
+ printf '`is_to_be_deleted`'
+ printf ') SELECT'
+ printf ' `binary_packages`.`id`,'
+ printf '`d_r`.`id`,'
+ printf 'NOW(),'
+ printf '1'
+ printf ' FROM'
+ printf ' %s' "${query_delete_packages}"
+ mysql_join_binary_packages_in_repositories_repositories
+ # shellcheck disable=SC2154
+ printf ' AND `repositories`.`stability` in (%s,%s)' \
+ "${repository_stability_ids__unbuilt}" \
+ "${repository_stability_ids__virtual}"
+ mysql_join_build_assignments_architectures '' 'ba_a'
+ printf ' JOIN `repositories` AS `d_r`'
+ printf ' ON ('
+ # arch-specific build_assignments must match exactly
+ printf '`d_r`.`architecture`=`ba_a`.`id`'
+ # "any" build_assignments build for all architectures
+ printf ' OR `ba_a`.`name`="any"'
+ printf ')'
+ # shellcheck disable=SC2154
+ printf ' AND `d_r`.`stability`=%s;\n' \
+ "${repository_stability_ids__forbidden}"
- printf 'COMMIT;\n'
+ printf 'COMMIT;\n'
- printf 'DELETE `binary_packages_in_repositories`'
- printf ' FROM %s' "${query_delete_packages}"
- mysql_join_binary_packages_in_repositories_repositories
- # shellcheck disable=SC2154
- printf ' AND `repositories`.`stability` in (%s,%s);\n' \
- "${repository_stability_ids__unbuilt}" \
- "${repository_stability_ids__virtual}"
+ printf 'DELETE `binary_packages_in_repositories`'
+ printf ' FROM %s' "${query_delete_packages}"
+ mysql_join_binary_packages_in_repositories_repositories
+ # shellcheck disable=SC2154
+ printf ' AND `repositories`.`stability` in (%s,%s);\n' \
+ "${repository_stability_ids__unbuilt}" \
+ "${repository_stability_ids__virtual}"
- printf 'COMMIT;\n'
+ printf 'COMMIT;\n'
- # other packages are marked as `is_to_be_deleted`
- printf 'UPDATE %s' "${query_delete_packages}"
- printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
- } | \
- mysql_run_query
- fi
+ # other packages are marked as `is_to_be_deleted`
+ printf 'UPDATE %s' "${query_delete_packages}"
+ printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
+ } | \
+ mysql_run_query
}
something_new=false
@@ -330,12 +312,11 @@ echo 'Check modified packages from the last update, and put them to the build li
mysql_join_upstream_repositories_git_repositories
printf ' WHERE `package_sources`.`pkgbase` IN ('
# shellcheck disable=SC2154
- git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \
- sed -n '
- s/^-\([^-]\)/\1/
- T
- s/\s*#.*$//
- p
+ git -C "${repo_paths__archlinux32}" diff --no-renames --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+ sed '
+ s@^D\tblacklist/[^/]\+/[^/]\+/@@
+ t
+ d
' | \
base64_encode_each | \
sed '
@@ -444,6 +425,7 @@ echo 'Check modified packages from the last update, and put them to the build li
fi
fi
if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
+
# shellcheck disable=SC2016
{
# delete old build assignment and associated binary packages
@@ -476,6 +458,7 @@ echo 'Check modified packages from the last update, and put them to the build li
)"
} | \
mysql_run_query
+ printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages"
# shellcheck disable=SC2154
printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2
mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
@@ -487,89 +470,23 @@ echo 'Check modified packages from the last update, and put them to the build li
exit 1
done
-echo 'apply blacklisting'
-echo ' - generate list of packages'
-# ignore blacklisted packages and dependent packages
-# this is the first time when all the information is available and up to date
-
-{
+if [ -s "${tmp_dir}/modified-packages" ]; then
+ echo 'Delete black-listed packages.'
+ sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages"
+ # extract black-listed packages
git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
- tar -Ox 'blacklist' | \
+ tar -t 'blacklist' | \
sed '
- s/\s*#.*$//
- /^\S\+\s\+\S\+$/!d
- '
- if [ -n "${test_exclusion}" ]; then
- echo "${test_exclusion}"
- fi
- # shellcheck disable=SC2016
- {
- printf 'SELECT DISTINCT "any",`package_sources`.`pkgbase`'
- printf ' FROM `package_sources`'
- printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
- } | \
- mysql_run_query
-} | \
- tr ' ' '\t' | \
- sort -u > \
- "${tmp_dir}/black-listed"
-
-# shellcheck disable=SC2016
-{
- printf 'CREATE TEMPORARY TABLE `blacklist` ('
- printf '`arch` VARCHAR(16),'
- printf '`pkgbase` VARCHAR(64),'
- printf 'UNIQUE KEY `content`(`arch`,`pkgbase`)'
- printf ');\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `blacklist` (`arch`,`pkgbase`);\n' \
- "${tmp_dir}/black-listed"
- printf 'CALL `blacklist_packages`;\n'
-} | \
- mysql_run_query | \
- sort -u | \
- sponge "${tmp_dir}/black-listed"
-
-deletion_list_count=$(
- grep -cv '^i486\s' < \
- "${tmp_dir}/black-listed"
-)
-if [ "${deletion_list_count}" -gt 2000 ]; then
- >&2 printf 'There are %s > 2000 packages on the deletion-list. This does not seem right.\n' \
- "${deletion_list_count}"
- if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
- # shellcheck disable=SC2119
- {
- printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
- for repo_name in ${repo_names}; do
- eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"'
- eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"'
- # shellcheck disable=SC2154
- if [ "${old_revision}" != "${new_revision}" ]; then
- printf '%s: %s -> %s\n' \
- "${repo_name}" \
- "${old_revision}" \
- "${new_revision}"
- fi
- done
- } | \
- tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
- irc_say
- fi
- exit 3
-fi
-rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-
-echo ' - delete packages'
-
-while read -r arch package; do
- delete_package "${arch}" "${package}"
-done < \
- "${tmp_dir}/black-listed"
-
-if [ -n "${test_exclusion}" ]; then
- # TODO: reimplement test_exclusion with information from the database
- >&2 echo 'sry, not yet done.'
- exit 0
+ s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
+ t
+ d
+ ' | \
+ sort -u | \
+ sort -k2,2 | \
+ join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - | \
+ while read -r arch pkgbase; do
+ delete_package "${arch}" "${pkgbase}"
+ done
fi
echo 'Done - mark decisions as final.'