summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2018-07-23 15:48:40 +0200
committerErich Eckner <git@eckner.net>2018-07-23 15:48:40 +0200
commit86207d643507dcc1f1a40d7693b1e21c2acdddda (patch)
tree66bef6f965b84109978b6b6a768a0495fdcce77d
parent1536ba90d8bc87f057947cded87497337fb1c5a3 (diff)
downloadbuilder-86207d643507dcc1f1a40d7693b1e21c2acdddda.tar.xz
bin/get-package-updates: use 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`' instead of 'INSERT IGNORE INTO `%s` VALUES ...'
-rwxr-xr-xbin/get-package-updates176
1 files changed, 76 insertions, 100 deletions
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1b3bd68..724e261 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -207,7 +207,13 @@ if ! verbose_flock -s ${block_flag} 8; then
exit
fi
-trap mysql_cleanup EXIT
+cleanup() {
+ mysql_cleanup
+ rm -rf --one-file-system "${tmp_dir:?}"
+}
+tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
+
+trap cleanup EXIT
# shellcheck disable=SC2119
mysql_cleanup
@@ -422,107 +428,81 @@ echo 'apply blacklisting'
# ignore blacklisted packages and dependent packages
# this is the first time when all the information is available and up to date
-# TODO: use 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`' instead of
-# "INSERT IGNORE INTO `%s` VALUES ..."
-
-black_listed=''
-black_listed_new=$(
+touch "${tmp_dir}/black-listed"
+{
+ git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+ tar -Ox 'blacklist' | \
+ sed '
+ s/\s*#.*$//
+ /^\s*$/d
+ '
+ if [ -n "${test_exclusion}" ]; then
+ echo "${test_exclusion}"
+ fi
+ # shellcheck disable=SC2016
{
- {
- git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
- tar -Ox 'blacklist' | \
- sed '
- s/\s*#.*$//
- /^\s*$/d
- '
- if [ -n "${test_exclusion}" ]; then
- echo "${test_exclusion}"
- fi
- } | \
- base64_encode_each
- # shellcheck disable=SC2016
- {
- printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")'
- printf ' FROM `package_sources`'
- printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
- } | \
- mysql_run_query
+ printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+ printf ' FROM `package_sources`'
+ printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
} | \
- sort -u
-)
+ mysql_run_query
+} | \
+ sort -u > \
+ "${tmp_dir}/black-listed-new"
-while [ -n "${black_listed_new}" ]; do
- black_listed=$(
- printf '%s\n' "${black_listed}" "${black_listed_new}" | \
- grep -vxF '' | \
- sort -u
- )
- black_listed_new=$(
- {
- printf '%s\n' "${black_listed}" "${black_listed}"
+while [ -s "${tmp_dir}/black-listed-new" ]; do
+ cat \
+ "${tmp_dir}/black-listed-new" \
+ "${tmp_dir}/black-listed" | \
+ sort -u | \
+ sponge "${tmp_dir}/black-listed"
+ {
+ sed 'p' "${tmp_dir}/black-listed"
- # shellcheck disable=SC2016
- {
- printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
- printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
- printf '%s\n' "${black_listed}" | \
- sort -u | \
- sed '
- s/^/(from_base64("/
- s/$/")),/
- $ s/,$/;/
- '
- printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
- printf ' FROM `package_sources` AS `a_ps`'
- mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
- mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
- mysql_join_binary_packages_dependencies 'a_bp'
- mysql_join_dependencies_dependency_types
- printf ' AND (`dependency_types`.`relevant_for_building`'
- printf ' OR `dependency_types`.`relevant_for_binary_packages`)'
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `bl` (`pkgbase`);\n' \
+ "${tmp_dir}/black-listed"
+ printf 'SELECT `a_ps`.`pkgbase`'
+ printf ' FROM `package_sources` AS `a_ps`'
+ mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
+ mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
+ mysql_join_binary_packages_dependencies 'a_bp'
+ mysql_join_dependencies_dependency_types
+ printf ' AND (`dependency_types`.`relevant_for_building`'
+ printf ' OR `dependency_types`.`relevant_for_binary_packages`)'
+ printf ' WHERE NOT EXISTS ('
+ printf ' SELECT 1'
+ printf ' FROM `install_target_providers`'
printf ' WHERE NOT EXISTS ('
printf ' SELECT 1'
- printf ' FROM `install_target_providers`'
- printf ' WHERE NOT EXISTS ('
- printf ' SELECT 1'
- printf ' FROM `bl`'
- printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
- mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
- mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
- printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
- printf ')'
- printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
- printf ') AND EXISTS ('
- # TODO: This should be corrected at the root: automatic install targets, which are bogus should
- # not be added in the first place - but how do we detect that? -> add exceptions!
- printf ' SELECT 1'
- printf ' FROM `install_target_providers`'
- printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
- printf ');\n'
- } | \
- mysql_run_query | \
- sort -u
+ printf ' FROM `bl`'
+ printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
+ mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
+ mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
+ printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
+ printf ')'
+ printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ') AND EXISTS ('
+ # TODO: This should be corrected at the root: automatic install targets, which are bogus should
+ # not be added in the first place - but how do we detect that? -> add exceptions!
+ printf ' SELECT 1'
+ printf ' FROM `install_target_providers`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ');\n'
} | \
- grep -vxF '' | \
- sort | \
- uniq -u
- )
+ mysql_run_query | \
+ sort -u
+ } | \
+ sort | \
+ uniq -u | \
+ sponge "${tmp_dir}/black-listed-new"
done
-black_listed=$(
- printf '%s\n' "${black_listed}" | \
- while read -r line; do
- printf '%s' \
- "${line}" | \
- base64 -d
- printf '\n'
- done
-)
-
deletion_list_count=$(
- # shellcheck disable=SC2086
- printf '%s\n' ${black_listed} | \
- wc -l
+ wc -l < \
+ "${tmp_dir}/black-listed"
)
if [ "${deletion_list_count}" -gt 1000 ]; then
>&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
@@ -550,14 +530,10 @@ if [ "${deletion_list_count}" -gt 1000 ]; then
fi
rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-echo "${black_listed}" | \
- while read -r package; do
- if [ -z "${package}" ]; then
- continue
- fi
- delete_package "${package}"
- done
-
+while read -r package; do
+ delete_package "${package}"
+done < \
+ "${tmp_dir}/black-listed"
if [ -n "${test_exclusion}" ]; then
# TODO: reimplement test_exclusion with information from the database