From 6f658664491b2dfdcc9a3abcbd27221f739d4851 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 25 Jan 2018 10:10:48 +0100 Subject: bin/get-package-updates: keep track of dependency loops in database --- bin/get-package-updates | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 801aadd..e01f3dd 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -556,6 +556,9 @@ else rm "${work_dir}/tsort.error" fi +# update loop list in database +mysql_find_build_assignment_loops + deletion_list_count=$( wc -l < \ "${work_dir}/deletion-list.new" -- cgit v1.2.3-54-g00ecf From a20c7b23be889cbfaef8983e6bc14db80b369530 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 26 Jan 2018 16:25:08 +0100 Subject: bin/get-package-updates: remove done TODO --- bin/get-package-updates | 2 -- 1 file changed, 2 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index e01f3dd..320cec1 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -9,8 +9,6 @@ # TODO: Find out, why sometimes package updates are missed. -# TODO: mark loops in mysql database - # TODO: test_exclusion does not yet cooperate with the database # shellcheck disable=SC2016 -- cgit v1.2.3-54-g00ecf From ab09c0383c10374ddcc587f60f36a88daabe9b42 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 28 Jan 2018 20:49:10 +0100 Subject: bin/get-package-updates: talk to irc channel if too many packages will be blacklisted --- bin/get-package-updates | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 320cec1..ce3de0a 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -564,8 +564,24 @@ deletion_list_count=$( if [ "${deletion_list_count}" -gt 1000 ]; then >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ "${deletion_list_count}" + if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then + { + printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" + for repo_name in ${repo_names}; do + if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then + printf '%s: %s -> %s\n' \ + "${repo_name}" \ + "$(cat "${work_dir}/${repo_name}.revision")" \ + "$(cat "${work_dir}/${repo_name}.revision.new")" + fi + done + } | \ + tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ + sponge "${irc_dir}/#archlinux-ports/in" + fi exit 3 fi +rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" # Move the .new-files to the actual files -- cgit v1.2.3-54-g00ecf From b2cf8dc8a879c626453879dc4d2f175d68dbfd2f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 29 Jan 2018 13:42:01 +0100 Subject: bin/get-package-updates: add todo --- bin/get-package-updates | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index ce3de0a..d8ecd64 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -84,6 +84,8 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi +# TODO: Should new deletion-list packages be recorded in the database, too? + # delete_package package # mark $package for deletion delete_package() { -- cgit v1.2.3-54-g00ecf From 6c9b41e2ce003a1920daa2ca7b43365d99fa29dc Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 30 Jan 2018 10:22:30 +0100 Subject: bin/get-package-updates: fix typo in comment --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index d8ecd64..069277a 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -289,7 +289,7 @@ done | \ # shellcheck disable=SC2016 { # delete old build assignment and associated binary packages - # which are not yet build or on the deletion list + # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`' printf ' FROM `binary_packages`' printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ -- cgit v1.2.3-54-g00ecf From f06d489ccfae3607d0b5bf442b38fe1ae81b62d6 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 31 Jan 2018 13:44:37 +0100 Subject: bin/get-package-updates: read lib32-* packages from database --- bin/get-package-updates | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 069277a..55efee4 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -343,9 +343,14 @@ black_listed_new=$( if [ -n "${test_exclusion}" ]; then echo "${test_exclusion}" fi - find "${work_dir}/package-infos" -maxdepth 1 -name 'lib32-*' -printf '%f\n' | \ - sed 's|\(\.[^.]\+\)\{4\}$||' | \ - uniq + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT `package_sources`.`pkgbase`' + printf ' FROM `package_sources`' + printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' } | \ sort -u ) -- cgit v1.2.3-54-g00ecf From 8d3d7cabdad3b873722064632968f3ce916b3e99 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 11:51:58 +0100 Subject: add many mysql-TODOs --- bin/build-master-status | 2 ++ bin/build-master-status-from-mysql | 2 ++ bin/build-packages | 2 ++ bin/calculate-dependent-packages | 4 ++++ bin/check-bugtracker | 2 ++ bin/cleanup | 2 ++ bin/db-update | 6 ++---- bin/get-package-updates | 2 ++ bin/interpret-mail | 2 ++ bin/prioritize-build-list | 2 ++ bin/return-assignment | 2 ++ bin/sanity-check | 2 ++ bin/seed-build-list | 2 ++ bin/show-dependencies | 2 ++ bin/why-dont-you | 2 ++ 15 files changed, 32 insertions(+), 4 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status b/bin/build-master-status index 1672789..b117e4e 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: replace by build-master-status-from-mysql + usage() { >&2 echo '' >&2 echo 'build-master-status: report about status of build master' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 44e07c1..16353dd 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: add all (necessary) features from build-master-status + tmp_dir=$(mktemp -d 'tmp.build-master-status-from-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT diff --git a/bin/build-packages b/bin/build-packages index 81afa7f..94292c7 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -10,6 +10,8 @@ # TODO: report back in regular intervals, so the build master can # abort if the package is outdated and/or someone else finished it. +# TODO: build other "architectures", too (pentium4, i486) + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/calculate-dependent-packages b/bin/calculate-dependent-packages index bfb87a1..3208e61 100755 --- a/bin/calculate-dependent-packages +++ b/bin/calculate-dependent-packages @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: save information in database + +# TODO: read information from database + if [ -s "${work_dir}/build-master-sanity" ]; then >&2 echo 'Build master is not sane.' exit diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 46e78db..6b304e3 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -8,6 +8,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + tmp_dir=$(mktemp -d 'tmp.check-bugtracker.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT diff --git a/bin/cleanup b/bin/cleanup index 78daece..15b6fc9 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: clean database, too + # we only clean if run interactive or if no one is logged in if ! tty -s && \ [ -n "$(users)" ]; then diff --git a/bin/db-update b/bin/db-update index fe45f41..35fd223 100755 --- a/bin/db-update +++ b/bin/db-update @@ -20,11 +20,9 @@ # track packages in stable, too # know _exactly_ in which repository the packages are -# TODO: fully integrate database (in other scripts too) - e.g. rely on -# info in db instead of state-files +# TODO: save information in database -# TODO: add possibility to manually clone a (binary) package (e.g. -# for build-support) - this should also be possible via email +# TODO: read information from database # shellcheck disable=SC2039 # shellcheck source=conf/default.conf diff --git a/bin/get-package-updates b/bin/get-package-updates index 55efee4..91fb6be 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -11,6 +11,8 @@ # TODO: test_exclusion does not yet cooperate with the database +# TODO: read information from database + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/interpret-mail b/bin/interpret-mail index d16c07f..03336c2 100755 --- a/bin/interpret-mail +++ b/bin/interpret-mail @@ -5,6 +5,8 @@ # TODO: enable email interface to delete packages +# TODO: read information from database (?) + if [ $# -ne 0 ]; then >&2 echo '' >&2 echo 'usage: interpret-mail' diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index 06d4c06..5655223 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -3,6 +3,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: save information in database + if [ $# -ne 1 ]; then echo 'Usage: prioritize-build-list pkg-regex' echo ' moves packages matching pkg-regex to front of build list' diff --git a/bin/return-assignment b/bin/return-assignment index 9de8add..42393ac 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -16,6 +16,8 @@ # TODO: sign database +# TODO: read information from database + # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" diff --git a/bin/sanity-check b/bin/sanity-check index 8736d47..e31a5ba 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + usage() { >&2 echo '' >&2 echo 'sanity-check [options] [checks]: check sanity of build master' diff --git a/bin/seed-build-list b/bin/seed-build-list index cd3152d..08f6a42 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -5,6 +5,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + # shellcheck disable=SC2016 usage() { >&2 echo '' diff --git a/bin/show-dependencies b/bin/show-dependencies index efdf745..ee7db19 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -3,6 +3,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + # Create a lock file. if [ $# -eq 0 ]; then diff --git a/bin/why-dont-you b/bin/why-dont-you index 2e3ea86..830048b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -7,6 +7,8 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: read information from database + action="$1" shift -- cgit v1.2.3-54-g00ecf From 879c73de4704e8531e0434316de09b184f5a43dc Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 2 Feb 2018 10:44:52 +0100 Subject: bin/get-package-updates: update todo --- bin/get-package-updates | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 91fb6be..fc66ec6 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -86,7 +86,8 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# TODO: Should new deletion-list packages be recorded in the database, too? +# TODO: New deletion-list packages should be recorded in the database, +# too - including all binary information (especially dependencies!) # delete_package package # mark $package for deletion -- cgit v1.2.3-54-g00ecf From ac8df27bd6c262f917207a1959f26dff74d52aa0 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Feb 2018 09:00:25 +0100 Subject: bin/get-package-updates: style --- bin/get-package-updates | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index fc66ec6..1ad5ce4 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -474,8 +474,10 @@ while [ -n "${black_listed_new}" ]; do done echo "${black_listed}" | \ while read -r package; do - [ -n "${package}" ] && \ - delete_package "${package}" + if [ -z "${package}" ]; then + continue + fi + delete_package "${package}" done sort -u "${work_dir}/deletion-list.new" | \ -- cgit v1.2.3-54-g00ecf From 85f10d9ce376f8c921abd6f0266d5da62bb7c791 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Feb 2018 09:12:30 +0100 Subject: bin/get-package-updates: clarify todo --- bin/get-package-updates | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 1ad5ce4..ffbd390 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -86,8 +86,12 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# TODO: New deletion-list packages should be recorded in the database, -# too - including all binary information (especially dependencies!) +# TODO: How should new deletion-list packages be handled? +# - packages deleted upstream should be marked as to-be-deleted if +# existent and otherwise be ignored +# - packages deleted due to black listing should be marked as black +# listed - and thus as to-be-deleted, but they should remain in the +# database after deletion! # delete_package package # mark $package for deletion -- cgit v1.2.3-54-g00ecf From 8aca660449fde7a425e8d7b0af49955ec90476fc Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 6 Feb 2018 09:11:50 +0100 Subject: bin/get-package-updates: add TODO --- bin/get-package-updates | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index ffbd390..7b394c0 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -13,6 +13,9 @@ # TODO: read information from database +# TODO: correctly handle if pkgbase of a split package is renamed, e.g.: +# $a -> ($a,$b) ==> $b -> ($a,$b) + # shellcheck disable=SC2016 usage() { >&2 echo '' -- cgit v1.2.3-54-g00ecf From 668b94e67649d45d3bd7dacf075dab14428ce24b Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 6 Feb 2018 10:35:33 +0100 Subject: bin/get-package-updates: use new column `is_to_be_deleted` in `binary_packages` --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 7b394c0..f451991 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,7 +110,7 @@ delete_package() { 'build_assignments' 'build_assignments' 'build_assignment' printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ 'package_sources' 'package_sources' 'build_assignments' 'package_source' - printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' + printf ' SET `binary_packages`.`is_to_be_deleted`=1' printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" -- cgit v1.2.3-54-g00ecf From 86823d203d23536386d7c01356e2a5a6df810adf Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 6 Feb 2018 13:48:56 +0100 Subject: bin/get-package-updates: build-list packages should be "deleted" immediately --- bin/get-package-updates | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index f451991..180e0ae 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -104,15 +104,26 @@ delete_package() { sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" # shellcheck disable=SC2016 { + # packages from the build-list go straight to the deletion-list printf 'UPDATE `binary_packages`' printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ 'repositories' 'repositories' 'repository' \ 'build_assignments' 'build_assignments' 'build_assignment' printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ 'package_sources' 'package_sources' 'build_assignments' 'package_source' - printf ' SET `binary_packages`.`is_to_be_deleted`=1' + printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' printf ' WHERE `repositories`.`name`="build-list"' - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");' \ + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ + "$(printf '%s' "$1" | base64 -w0)" + # other packages are marked as `is_to_be_deleted` + printf 'UPDATE `binary_packages`' + printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ + 'repositories' 'repositories' 'repository' \ + 'build_assignments' 'build_assignments' 'build_assignment' + printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ + 'package_sources' 'package_sources' 'build_assignments' 'package_source' + printf ' SET `binary_packages`.`is_to_be_deleted`=1' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ ${mysql_command} -- cgit v1.2.3-54-g00ecf From fd4fbd34ec608f22394a6367ced9bdcf3b13ce6a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Feb 2018 10:46:37 +0100 Subject: bin/get-package-updates: removing a package from the deletion list should remove is_to_be_deleted markers, too --- bin/get-package-updates | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 180e0ae..8c758c5 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -323,6 +323,18 @@ done | \ base64 -w0 )" printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n' + # remove is-to-be-deleted marker from old binary packages + printf 'UPDATE `binary_packages`' + printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ + 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ + 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ + 'repositories' 'repositories' 'binary_packages' 'repository' + printf ' SET `is_to_be_deleted`=0' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")\;' \ + "$( + printf '%s' "${package}" | \ + base64 -w0 + )" } | \ ${mysql_command} mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" -- cgit v1.2.3-54-g00ecf From 5c5b3b2ef97347ba70458ea02989e838b5c76393 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Feb 2018 11:31:14 +0100 Subject: bin/get-package-updates: fix typo in mysql query --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 8c758c5..e57d72b 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -330,7 +330,7 @@ done | \ 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ 'repositories' 'repositories' 'binary_packages' 'repository' printf ' SET `is_to_be_deleted`=0' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")\;' \ + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$( printf '%s' "${package}" | \ base64 -w0 -- cgit v1.2.3-54-g00ecf From 54f1d508044492fdd4410defbc94963bd0608d63 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Feb 2018 14:10:07 +0100 Subject: bin/get-package-updates: take dependency information for black listing from database --- bin/get-package-updates | 226 +++++++++++++++++++----------------------------- 1 file changed, 87 insertions(+), 139 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index e57d72b..4fcefd9 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -367,18 +367,21 @@ echo 'apply blacklisting' black_listed='' black_listed_new=$( { - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \ - tar -Ox 'blacklist' | \ - sed ' - s/\s*#.*$// - /^\s*$/d - ' - if [ -n "${test_exclusion}" ]; then - echo "${test_exclusion}" - fi + { + git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \ + tar -Ox 'blacklist' | \ + sed ' + s/\s*#.*$// + /^\s*$/d + ' + if [ -n "${test_exclusion}" ]; then + echo "${test_exclusion}" + fi + } | \ + base64_encode_each # shellcheck disable=SC2016 { - printf 'SELECT DISTINCT `package_sources`.`pkgbase`' + printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")' printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ @@ -388,120 +391,91 @@ black_listed_new=$( sort -u ) -find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' -printf '%f\n' | \ - grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.builds$' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1| - ' | \ - sort -u | \ - sort -k4,4 > \ - "${tmp_dir}/packages" - -{ - uniq -f3 -u "${tmp_dir}/packages" | \ - awk '{print $4 " " $4 "." $1 "." $2 "." $3}' - uniq -f3 -D "${tmp_dir}/packages" | \ - uniq -f3 --group=append | \ - while read -r rev mod_rev repo pkg; do - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - # shellcheck disable=SC2031 - printf '%s %s.%s.%s.%s\n' \ - "${opkg}" \ - "${opkg}" \ - "$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} | \ - sort -u | \ - find_newest_of_git_revisions - )" \ - "$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} | \ - sort -u | \ - find_newest_of_git_revisions - )" \ - "${orepo}" | \ - grep '\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+$' - revs='' - mod_revs='' - continue - fi - revs=$( - printf '%s\n' \ - ${revs} \ - "${rev}" - ) - mod_revs=$( - printf '%s\n' \ - ${mod_revs} \ - "${mod_rev}" - ) - opkg="${pkg}" - orepo="${repo}" - done -} | \ - sort -k1,1 > \ - "${tmp_dir}/newest-revisions" - -sed ' - s|^\S\+ |'"${work_dir}"'/package-infos/| - s|$|.builds| -' "${tmp_dir}/newest-revisions" | \ - xargs -r cat | \ - sort | \ - uniq -c > \ - "${work_dir}/built-packages" - -sed ' - s|^\S\+ |'"${work_dir}"'/package-infos/| - s|$|.build-depends| -' "${tmp_dir}/newest-revisions" | \ - xargs -r grep -HF '' | \ - sed ' - s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2| - s|^.*/|| - ' | \ - sort -k2,2 > \ - "${work_dir}/newest-dependencies" - while [ -n "${black_listed_new}" ]; do black_listed=$( - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} ${black_listed_new} | \ + printf '%s\n' "${black_listed}" "${black_listed_new}" | \ + grep -vxF '' | \ sort -u ) black_listed_new=$( { - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} | \ - sort -k1,1 | \ - join -j1 -o 2.2 - "${tmp_dir}/newest-revisions" | \ - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' | \ - xargs -r cat | \ - sort | \ - uniq -c - cat "${work_dir}/built-packages" + printf '%s\n' "${black_listed}" "${black_listed}" + + # shellcheck disable=SC2016 + { + printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' + printf 'INSERT INTO `bl` (`pkgbase`) VALUES ' + printf '(from_base64("%s")),\n' "${black_listed}" | \ + sort -u | \ + sed ' + $ s/,$/;/ + ' + printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' + printf ' FROM `package_sources` AS `a_ps`' + printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`' + printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`' + printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`' + printf ' WHERE NOT EXISTS (' + printf ' SELECT *' + printf ' FROM `install_target_providers`' + printf ' WHERE NOT EXISTS (' + printf ' SELECT *' + printf ' FROM `bl`' + printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' + printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`' + printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`' + printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' + printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ')' + printf ');\n' + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + sort -u } | \ - sort | \ - uniq -d | \ - awk '{print $2}' | \ - sort -k1,1 | \ - join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \ - sort -u - ) - black_listed_new=$( - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \ + grep -vxF '' | \ sort | \ uniq -u ) done + +black_listed=$( + printf '%s\n' "${black_listed}" | \ + while read -r line; do + printf '%s' \ + "${line}" | \ + base64 -d + printf '\n' + done +) + +deletion_list_count=$( + # shellcheck disable=SC2086 + printf '%s\n' ${black_listed} | \ + wc -l +) +if [ "${deletion_list_count}" -gt 1000 ]; then + >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ + "${deletion_list_count}" + if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then + { + printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" + for repo_name in ${repo_names}; do + if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then + printf '%s: %s -> %s\n' \ + "${repo_name}" \ + "$(cat "${work_dir}/${repo_name}.revision")" \ + "$(cat "${work_dir}/${repo_name}.revision.new")" + fi + done + } | \ + tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ + sponge "${irc_dir}/#archlinux-ports/in" + fi + exit 3 +fi +rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" + echo "${black_listed}" | \ while read -r package; do if [ -z "${package}" ]; then @@ -599,32 +573,6 @@ fi # update loop list in database mysql_find_build_assignment_loops -deletion_list_count=$( - wc -l < \ - "${work_dir}/deletion-list.new" -) -if [ "${deletion_list_count}" -gt 1000 ]; then - >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ - "${deletion_list_count}" - if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then - { - printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" - for repo_name in ${repo_names}; do - if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then - printf '%s: %s -> %s\n' \ - "${repo_name}" \ - "$(cat "${work_dir}/${repo_name}.revision")" \ - "$(cat "${work_dir}/${repo_name}.revision.new")" - fi - done - } | \ - tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux-ports/in" - fi - exit 3 -fi -rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" - # Move the .new-files to the actual files rm -rf --one-file-system "${work_dir}/build-list.loops" -- cgit v1.2.3-54-g00ecf From 765f9b249eed27f2b441107de8e2ebcbcbf874b4 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Feb 2018 16:36:00 +0100 Subject: bin/get-package-updates: create metadata for _new_ build list packages --- bin/get-package-updates | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 4fcefd9..a98c9ee 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -355,11 +355,48 @@ echo 'Extract dependencies of packages.' mkdir -p "${work_dir}/package-infos" while read -r package git_revision mod_git_revision repository; do - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - done < "${work_dir}/build-list.new" +{ + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT ' + printf 'replace(to_base64(`package_sources`.`%s`),"\\n",""),' \ + 'pkgbase' 'git_revision' 'mod_git_revision' + printf 'replace(to_base64(`upstream_repositories`.`name`),"\\n","")' + printf ' FROM `binary_packages`' + printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ + 'repositories' 'repositories' 'binary_packages' 'repository' \ + 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ + 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ + 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + printf ' WHERE `repositories`.`name` in ("build-list","deletion-list")' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + ' | \ + while read -r a b c d; do + printf '%s ' \ + "$(printf '%s' "${a}" | base64 -d)" \ + "$(printf '%s' "${b}" | base64 -d)" \ + "$(printf '%s' "${c}" | base64 -d)" \ + "$(printf '%s' "${d}" | base64 -d)" | \ + sed 's/ /\n/' + done | \ + sed ' + y/\t/ / + p + ' + cat "${work_dir}/build-list.new" +} | \ + sort | \ + uniq -u | \ + while read -r package git_revision mod_git_revision repository; do + mysql_generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" + done + echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date -- cgit v1.2.3-54-g00ecf From 2f06112371038694b6cac19e7619c8261e7baf1e Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Feb 2018 17:04:19 +0100 Subject: bin/get-package-updates: reorder pipe --- bin/get-package-updates | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index a98c9ee..cfabace 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -376,6 +376,7 @@ done < "${work_dir}/build-list.new" ${mysql_command} --raw --batch | \ sed ' 1d + y/\t/ / ' | \ while read -r a b c d; do printf '%s ' \ @@ -385,10 +386,8 @@ done < "${work_dir}/build-list.new" "$(printf '%s' "${d}" | base64 -d)" | \ sed 's/ /\n/' done | \ - sed ' - y/\t/ / - p - ' + grep -vxF '' | \ + sed 'p' cat "${work_dir}/build-list.new" } | \ sort | \ -- cgit v1.2.3-54-g00ecf From d1281c9b6f591201878aa092fdb4a2234cf8d31f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Feb 2018 14:05:11 +0100 Subject: bin/get-package-updates: bugfix black listing --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index cfabace..a8dc710 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -461,8 +461,8 @@ while [ -n "${black_listed_new}" ]; do printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`' printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`' printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' - printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ')' + printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ ${mysql_command} --raw --batch | \ -- cgit v1.2.3-54-g00ecf From 98e89a079be0423449481606c4fc63bb9f9087c7 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Feb 2018 14:18:19 +0100 Subject: bin/mysql-functions: mysql_generate_package_metadata has more obligatory parameters now --- bin/bootstrap-mysql | 2 +- bin/get-package-updates | 40 +--------------------------------------- bin/mysql-functions | 39 +++++++++++++++++++-------------------- bin/seed-build-list | 2 +- 4 files changed, 22 insertions(+), 61 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 5acc2e2..1a50496 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -475,7 +475,7 @@ find "${work_dir}/package-states" \ tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do - mysql_generate_package_metadata "${state_file}" + mysql_generate_package_metadata 'build-list' "${state_file}" done # shellcheck disable=SC2016 diff --git a/bin/get-package-updates b/bin/get-package-updates index a8dc710..8322e6d 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -337,7 +337,7 @@ done | \ )" } | \ ${mysql_command} - mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" + mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi @@ -358,44 +358,6 @@ while read -r package git_revision mod_git_revision repository; do generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" done < "${work_dir}/build-list.new" -{ - # shellcheck disable=SC2016 - { - printf 'SELECT DISTINCT ' - printf 'replace(to_base64(`package_sources`.`%s`),"\\n",""),' \ - 'pkgbase' 'git_revision' 'mod_git_revision' - printf 'replace(to_base64(`upstream_repositories`.`name`),"\\n","")' - printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' - printf ' WHERE `repositories`.`name` in ("build-list","deletion-list")' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ - while read -r a b c d; do - printf '%s ' \ - "$(printf '%s' "${a}" | base64 -d)" \ - "$(printf '%s' "${b}" | base64 -d)" \ - "$(printf '%s' "${c}" | base64 -d)" \ - "$(printf '%s' "${d}" | base64 -d)" | \ - sed 's/ /\n/' - done | \ - grep -vxF '' | \ - sed 'p' - cat "${work_dir}/build-list.new" -} | \ - sort | \ - uniq -u | \ - while read -r package git_revision mod_git_revision repository; do - mysql_generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - done - echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date diff --git a/bin/mysql-functions b/bin/mysql-functions index 64f1915..19fb773 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -190,15 +190,16 @@ mysql_add_install_target() { ${mysql_command} } -# mysql_generate_package_metadata $package $git_revision $mod_git_revision $repository +# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository # or -# mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# if sub_pkgrel should be determined automatically and the package is on the build-list +# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository +# if sub_pkgrel should be determined automatically # and # mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository # or # mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository -# if $sub_pkgrel should be forced and the package is currently in $current_repository +# if $sub_pkgrel should be forced + # generate the meta data of a package (dependencies, built packages, ...) in the database mysql_generate_package_metadata() { @@ -207,10 +208,6 @@ mysql_generate_package_metadata() { case "$1" in ''|*[!0-9]*) unset forced_sub_pkgrel - current_repository=$( - printf 'build-list' | \ - base64 -w0 - ) ;; *) forced_sub_pkgrel=$( @@ -218,21 +215,12 @@ mysql_generate_package_metadata() { base64 -w0 ) shift - current_repository=$( - printf '%s' "$1" | \ - base64 -w0 - ) - shift ;; esac - package="$1" - git_revision="$2" - mod_git_revision="$3" - repository="$4" - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + current_repository="$1" + package="$2" - if [ $# -eq 1 ]; then + if [ $# -eq 2 ]; then # second form repository="${package##*.}" package="${package%.*}" @@ -240,8 +228,15 @@ mysql_generate_package_metadata() { package="${package%.*}" git_revision="${package##*.}" package="${package%.*}" + else + git_revision="$3" + mod_git_revision="$4" + repository="$5" fi + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + printf '.' >&2 if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" @@ -276,6 +271,10 @@ mysql_generate_package_metadata() { printf '.' >&2 # now we encode everything in base64 + current_repository=$( + printf '%s' "${current_repository}" | \ + base64 -w0 + ) pkgbase=$( printf '%s' "${pkgbase}" | \ base64 -w0 diff --git a/bin/seed-build-list b/bin/seed-build-list index e17242c..b049560 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -305,7 +305,7 @@ if ${update}; then tee -a "${work_dir}/build-list" while read -r git_revision mod_git_revision repository pkgbase; do - mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" done < \ "${tmp_dir}/build-list.new" -- cgit v1.2.3-54-g00ecf From b8c6ba933adb5f4bad3ac28864a61d7e3b7166c1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 11 Feb 2018 14:10:09 +0100 Subject: bin/get-package-updates: fix black listing mysql query --- bin/get-package-updates | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 8322e6d..612294e 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -403,9 +403,11 @@ while [ -n "${black_listed_new}" ]; do { printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' printf 'INSERT INTO `bl` (`pkgbase`) VALUES ' - printf '(from_base64("%s")),\n' "${black_listed}" | \ + printf '%s\n' "${black_listed}" | \ sort -u | \ sed ' + s/^/(from_base64("/ + s/$/")),/ $ s/,$/;/ ' printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' -- cgit v1.2.3-54-g00ecf From 1ed82cccb9c46a715f033e1f83a7681b2d819d90 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 11 Feb 2018 14:39:46 +0100 Subject: bin/get-package-updates: add todo --- bin/get-package-updates | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 612294e..bddcf12 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -16,6 +16,8 @@ # TODO: correctly handle if pkgbase of a split package is renamed, e.g.: # $a -> ($a,$b) ==> $b -> ($a,$b) +# TODO: keep database clean in case of abort + # shellcheck disable=SC2016 usage() { >&2 echo '' -- cgit v1.2.3-54-g00ecf From 5f7a55ba4f3145533010b17320aa14a584813724 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 11 Feb 2018 21:09:40 +0100 Subject: bin/get-package-updates: create metadata (in files) less thorough --- bin/get-package-updates | 1 + 1 file changed, 1 insertion(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index bddcf12..d752106 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -340,6 +340,7 @@ done | \ } | \ ${mysql_command} mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" + generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi -- cgit v1.2.3-54-g00ecf From d1bb013a5072bf9c9ef4846331e7331e3667b943 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Feb 2018 22:31:49 +0100 Subject: bin/get-package-updates: delay update of build-list in database --- bin/get-package-updates | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index d752106..07e88a2 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -185,6 +185,17 @@ if ! flock -s ${block_flag} 8; then exit fi +# shellcheck disable=SC2016 +{ + printf 'DELETE FROM `binary_packages`' + printf ' WHERE `binary_packages`.`repository`=(' + printf 'SELECT `repositories`.`id`' + printf ' FROM `repositories`' + printf ' WHERE `repositories`.`name`="to-be-decided"' + printf ')' +} | \ + ${mysql_command} + echo 'Check modified packages from the last update, and put them to the build list.' # Check modified packages from the last update, and put them to the build list. @@ -573,6 +584,20 @@ else rm "${work_dir}/tsort.error" fi +# shellcheck disable=SC2016 +{ + printf 'UPDATE `binary_packages`' + printf ' JOIN `repositories` AS `from_repo`' + printf ' ON `binary_packages`.`repository`=`from_repo`.`id`' + printf ' SET `repository`=(' + printf 'SELECT `to_repo`.`id`' + printf ' FROM `repositories` AS `to_repo`' + printf ' WHERE `to_repo`.`name`="build-list"' + printf ')' + printf ' WHERE `from_repo`.`name`="to-be-decided";\n' +} | \ + ${mysql_command} + # update loop list in database mysql_find_build_assignment_loops -- cgit v1.2.3-54-g00ecf From fd342d4b92aa6881df2903804829a54100b83367 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Feb 2018 22:53:40 +0100 Subject: bin/get-package-updates: actually put new packages into "to-be-decided" instead of "build-list" --- bin/get-package-updates | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 07e88a2..533f14f 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -106,7 +106,7 @@ delete_package() { sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" # shellcheck disable=SC2016 { - # packages from the build-list go straight to the deletion-list + # packages from the build-list/to-be-decided go straight to the deletion-list printf 'UPDATE `binary_packages`' printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ 'repositories' 'repositories' 'repository' \ @@ -114,7 +114,7 @@ delete_package() { printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ 'package_sources' 'package_sources' 'build_assignments' 'package_source' printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' - printf ' WHERE `repositories`.`name`="build-list"' + printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$(printf '%s' "$1" | base64 -w0)" # other packages are marked as `is_to_be_deleted` @@ -350,7 +350,7 @@ done | \ )" } | \ ${mysql_command} - mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" + mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi -- cgit v1.2.3-54-g00ecf From 85e1b4ef9f0da06b6bcb2a3c8e550fdd1bb8b794 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 13 Feb 2018 10:38:17 +0100 Subject: bin/get-package-updates: test_exclusion now cooperates with the database --- bin/get-package-updates | 57 ++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 27 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 533f14f..465f896 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -9,8 +9,6 @@ # TODO: Find out, why sometimes package updates are missed. -# TODO: test_exclusion does not yet cooperate with the database - # TODO: read information from database # TODO: correctly handle if pkgbase of a split package is renamed, e.g.: @@ -104,31 +102,36 @@ delete_package() { echo "$1" >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - # shellcheck disable=SC2016 - { - # packages from the build-list/to-be-decided go straight to the deletion-list - printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' - printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' - printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' - printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ - "$(printf '%s' "$1" | base64 -w0)" - # other packages are marked as `is_to_be_deleted` - printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' - printf ' SET `binary_packages`.`is_to_be_deleted`=1' - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ - "$(printf '%s' "$1" | base64 -w0)" - } | \ - ${mysql_command} + # TODO: Once we want to rely on the database for test_exclusion, we + # need to run the command below unconditionally, but with some + # changes, so we can easily revert. + if [ -z "${test_exclusion}" ]; then + # shellcheck disable=SC2016 + { + # packages from the build-list/to-be-decided go straight to the deletion-list + printf 'UPDATE `binary_packages`' + printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ + 'repositories' 'repositories' 'repository' \ + 'build_assignments' 'build_assignments' 'build_assignment' + printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ + 'package_sources' 'package_sources' 'build_assignments' 'package_source' + printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' + printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ + "$(printf '%s' "$1" | base64 -w0)" + # other packages are marked as `is_to_be_deleted` + printf 'UPDATE `binary_packages`' + printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ + 'repositories' 'repositories' 'repository' \ + 'build_assignments' 'build_assignments' 'build_assignment' + printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ + 'package_sources' 'package_sources' 'build_assignments' 'package_source' + printf ' SET `binary_packages`.`is_to_be_deleted`=1' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ + "$(printf '%s' "$1" | base64 -w0)" + } | \ + ${mysql_command} + fi } # create tmp_dir and trap -- cgit v1.2.3-54-g00ecf From a9f0449be9a12ffd6842671eee49fde40b3196f3 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 13 Feb 2018 10:48:25 +0100 Subject: bin/get-package-updates: cleanup db in trap on exit --- bin/get-package-updates | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 465f896..abd380e 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -135,8 +135,20 @@ delete_package() { } # create tmp_dir and trap +clean_up() { + rm -rf --one-file-system "${tmp_dir}" + # shellcheck disable=SC2016 + { + printf 'DELETE FROM `binary_packages`' + printf ' WHERE `binary_packages`.`repository`=(' + printf 'SELECT `repositories`.`id` FROM `repositories`' + printf ' WHERE `repositories`.`name`="to-be-decided"' + printf ');\n' + } | \ + ${mysql_command} +} tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) -trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT +trap 'clean_up' EXIT # Update git repositories (official packages, community packages and the repository of package customizations). -- cgit v1.2.3-54-g00ecf From b7b8706cf9221a880f79262d98d507f6e64c3d07 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Feb 2018 14:20:31 +0100 Subject: bin/get-package-updates: update git hashes in database, too --- bin/get-package-updates | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index abd380e..0b4d4f5 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -601,6 +601,15 @@ fi # shellcheck disable=SC2016 { + # update hashes of repositories in mysql database + for repo in ${repo_names}; do + printf 'UPDATE `git_repositories`' + printf ' SET `git_repositories`.`head`=from_base64("%s")' \ + "$(eval 'printf '"'"'%s'"'"' "${repo_revisions__'"${repo}"'}" | base64 -w0')" + printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${repo}" | base64 -w0)" + done + # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages`' printf ' JOIN `repositories` AS `from_repo`' printf ' ON `binary_packages`.`repository`=`from_repo`.`id`' -- cgit v1.2.3-54-g00ecf From a6fc54fea6be7bb1e5a64a8429d8832d8456b19f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Feb 2018 00:20:56 +0100 Subject: bin/get-package-updates: repair blacklisting --- bin/get-package-updates | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 0b4d4f5..67e3f73 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -456,6 +456,12 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' printf ')' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ') AND EXISTS (' + # TODO: This should be corrected at the root: automatic install targets, which are bogus should + # not be added in the first place - but how do we detect that? + printf ' SELECT *' + printf ' FROM `install_target_providers`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ ${mysql_command} --raw --batch | \ -- cgit v1.2.3-54-g00ecf From ab3d96872cdac3b814592b55ca3a0cb159d626be Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Feb 2018 13:17:05 +0100 Subject: bin/get-package-updates: remove safety net which calls "generate_package_metadata" on _all_ packages --- bin/get-package-updates | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 67e3f73..89b09df 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -377,16 +377,6 @@ done | \ sort -u "${work_dir}/deletion-list.new" | \ sponge "${work_dir}/deletion-list.new" -echo 'Extract dependencies of packages.' - -# First, we extract the dependencies of each package. - -mkdir -p "${work_dir}/package-infos" - -while read -r package git_revision mod_git_revision repository; do - generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" -done < "${work_dir}/build-list.new" - echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date -- cgit v1.2.3-54-g00ecf From d2e0ef2a558e27e07150f239d91e4c29ce822426 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 16 Feb 2018 11:14:03 +0100 Subject: bin/mysql-functions: mysql_join_*_* new --- bin/bootstrap-mysql | 16 +--- bin/build-master-status-from-mysql | 65 ++++++-------- bin/check-bugtracker | 2 +- bin/common-functions | 5 +- bin/copy-to-build-support | 6 +- bin/db-update | 4 +- bin/get-assignment | 34 ++++---- bin/get-package-updates | 43 ++++----- bin/modify-package-state | 21 ++--- bin/mysql-functions | 174 +++++++++++++++++++++++++------------ bin/prioritize-build-list | 7 +- bin/return-assignment | 40 ++++----- bin/seed-build-list | 15 ++-- bin/show-dependencies | 7 +- bin/why-dont-you | 23 +++-- 15 files changed, 243 insertions(+), 219 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 322e0a1..30c260a 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -512,9 +512,8 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ ' | \ while read -r state_file pkgbase git_revision mod_git_revision repository; do printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ "$( tr -d '\n' < \ @@ -563,10 +562,9 @@ grep '^\('"$( "${tmp_dir}/new-stable-packages" if [ -s "${tmp_dir}/new-stable-packages" ]; then + # shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'CREATE TEMPORARY TABLE `stable_packages` (' - # shellcheck disable=SC2016 printf '`%s` %s,' \ 'pkgname' 'VARCHAR(64)' \ 'epoch' 'MEDIUMINT' \ @@ -576,9 +574,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then 'architecture' 'VARCHAR(16)' \ 'repository' 'VARCHAR(64)' \ 'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT' - # shellcheck disable=SC2016 printf 'PRIMARY KEY (`build_assignment`));\n' - # shellcheck disable=SC2016 sed ' 1~10 ! b not_start s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/ @@ -589,28 +585,22 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then :end s/,$/;/ ' "${tmp_dir}/new-stable-packages" - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' printf ' SELECT ' - # shellcheck disable=SC2016 printf '`stable_packages`.`%s`,' \ 'pkgname' \ 'epoch' \ 'pkgver' \ 'pkgrel' \ 'sub_pkgrel' - # shellcheck disable=SC2016 printf '`%s`.`id`,' \ 'architectures' \ 'repositories' - # shellcheck disable=SC2016 printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' - # shellcheck disable=SC2016 printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ 'repositories' 'repository' 'repositories' \ 'architectures' 'architecture' 'architectures' printf ';\n' - # shellcheck disable=SC2016 printf 'DROP TABLE `stable_packages`;\n' } | \ ${mysql_command} diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7a6605a..ea8d0f1 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -21,11 +21,10 @@ fi { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_upstream_repositories + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ ${mysql_command} --batch | \ @@ -45,10 +44,9 @@ sort "${work_dir}/build-list" > \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } | \ ${mysql_command} --raw --batch | \ sed ' @@ -68,10 +66,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin { printf 'SELECT DISTINCT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ @@ -89,9 +86,8 @@ sort "${work_dir}/deletion-list" > \ { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'build_dependency_loops' 'build_dependency_loops' 'build_assignment' 'build_assignments' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ ${mysql_command} --batch | \ @@ -118,10 +114,9 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -154,10 +149,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE NOT `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -188,15 +182,13 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'mod_git_revision' printf '`upstream_repositories`.`name`,`install_targets`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'dependencies' 'binary_packages' 'dependencies' 'dependent' \ - 'dependency_types' 'dependency_types' 'dependencies' 'dependency_type' - printf ' AND `dependency_types`.`relevant_for_building`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'install_targets' 'install_targets' 'dependencies' 'depending_on' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + mysql_join_dependencies_install_targets + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ ${mysql_command} --raw --batch | \ sed ' @@ -214,10 +206,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages } | \ ${mysql_command} --raw --batch | \ sed ' diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 72ab133..d6e92e5 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -31,7 +31,7 @@ bug_list=$( while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_binary_packages_repositories printf ' SET `has_issues`=%s' \ "${has_issues%:*}" printf ' WHERE `repositories`.`stability`=%s' \ diff --git a/bin/common-functions b/bin/common-functions index d7f3e41..a84271b 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -478,9 +478,8 @@ remove_old_package_versions() { printf '`binary_packages`.`sub_pkgrel`' printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'architectures' 'architectures' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' # name must match diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 4710012..808d041 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -31,8 +31,8 @@ sed -n ' { printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ @@ -89,7 +89,7 @@ sed -n ' # shellcheck disable=SC2086 printf ',`binary_packages`.`%s`' ${to_copy} printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`id`=%s;\n' \ "${id}" diff --git a/bin/db-update b/bin/db-update index e5ea193..b830981 100755 --- a/bin/db-update +++ b/bin/db-update @@ -325,8 +325,8 @@ move_packages() { while read -r package; do while read -r part; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo` ON `binary_packages`.`repository`=`from_repo`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' + mysql_join_binary_packages_architectures printf ' SET `binary_packages`.`repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` as `to_repo`' diff --git a/bin/get-assignment b/bin/get-assignment index a7eec2d..1edcffb 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -45,11 +45,10 @@ hand_out_assignment() { 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `package_sources`.`%s`=from_base64("%s")' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -80,8 +79,8 @@ hand_out_assignment() { printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -94,8 +93,8 @@ hand_out_assignment() { "$(printf '%s' "${slave}" | base64 -w0)" printf 'UPDATE `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`priority`=0' printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ @@ -150,10 +149,9 @@ currently_building=$( { printf 'SELECT ' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_slaves' 'build_slaves' 'currently_building' 'build_assignments' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' @@ -189,9 +187,8 @@ next_building=$( mysql_query_is_part_of_loop '`build_assignments`.`id`' printf ' AS `part_of_loop`,' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND NOT EXISTS (' printf ' SELECT *' @@ -231,9 +228,8 @@ count_pending=$( { printf 'SELECT count(*)' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' diff --git a/bin/get-package-updates b/bin/get-package-updates index 89b09df..e81d128 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,22 +110,18 @@ delete_package() { { # packages from the build-list/to-be-decided go straight to the deletion-list printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$(printf '%s' "$1" | base64 -w0)" # other packages are marked as `is_to_be_deleted` printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`is_to_be_deleted`=1' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" @@ -341,10 +337,9 @@ done | \ # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${package}" | \ @@ -353,10 +348,9 @@ done | \ printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n' # remove is-to-be-deleted marker from old binary packages printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' SET `is_to_be_deleted`=0' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$( @@ -431,9 +425,9 @@ while [ -n "${black_listed_new}" ]; do ' printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' printf ' FROM `package_sources` AS `a_ps`' - printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`' - printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`' - printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_binary_packages_dependencies 'a_bp' printf ' WHERE NOT EXISTS (' printf ' SELECT *' printf ' FROM `install_target_providers`' @@ -441,8 +435,8 @@ while [ -n "${black_listed_new}" ]; do printf ' SELECT *' printf ' FROM `bl`' printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' - printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`' - printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' printf ')' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' @@ -607,8 +601,7 @@ fi done # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo`' - printf ' ON `binary_packages`.`repository`=`from_repo`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' printf ' SET `repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` AS `to_repo`' diff --git a/bin/modify-package-state b/bin/modify-package-state index b2d3133..b0ba53e 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -136,10 +136,9 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_architectures ) # shellcheck disable=SC2016,SC2031 selector=$( @@ -161,12 +160,11 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities ) # shellcheck disable=SC2016 selector=$( @@ -238,8 +236,7 @@ while read -r package reason; do printf 'UPDATE `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b`' printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories`' - printf ' ON `b`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'b' printf ' SET `b`.`is_tested`=1' printf ' WHERE `a`.`is_tested`' printf ' AND NOT `b`.`is_tested`' diff --git a/bin/mysql-functions b/bin/mysql-functions index 7f2cdab..6950ee8 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -131,7 +131,6 @@ mysql_show_binary_package() { shift done - local joint { printf 'SELECT' printf ' `%s`.`%s`,' \ @@ -146,17 +145,11 @@ mysql_show_binary_package() { 'package_sources' 'mod_git_revision' \ 'upstream_repositories' 'name' printf ' FROM `binary_packages`' - for joint in \ - 'architectures:binary_packages:architecture' \ - 'package_sources:binary_packages:package_source' \ - 'repositories:binary_packages:repository' \ - 'upstream_repositories:package_sources:upstream_package_repository'; do - printf ' JOIN `%s` ON `%s`.`id` =' \ - "${joint%%:*}" "${joint%%:*}" - joint="${joint#*:}" - printf ' `%s`.`%s`' \ - "${joint%:*}" "${joint#*:}" - done + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'binary_packages' 'pkgname' "${pkgname}" \ @@ -434,8 +427,8 @@ mysql_generate_package_metadata() { # build-list or in $current_repository (beware of split # packages!) printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -448,7 +441,7 @@ mysql_generate_package_metadata() { "${current_repository}" # max(sub_pkgrel)+1 printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -496,11 +489,9 @@ mysql_generate_package_metadata() { printf ' `%s` JOIN' \ 'repositories' \ 'architectures' \ - 'build_assignments' \ - 'package_sources' - printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`' - printf ' JOIN `upstream_repositories`' - printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' + 'build_assignments' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'repositories' 'name' "${current_repository}" \ @@ -513,7 +504,6 @@ mysql_generate_package_metadata() { } | \ sed ' s|,)|)|g - s|JOIN ON|ON|g s| AND;$|;| ' >> \ "${temp_dir}/add-binary-packages-command" @@ -586,8 +576,7 @@ mysql_generate_package_metadata() { ;; esac printf ' `binary_packages`' - printf ' JOIN `architectures`' - printf ' ON `binary_packages`.`architecture` = `architectures`.`id`' + mysql_join_binary_packages_architectures printf ' JOIN `install_targets`' printf ' JOIN `%s`' "${link}" printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" @@ -649,9 +638,9 @@ mysql_sanity_check() { { printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' printf ' FROM `binary_packages`' - printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`' - printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`' + mysql_join_binary_packages_build_assignments + mysql_join_binary_packages_architectures '' 'bp_arch' + mysql_join_build_assignments_architectures '' 'ba_arch' printf ' WHERE `bp_arch`.`name`!="any"' printf ' AND `ba_arch`.`name`="any";\n' printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' @@ -660,14 +649,13 @@ mysql_sanity_check() { printf ' ON `a`.`pkgname`=`b`.`pkgname`' printf ' AND `a`.`repository`=`b`.`repository`' printf ' AND `a`.`id`!=`b`.`id`' - printf ' JOIN `repositories`' - printf ' ON `a`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'a' printf ' WHERE `repositories`.`name`="build-list";\n' printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' printf ' FROM `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`' - printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`' + mysql_join_binary_packages_repositories 'a' 'arep' + mysql_join_binary_packages_repositories 'b' 'brep' printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' printf ' AND `%srep`.`name`="build-list"' \ 'a' 'b' @@ -700,8 +688,8 @@ mysql_sanity_check() { { printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ ${mysql_command} --batch | \ @@ -728,12 +716,11 @@ mysql_find_build_assignment_loops() { { printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' printf ' FROM `dependencies`' - printf ' JOIN `install_target_providers` ON `dependencies`.`depending_on`=`install_target_providers`.`install_target`' - printf ' JOIN `binary_packages` `packages_dependent` ON `dependencies`.`dependent`=`packages_dependent`.`id`' - printf ' JOIN `binary_packages` `packages_dependency` ON `install_target_providers`.`package`=`packages_dependency`.`id`' - printf ' JOIN `repositories` `repositories_%s` ON `packages_%s`.`repository`=`repositories_%s`.`id`' \ - 'dependent' 'dependent' 'dependent' \ - 'dependency' 'dependency' 'dependency' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'packages_dependency' + mysql_join_dependencies_binary_packages '' 'packages_dependent' + mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' + mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ ${mysql_command} --raw --batch | \ @@ -836,8 +823,8 @@ mysql_cleanup() { printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' printf ');\n' printf 'UPDATE `build_slaves`' - printf ' JOIN `binary_packages` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_build_slaves_binary_packages + mysql_join_binary_packages_repositories printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ @@ -849,12 +836,11 @@ mysql_cleanup() { mysql_query_has_pending_dependencies() { printf 'EXISTS (' printf 'SELECT * FROM `binary_packages` as `to_dos`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \ - 'repositories` AS `to_do_repos' 'to_do_repos' 'id' 'to_dos' 'repository' \ - 'dependencies' 'to_dos' 'id' 'dependencies' 'dependent' \ - 'install_target_providers' 'dependencies' 'depending_on' 'install_target_providers' 'install_target' \ - 'binary_packages` AS `bin_deps' 'bin_deps' 'id' 'install_target_providers' 'package' \ - 'repositories` AS `dep_repos' 'dep_repos' 'id' 'bin_deps' 'repository' + mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' + mysql_join_binary_packages_dependencies 'to_dos' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'bin_deps' + mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' printf ' WHERE' printf ' `%s`.`name`="build-list" AND' \ 'dep_repos' 'to_do_repos' @@ -884,9 +870,8 @@ mysql_query_select_pkgbase_and_revision() { 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } # mysql_repair_binary_packages_without_build_assignment @@ -900,7 +885,7 @@ mysql_repair_binary_packages_without_build_assignment() { '`binary_packages`.`pkgname`' \ '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ ${mysql_command} --raw --batch | \ @@ -954,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' LIMIT 1;\n' printf 'UPDATE `binary_packages`' printf ' JOIN `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_binary_packages_build_assignments printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" @@ -980,9 +965,8 @@ mysql_remove_duplicate_build_order() { 'pkgver' 'pkgver' \ 'pkgrel' 'pkgrel' printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \ - 'orep' 'orep' 'old' \ - 'nrep' 'nrep' 'new' + mysql_join_binary_packages_repositories 'old' 'orep' + mysql_join_binary_packages_repositories 'new' 'nrep' printf ' WHERE `orep`.`name`!="to-be-decided"' printf ' AND `nrep`.`name`="to-be-decided";\n' printf 'UPDATE IGNORE `dependencies`' @@ -1004,6 +988,9 @@ mysql_remove_duplicate_build_order() { ${mysql_command} } +# mysql_package_name_query +# print a mysql query of the full name of a package file + mysql_package_name_query() { printf 'CONCAT(' printf '`binary_packages`.`pkgname`,"-",' @@ -1014,3 +1001,82 @@ mysql_package_name_query() { printf '`architectures`.`name`,".pkg.tar.xz"' printf ')' } + +# mysql_join_*_* +# print 'JOIN' part of mysql query to connect the respective tables +# these functions take 2 optional arguments, acting as aliases for +# the tables + +# mysql_join__generic $table_a $column_a $table_b $column_b +# create mysql_join_${table_a}_${table_b}() function + +mysql_join__generic() { + eval "$( + printf 'mysql_join_%s_%s() {\n' "$1" "$3" + printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' + printf ' fi\n' + printf ' if [ -n "$1" ]; then\n' + printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" + printf ' else\n' + printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" + printf ' fi\n' + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" + printf ' else\n' + printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" + printf ' fi\n' + printf '}\n' + )" +} + +for link in \ + 'binary_packages:architecture:architectures' \ + 'binary_packages:repository:repositories' \ + 'binary_packages:build_assignment:build_assignments' \ + \ + 'build_assignments:architecture:architectures' \ + 'build_assignments:package_source:package_sources' \ + \ + 'build_dependency_loops:build_assignment:build_assignments' \ + 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ + \ + 'build_slaves:currently_building:build_assignments' \ + 'build_slaves:currently_building build_assignment:binary_packages' \ + \ + 'dependencies:depending_on:install_targets' \ + 'dependencies:dependent:binary_packages' \ + 'dependencies:dependency_type:dependency_types' \ + \ + 'failed_builds:reason:fail_reason' \ + 'failed_builds:build_assignment:build_assignments' \ + 'failed_builds:build_slave:build_slaves' \ + \ + 'install_target_providers:package:binary_packages' \ + 'install_target_providers:install_target:install_targets' \ + 'install_target_providers:install_target depending_on:dependencies' \ + \ + 'package_sources:upstream_package_repository:upstream_repositories' \ + \ + 'repositories:stability:repository_stabilities' \ + \ + 'upstream_repositories:git_repository:git_repositories'; do +# A join for these cannot be done, because it's not clear on what to join: +# 'repository_stability_relations:more_stable:repository_stabilities' +# 'repository_stability_relations:less_stable:repository_stabilities' + + table_b="${link##*:}" + table_a="${link%:*}" + column_b="${table_a##*:}" + table_a="${table_a%:*}" + column_a="${column_b% *}" + if [ "${column_a}" = "${column_b}" ]; then + column_b='id' + else + column_b="${column_b##* }" + fi + + mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" + mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" +done diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index d611e79..e8dcf0a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -32,10 +32,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_package_sources + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' SET `build_assignments`.`priority`=(' printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1' printf ' FROM (' diff --git a/bin/return-assignment b/bin/return-assignment index 7b2dec3..995eae4 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -51,12 +51,11 @@ if [ "$5" = 'ERROR' ]; then infos=$( { printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( # shellcheck disable=SC2154 @@ -157,7 +156,7 @@ if [ "$5" = 'ERROR' ]; then printf 'DROP TABLE `failures`;\n' fi printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' + mysql_join_build_assignments_build_slaves printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( @@ -267,12 +266,11 @@ clean_up_lock_file() { if ! { printf 'SELECT count(*)' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'build_slaves' 'build_assignments' 'build_slaves' 'currently_building' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_build_slaves + mysql_join_binary_packages_repositories + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -396,9 +394,9 @@ package_errors=$( mysql_package_name_query printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_build_slaves + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ @@ -476,9 +474,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `repositories` AS `old_repo` ON `binary_packages`.`repository`=`old_repo`.`id`' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories '' 'old_repo' printf ' SET' printf ' `build_assignments`.`is_broken`=0,' printf ' `build_assignments`.`priority`=0,' @@ -507,8 +505,8 @@ fi printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n' printf 'INSERT INTO `loops_to_delete`' printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_dependency_loops`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_build_dependency_loops_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE NOT `repositories`.`name` = "build-list";\n' printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS (' printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`' diff --git a/bin/seed-build-list b/bin/seed-build-list index bfe975e..5ce9037 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -123,7 +123,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf 'SELECT ' mysql_package_name_query printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures } | \ ${mysql_command} --raw --batch | \ sed ' @@ -161,7 +161,7 @@ fi printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)' printf ' SELECT DISTINCT `binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n' } | \ sponge "${tmp_dir}/ignore-packages" @@ -253,7 +253,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '`pkgbases`.`repository`' printf ' FROM `pkgbases`' printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`' - printf ' JOIN `git_repositories` ON `git_repositories`.`id`=`upstream_repositories`.`git_repository`' + mysql_join_upstream_repositories_git_repositories printf ';\n' printf 'SELECT ' printf '`package_sources`.`pkgbase`,' @@ -264,11 +264,10 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '),' printf '`upstream_repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'git_repositories' 'git_repositories' 'upstream_repositories' 'git_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_git_repositories printf ' WHERE (' if [ -s "${tmp_dir}/package-regexes" ]; then grep -vxF '' "${tmp_dir}/package-regexes" | \ diff --git a/bin/show-dependencies b/bin/show-dependencies index 0a7a1fd..bb1ae37 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -13,7 +13,7 @@ if [ $# -eq 0 ]; then { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `build_assignments` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_package_sources_build_assignments printf ' WHERE `build_assignments`.`is_broken`' } | \ ${mysql_command} --raw --batch | \ @@ -28,9 +28,8 @@ if [ $# -eq 0 ]; then { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' ORDER BY `binary_packages`.`pkgname`' # Why can't mysql order in the same way "sort" does! } | \ diff --git a/bin/why-dont-you b/bin/why-dont-you index e9922b7..d95f2a7 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -57,10 +57,9 @@ case "${action}" in # package_source, build_assignment, binary_package, repostory printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`' printf ' FROM `package_sources` AS `tb_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `tb_ba' 'tb_ba' 'package_source' 'tb_ps' \ - 'binary_packages` AS `tb_bin' 'tb_bin' 'build_assignment' 'tb_ba' \ - 'repositories` AS `tb_rep' 'tb_bin' 'repository' 'tb_rep' + mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba' + mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin' + mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep' printf ' WHERE `tb_rep`.`name`="build-list"' printf ') AS `to_build`' printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`' @@ -70,16 +69,14 @@ case "${action}" in # potential dependencies printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`' printf ' FROM `package_sources` AS `dep_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `dep_ba' 'dep_ba' 'package_source' 'dep_ps' \ - 'binary_packages` AS `dep_bin' 'dep_bin' 'build_assignment' 'dep_ba' \ - 'repositories` AS `dep_rep' 'dep_bin' 'repository' 'dep_rep' \ - 'install_target_providers' 'install_target_providers' 'package' 'dep_bin' - # starting from the line above, we have some additional joins, + mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba' + mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin' + mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep' + # now we have some (=3) additional joins, # because we are interested in dependency relations to `to_build` - printf ' JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'dependency_types' 'dependencies' 'dependency_type' 'dependency_types' + mysql_join_binary_packages_install_target_providers 'dep_bin' + mysql_join_install_target_providers_dependencies + mysql_join_dependencies_dependency_types printf ' WHERE `dep_rep`.`name`="build-list"' printf ' AND `dependency_types`.`relevant_for_building`' printf ') AS `deps`' -- cgit v1.2.3-54-g00ecf From 33f10c6217830fd2430429da1d7f2d7a451522be Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sat, 17 Feb 2018 20:27:43 +0100 Subject: bin/get-package-updates: save correct revisions in database --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index e81d128..d516a96 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -595,7 +595,7 @@ fi for repo in ${repo_names}; do printf 'UPDATE `git_repositories`' printf ' SET `git_repositories`.`head`=from_base64("%s")' \ - "$(eval 'printf '"'"'%s'"'"' "${repo_revisions__'"${repo}"'}" | base64 -w0')" + "$(eval 'printf '"'"'%s'"'"' "${new_repo_revisions__'"${repo}"'}" | base64 -w0')" printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repo}" | base64 -w0)" done -- cgit v1.2.3-54-g00ecf From 89745907f5137c4de7ed7dfc50646b066cd8fa86 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 18 Feb 2018 08:27:48 +0100 Subject: bin/get-package-updates: clarify comment --- bin/get-package-updates | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index d516a96..97422f5 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -611,7 +611,8 @@ fi } | \ ${mysql_command} -# update loop list in database +# update loop list in database (beware, the packages are expected to be in "build-list", +# not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". mysql_find_build_assignment_loops # Move the .new-files to the actual files -- cgit v1.2.3-54-g00ecf From fd812f6973156991a4462ef996161730dfad9e78 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 18 Feb 2018 20:34:43 +0100 Subject: bin/get-package-updates: remove useless cat - we have this in a variable already --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 97422f5..1ce72e5 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -379,7 +379,7 @@ black_listed='' black_listed_new=$( { { - git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision.new")" -- 'blacklist' | \ + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ tar -Ox 'blacklist' | \ sed ' s/\s*#.*$// -- cgit v1.2.3-54-g00ecf From f05923eda5d3a307f7fbfb44e451fd9441222aae Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sun, 18 Feb 2018 21:03:16 +0100 Subject: bin/get-package-updates: use mysql_cleanup --- bin/get-package-updates | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 1ce72e5..70f62e8 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -196,16 +196,8 @@ if ! flock -s ${block_flag} 8; then exit fi -# shellcheck disable=SC2016 -{ - printf 'DELETE FROM `binary_packages`' - printf ' WHERE `binary_packages`.`repository`=(' - printf 'SELECT `repositories`.`id`' - printf ' FROM `repositories`' - printf ' WHERE `repositories`.`name`="to-be-decided"' - printf ')' -} | \ - ${mysql_command} +# shellcheck disable=SC2119 +mysql_cleanup echo 'Check modified packages from the last update, and put them to the build list.' -- cgit v1.2.3-54-g00ecf From f22acc6a42126250fb0aa1844e20ec0abc6210b3 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 27 Feb 2018 07:13:45 +0100 Subject: bin/get-package-updates: add TODO --- bin/get-package-updates | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 70f62e8..1accadf 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -16,6 +16,9 @@ # TODO: keep database clean in case of abort +# TODO: It seems, updating present build assignments in the database is borken +# ... they seem to get deleted + # shellcheck disable=SC2016 usage() { >&2 echo '' -- cgit v1.2.3-54-g00ecf From cbafb5816805fb00f3d309e99e4477d17e0607d4 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 27 Feb 2018 09:56:56 +0100 Subject: bin/get-package-updates: fix typo in comment --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 1accadf..2fd0993 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -16,7 +16,7 @@ # TODO: keep database clean in case of abort -# TODO: It seems, updating present build assignments in the database is borken +# TODO: It seems, updating present build assignments in the database is broken # ... they seem to get deleted # shellcheck disable=SC2016 -- cgit v1.2.3-54-g00ecf From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-54-g00ecf From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:05:51 +0100 Subject: use "mysql -N" instead of cumbersome sedding --- bin/build-master-status | 5 ++--- bin/build-master-status-from-mysql | 46 ++++++++++---------------------------- bin/check-bugtracker | 5 ++--- bin/copy-to-build-support | 7 ++---- bin/db-update | 14 ++++-------- bin/get-assignment | 16 +++++-------- bin/get-package-updates | 6 ++--- bin/modify-package-state | 3 +-- bin/return-assignment | 20 +++++------------ bin/seed-build-list | 13 ++++------- bin/show-dependencies | 3 +-- bin/why-dont-you | 7 ++---- lib/common-functions | 7 ++---- lib/mysql-functions | 19 +++++----------- 14 files changed, 51 insertions(+), 120 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status b/bin/build-master-status index 9be7823..deb58da 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,10 +562,9 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d - 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index fa54177..d1985e7 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,11 +27,8 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" sort "${work_dir}/build-list" > \ @@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \ @@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" { @@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then # TODO: consider versions dependencies, too diff --git a/bin/check-bugtracker b/bin/check-bugtracker index f15757b..156f438 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,8 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' @@ -61,5 +60,5 @@ bug_list=$( tr '\n' ',' printf '"");\n' done - done | tee /dev/stderr | \ + done | \ mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 0b87107..e447596 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,11 +44,8 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${id}" ]; then continue diff --git a/bin/db-update b/bin/db-update index 534c245..78ff622 100755 --- a/bin/db-update +++ b/bin/db-update @@ -457,11 +457,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ sort -u @@ -496,11 +493,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ sort -u diff --git a/bin/get-assignment b/bin/get-assignment index bc67fab..964c6b7 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,11 +58,8 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' { # shellcheck disable=SC2154 @@ -157,8 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ -n "${currently_building}" ]; then @@ -214,9 +210,8 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ + mysql_run_query -N --batch --raw | \ sed ' - 1d y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ ' @@ -238,8 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index 11f384c..e855b19 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,8 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch } | \ sort -u ) @@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 8c5fac7..68b2384 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,8 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index ed29f34..a7e6623 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' @@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -283,9 +279,8 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ - grep -vxF '0'; then + mysql_run_query -N --raw --batch | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi @@ -441,10 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - ' + mysql_run_query -N --raw --batch } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index b9ae193..def9d56 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ @@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sort -u | \ - sed ' - y/\t/ / - / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d - ' | \ + tr '\t' ' ' | \ if ${update}; then # always block if locked exec 9> "${build_list_lock_file}" diff --git a/bin/show-dependencies b/bin/show-dependencies index 9e94fe3..ed7ca87 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,9 +10,8 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/\s.*$// ' | \ sort -u diff --git a/bin/why-dont-you b/bin/why-dont-you index b5d73c5..35b9419 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,11 +85,8 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' / NULL \S\+$/ b multi-dep diff --git a/lib/common-functions b/lib/common-functions index 740c266..ba434ad 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,11 +501,8 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ shrink_version 4 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 6cbc502..3e7e19f 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -668,9 +668,8 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - /^CONCAT("/d s,^,, s,$,, ' @@ -699,9 +698,8 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s,\t,/, s,\t,-, s,\t,:, @@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ { @@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 16:30:10 +0100 Subject: make -N --raw --batch the default arguments to mysql, explicitely override them if required --- bin/build-master-status | 2 +- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 2 +- bin/copy-to-build-support | 2 +- bin/db-update | 4 ++-- bin/get-assignment | 8 ++++---- bin/get-package-updates | 4 ++-- bin/modify-package-state | 2 +- bin/return-assignment | 8 ++++---- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/why-dont-you | 2 +- lib/common-functions | 2 +- lib/mysql-functions | 18 +++++++++--------- 14 files changed, 40 insertions(+), 40 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status b/bin/build-master-status index deb58da..006750b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,7 +562,7 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7cf1b95..a72d396 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" @@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" @@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" @@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 156f438..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index e447596..b3decd4 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${id}" ]; then diff --git a/bin/db-update b/bin/db-update index 5efa220..5756494 100755 --- a/bin/db-update +++ b/bin/db-update @@ -467,7 +467,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ @@ -503,7 +503,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ diff --git a/bin/get-assignment b/bin/get-assignment index 964c6b7..2df3059 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' { @@ -154,7 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ -n "${currently_building}" ]; then @@ -210,7 +210,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw | \ + mysql_run_query | \ sed ' y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ @@ -233,7 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index e855b19..a95bd94 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -u ) @@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 68b2384..9b0571b 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index a7e6623..717a30e 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then @@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query -N --raw --batch + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -279,7 +279,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 @@ -436,7 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index def9d56..d49a8f3 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' @@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u | \ tr '\t' ' ' | \ if ${update}; then diff --git a/bin/show-dependencies b/bin/show-dependencies index ed7ca87..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/\s.*$// ' | \ @@ -177,7 +177,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/why-dont-you b/bin/why-dont-you index 35b9419..64dfa1b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' diff --git a/lib/common-functions b/lib/common-functions index 0f75cb8..c7a64fe 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 9d705d2..26d3db8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,17 +26,17 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then - ${mysql_command} "$@" + ${mysql_command} -N --raw --batch "$@" else { - printf '%s %s: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ | tee -a "${work_dir}/mysql.stdin" \ | tee -a "${work_dir}/mysql.stdout" \ >> "${work_dir}/mysql.stderr" tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ + | ${mysql_command} -N --raw --batch "$@" \ 2>> "${work_dir}/mysql.stderr" \ | tee -a "${work_dir}/mysql.stdout" for s in \ @@ -193,7 +193,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - mysql_run_query --html + mysql_run_query --html --column-names } # mysql_add_install_target $install_target @@ -693,7 +693,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,^,, s,$,, @@ -723,7 +723,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,\t,/, s,\t,-, @@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ @@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From b61c5f236ced1841b84c7d24feedad500b685509 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 20:01:15 +0100 Subject: bin/get-package-updates: remove broken, blocked and locked marker files of build assignments which get deleted --- bin/get-package-updates | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index a95bd94..1e0502f 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -105,6 +105,9 @@ delete_package() { echo "$1" >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" + find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ + -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(blocked\|locked\)' \ + -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. -- cgit v1.2.3-54-g00ecf From f92358c7654a251016f251519ddf863d9637fef3 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 20:12:26 +0100 Subject: bin/get-package-updates: actually remove "broken" marker files ,too --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 1e0502f..ffcb078 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -106,7 +106,7 @@ delete_package() { "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(blocked\|locked\)' \ + -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \ -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some -- cgit v1.2.3-54-g00ecf From 9ede9a11566506384d5ce13736a9153313a14535 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Mar 2018 16:09:53 +0100 Subject: bin/get-package-updates: do not remove to-be-decided packages (of other running instance!) if we cannot acquire a lock --- bin/get-package-updates | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index ffcb078..5285646 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -16,9 +16,6 @@ # TODO: keep database clean in case of abort -# TODO: It seems, updating present build assignments in the database is broken -# ... they seem to get deleted - # shellcheck disable=SC2016 usage() { >&2 echo '' @@ -136,22 +133,6 @@ delete_package() { fi } -# create tmp_dir and trap -clean_up() { - rm -rf --one-file-system "${tmp_dir}" - # shellcheck disable=SC2016 - { - printf 'DELETE FROM `binary_packages`' - printf ' WHERE `binary_packages`.`repository`=(' - printf 'SELECT `repositories`.`id` FROM `repositories`' - printf ' WHERE `repositories`.`name`="to-be-decided"' - printf ');\n' - } | \ - mysql_run_query -} -tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) -trap 'clean_up' EXIT - # Update git repositories (official packages, community packages and the repository of package customizations). for repo in ${repo_names}; do @@ -202,6 +183,8 @@ if ! flock -s ${block_flag} 8; then exit fi +trap mysql_cleanup EXIT + # shellcheck disable=SC2119 mysql_cleanup -- cgit v1.2.3-54-g00ecf From fe3ae1c23523c824fb3fe1e7904c24b949b7ce6f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Mar 2018 16:14:37 +0100 Subject: bin/get-package-updates: clarify comment --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 5285646..0276b7e 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -1,6 +1,6 @@ #!/bin/sh -# check for packages that need to be built, and build a list in the proper build order +# check for packages that need to be built # Details: # https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates -- cgit v1.2.3-54-g00ecf From e40900974a9671651e0b18f4c9367e498157814c Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Mar 2018 11:40:07 +0100 Subject: bin/get-package-updates: only put packages on the deletion-list which are known in the database --- bin/get-package-updates | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 0276b7e..b48660d 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -99,7 +99,16 @@ fi # delete_package package # mark $package for deletion delete_package() { - echo "$1" >> \ + # shellcheck disable=SC2016 + { + printf 'SELECT DISTINCT `package_sources`.`pkgbase`' + printf ' FROM `package_sources`' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ + "$(printf '%s' "$1" | base64 -w0)" + } | \ + mysql_run_query >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ -- cgit v1.2.3-54-g00ecf From 329bf724368aed075f28002cf2f1d40d8f325de8 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 06:38:15 +0100 Subject: bin/get-package-updates: remove old locked and blocked marker of _updated_ packages, too --- bin/get-package-updates | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index b48660d..081f457 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -317,6 +317,9 @@ done | \ if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" + find "${work_dir}/package-states" -regextype grep \ + -regex ".*/$(str_to_regex "${package}")\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)" \ + -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" -- cgit v1.2.3-54-g00ecf From cd4463bfdf66882031af0f2aed430d803ad1618a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 08:37:29 +0100 Subject: bin/get-package-updates: satisfy shellcheck --- bin/get-package-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 081f457..1cf8782 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -318,7 +318,7 @@ done | \ # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" find "${work_dir}/package-states" -regextype grep \ - -regex ".*/$(str_to_regex "${package}")\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)" \ + -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \ -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ -- cgit v1.2.3-54-g00ecf From 6a69a4c9f2b38b10c26202847f8e64e47a0306fe Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 19 Mar 2018 22:10:52 +0100 Subject: #archlinux-ports -> #archlinux32 --- bin/build-master-status-from-mysql | 2 +- bin/get-package-updates | 2 +- bin/ii-connect | 6 +++--- bin/return-assignment | 4 ++-- bin/sanity-check | 4 ++-- lib/mysql-functions | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 3918489..2c33708 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then printf 'girls: my database is dirty again ...\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" echo 'build master is insane' > \ "${work_dir}/build-master-sanity" fi diff --git a/bin/get-package-updates b/bin/get-package-updates index 1cf8782..d3cb140 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi exit 3 fi diff --git a/bin/ii-connect b/bin/ii-connect index 973b227..cf332c0 100755 --- a/bin/ii-connect +++ b/bin/ii-connect @@ -19,12 +19,12 @@ if tail -n1 "${irc_dir}/nickserv/out" 2> /dev/null | \ sponge "${irc_dir}/nickserv/in" fi -# join #archlinux-ports if not yet done +# join #archlinux32 if not yet done if ! grep ' buildmaster\.archlinux32\.org .* buildmaster$' "${irc_dir}/out" | \ tail -n1 | \ - grep -q ' #archlinux-ports '; then + grep -q ' #archlinux32 '; then { - echo '/j #archlinux-ports' + echo '/j #archlinux32' echo '/WHO buildmaster' } | \ sponge "${irc_dir}/in" diff --git a/bin/return-assignment b/bin/return-assignment index a61460e..417654b 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -222,7 +222,7 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux-ports/in" ]; then + if [ -p "${irc_dir}/#archlinux32/in" ]; then { printf '%s is broken (says %s).' \ "$1" \ @@ -235,7 +235,7 @@ if [ "$5" = 'ERROR' ]; then fi printf '\n' } | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi fi diff --git a/bin/sanity-check b/bin/sanity-check index b3e02aa..d98332c 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -22,7 +22,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -432,5 +432,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux-ports/in" + sponge "${irc_dir}/#archlinux32/in" fi diff --git a/lib/mysql-functions b/lib/mysql-functions index b9ae4ab..0b21bdf 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -83,7 +83,7 @@ mysql_run_query() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_file##*/}" \ - | sponge "${irc_dir}/#archlinux-ports/in" + | sponge "${irc_dir}/#archlinux32/in" fi echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" -- cgit v1.2.3-54-g00ecf From bd5173dc0c845e0e01de8aed01bd85b0e6646344 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 10:33:35 +0100 Subject: lib/common-functions: irc_say new --- bin/build-master-status-from-mysql | 2 +- bin/get-package-updates | 2 +- bin/return-assignment | 28 +++++++++++++--------------- bin/sanity-check | 4 ++-- lib/common-functions | 8 ++++++++ lib/mysql-functions | 2 +- 6 files changed, 26 insertions(+), 20 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 729375f..e4dd313 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -249,7 +249,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ if [ -s "${webserver_directory}/mysql-sanity.html" ] && \ [ ! -s "${work_dir}/build-master-sanity" ]; then printf 'oh girls, my database is dirty again ...\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say echo 'build master is insane' > \ "${work_dir}/build-master-sanity" fi diff --git a/bin/get-package-updates b/bin/get-package-updates index d3cb140..a6c3f32 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -480,7 +480,7 @@ if [ "${deletion_list_count}" -gt 1000 ]; then done } | \ tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi exit 3 fi diff --git a/bin/return-assignment b/bin/return-assignment index 417654b..c259f1c 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -222,21 +222,19 @@ if [ "$5" = 'ERROR' ]; then fi ) - if [ -p "${irc_dir}/#archlinux32/in" ]; then - { - printf '%s is broken (says %s).' \ - "$1" \ - "${slave}" - if [ -n "${rescheduled_packages}" ]; then - printf -- ' - I rescheduled:' - # shellcheck disable=SC2086 - printf ' %s,' ${rescheduled_packages} | \ - sed 's/,$/./' - fi - printf '\n' - } | \ - sponge "${irc_dir}/#archlinux32/in" - fi + { + printf '%s is broken (says %s).' \ + "$1" \ + "${slave}" + if [ -n "${rescheduled_packages}" ]; then + printf -- ' - I rescheduled:' + # shellcheck disable=SC2086 + printf ' %s,' ${rescheduled_packages} | \ + sed 's/,$/./' + fi + printf '\n' + } | \ + irc_say fi exit 0 diff --git a/bin/sanity-check b/bin/sanity-check index d98332c..9615ac7 100755 --- a/bin/sanity-check +++ b/bin/sanity-check @@ -22,7 +22,7 @@ usage() { i_am_insane() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION goes insane.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi echo 'build master is insane' > \ "${work_dir}/build-master-sanity" @@ -432,5 +432,5 @@ done if [ -f "${work_dir}/build-master-sanity" ]; then rm "${work_dir}/build-master-sanity" printf '\001ACTION resumes sanity.\001\n' | \ - sponge "${irc_dir}/#archlinux32/in" + irc_say fi diff --git a/lib/common-functions b/lib/common-functions index 1abe7bc..aa32409 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -1390,3 +1390,11 @@ extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() { epoch="${epoch%%:*}" fi } + +# irc_say +# say content of stdin in irc channel +irc_say() { + if [ -p "${irc_dir}/#archlinux32/in" ]; then + sponge "${irc_dir}/#archlinux32/in" + fi +} diff --git a/lib/mysql-functions b/lib/mysql-functions index 0b21bdf..801eca2 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -83,7 +83,7 @@ mysql_run_query() { if [ ! -s "${work_dir}/build-master-sanity" ]; then printf '\001ACTION failed to execute a mysql query - can you have a look at "%s"?.\001\n' \ "${query_file##*/}" \ - | sponge "${irc_dir}/#archlinux32/in" + | irc_say fi echo 'A mysql query failed.' > \ "${work_dir}/build-master-sanity" -- cgit v1.2.3-54-g00ecf From b7f0199b15fb07d2532b12c627bf400783e7a115 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 14:35:12 +0100 Subject: work/build-list.loops decomissioned --- bin/build-master-status-from-mysql | 21 ------------------- bin/get-assignment | 6 ------ bin/get-package-updates | 42 ++------------------------------------ bin/return-assignment | 18 ---------------- conf/default.conf | 1 - 5 files changed, 2 insertions(+), 86 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 5f27d89..edabc1e 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -72,23 +72,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin sort "${work_dir}/deletion-list" > \ "${tmp_dir}/deletion-list.file" -# shellcheck disable=SC2016 -{ - printf 'SELECT `package_sources`.`pkgbase`' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_build_dependency_loops - printf ';\n' -} | \ - mysql_run_query | \ - sort > \ - "${tmp_dir}/loops.mysql" -find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ - -regex '.*/loop_[0-9]\+' \ - -exec 'cat' '{}' \; | \ - sort > \ - "${tmp_dir}/loops.file" - # shellcheck disable=SC2016 { printf 'SELECT ' @@ -220,10 +203,6 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ "${tmp_dir}/deletion-list.file" \ "${tmp_dir}/deletion-list.mysql" || \ true - diff -u \ - "${tmp_dir}/loops.file" \ - "${tmp_dir}/loops.mysql" >&2 || \ - true diff -u \ "${tmp_dir}/tested.file" \ "${tmp_dir}/tested.mysql" || \ diff --git a/bin/get-assignment b/bin/get-assignment index 2df3059..3357fc6 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -103,12 +103,6 @@ hand_out_assignment() { } | \ mysql_run_query - # lock every loop this package breaks - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -exec touch '{}.locked' \; - exit 0 } diff --git a/bin/get-package-updates b/bin/get-package-updates index a6c3f32..69c5e9b 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -528,7 +528,7 @@ echo 'Now actually sort it.' { # this part will have the correct build order, but all the infos are missing - tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \ + tsort "${work_dir}/build-order" | \ nl -ba | \ awk '{print $1 " not-git also-not-git whatever " $2}' # this part has all the infos, but possibly the wrong order @@ -543,43 +543,6 @@ echo 'Now actually sort it.' awk '{print $5 " " $7 " " $8 " " $9}' > \ "${work_dir}/build-list.new.new" -rm --one-file-system -rf "${work_dir}/build-list.loops.new" -mkdir "${work_dir}/build-list.loops.new" - -if [ -s "${work_dir}/tsort.error" ]; then - >&2 echo 'WARNING: There is a dependency cycle!' - >&2 cat "${work_dir}/tsort.error" - >&2 echo - >&2 echo 'I will continue anyway.' - # save loops in separate files each, so breaking them is easier - awk ' - /^tsort: \S+: input contains a loop:$/{ - n++; - getline - } - { - print $2 >"'"${work_dir}"'/build-list.loops.new/loop_" n - } - ' "${work_dir}/tsort.error" - - # remove lines from loop files which are no packages - find "${work_dir}/build-list.loops.new" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -printf '%p\n' | \ - while read -r loop; do - { - sort -u "${loop}" - cut -d' ' -f1 "${work_dir}/build-list.new.new" | \ - sort -u - } | \ - sort | \ - uniq -d | \ - sponge "${loop}" - done -else - rm "${work_dir}/tsort.error" -fi - # shellcheck disable=SC2016 { # update hashes of repositories in mysql database @@ -608,9 +571,8 @@ mysql_find_build_assignment_loops # Move the .new-files to the actual files -rm -rf --one-file-system "${work_dir}/build-list.loops" { - printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list" + printf '%s\n' "build-list.new" "build-list" "deletion-list" # shellcheck disable=SC2086 printf '%s.revision\n' ${repo_names} } | \ diff --git a/bin/return-assignment b/bin/return-assignment index c259f1c..b66244b 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -169,18 +169,6 @@ if [ "$5" = 'ERROR' ]; then if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - # unlock every loop this package would have broken and which is not - # broken by another locked package - locked_packages=$( - find "${work_dir}/package-states/" -maxdepth 1 -name '*.locked' -printf '%f\n' | \ - sed 's@^\(.\+\)\.\([0-9a-f]\{40\}\.\)\{2\}[^.]\+\.locked$@\1@' - ) - find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -not -exec grep -qxF "${locked_packages}" '{}' \; \ - -exec rm '{}.locked' \; - # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { @@ -538,12 +526,6 @@ find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ -exec rm '{}' \; -# remove all loops which are broken by this package -find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ - -regex '.*/loop_[0-9]\+' \ - -exec grep -qxF "$1" '{}' \; \ - -exec rm '{}.locked' \; - # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" diff --git a/conf/default.conf b/conf/default.conf index 7e7ed3d..b508deb 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -135,7 +135,6 @@ fi mkdir -p "${work_dir}" touch "${work_dir}/build-list" touch "${work_dir}/deletion-list" -mkdir -p "${work_dir}/build-list.loops" for repo in ${repo_names}; do -- cgit v1.2.3-54-g00ecf From 944da1b00e72f35fd758951a7b25bbb61f43b9fb Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 21 Mar 2018 10:23:52 +0100 Subject: bin/get-package-updates: work/build-order decomissioned --- bin/get-package-updates | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 69c5e9b..ce9d57a 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -506,42 +506,7 @@ if [ -n "${test_exclusion}" ]; then exit 0 fi -# Now we create the partial order. - -while read -r package git_revision mod_git_revision repository; do - # add "$pkgname -> $build-target" to build-order list - sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds" - # add "$dependency -> $pkgname" to build-order list - sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends" - # add "base/base-devel -> $pkgname" to build-order list - printf '%s '"${package}"'\n' 'base' 'base-devel' -done \ - < "${work_dir}/build-list.new" \ - > "${work_dir}/build-order" - -if grep -vq '^\S\+ \S\+$' "${work_dir}/build-order"; then - >&2 echo 'Created partial order file "build-order" is invalid.' - exit 2 -fi - -echo 'Now actually sort it.' - -{ - # this part will have the correct build order, but all the infos are missing - tsort "${work_dir}/build-order" | \ - nl -ba | \ - awk '{print $1 " not-git also-not-git whatever " $2}' - # this part has all the infos, but possibly the wrong order - awk '{print "0 " $2 " " $3 " " $4 " " $1}' "${work_dir}/build-list.new" -} | \ - sort -k5,5 -k1nr | \ - # now, we have the correct order and the infos, but in adjacent lines - uniq -f4 -D | \ - sed '/^0 /d;N;s|\n| |' | \ - # now in one line, each - sort -k1n,1 | \ - awk '{print $5 " " $7 " " $8 " " $9}' > \ - "${work_dir}/build-list.new.new" +echo 'Done - mark decisions as final.' # shellcheck disable=SC2016 { @@ -572,7 +537,7 @@ mysql_find_build_assignment_loops # Move the .new-files to the actual files { - printf '%s\n' "build-list.new" "build-list" "deletion-list" + printf '%s\n' "build-list" "deletion-list" # shellcheck disable=SC2086 printf '%s.revision\n' ${repo_names} } | \ -- cgit v1.2.3-54-g00ecf From 4f5f0fafc4be959a9a05adac8bf5162a7c5bab46 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 14:18:41 +0100 Subject: lib/common-functions: remove generate_package_metadata, delete_old_metadata - this information is now in the database --- bin/cleanup | 3 - bin/get-package-updates | 1 - bin/seed-build-list | 1 - lib/common-functions | 228 +----------------------------------------------- 4 files changed, 1 insertion(+), 232 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/cleanup b/bin/cleanup index 15b6fc9..02ac98b 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -99,9 +99,6 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \ -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \ -delete -# remove old package meta data -delete_old_metadata - # remove dependency graphs of packages on the deletion list sed ' s|^|'"${webserver_directory}"'/graphs/| diff --git a/bin/get-package-updates b/bin/get-package-updates index ce9d57a..a36cb8a 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -353,7 +353,6 @@ done | \ } | \ mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" - generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi diff --git a/bin/seed-build-list b/bin/seed-build-list index 0db9cdf..41707dd 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -373,7 +373,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2 printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ sed 's/ $/\n/' - generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \ "${work_dir}/build-list" diff --git a/lib/common-functions b/lib/common-functions index e3aab4c..0a5ef1c 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -6,7 +6,7 @@ # TODO: have full information (currently in files) in database -# TODO: remove state files / metadata files +# TODO: remove state files if [ -z "${base_dir}" ]; then # just to make shellcheck happy @@ -123,232 +123,6 @@ find_git_repository_to_package_repository() { } -# generate_package_metadata $package $git_revision $mod_git_revision $repository -# or -# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# generate the meta data files of a package (dependencies, built packages, ...) - -generate_package_metadata() { - - local package="$1" - local git_revision="$2" - local mod_git_revision="$3" - local repository="$4" - local file_prefix - local file - local PKGBUILD - - if [ $# -eq 1 ]; then - # second form - repository="${package##*.}" - package="${package%.*}" - mod_git_revision="${package##*.}" - package="${package%.*}" - git_revision="${package##*.}" - package="${package%.*}" - fi - - file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}" - - if [ -e "${file_prefix}.builds" ] && \ - [ -e "${file_prefix}.build-depends" ] && \ - [ -e "${file_prefix}.run-depends" ] && \ - [ -e "${file_prefix}.groups" ] && \ - [ -e "${file_prefix}.packages" ]; then - return 0 - fi - - if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then - printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO" - exit 1 - fi - if [ ! -s "${file_prefix}.SRCINFO" ]; then - >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO" - exit 1 - fi - - # otherwise this just calls for trouble - sed -i ' - /^[^=]*=\s*$/d - s/_i686\(\s*=\)/\1/ - ' "${file_prefix}.SRCINFO" - - # extract "groups" = groups \cup provides - grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.groups" - - # extract "packages" = pkgname - grep '^pkgname = ' "${file_prefix}.SRCINFO" | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u > \ - "${file_prefix}.packages" - - # extract "builds" = provides \cup pkgname \cup groups - cat "${file_prefix}.groups" "${file_prefix}.packages" | \ - sort -u > \ - "${file_prefix}.builds" - - # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' 'base-devel' - sed -n "$( - printf '/^\t%s = /p\n' \ - 'depends' \ - 'makedepends' - )" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.build-depends" - - # extract "run-depends" = depends \cup \{ base \} \setminus "builds" - { - { - printf 'all_depend = %s\n' 'base' - sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO" - } | \ - cut -d= -f2 | \ - sed 's|^\s\+||; s|[<>]$||' | \ - sort -u - sed 'p' "${file_prefix}.builds" - } | \ - sort | \ - uniq -u > \ - "${file_prefix}.run-depends" - - rm "${file_prefix}.SRCINFO" - -} - -# delete_old_metadata -# delete old (=unneeded) meta data of packages - -delete_old_metadata() { - - local current_metadata - - current_metadata=$( - find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.[^.]\+$|| - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3| - ' | \ - sort -u - ) - - ( # the new shell is intentional - # what we have - echo "${current_metadata}" - - # package-states should stay - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \ - sed ' - s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3| - ' | \ - sort -u | \ - sed 'p' - - # build-list items should stay - sed 'p' "${work_dir}/build-list" - - tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - - echo "${current_metadata}" | \ - sort -k1,1 > \ - "${tmp_dir}/current-metadata" - - # the newest of the following should stay: - { - # deletion-list items - cat "${work_dir}/deletion-list" - # all packages in the repos - for repo in ${repo_names}; do - eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \ - tar -t | \ - sed ' - s|/$|| - /\//d - ' - done - } | \ - sort -u | \ - join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \ - sort -k4,4 > \ - "${tmp_dir}/find-newest-revisions" - - uniq -uf3 < \ - "${tmp_dir}/find-newest-revisions" | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - - uniq -Df3 < \ - "${tmp_dir}/find-newest-revisions" | \ - uniq --group=append -f3 | \ - { - revs='' - mod_revs='' - opkg='' - orepo='' - while read -r rev mod_rev repo pkg; do - - if [ -z "${rev}" ] && \ - [ -z "${mod_rev}" ] && \ - [ -z "${repo}" ] && \ - [ -z "${pkg}" ]; then - - printf '%s %s %s %s\n' \ - "$( - printf '%s\n' ${revs} | \ - find_newest_of_git_revisions - )" \ - "$( - printf '%s\n' ${mod_revs} | \ - find_newest_of_git_revisions - )" \ - "${orepo}" \ - "${opkg}" - - revs='' - mod_revs='' - orepo='' - opkg='' - continue - fi - revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${revs} ${rev} | \ - sort -u - ) - mod_revs=$( - # shellcheck disable=SC2086 - printf '%s\n' ${mod_revs} ${mod_rev} | \ - sort -u - ) - orepo="${repo}" - opkg="${pkg}" - done - } | \ - awk '{print $4 " " $1 " " $2 " " $3}' | \ - sed 'p' - ) | \ - sort | \ - uniq -u | \ - while read -r pkg rev mod_rev repo; do - rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."* - done -} - # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository # print which (stable) repository a package belongs to -- cgit v1.2.3-54-g00ecf From c772be57eca3a2176ac3931c02fb582f85c0d575 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:19:23 +0100 Subject: decommission package-states directory - we rely on the database now --- bin/bootstrap-mysql | 60 ++++------------------------------------------ bin/cleanup | 20 ---------------- bin/db-update | 62 ------------------------------------------------ bin/get-assignment | 27 +++------------------ bin/get-package-updates | 6 ----- bin/modify-package-state | 38 ----------------------------- bin/return-assignment | 38 ----------------------------- lib/common-functions | 4 ---- 8 files changed, 7 insertions(+), 248 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4c1938f..fc347d7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: this should become some sort of mysql dump to bootstrap the +# tables and stored functions only - when the database is gone, we're +# pretty much screwed anyway :-D + tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT @@ -792,68 +796,12 @@ fi } | \ mysql_run_query -find "${work_dir}/package-states" \ - -maxdepth 1 \ - -mindepth 1 \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed ' - 1!d - s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@ - s@^\S*/@@ - s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 / - s/ \([0-9]\+\) / \1.0 / - s/ [0-9]\+\.\([0-9]\+\) / \1 / - ' {} \; | \ - while read -r state_file sub_pkgrel arch package_file; do - printf '%s ' "${state_file}" - repository=$( - grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \ - cut -d' ' -f1 - ) - if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then - printf 'not found exactly once on the master mirror.\n' - >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}" - continue - fi - mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}" - printf ' ok\n' - done - tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do mysql_generate_package_metadata 'build-list' "${state_file}" done -# shellcheck disable=SC2016 -find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ - sed ' - s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3| - ' | \ - while read -r state_file pkgbase git_revision mod_git_revision repository; do - printf 'UPDATE `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ - "$( - tr -d '\n' < \ - "${state_file}" | \ - base64 -w0 - )" - printf ' WHERE' - printf ' `package_sources`.`%s`=from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" - printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${repository}" | base64 -w0)" - done | \ - mysql_run_query - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/cleanup b/bin/cleanup index a0bf790..7aaa39f 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# remove blocked/broken/locked markes of packages not on the buildlist anymore - -{ - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5| - ' - awk '{ - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked" - }' "${work_dir}/build-list" -} | \ - sort -k3 | \ - uniq -uf2 | \ - grep '^state ' | \ - awk '{print $2}' | \ - sed "s|^|${work_dir}/package-states/|" | \ - xargs -rn1 rm - # remove logs where package is not broken/locked anymore { diff --git a/bin/db-update b/bin/db-update index 6ecf9c9..6722f24 100755 --- a/bin/db-update +++ b/bin/db-update @@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do mysql_join_repositories_repository_stabilities printf ' AND `repository_stabilities`.`name`="%s"' \ "${source_stability}" - # TODO: we may relax this condition, if we a) got rid of state files - # or b) improved keeping them in sync (*.testing -> /dev/null is not - # implemented, only .tested -> /dev/null) - if [ ! "${source_stability}" = 'staging' ]; then - printf ' AND `binary_packages`.`is_tested`' - fi mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories @@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query - - sed_apply_rm=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/rm" - ) - - sed_apply_remove_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/mv" - ) - - sed_apply_move_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/{\n' "$(str_to_regex "${pkg}")" - printf 'w %%s\n' - printf 'd\n' - printf '}\n' - done < \ - "${tmp_dir}/mv" - ) - - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed -i "${sed_apply_rm}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.tested' \ - -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.done' \ - -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \ - -printf '%p\n' | \ - while read -r sf; do - sed -i "$( - printf '%s\n' "${sed_apply_move_part_of_mv}" | \ - sed ' - s,%s,'"${sf%.done}.testing"',g - ' - )" "${sf}" - done - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -type f \ - -not -exec test -s '{}' \; \ - -delete done trigger_mirror_refreshs diff --git a/bin/get-assignment b/bin/get-assignment index 3357fc6..a82e1f8 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -18,17 +18,8 @@ # TODO: honor manual build order of tool-chain: # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc -mkdir -p "${work_dir}/package-states" - hand_out_assignment() { - # we don't care anymore if an older version of this package was - # "locked" or "broken" - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \ - -delete - # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { @@ -57,21 +48,7 @@ hand_out_assignment() { printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' - } | \ - mysql_run_query | \ - tr '\t' ' ' - { - # shellcheck disable=SC2154 - echo "${slave}" - if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - cat "${work_dir}/package-states/$1.$2.$3.$4.locked" - fi - } | \ - sort -u | \ - sponge "${work_dir}/package-states/$1.$2.$3.$4.locked" - # shellcheck disable=SC2016 - { printf 'UPDATE `build_slaves`' printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' @@ -86,6 +63,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ')' + # shellcheck disable=SC2154 printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -101,7 +79,8 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - mysql_run_query + mysql_run_query | \ + tr '\t' ' ' exit 0 diff --git a/bin/get-package-updates b/bin/get-package-updates index a36cb8a..3e9edf9 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -111,9 +111,6 @@ delete_package() { mysql_run_query >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \ - -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -317,9 +314,6 @@ done | \ if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -regextype grep \ - -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \ - -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" diff --git a/bin/modify-package-state b/bin/modify-package-state index 76b8c1e..4a2f5ac 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -243,46 +243,8 @@ while read -r package reason; do { printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" printf ';\n' - # TODO: remove the below part, once the database if fully functional - printf 'UPDATE `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b`' - printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - mysql_join_binary_packages_repositories 'b' - printf ' SET `b`.`is_tested`=1' - printf ' WHERE `a`.`is_tested`' - printf ' AND NOT `b`.`is_tested`' - printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ mysql_run_query - case "${action}" in - 'block') - printf '%s' "${reason}" > \ - "${work_dir}/package-states/${package}.blocked" - ;; - 'unblock') - rm -f "${work_dir}/package-states/${package}.blocked" - ;; - 'faulty') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.tested' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ed}ing" - done - ;; - 'tested') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.testing' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ing}ed" - done - ;; - *) - >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" - exit 42 - ;; - esac printf '%s %s\n' "${action}" "${package}" fi done \ diff --git a/bin/return-assignment b/bin/return-assignment index 02bd491..da9e762 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then '*.build-log.gz' ) - # shellcheck disable=SC2154 - echo "${slave}" >> \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then @@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then } | \ mysql_run_query - # shellcheck disable=SC2154 - sed -i ' - /^'"$(str_to_regex "${slave}")"'$/d - ' "${work_dir}/package-states/$1.$2.$3.$4.locked" - if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - fi - # release lock on build-list - otherwise seed-build-list won't run flock -u 9 @@ -517,24 +497,9 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove old state files (these should be only "done" markers, but -# actually we don't care what it is) - as long as it's not "testing" or "tested" -find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -not -name '*.testing' \ - -not -name '*.tested' \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ - -exec rm '{}' \; - # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" -# remove package lock file -if ! [ "${destination}" = 'build-support' ]; then - # shellcheck disable=SC2086 - printf '%s\n' ${packages} > \ - "${work_dir}/package-states/$1.$2.$3.$4.done" -fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' @@ -578,6 +543,3 @@ fi printf 'DROP TABLE `loops_to_delete`;\n' } | \ mysql_run_query -rm -f \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/lib/common-functions b/lib/common-functions index 0a5ef1c..f66ccd1 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -4,10 +4,6 @@ # shellcheck disable=SC2039 -# TODO: have full information (currently in files) in database - -# TODO: remove state files - if [ -z "${base_dir}" ]; then # just to make shellcheck happy . 'conf/default.conf' -- cgit v1.2.3-54-g00ecf From b3023cfe9c2d4e8319885e22a575bd1a522b3f6c Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:42:11 +0100 Subject: decommission build-list file --- bin/bootstrap-mysql | 6 ------ bin/get-assignment | 9 --------- bin/get-package-updates | 33 ++++----------------------------- bin/prioritize-build-list | 6 ------ bin/return-assignment | 3 --- bin/seed-build-list | 2 -- conf/default.conf | 1 - 7 files changed, 4 insertions(+), 56 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index fc347d7..817989d 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -796,12 +796,6 @@ fi } | \ mysql_run_query -tr ' ' '.' < \ - "${work_dir}/build-list" | \ - while read -r state_file; do - mysql_generate_package_metadata 'build-list' "${state_file}" - done - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/get-assignment b/bin/get-assignment index a82e1f8..7ac749f 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -20,15 +20,6 @@ hand_out_assignment() { - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'SELECT ' diff --git a/bin/get-package-updates b/bin/get-package-updates index 3e9edf9..dcc9f3b 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,7 +110,6 @@ delete_package() { } | \ mysql_run_query >> \ "${work_dir}/deletion-list.new" - sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -201,9 +200,6 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -cp \ - "${work_dir}/build-list" \ - "${work_dir}/build-list.new" cp \ "${work_dir}/deletion-list" \ "${work_dir}/deletion-list.new" @@ -223,26 +219,9 @@ for repo in ${repo_names}; do git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" fi } | \ - # Packages which are already on the build list should receive a git_revision bump if _any_ file changed. - # Thus, we rename any file "PKGBUILD" to trigger the successive logic. - if [ "${repo}" = 'archlinux32' ]; then - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t[^/]\\+/| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - else - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - fi | \ # only track changes in PKGBUILDs + # TODO: However, packages which are already on the build list should + # receive a git_revision bump if _any_ file changed. grep '/PKGBUILD$' | \ if [ "${repo}" = "archlinux32" ]; then # modify the directory structure from the modifiaction-repository @@ -312,11 +291,6 @@ done | \ fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - # new or modified PKGBUILD - sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - # shellcheck disable=SC2154 - echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ - "${work_dir}/build-list.new" sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new" # shellcheck disable=SC2016 { @@ -346,6 +320,7 @@ done | \ )" } | \ mysql_run_query + # shellcheck disable=SC2154 mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi @@ -530,7 +505,7 @@ mysql_find_build_assignment_loops # Move the .new-files to the actual files { - printf '%s\n' "build-list" "deletion-list" + printf '%s\n' "deletion-list" # shellcheck disable=SC2086 printf '%s.revision\n' ${repo_names} } | \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index 1ff3d5a..db828ae 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -23,12 +23,6 @@ if ! flock -s -n 8; then exit 1 fi -{ - sed -n "/^$1/p" "${work_dir}/build-list" - sed "/^$1/d" "${work_dir}/build-list" -} | \ - sponge "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' diff --git a/bin/return-assignment b/bin/return-assignment index da9e762..a2669f6 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -497,9 +497,6 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove package from build list -sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' diff --git a/bin/seed-build-list b/bin/seed-build-list index 41707dd..ae08d57 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -374,8 +374,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ sed 's/ $/\n/' mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" - printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \ - "${work_dir}/build-list" printf '\n' >&2 done diff --git a/conf/default.conf b/conf/default.conf index b508deb..e8e0169 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then fi mkdir -p "${work_dir}" -touch "${work_dir}/build-list" touch "${work_dir}/deletion-list" for repo in ${repo_names}; do -- cgit v1.2.3-54-g00ecf From ba6f43ca62a719d075cd23ccba35dbbab5f43951 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:45:47 +0100 Subject: decommission deletion-list --- bin/build-master-status-from-mysql | 20 -------------------- bin/cleanup | 7 ------- bin/get-package-updates | 35 ++++------------------------------- conf/default.conf | 1 - 4 files changed, 4 insertions(+), 59 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index c6df53b..962b12e 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -20,28 +20,8 @@ fi # shellcheck disable=SC2119 mysql_cleanup -# shellcheck disable=SC2016 -{ - printf 'SELECT DISTINCT `package_sources`.`pkgbase`' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_binary_packages - mysql_join_binary_packages_repositories - printf ' WHERE `repositories`.`name`="deletion-list"' - printf ' OR `binary_packages`.`is_to_be_deleted`;\n' -} | \ - mysql_run_query | \ - sort > \ - "${tmp_dir}/deletion-list.mysql" -sort "${work_dir}/deletion-list" > \ - "${tmp_dir}/deletion-list.file" - { mysql_sanity_check || true - diff -u \ - "${tmp_dir}/deletion-list.file" \ - "${tmp_dir}/deletion-list.mysql" || \ - true } | \ sed ' s,^-.*$,\0, diff --git a/bin/cleanup b/bin/cleanup index 7aaa39f..c9a8d2b 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -69,11 +69,4 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \ -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \ -delete -# remove dependency graphs of packages on the deletion list -sed ' - s|^|'"${webserver_directory}"'/graphs/| - s|$|.png| -' "${work_dir}/deletion-list" | \ - xargs -rn1 rm -f - exit 0 diff --git a/bin/get-package-updates b/bin/get-package-updates index dcc9f3b..f4c53b4 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -99,17 +99,6 @@ fi # delete_package package # mark $package for deletion delete_package() { - # shellcheck disable=SC2016 - { - printf 'SELECT DISTINCT `package_sources`.`pkgbase`' - printf ' FROM `package_sources`' - mysql_join_package_sources_build_assignments - mysql_join_build_assignments_binary_packages - printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ - "$(printf '%s' "$1" | base64 -w0)" - } | \ - mysql_run_query >> \ - "${work_dir}/deletion-list.new" # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -200,10 +189,6 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -cp \ - "${work_dir}/deletion-list" \ - "${work_dir}/deletion-list.new" - for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"' @@ -291,7 +276,6 @@ done | \ fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new" # shellcheck disable=SC2016 { # delete old build assignment and associated binary packages @@ -329,9 +313,6 @@ done | \ exit 1 done -sort -u "${work_dir}/deletion-list.new" | \ - sponge "${work_dir}/deletion-list.new" - echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date @@ -462,15 +443,10 @@ echo "${black_listed}" | \ delete_package "${package}" done -sort -u "${work_dir}/deletion-list.new" | \ - sponge "${work_dir}/deletion-list.new" - if [ -n "${test_exclusion}" ]; then - # we should not actually update the build-list et. al, but solely print this difference: - if diff --color -u "${work_dir}/deletion-list" "${work_dir}/deletion-list.new"; then - printf 'If you put "%s" on the blacklist, no additional packages will end up on the deletion list.\n' "${test_exclusion}" - fi + # TODO: reimplement test_exclusion with information from the database + >&2 echo 'sry, not yet done.' exit 0 fi @@ -504,11 +480,8 @@ mysql_find_build_assignment_loops # Move the .new-files to the actual files -{ - printf '%s\n' "deletion-list" - # shellcheck disable=SC2086 - printf '%s.revision\n' ${repo_names} -} | \ +# shellcheck disable=SC2086 +printf '%s.revision\n' ${repo_names} | \ while read -r file; do mv "${work_dir}/${file}.new" "${work_dir}/${file}" done diff --git a/conf/default.conf b/conf/default.conf index e8e0169..11483a0 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then fi mkdir -p "${work_dir}" -touch "${work_dir}/deletion-list" for repo in ${repo_names}; do -- cgit v1.2.3-54-g00ecf From a3888e3b4c363d7159e0274e75ecc7c7a9f3be1b Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 10:37:08 +0100 Subject: bin/get-package-updates: save black-list(-reason) in database --- bin/get-package-updates | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index f4c53b4..1b01087 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -454,6 +454,30 @@ echo 'Done - mark decisions as final.' # shellcheck disable=SC2016 { + # save blacklist into database + printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64), `reason` TEXT);\n' + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ + tar -Ox 'blacklist' | \ + sed -n ' + s/^\(\S\+\)\s*#\s*/\1 / + T + p + ' | \ + while read -r pkgbase reason; do + printf '(from_base64("%s"),from_base64("%s")),\n' \ + "$(printf '%s' "${pkgbase}" | base64 -w0)" \ + "$(printf '%s' "${reason}" | base64 -w0)" + done | \ + sed ' + 1 i INSERT INTO `bl` (`pkgbase`,`reason`) VALUES + $ s/,$/;/ + ' + printf 'UPDATE `build_assignments`' + mysql_join_build_assignments_package_sources + printf ' LEFT JOIN `bl` ON `package_sources`.`pkgbase`=`bl`.`pkgbase`' + printf ' SET `build_assignments`.`is_black_listed`=`bl`.`reason`;\n' + printf 'DROP TEMPORARY TABLE `bl`;\n' + # update hashes of repositories in mysql database for repo in ${repo_names}; do printf 'UPDATE `git_repositories`' -- cgit v1.2.3-54-g00ecf From 9d7f5d9f22c633381612bb2a1fbc036e54b40254 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 11:19:58 +0100 Subject: bin/get-package-updates: use revisions from database instead of files --- bin/get-package-updates | 70 +++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 37 deletions(-) (limited to 'bin/get-package-updates') diff --git a/bin/get-package-updates b/bin/get-package-updates index 1b01087..cf2ea6c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -127,31 +127,30 @@ delete_package() { fi } -# Update git repositories (official packages, community packages and the repository of package customizations). +something_new=false for repo in ${repo_names}; do eval repo_path='"${repo_paths__'"${repo}"'}"' + # Update git repositories (official packages, community packages and the repository of package customizations). git -C "${repo_path}" fetch origin master:master -done - -# Read previous git revision numbers from files. - -something_new=false - -for repo in ${repo_names}; do + # read previous git revision numbers from database. + # shellcheck disable=SC2016 eval "old_repo_revisions__${repo}='$( - cat "${work_dir}/${repo}.revision" 2> /dev/null || \ - echo NONE + { + printf 'SELECT `git_repositories`.`head` FROM `git_repositories`' + printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \ + "$(printf '%s' "${repo}" | base64 -w0)" + } | \ + mysql_run_query )'" - eval repo_path='"${repo_paths__'"${repo}"'}"' - eval "new_repo_revisions__${repo}='$( - if ${pull}; then + # determine new git revision + if ${pull}; then + eval "new_repo_revisions__${repo}='$( git -C "${repo_path}" rev-parse HEAD - else - cat "${work_dir}/${repo}.revision" - fi | \ - tee "${work_dir}/${repo}.revision.new" - )'" + )'" + else + eval 'new_repo_revisions__'"${repo}"'="${old_repo_revisions__'"${repo}"'}"' + fi if ! eval '[ "${new_repo_revisions__'"${repo}"'}" = "${old_repo_revisions__'"${repo}"'}" ]'; then something_new=true fi @@ -241,24 +240,26 @@ done | \ while read -r mode package git_revision repository; do if [ "${mode}" = 'D' ]; then # deleted PKGBUILD - git_revision=$(cat "${work_dir}/archlinux32.revision.new") + # shellcheck disable=SC2154 + git_revision="${new_repo_revisions__archlinux32}" found_package=false for repository in ${repo_names}; do eval 'repo_path="${repo_paths__'"${repository}"'}"' if [ "${repository}" = "archlinux32" ]; then - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" 2> /dev/null | \ + if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \ tar -t 2> /dev/null | \ grep -q "/$(str_to_regex "${package}")/PKGBUILD$"; then found_package=true break; fi else - if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision.new")" -- "${package}/repos" 2> /dev/null | \ - tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \ - cut -d/ -f3 | \ - grep -v 'staging\|testing\|-unstable' | \ - grep -vq -- '-i686$'; then - git_revision=$(cat "${work_dir}/${repository}.revision.new") + # shellcheck disable=SC2154 + if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos" 2> /dev/null | ' \ + 'tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | ' \ + 'cut -d/ -f3 | ' \ + 'grep -v '"'"'staging\|testing\|-unstable'"'"' | ' \ + 'grep -vq -- '"'"'-i686$'"'"; then + eval 'git_revision="${new_repo_revisions__'"${repository}"'}"' found_package=true break; fi @@ -420,11 +421,14 @@ if [ "${deletion_list_count}" -gt 1000 ]; then { printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}" for repo_name in ${repo_names}; do - if ! diff -q "${work_dir}/${repo_name}.revision" "${work_dir}/${repo_name}.revision.new" >/dev/null; then + eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"' + eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"' + # shellcheck disable=SC2154 + if [ "${old_revision}" != "${new_revision}" ]; then printf '%s: %s -> %s\n' \ "${repo_name}" \ - "$(cat "${work_dir}/${repo_name}.revision")" \ - "$(cat "${work_dir}/${repo_name}.revision.new")" + "${old_revision}" \ + "${new_revision}" fi done } | \ @@ -502,14 +506,6 @@ echo 'Done - mark decisions as final.' # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". mysql_find_build_assignment_loops -# Move the .new-files to the actual files - -# shellcheck disable=SC2086 -printf '%s.revision\n' ${repo_names} | \ - while read -r file; do - mv "${work_dir}/${file}.new" "${work_dir}/${file}" - done - # Remove the lock file rm -f "${build_list_lock_file}" -- cgit v1.2.3-54-g00ecf