From 4bc90d4dbd1d0ea28ae8da27b893f01ce5ba1e32 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 25 Jan 2018 09:28:56 +0100 Subject: bin/copy-to-build-support new for copying packages into build-support --- bin/copy-to-build-support | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 bin/copy-to-build-support (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support new file mode 100755 index 0000000..85d73e5 --- /dev/null +++ b/bin/copy-to-build-support @@ -0,0 +1,48 @@ +#!/bin/sh + +# copy the given package(s) into build-support + +# shellcheck source=conf/default.conf +. "${0%/*}/../conf/default.conf" + +# shellcheck disable=SC2016 +{ + to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' + printf '%s\n' "$@" | \ + sed -n ' + s/.\+/\0 \0/ + T + s/\.pkg\.tar\.xz$// + s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ + s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ + s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ + p + ' | \ + while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do + + # TODO: move packages on master mirror, too! + + printf 'INSERT IGNORE INTO `binary_packages`' + printf ' (`repository`' + # shellcheck disable=SC2086 + printf ',`%s`' ${to_copy} + printf ')' + printf ' SELECT' + printf ' (SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="build-support")' + # shellcheck disable=SC2086 + printf ',`binary_packages`.`%s`' ${to_copy} + printf ' FROM `binary_packages`' + printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" + printf ' `architectures`.`name`=from_base64("%s")' \ + "$(printf '%s' "${architecture}" | base64 -w0)" + printf ';\n' + done +} | \ + ${mysql_command} -- cgit v1.2.3-54-g00ecf From fbe2bc8a42570944d2b5fc8471bc43c568bdf3c2 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 16:18:10 +0100 Subject: bin/copy-to-build-support: copies packages on master mirror, to --- bin/copy-to-build-support | 87 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 10 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 85d73e5..f9eb53c 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -5,6 +5,9 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX") +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + # shellcheck disable=SC2016 { to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' @@ -20,7 +23,53 @@ ' | \ while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do - # TODO: move packages on master mirror, too! + id=$( + { + printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' + printf ' FROM `binary_packages`' + printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" + printf ' `architectures`.`name`=from_base64("%s")' \ + "$(printf '%s' "${architecture}" | base64 -w0)" + printf ' LIMIT 1;\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + y/\t/ / + ' + ) + if [ -z "${id}" ]; then + continue + fi + repository="${id#* }" + id="${id%% *}" + + for suffix in '' '.sig'; do + printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \ + "${repository}" \ + "${package}" \ + "${suffix}" \ + "${package}" \ + "${suffix}" + done >> \ + "${tmp_dir}/sftp-command" + printf '%s/i686/%s/%s\n' \ + "${master_mirror_rsync_directory}" \ + "${repository}" \ + "${package}" | \ + sed ' + p + s/$/.sig/ + ' >> \ + "${tmp_dir}/to-copy" printf 'INSERT IGNORE INTO `binary_packages`' printf ' (`repository`' @@ -34,15 +83,33 @@ printf ' FROM `binary_packages`' printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ - 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ - 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ - 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ - 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" - printf ' `architectures`.`name`=from_base64("%s")' \ - "$(printf '%s' "${architecture}" | base64 -w0)" - printf ';\n' + printf ' `binary_packages`.`id`=%s;\n' \ + "${id}" done } | \ ${mysql_command} + +${master_mirror_rsync_command} \ + "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \ + "${master_mirror_rsync_directory}/i686/build-support/build-support.files."* \ + "${tmp_dir}/" + +if [ -s "${tmp_dir}/to-copy" ]; then + mkdir "${tmp_dir}/transit/" + # shellcheck disable=SC2046 + ${master_mirror_rsync_command} \ + $(cat "${tmp_dir}/to-copy") \ + "${tmp_dir}/transit/" + repo-add "${tmp_dir}/build-support.db.tar.gz" \ + "${tmp_dir}/transit/"*".pkg.tar.xz" +fi + +if [ -s "${tmp_dir}/sftp-command" ]; then + ${master_mirror_sftp_command} < \ + "${tmp_dir}/sftp-command" +fi + +${master_mirror_rsync_command} \ + "${tmp_dir}/build-support.db."* \ + "${tmp_dir}/build-support.files."* \ + "${master_mirror_rsync_directory}/i686/build-support/" -- cgit v1.2.3-54-g00ecf From 75eee3c091714d13638afdfadc79b8d29537f22f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 16:35:05 +0100 Subject: bin/copy-to-build-support: removing of old build-support packages started --- bin/copy-to-build-support | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index f9eb53c..8a25f05 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -52,6 +52,8 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT repository="${id#* }" id="${id%% *}" + printf '%s' "${package}" >> \ + "${tmp_dir}/packages" for suffix in '' '.sig'; do printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \ "${repository}" \ @@ -113,3 +115,11 @@ ${master_mirror_rsync_command} \ "${tmp_dir}/build-support.db."* \ "${tmp_dir}/build-support.files."* \ "${master_mirror_rsync_directory}/i686/build-support/" + +# TODO: This does not work correctly yet! +if [ -s "${tmp_dir}/packages" ]; then + while read -r package; do + remove_old_package_versions 'i686' 'build-support' "${package}" + done < \ + "${tmp_dir}/packages" +fi -- cgit v1.2.3-54-g00ecf From ec0fadd3426af36f55ac8d7082e4b2dca957ae56 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 2 Feb 2018 09:42:38 +0100 Subject: bin/copy-to-build-support: finshed, should work as defined --- bin/copy-to-build-support | 135 ++++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 65 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 8a25f05..2fa43d1 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -8,71 +8,72 @@ tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX") trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -# shellcheck disable=SC2016 -{ - to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' - printf '%s\n' "$@" | \ - sed -n ' - s/.\+/\0 \0/ - T - s/\.pkg\.tar\.xz$// - s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ - s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ - s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ - p - ' | \ - while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do +to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' +printf '%s\n' "$@" | \ + sed -n ' + s/.\+/\0 \0/ + T + s/\.pkg\.tar\.xz$// + s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ + s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ + s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ + p + ' | \ + while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do - id=$( - { - printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' - printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' WHERE' - printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ - 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ - 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ - 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ - 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ - 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" - printf ' `architectures`.`name`=from_base64("%s")' \ - "$(printf '%s' "${architecture}" | base64 -w0)" - printf ' LIMIT 1;\n' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - 1d - y/\t/ / - ' - ) - if [ -z "${id}" ]; then - continue - fi - repository="${id#* }" - id="${id%% *}" + # shellcheck disable=SC2016 + id=$( + { + printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' + printf ' FROM `binary_packages`' + printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + printf ' WHERE' + printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ + 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ + 'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \ + 'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \ + 'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" \ + 'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" + printf ' `architectures`.`name`=from_base64("%s")' \ + "$(printf '%s' "${architecture}" | base64 -w0)" + printf ' LIMIT 1;\n' + } | \ + ${mysql_command} --raw --batch | \ + sed ' + 1d + y/\t/ / + ' + ) + if [ -z "${id}" ]; then + continue + fi + repository="${id#* }" + id="${id%% *}" - printf '%s' "${package}" >> \ - "${tmp_dir}/packages" - for suffix in '' '.sig'; do - printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \ - "${repository}" \ - "${package}" \ - "${suffix}" \ - "${package}" \ - "${suffix}" - done >> \ - "${tmp_dir}/sftp-command" - printf '%s/i686/%s/%s\n' \ - "${master_mirror_rsync_directory}" \ + printf '%s\n' "${package}" >> \ + "${tmp_dir}/packages" + for suffix in '' '.sig'; do + printf 'ln "i686/%s/%s%s" "i686/build-support/%s%s"\n' \ "${repository}" \ - "${package}" | \ - sed ' - p - s/$/.sig/ - ' >> \ - "${tmp_dir}/to-copy" + "${package}" \ + "${suffix}" \ + "${package}" \ + "${suffix}" + done >> \ + "${tmp_dir}/sftp-command" + printf '%s/i686/%s/%s\n' \ + "${master_mirror_rsync_directory}" \ + "${repository}" \ + "${package}" | \ + sed ' + p + s/$/.sig/ + ' >> \ + "${tmp_dir}/to-copy" + # shellcheck disable=SC2016 + { printf 'INSERT IGNORE INTO `binary_packages`' printf ' (`repository`' # shellcheck disable=SC2086 @@ -87,9 +88,9 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT printf ' WHERE' printf ' `binary_packages`.`id`=%s;\n' \ "${id}" - done -} | \ - ${mysql_command} + } >> \ + "${tmp_dir}/mysql-command" + done ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/build-support.db."* \ @@ -116,7 +117,11 @@ ${master_mirror_rsync_command} \ "${tmp_dir}/build-support.files."* \ "${master_mirror_rsync_directory}/i686/build-support/" -# TODO: This does not work correctly yet! +if [ -s "${tmp_dir}/mysql-command" ]; then + ${mysql_command} < \ + "${tmp_dir}/mysql-command" +fi + if [ -s "${tmp_dir}/packages" ]; then while read -r package; do remove_old_package_versions 'i686' 'build-support' "${package}" -- cgit v1.2.3-54-g00ecf From fdf371775ebbc5b70fc0df755259b8a7acce41d3 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 2 Feb 2018 09:53:40 +0100 Subject: bin/copy-to-build-support: expect packages in file instead of arguments --- bin/copy-to-build-support | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 2fa43d1..4710012 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -5,20 +5,25 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +if [ "$#" -ne 1 ]; then + >&2 echo 'usage:' + >&2 echo ' copy-to-build-support package-list' + exit 2 +fi + tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX") trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT to_copy='build_assignment epoch pkgver pkgrel sub_pkgrel has_issues is_tested pkgname architecture' -printf '%s\n' "$@" | \ - sed -n ' - s/.\+/\0 \0/ - T - s/\.pkg\.tar\.xz$// - s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ - s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ - s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ - p - ' | \ +sed -n ' + s/.\+/\0 \0/ + T + s/\.pkg\.tar\.xz$// + s/\(-[0-9]\+\)\(-[^- ]\+\)$/\1.0\2/ + s/-\([^-: ]\+\)\(\(-[^- ]\+\)\{2\}\)$/-0:\1\2/ + s/-\([^-: ]\+\):\([^-: ]\+\)-\([^-. ]\+\).\([^-. ]\+\)-\([^- ]\+\)$/ \1 \2 \3 \4 \5/ + p +' "$1" | \ while read -r package pkgname epoch pkgver pkgrel sub_pkgrel architecture; do # shellcheck disable=SC2016 @@ -128,3 +133,8 @@ if [ -s "${tmp_dir}/packages" ]; then done < \ "${tmp_dir}/packages" fi + +if [ -w "$1" ]; then + cat "${tmp_dir}/packages" > \ + "$1" +fi -- cgit v1.2.3-54-g00ecf From d2e0ef2a558e27e07150f239d91e4c29ce822426 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 16 Feb 2018 11:14:03 +0100 Subject: bin/mysql-functions: mysql_join_*_* new --- bin/bootstrap-mysql | 16 +--- bin/build-master-status-from-mysql | 65 ++++++-------- bin/check-bugtracker | 2 +- bin/common-functions | 5 +- bin/copy-to-build-support | 6 +- bin/db-update | 4 +- bin/get-assignment | 34 ++++---- bin/get-package-updates | 43 ++++----- bin/modify-package-state | 21 ++--- bin/mysql-functions | 174 +++++++++++++++++++++++++------------ bin/prioritize-build-list | 7 +- bin/return-assignment | 40 ++++----- bin/seed-build-list | 15 ++-- bin/show-dependencies | 7 +- bin/why-dont-you | 23 +++-- 15 files changed, 243 insertions(+), 219 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 322e0a1..30c260a 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -512,9 +512,8 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ ' | \ while read -r state_file pkgbase git_revision mod_git_revision repository; do printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ "$( tr -d '\n' < \ @@ -563,10 +562,9 @@ grep '^\('"$( "${tmp_dir}/new-stable-packages" if [ -s "${tmp_dir}/new-stable-packages" ]; then + # shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'CREATE TEMPORARY TABLE `stable_packages` (' - # shellcheck disable=SC2016 printf '`%s` %s,' \ 'pkgname' 'VARCHAR(64)' \ 'epoch' 'MEDIUMINT' \ @@ -576,9 +574,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then 'architecture' 'VARCHAR(16)' \ 'repository' 'VARCHAR(64)' \ 'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT' - # shellcheck disable=SC2016 printf 'PRIMARY KEY (`build_assignment`));\n' - # shellcheck disable=SC2016 sed ' 1~10 ! b not_start s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/ @@ -589,28 +585,22 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then :end s/,$/;/ ' "${tmp_dir}/new-stable-packages" - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' printf ' SELECT ' - # shellcheck disable=SC2016 printf '`stable_packages`.`%s`,' \ 'pkgname' \ 'epoch' \ 'pkgver' \ 'pkgrel' \ 'sub_pkgrel' - # shellcheck disable=SC2016 printf '`%s`.`id`,' \ 'architectures' \ 'repositories' - # shellcheck disable=SC2016 printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' - # shellcheck disable=SC2016 printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ 'repositories' 'repository' 'repositories' \ 'architectures' 'architecture' 'architectures' printf ';\n' - # shellcheck disable=SC2016 printf 'DROP TABLE `stable_packages`;\n' } | \ ${mysql_command} diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7a6605a..ea8d0f1 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -21,11 +21,10 @@ fi { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_upstream_repositories + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ ${mysql_command} --batch | \ @@ -45,10 +44,9 @@ sort "${work_dir}/build-list" > \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } | \ ${mysql_command} --raw --batch | \ sed ' @@ -68,10 +66,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin { printf 'SELECT DISTINCT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ @@ -89,9 +86,8 @@ sort "${work_dir}/deletion-list" > \ { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'build_dependency_loops' 'build_dependency_loops' 'build_assignment' 'build_assignments' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ ${mysql_command} --batch | \ @@ -118,10 +114,9 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -154,10 +149,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE NOT `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -188,15 +182,13 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'mod_git_revision' printf '`upstream_repositories`.`name`,`install_targets`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'dependencies' 'binary_packages' 'dependencies' 'dependent' \ - 'dependency_types' 'dependency_types' 'dependencies' 'dependency_type' - printf ' AND `dependency_types`.`relevant_for_building`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'install_targets' 'install_targets' 'dependencies' 'depending_on' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + mysql_join_dependencies_install_targets + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ ${mysql_command} --raw --batch | \ sed ' @@ -214,10 +206,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages } | \ ${mysql_command} --raw --batch | \ sed ' diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 72ab133..d6e92e5 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -31,7 +31,7 @@ bug_list=$( while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_binary_packages_repositories printf ' SET `has_issues`=%s' \ "${has_issues%:*}" printf ' WHERE `repositories`.`stability`=%s' \ diff --git a/bin/common-functions b/bin/common-functions index d7f3e41..a84271b 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -478,9 +478,8 @@ remove_old_package_versions() { printf '`binary_packages`.`sub_pkgrel`' printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'architectures' 'architectures' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' # name must match diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 4710012..808d041 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -31,8 +31,8 @@ sed -n ' { printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ @@ -89,7 +89,7 @@ sed -n ' # shellcheck disable=SC2086 printf ',`binary_packages`.`%s`' ${to_copy} printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`id`=%s;\n' \ "${id}" diff --git a/bin/db-update b/bin/db-update index e5ea193..b830981 100755 --- a/bin/db-update +++ b/bin/db-update @@ -325,8 +325,8 @@ move_packages() { while read -r package; do while read -r part; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo` ON `binary_packages`.`repository`=`from_repo`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' + mysql_join_binary_packages_architectures printf ' SET `binary_packages`.`repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` as `to_repo`' diff --git a/bin/get-assignment b/bin/get-assignment index a7eec2d..1edcffb 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -45,11 +45,10 @@ hand_out_assignment() { 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `package_sources`.`%s`=from_base64("%s")' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -80,8 +79,8 @@ hand_out_assignment() { printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -94,8 +93,8 @@ hand_out_assignment() { "$(printf '%s' "${slave}" | base64 -w0)" printf 'UPDATE `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`priority`=0' printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ @@ -150,10 +149,9 @@ currently_building=$( { printf 'SELECT ' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_slaves' 'build_slaves' 'currently_building' 'build_assignments' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' @@ -189,9 +187,8 @@ next_building=$( mysql_query_is_part_of_loop '`build_assignments`.`id`' printf ' AS `part_of_loop`,' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND NOT EXISTS (' printf ' SELECT *' @@ -231,9 +228,8 @@ count_pending=$( { printf 'SELECT count(*)' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' diff --git a/bin/get-package-updates b/bin/get-package-updates index 89b09df..e81d128 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,22 +110,18 @@ delete_package() { { # packages from the build-list/to-be-decided go straight to the deletion-list printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$(printf '%s' "$1" | base64 -w0)" # other packages are marked as `is_to_be_deleted` printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`is_to_be_deleted`=1' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" @@ -341,10 +337,9 @@ done | \ # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${package}" | \ @@ -353,10 +348,9 @@ done | \ printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n' # remove is-to-be-deleted marker from old binary packages printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' SET `is_to_be_deleted`=0' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$( @@ -431,9 +425,9 @@ while [ -n "${black_listed_new}" ]; do ' printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' printf ' FROM `package_sources` AS `a_ps`' - printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`' - printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`' - printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_binary_packages_dependencies 'a_bp' printf ' WHERE NOT EXISTS (' printf ' SELECT *' printf ' FROM `install_target_providers`' @@ -441,8 +435,8 @@ while [ -n "${black_listed_new}" ]; do printf ' SELECT *' printf ' FROM `bl`' printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' - printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`' - printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' printf ')' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' @@ -607,8 +601,7 @@ fi done # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo`' - printf ' ON `binary_packages`.`repository`=`from_repo`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' printf ' SET `repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` AS `to_repo`' diff --git a/bin/modify-package-state b/bin/modify-package-state index b2d3133..b0ba53e 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -136,10 +136,9 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_architectures ) # shellcheck disable=SC2016,SC2031 selector=$( @@ -161,12 +160,11 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities ) # shellcheck disable=SC2016 selector=$( @@ -238,8 +236,7 @@ while read -r package reason; do printf 'UPDATE `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b`' printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories`' - printf ' ON `b`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'b' printf ' SET `b`.`is_tested`=1' printf ' WHERE `a`.`is_tested`' printf ' AND NOT `b`.`is_tested`' diff --git a/bin/mysql-functions b/bin/mysql-functions index 7f2cdab..6950ee8 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -131,7 +131,6 @@ mysql_show_binary_package() { shift done - local joint { printf 'SELECT' printf ' `%s`.`%s`,' \ @@ -146,17 +145,11 @@ mysql_show_binary_package() { 'package_sources' 'mod_git_revision' \ 'upstream_repositories' 'name' printf ' FROM `binary_packages`' - for joint in \ - 'architectures:binary_packages:architecture' \ - 'package_sources:binary_packages:package_source' \ - 'repositories:binary_packages:repository' \ - 'upstream_repositories:package_sources:upstream_package_repository'; do - printf ' JOIN `%s` ON `%s`.`id` =' \ - "${joint%%:*}" "${joint%%:*}" - joint="${joint#*:}" - printf ' `%s`.`%s`' \ - "${joint%:*}" "${joint#*:}" - done + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'binary_packages' 'pkgname' "${pkgname}" \ @@ -434,8 +427,8 @@ mysql_generate_package_metadata() { # build-list or in $current_repository (beware of split # packages!) printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -448,7 +441,7 @@ mysql_generate_package_metadata() { "${current_repository}" # max(sub_pkgrel)+1 printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -496,11 +489,9 @@ mysql_generate_package_metadata() { printf ' `%s` JOIN' \ 'repositories' \ 'architectures' \ - 'build_assignments' \ - 'package_sources' - printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`' - printf ' JOIN `upstream_repositories`' - printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' + 'build_assignments' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'repositories' 'name' "${current_repository}" \ @@ -513,7 +504,6 @@ mysql_generate_package_metadata() { } | \ sed ' s|,)|)|g - s|JOIN ON|ON|g s| AND;$|;| ' >> \ "${temp_dir}/add-binary-packages-command" @@ -586,8 +576,7 @@ mysql_generate_package_metadata() { ;; esac printf ' `binary_packages`' - printf ' JOIN `architectures`' - printf ' ON `binary_packages`.`architecture` = `architectures`.`id`' + mysql_join_binary_packages_architectures printf ' JOIN `install_targets`' printf ' JOIN `%s`' "${link}" printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" @@ -649,9 +638,9 @@ mysql_sanity_check() { { printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' printf ' FROM `binary_packages`' - printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`' - printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`' + mysql_join_binary_packages_build_assignments + mysql_join_binary_packages_architectures '' 'bp_arch' + mysql_join_build_assignments_architectures '' 'ba_arch' printf ' WHERE `bp_arch`.`name`!="any"' printf ' AND `ba_arch`.`name`="any";\n' printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' @@ -660,14 +649,13 @@ mysql_sanity_check() { printf ' ON `a`.`pkgname`=`b`.`pkgname`' printf ' AND `a`.`repository`=`b`.`repository`' printf ' AND `a`.`id`!=`b`.`id`' - printf ' JOIN `repositories`' - printf ' ON `a`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'a' printf ' WHERE `repositories`.`name`="build-list";\n' printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' printf ' FROM `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`' - printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`' + mysql_join_binary_packages_repositories 'a' 'arep' + mysql_join_binary_packages_repositories 'b' 'brep' printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' printf ' AND `%srep`.`name`="build-list"' \ 'a' 'b' @@ -700,8 +688,8 @@ mysql_sanity_check() { { printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ ${mysql_command} --batch | \ @@ -728,12 +716,11 @@ mysql_find_build_assignment_loops() { { printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' printf ' FROM `dependencies`' - printf ' JOIN `install_target_providers` ON `dependencies`.`depending_on`=`install_target_providers`.`install_target`' - printf ' JOIN `binary_packages` `packages_dependent` ON `dependencies`.`dependent`=`packages_dependent`.`id`' - printf ' JOIN `binary_packages` `packages_dependency` ON `install_target_providers`.`package`=`packages_dependency`.`id`' - printf ' JOIN `repositories` `repositories_%s` ON `packages_%s`.`repository`=`repositories_%s`.`id`' \ - 'dependent' 'dependent' 'dependent' \ - 'dependency' 'dependency' 'dependency' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'packages_dependency' + mysql_join_dependencies_binary_packages '' 'packages_dependent' + mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' + mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ ${mysql_command} --raw --batch | \ @@ -836,8 +823,8 @@ mysql_cleanup() { printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' printf ');\n' printf 'UPDATE `build_slaves`' - printf ' JOIN `binary_packages` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_build_slaves_binary_packages + mysql_join_binary_packages_repositories printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ @@ -849,12 +836,11 @@ mysql_cleanup() { mysql_query_has_pending_dependencies() { printf 'EXISTS (' printf 'SELECT * FROM `binary_packages` as `to_dos`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \ - 'repositories` AS `to_do_repos' 'to_do_repos' 'id' 'to_dos' 'repository' \ - 'dependencies' 'to_dos' 'id' 'dependencies' 'dependent' \ - 'install_target_providers' 'dependencies' 'depending_on' 'install_target_providers' 'install_target' \ - 'binary_packages` AS `bin_deps' 'bin_deps' 'id' 'install_target_providers' 'package' \ - 'repositories` AS `dep_repos' 'dep_repos' 'id' 'bin_deps' 'repository' + mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' + mysql_join_binary_packages_dependencies 'to_dos' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'bin_deps' + mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' printf ' WHERE' printf ' `%s`.`name`="build-list" AND' \ 'dep_repos' 'to_do_repos' @@ -884,9 +870,8 @@ mysql_query_select_pkgbase_and_revision() { 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } # mysql_repair_binary_packages_without_build_assignment @@ -900,7 +885,7 @@ mysql_repair_binary_packages_without_build_assignment() { '`binary_packages`.`pkgname`' \ '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ ${mysql_command} --raw --batch | \ @@ -954,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' LIMIT 1;\n' printf 'UPDATE `binary_packages`' printf ' JOIN `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_binary_packages_build_assignments printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" @@ -980,9 +965,8 @@ mysql_remove_duplicate_build_order() { 'pkgver' 'pkgver' \ 'pkgrel' 'pkgrel' printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \ - 'orep' 'orep' 'old' \ - 'nrep' 'nrep' 'new' + mysql_join_binary_packages_repositories 'old' 'orep' + mysql_join_binary_packages_repositories 'new' 'nrep' printf ' WHERE `orep`.`name`!="to-be-decided"' printf ' AND `nrep`.`name`="to-be-decided";\n' printf 'UPDATE IGNORE `dependencies`' @@ -1004,6 +988,9 @@ mysql_remove_duplicate_build_order() { ${mysql_command} } +# mysql_package_name_query +# print a mysql query of the full name of a package file + mysql_package_name_query() { printf 'CONCAT(' printf '`binary_packages`.`pkgname`,"-",' @@ -1014,3 +1001,82 @@ mysql_package_name_query() { printf '`architectures`.`name`,".pkg.tar.xz"' printf ')' } + +# mysql_join_*_* +# print 'JOIN' part of mysql query to connect the respective tables +# these functions take 2 optional arguments, acting as aliases for +# the tables + +# mysql_join__generic $table_a $column_a $table_b $column_b +# create mysql_join_${table_a}_${table_b}() function + +mysql_join__generic() { + eval "$( + printf 'mysql_join_%s_%s() {\n' "$1" "$3" + printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' + printf ' fi\n' + printf ' if [ -n "$1" ]; then\n' + printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" + printf ' else\n' + printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" + printf ' fi\n' + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" + printf ' else\n' + printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" + printf ' fi\n' + printf '}\n' + )" +} + +for link in \ + 'binary_packages:architecture:architectures' \ + 'binary_packages:repository:repositories' \ + 'binary_packages:build_assignment:build_assignments' \ + \ + 'build_assignments:architecture:architectures' \ + 'build_assignments:package_source:package_sources' \ + \ + 'build_dependency_loops:build_assignment:build_assignments' \ + 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ + \ + 'build_slaves:currently_building:build_assignments' \ + 'build_slaves:currently_building build_assignment:binary_packages' \ + \ + 'dependencies:depending_on:install_targets' \ + 'dependencies:dependent:binary_packages' \ + 'dependencies:dependency_type:dependency_types' \ + \ + 'failed_builds:reason:fail_reason' \ + 'failed_builds:build_assignment:build_assignments' \ + 'failed_builds:build_slave:build_slaves' \ + \ + 'install_target_providers:package:binary_packages' \ + 'install_target_providers:install_target:install_targets' \ + 'install_target_providers:install_target depending_on:dependencies' \ + \ + 'package_sources:upstream_package_repository:upstream_repositories' \ + \ + 'repositories:stability:repository_stabilities' \ + \ + 'upstream_repositories:git_repository:git_repositories'; do +# A join for these cannot be done, because it's not clear on what to join: +# 'repository_stability_relations:more_stable:repository_stabilities' +# 'repository_stability_relations:less_stable:repository_stabilities' + + table_b="${link##*:}" + table_a="${link%:*}" + column_b="${table_a##*:}" + table_a="${table_a%:*}" + column_a="${column_b% *}" + if [ "${column_a}" = "${column_b}" ]; then + column_b='id' + else + column_b="${column_b##* }" + fi + + mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" + mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" +done diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index d611e79..e8dcf0a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -32,10 +32,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_package_sources + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' SET `build_assignments`.`priority`=(' printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1' printf ' FROM (' diff --git a/bin/return-assignment b/bin/return-assignment index 7b2dec3..995eae4 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -51,12 +51,11 @@ if [ "$5" = 'ERROR' ]; then infos=$( { printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( # shellcheck disable=SC2154 @@ -157,7 +156,7 @@ if [ "$5" = 'ERROR' ]; then printf 'DROP TABLE `failures`;\n' fi printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' + mysql_join_build_assignments_build_slaves printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( @@ -267,12 +266,11 @@ clean_up_lock_file() { if ! { printf 'SELECT count(*)' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'build_slaves' 'build_assignments' 'build_slaves' 'currently_building' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_build_slaves + mysql_join_binary_packages_repositories + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -396,9 +394,9 @@ package_errors=$( mysql_package_name_query printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_build_slaves + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ @@ -476,9 +474,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `repositories` AS `old_repo` ON `binary_packages`.`repository`=`old_repo`.`id`' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories '' 'old_repo' printf ' SET' printf ' `build_assignments`.`is_broken`=0,' printf ' `build_assignments`.`priority`=0,' @@ -507,8 +505,8 @@ fi printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n' printf 'INSERT INTO `loops_to_delete`' printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_dependency_loops`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_build_dependency_loops_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE NOT `repositories`.`name` = "build-list";\n' printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS (' printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`' diff --git a/bin/seed-build-list b/bin/seed-build-list index bfe975e..5ce9037 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -123,7 +123,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf 'SELECT ' mysql_package_name_query printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures } | \ ${mysql_command} --raw --batch | \ sed ' @@ -161,7 +161,7 @@ fi printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)' printf ' SELECT DISTINCT `binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n' } | \ sponge "${tmp_dir}/ignore-packages" @@ -253,7 +253,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '`pkgbases`.`repository`' printf ' FROM `pkgbases`' printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`' - printf ' JOIN `git_repositories` ON `git_repositories`.`id`=`upstream_repositories`.`git_repository`' + mysql_join_upstream_repositories_git_repositories printf ';\n' printf 'SELECT ' printf '`package_sources`.`pkgbase`,' @@ -264,11 +264,10 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '),' printf '`upstream_repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'git_repositories' 'git_repositories' 'upstream_repositories' 'git_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_git_repositories printf ' WHERE (' if [ -s "${tmp_dir}/package-regexes" ]; then grep -vxF '' "${tmp_dir}/package-regexes" | \ diff --git a/bin/show-dependencies b/bin/show-dependencies index 0a7a1fd..bb1ae37 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -13,7 +13,7 @@ if [ $# -eq 0 ]; then { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `build_assignments` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_package_sources_build_assignments printf ' WHERE `build_assignments`.`is_broken`' } | \ ${mysql_command} --raw --batch | \ @@ -28,9 +28,8 @@ if [ $# -eq 0 ]; then { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' ORDER BY `binary_packages`.`pkgname`' # Why can't mysql order in the same way "sort" does! } | \ diff --git a/bin/why-dont-you b/bin/why-dont-you index e9922b7..d95f2a7 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -57,10 +57,9 @@ case "${action}" in # package_source, build_assignment, binary_package, repostory printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`' printf ' FROM `package_sources` AS `tb_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `tb_ba' 'tb_ba' 'package_source' 'tb_ps' \ - 'binary_packages` AS `tb_bin' 'tb_bin' 'build_assignment' 'tb_ba' \ - 'repositories` AS `tb_rep' 'tb_bin' 'repository' 'tb_rep' + mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba' + mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin' + mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep' printf ' WHERE `tb_rep`.`name`="build-list"' printf ') AS `to_build`' printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`' @@ -70,16 +69,14 @@ case "${action}" in # potential dependencies printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`' printf ' FROM `package_sources` AS `dep_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `dep_ba' 'dep_ba' 'package_source' 'dep_ps' \ - 'binary_packages` AS `dep_bin' 'dep_bin' 'build_assignment' 'dep_ba' \ - 'repositories` AS `dep_rep' 'dep_bin' 'repository' 'dep_rep' \ - 'install_target_providers' 'install_target_providers' 'package' 'dep_bin' - # starting from the line above, we have some additional joins, + mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba' + mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin' + mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep' + # now we have some (=3) additional joins, # because we are interested in dependency relations to `to_build` - printf ' JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'dependency_types' 'dependencies' 'dependency_type' 'dependency_types' + mysql_join_binary_packages_install_target_providers 'dep_bin' + mysql_join_install_target_providers_dependencies + mysql_join_dependencies_dependency_types printf ' WHERE `dep_rep`.`name`="build-list"' printf ' AND `dependency_types`.`relevant_for_building`' printf ') AS `deps`' -- cgit v1.2.3-54-g00ecf From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-54-g00ecf From 496c571cd233a8646351ba241c3119ceaf2b8063 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 09:05:51 +0100 Subject: use "mysql -N" instead of cumbersome sedding --- bin/build-master-status | 5 ++--- bin/build-master-status-from-mysql | 46 ++++++++++---------------------------- bin/check-bugtracker | 5 ++--- bin/copy-to-build-support | 7 ++---- bin/db-update | 14 ++++-------- bin/get-assignment | 16 +++++-------- bin/get-package-updates | 6 ++--- bin/modify-package-state | 3 +-- bin/return-assignment | 20 +++++------------ bin/seed-build-list | 13 ++++------- bin/show-dependencies | 3 +-- bin/why-dont-you | 7 ++---- lib/common-functions | 7 ++---- lib/mysql-functions | 19 +++++----------- 14 files changed, 51 insertions(+), 120 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/build-master-status b/bin/build-master-status index 9be7823..deb58da 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,10 +562,9 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d - 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / + 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' printf ';\n' printf 'INSERT INTO `td` (`file`,`line`,`description`) VALUES ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index fa54177..d1985e7 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,11 +27,8 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" sort "${work_dir}/build-list" > \ @@ -48,11 +45,8 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -printf '%f\n' | \ @@ -72,11 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -90,11 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -121,9 +107,8 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -156,9 +141,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ sort -u > \ @@ -190,11 +174,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" { @@ -210,11 +191,8 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then # TODO: consider versions dependencies, too diff --git a/bin/check-bugtracker b/bin/check-bugtracker index f15757b..156f438 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,8 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' @@ -61,5 +60,5 @@ bug_list=$( tr '\n' ',' printf '"");\n' done - done | tee /dev/stderr | \ + done | \ mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 0b87107..e447596 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,11 +44,8 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${id}" ]; then continue diff --git a/bin/db-update b/bin/db-update index 534c245..78ff622 100755 --- a/bin/db-update +++ b/bin/db-update @@ -457,11 +457,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ sort -u @@ -496,11 +493,8 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - p - ' + mysql_run_query -N --raw --batch | \ + sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ sort -u diff --git a/bin/get-assignment b/bin/get-assignment index bc67fab..964c6b7 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,11 +58,8 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' { # shellcheck disable=SC2154 @@ -157,8 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ -n "${currently_building}" ]; then @@ -214,9 +210,8 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query --batch --raw | \ + mysql_run_query -N --batch --raw | \ sed ' - 1d y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ ' @@ -238,8 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query --batch --raw | \ - sed '1d' + mysql_run_query -N --batch --raw ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index 11f384c..e855b19 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,8 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch } | \ sort -u ) @@ -443,8 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 8c5fac7..68b2384 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,8 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index ed29f34..a7e6623 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,11 +70,8 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' ) if [ -z "${infos}" ]; then >&2 echo 'You do not build this package (anymore) - move on.' @@ -111,8 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query --raw --batch | \ - sed '1d' + mysql_run_query -N --raw --batch ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -283,9 +279,8 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ - grep -vxF '0'; then + mysql_run_query -N --raw --batch | \ + grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi @@ -441,10 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - ' + mysql_run_query -N --raw --batch } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index b9ae193..def9d56 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,9 +125,8 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' } | \ @@ -206,8 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -294,12 +292,9 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sort -u | \ - sed ' - y/\t/ / - / [0-9a-f]\{40\} [0-9a-f]\{40\} /!d - ' | \ + tr '\t' ' ' | \ if ${update}; then # always block if locked exec 9> "${build_list_lock_file}" diff --git a/bin/show-dependencies b/bin/show-dependencies index 9e94fe3..ed7ca87 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,9 +10,8 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s/\s.*$// ' | \ sort -u diff --git a/bin/why-dont-you b/bin/why-dont-you index b5d73c5..35b9419 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,11 +85,8 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' / NULL \S\+$/ b multi-dep diff --git a/lib/common-functions b/lib/common-functions index 740c266..ba434ad 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,11 +501,8 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query --raw --batch | \ - sed ' - /^\S\+\sCONCAT(/d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ shrink_version 4 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 6cbc502..3e7e19f 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -668,9 +668,8 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - /^CONCAT("/d s,^,, s,$,, ' @@ -699,9 +698,8 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query --batch | \ + mysql_run_query -N --raw --batch | \ sed ' - 1d s,\t,/, s,\t,-, s,\t,:, @@ -730,11 +728,8 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ + mysql_run_query -N --raw --batch | \ + tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ { @@ -910,8 +905,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -939,8 +933,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query --raw --batch | \ - sed '1d' | \ + mysql_run_query -N --raw --batch | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From a13bfbacf27d09f7c8ea6b4b559b0980ee8957b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 7 Mar 2018 16:30:10 +0100 Subject: make -N --raw --batch the default arguments to mysql, explicitely override them if required --- bin/build-master-status | 2 +- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 2 +- bin/copy-to-build-support | 2 +- bin/db-update | 4 ++-- bin/get-assignment | 8 ++++---- bin/get-package-updates | 4 ++-- bin/modify-package-state | 2 +- bin/return-assignment | 8 ++++---- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/why-dont-you | 2 +- lib/common-functions | 2 +- lib/mysql-functions | 18 +++++++++--------- 14 files changed, 40 insertions(+), 40 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/build-master-status b/bin/build-master-status index deb58da..006750b 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -562,7 +562,7 @@ if ${web}; then # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' 1s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / ' diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7cf1b95..a72d396 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/build-list.mysql" @@ -45,7 +45,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort > \ "${tmp_dir}/locked.mysql" @@ -66,7 +66,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/deletion-list.mysql" sort "${work_dir}/deletion-list" > \ @@ -80,7 +80,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort > \ "${tmp_dir}/loops.mysql" find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ @@ -107,7 +107,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -141,7 +141,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ ' | \ @@ -174,7 +174,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/build-dependencies.mysql" @@ -191,7 +191,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ while read -r pkgbase git_revision mod_git_revision repository; do if [ -f "${work_dir}/package-infos/${pkgbase}.${git_revision}.${mod_git_revision}.${repository}.build-depends" ]; then diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 156f438..5bd05b4 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index e447596..b3decd4 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${id}" ]; then diff --git a/bin/db-update b/bin/db-update index 5efa220..5756494 100755 --- a/bin/db-update +++ b/bin/db-update @@ -467,7 +467,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_stabilize} | \ @@ -503,7 +503,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed 'p' # shellcheck disable=SC2086 printf '%s\n' ${packages_to_force_unstage} | \ diff --git a/bin/get-assignment b/bin/get-assignment index 964c6b7..2df3059 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' { @@ -154,7 +154,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ -n "${currently_building}" ]; then @@ -210,7 +210,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - mysql_run_query -N --batch --raw | \ + mysql_run_query | \ sed ' y/\t/ / s/^.* \(\S\+\( \S\+\)\{3\}\)$/\1/ @@ -233,7 +233,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - mysql_run_query -N --batch --raw + mysql_run_query ) if [ "${count_pending}" -eq 0 ]; then diff --git a/bin/get-package-updates b/bin/get-package-updates index e855b19..a95bd94 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -u ) @@ -442,7 +442,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u } | \ grep -vxF '' | \ diff --git a/bin/modify-package-state b/bin/modify-package-state index 68b2384..9b0571b 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qxF '1'; then # shellcheck disable=SC2016 { diff --git a/bin/return-assignment b/bin/return-assignment index a7e6623..717a30e 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' ) if [ -z "${infos}" ]; then @@ -108,7 +108,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - mysql_run_query -N --raw --batch + mysql_run_query ) for saved_build_log in ${saved_build_logs}; do printf '%s' "${fail_reason_identifiers}" | \ @@ -279,7 +279,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 @@ -436,7 +436,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - mysql_run_query -N --raw --batch + mysql_run_query } | \ sort -k2 | \ uniq -u -f1 diff --git a/bin/seed-build-list b/bin/seed-build-list index def9d56..d49a8f3 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ ' @@ -205,7 +205,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ @@ -292,7 +292,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sort -u | \ tr '\t' ' ' | \ if ${update}; then diff --git a/bin/show-dependencies b/bin/show-dependencies index ed7ca87..492f11b 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s/\s.*$// ' | \ @@ -177,7 +177,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/why-dont-you b/bin/why-dont-you index 35b9419..64dfa1b 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ sort -k7,7 -k6,6 -k5,5 | \ sed ' diff --git a/lib/common-functions b/lib/common-functions index 0f75cb8..c7a64fe 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ expand_version 4 | \ sort -k4V,4 -k3r,3 | \ diff --git a/lib/mysql-functions b/lib/mysql-functions index 9d705d2..26d3db8 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -26,17 +26,17 @@ base64_encode_each() { # wrapper function to query mysql mysql_run_query() { if [ -s "${work_dir}/build-master-sanity" ]; then - ${mysql_command} "$@" + ${mysql_command} -N --raw --batch "$@" else { - printf '%s %s: ' "$0" "${mysql_command} $*" + printf '%s %s: ' "$0" "${mysql_command} -N --raw --batch $*" date } \ | tee -a "${work_dir}/mysql.stdin" \ | tee -a "${work_dir}/mysql.stdout" \ >> "${work_dir}/mysql.stderr" tee -a "${work_dir}/mysql.stdin" \ - | ${mysql_command} "$@" \ + | ${mysql_command} -N --raw --batch "$@" \ 2>> "${work_dir}/mysql.stderr" \ | tee -a "${work_dir}/mysql.stdout" for s in \ @@ -193,7 +193,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - mysql_run_query --html + mysql_run_query --html --column-names } # mysql_add_install_target $install_target @@ -693,7 +693,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,^,, s,$,, @@ -723,7 +723,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ sed ' s,\t,/, s,\t,-, @@ -753,7 +753,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ tr '\t' ' ' | \ tsort 2>&1 >/dev/null | \ sed 's/^tsort:\s*//' | \ @@ -930,7 +930,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ while read -r id pkgname arch; do pkgname=$( printf '%s' "${pkgname}" | \ @@ -958,7 +958,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - mysql_run_query -N --raw --batch | \ + mysql_run_query | \ grep -qvxF '0'; then pkgbase="${pkgname}" fi -- cgit v1.2.3-54-g00ecf From 3c1f6750e8f906d8230266f904d69ce76d8909b9 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 21 Mar 2018 10:15:31 +0100 Subject: bin/copy-to-build-support: --wait new --- bin/copy-to-build-support | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) (limited to 'bin/copy-to-build-support') diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index b3decd4..404a2b9 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -5,12 +5,62 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# shellcheck disable=SC2016 +usage() { + >&2 echo '' + >&2 echo 'copy-to-build-support [options] package-list:' + >&2 echo ' copy the packages listed in package-list into [build-support]' + >&2 echo '' + >&2 echo 'possible options:' + >&2 echo ' -h|--help:' + >&2 echo ' Show this help and exit.' + >&2 echo ' -w|--wait:' + >&2 echo ' Wait for lock if necessary.' + [ -z "$1" ] && exit 1 || exit "$1" +} + +eval set -- "$( + getopt -o hw \ + --long help \ + --long wait \ + -n "$(basename "$0")" -- "$@" || \ + echo usage + )" + +wait_for_lock='-n' + +while true +do + case "$1" in + -h|--help) + usage 0 + ;; + -w|--wait) + wait_for_lock='' + ;; + --) + shift + break + ;; + *) + >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.' + exit 42 + ;; + esac + shift +done + if [ "$#" -ne 1 ]; then - >&2 echo 'usage:' - >&2 echo ' copy-to-build-support package-list' - exit 2 + >&2 echo 'No package-list was given.' + usage fi +exec 9> "${sanity_check_lock_file}" +flock -s ${wait_for_lock} 9 + +exec 8> "${package_database_lock_file}" +flock ${wait_for_lock} 8 + tmp_dir=$(mktemp -d "${work_dir}/tmp.copy-to-build-support.0.XXXXXXXXXX") trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT -- cgit v1.2.3-54-g00ecf