From 7cd9746ce2914bda810f2179f7dffe03a6707665 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 25 Jan 2018 10:15:00 +0100 Subject: bin/bootstrap-mysql: update operator names --- bin/bootstrap-mysql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 46678f7..c643fe4 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -338,10 +338,13 @@ fi ' ~/.ssh/authorized_keys | \ while read -r name key; do case "${name}" in - 'nlopc'*|'rechenknecht'|'buildknecht'*) + 'nlopc'*|'rechenknecht') operator='deep42thought' ;; - 'eurobuild') + 'buildknecht'*) + operator='deep42thought/vollzornbrot' + ;; + 'eurobuild3') operator='abaumann' ;; *) -- cgit v1.2.3-54-g00ecf From 1bef73485855314342c66d9bf43e3c71ffac6cbb Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 30 Jan 2018 14:00:25 +0100 Subject: bin/bootstrap-mysql: new dependency types --- bin/bootstrap-mysql | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index c643fe4..725b211 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -421,6 +421,8 @@ fi { printf '\n ("%s",%s,%s),' \ 'build' '1' '0' \ + 'check' '0' '0' \ + 'link' '0' '1' \ 'run' '1' '1' printf ';\n' } | \ -- cgit v1.2.3-54-g00ecf From c2ce72f0fb3218ba977e257508dbe8d19e811e27 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 31 Jan 2018 10:39:20 +0100 Subject: bin/bootstrap-mysql: more columns in `package_sources` --- bin/bootstrap-mysql | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 725b211..6c16a1a 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -99,6 +99,8 @@ if [ ! "$1" = 'slim' ]; then ' git_revision VARCHAR(40)' \ ' mod_git_revision VARCHAR(40)' \ ' upstream_package_repository SMALLINT :upstream_repositories' \ + ' uses_upstream BIT' \ + ' uses_modification BIT' \ ' UNIQUE content pkgbase git_revision mod_git_revision' \ 'upstream_repositories SMALLINT' \ ' name VARCHAR(64)' \ -- cgit v1.2.3-54-g00ecf From b36b0ca61efc66dc0bc4037f4fb5851d0d4d5059 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 31 Jan 2018 15:10:07 +0100 Subject: bin/bootstrap-mysql: set correct bug tracker categories --- bin/bootstrap-mysql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 6c16a1a..193b0eb 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -377,11 +377,11 @@ fi printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES' { printf '\n ("%s",%s),' \ - 'stable' '"Packages:Stable"' \ - 'testing' '"Packages:Testing"' \ + 'stable' '"Packages: Stable"' \ + 'testing' '"Packages: Testing"' \ 'staging' 'NULL' \ 'standalone' 'NULL' \ - 'unbuilt' '"Packages:Build-list"' \ + 'unbuilt' '"Packages: Build-list"' \ 'forbidden' 'NULL' printf ';\n' } | \ -- cgit v1.2.3-54-g00ecf From eec7bff763e119a05b3b16c2c1e7477eb381f86a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Feb 2018 14:12:20 +0100 Subject: bin/bootstrap-mysql,bin/mysql-functions: new column `priority` in `build_assignments` --- bin/bootstrap-mysql | 1 + bin/mysql-functions | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 193b0eb..ae51e25 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -82,6 +82,7 @@ if [ ! "$1" = 'slim' ]; then ' architecture SMALLINT :architectures' \ ' is_blocked VARCHAR(128) NULL' \ ' is_broken BIT' \ + ' priority SMALLINT' \ ' UNIQUE content package_source architecture' \ 'build_dependency_loops BIGINT' \ ' loop MEDIUMINT' \ diff --git a/bin/mysql-functions b/bin/mysql-functions index f3dcc5b..a17f7a2 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -310,8 +310,8 @@ mysql_generate_package_metadata() { fi } | \ while read -r arch; do - printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`)' - printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL' + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,NULL,0,0' printf ' FROM `architectures` JOIN `package_sources`' printf ' WHERE `architectures`.`name` = from_base64("%s")' \ "$( -- cgit v1.2.3-54-g00ecf From 25c4fc4327cd66e86121dd2740772fb6cd6bcc20 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Feb 2018 09:16:17 +0100 Subject: bin/bootstrap-mysql: create `todos` and `todo_links`, too --- bin/bootstrap-mysql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index ae51e25..a10b2d7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -125,7 +125,14 @@ if [ ! "$1" = 'slim' ]; then ' build_assignment BIGINT :build_assignments' \ ' date DATETIME' \ ' reason SMALLINT :fail_reasons' \ - ' log_file VARCHAR(512)' + ' log_file VARCHAR(512)' \ + 'todos MEDIUMINT' \ + ' file VARCHAR(64)' \ + ' line MEDIUMINT' \ + ' description VARCHAR(512)' + 'todo_links MEDIUMINT' \ + ' dependent MEDIUMINT' \ + ' depending_on MEDIUMINT' ) { -- cgit v1.2.3-54-g00ecf From 34c524246f2257b9cc94668fa07412ed44b09f40 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Feb 2018 14:26:12 +0100 Subject: bin/mysql-functions: mysql_repair_binary_packages_without_build_assignment started (and finished?) --- bin/bootstrap-mysql | 2 ++ bin/mysql-functions | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index a10b2d7..116b6c7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -587,3 +587,5 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then } | \ ${mysql_command} fi + +mysql_repair_binary_packages_without_build_assignment diff --git a/bin/mysql-functions b/bin/mysql-functions index d0861e7..1f2e882 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -771,3 +771,63 @@ mysql_query_select_pkgbase_and_revision() { 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' } + +# mysql_repair_binary_packages_without_build_assignment +# try to generate valid build assignments to binary packages without +# a valid one yet +mysql_repair_binary_packages_without_build_assignment() { + { + printf 'SELECT ' + printf '`binary_packages`.`id`' + printf ',replace(to_base64(%s),"\\n","")' \ + '`binary_packages`.`pkgname`' \ + '`architectures`.`name`' + printf ' FROM `binary_packages`' + printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + printf ' WHERE `binary_packages`.`build_assignment`<0' + } | \ + ${mysql_command} --raw --batch | \ + sed '1d' | \ + while read -r id pkgname arch; do + pkgname=$( + printf '%s' "${pkgname}" | \ + base64 -d + ) + pkgbase=$( + curl -Ss "$( + printf 'https://www.archlinux.org/packages/search/json/?name=%s' \ + "${pkgname}" + )" | \ + sed ' + s/^.*"results":\s*\[// + s/}\s*,\s*{/\n/g + ' | \ + grep '"pkgname":\s*"'"$(str_to_regex "${pkgname}")"'"' | \ + tr ',' '\n' | \ + grep '"pkgbase":' | \ + cut -d'"' -f4 + ) + if [ -z "${pkgbase}" ]; then + >&2 printf 'Could not find "%s" upstream.\n' "${pkgname}" + continue + fi + pkgbase=$( + printf '%s' "${pkgbase}" | \ + base64 -w0 + ) + printf 'INSERT IGNORE INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)' + printf ' SELECT `package_sources`.`id`,`architectures`.`id`,0,0,0' + printf ' FROM `package_sources`' + printf ' JOIN `architectures`' + printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' "${pkgbase}" + printf ' AND `architectures`.`name`=from_base64("%s")' "${arch}" + printf ' LIMIT 1;\n' + printf 'UPDATE `binary_packages`' + printf ' JOIN `build_assignments`' + printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' + printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' + printf ' WHERE `binary_packages`.`id`=%s' "${id}" + printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" + done | \ + ${mysql_command} +} -- cgit v1.2.3-54-g00ecf From 0a1661c353b65730bb905a17a165185dfbb32577 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 6 Feb 2018 10:34:28 +0100 Subject: bin/bootstrap-mysql, bin/mysql-functions: new column `is_to_be_deleted` in `binary_packages` --- bin/bootstrap-mysql | 5 +++-- bin/mysql-functions | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 116b6c7..5acc2e2 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -61,6 +61,7 @@ if [ ! "$1" = 'slim' ]; then ' sub_pkgrel MEDIUMINT' \ ' has_issues BIT' \ ' is_tested BIT' \ + ' is_to_be_deleted BIT' \ ' pkgname VARCHAR(64)' \ ' architecture SMALLINT :architectures' \ ' UNIQUE content build_assignment sub_pkgrel pkgname architecture repository' \ @@ -562,7 +563,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then s/,$/;/ ' "${tmp_dir}/new-stable-packages" # shellcheck disable=SC2016 - printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`)' + printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' printf ' SELECT ' # shellcheck disable=SC2016 printf '`stable_packages`.`%s`,' \ @@ -576,7 +577,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then 'architectures' \ 'repositories' # shellcheck disable=SC2016 - printf -- '-`build_assignment`,0,1 FROM `stable_packages`' + printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' # shellcheck disable=SC2016 printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ 'repositories' 'repository' 'repositories' \ diff --git a/bin/mysql-functions b/bin/mysql-functions index 6fb105f..c576260 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -90,10 +90,11 @@ mysql_add_binary_package() { { printf 'INSERT IGNORE INTO binary_packages' printf ' (' - printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' + printf '`%s`, ' 'sub_pkgrel' 'pkgname' 'package_source' 'repository' 'architecture' 'has_issues' 'is_tested' 'is_to_be_deleted' printf ') SELECT' printf ' from_base64("%s"), ' "${sub_pkgrel}" "${pkgname}" printf ' `%s`.`id`,' 'package_sources' 'repositories' 'architectures' + printf ' 0, 0, 0' printf ' FROM' printf ' `%s` JOIN' 'package_sources' 'repositories' 'architectures' printf ' `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' @@ -470,7 +471,8 @@ mysql_generate_package_metadata() { 'pkgname' \ 'sub_pkgrel' \ 'has_issues' \ - 'is_tested' + 'is_tested' \ + 'is_to_be_deleted' printf ') SELECT ' printf '`%s`.`id`,' \ 'build_assignments' \ @@ -481,7 +483,7 @@ mysql_generate_package_metadata() { "${pkgver}" \ "${pkgrel}" \ "${pkgname}" - printf '%s,0,0 FROM' \ + printf '%s,0,0,0 FROM' \ "${sub_pkgrel}" printf ' `%s` JOIN' \ 'repositories' \ -- cgit v1.2.3-54-g00ecf From 98e89a079be0423449481606c4fc63bb9f9087c7 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Feb 2018 14:18:19 +0100 Subject: bin/mysql-functions: mysql_generate_package_metadata has more obligatory parameters now --- bin/bootstrap-mysql | 2 +- bin/get-package-updates | 40 +--------------------------------------- bin/mysql-functions | 39 +++++++++++++++++++-------------------- bin/seed-build-list | 2 +- 4 files changed, 22 insertions(+), 61 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 5acc2e2..1a50496 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -475,7 +475,7 @@ find "${work_dir}/package-states" \ tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do - mysql_generate_package_metadata "${state_file}" + mysql_generate_package_metadata 'build-list' "${state_file}" done # shellcheck disable=SC2016 diff --git a/bin/get-package-updates b/bin/get-package-updates index a8dc710..8322e6d 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -337,7 +337,7 @@ done | \ )" } | \ ${mysql_command} - mysql_generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" + mysql_generate_package_metadata 'build-list' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi @@ -358,44 +358,6 @@ while read -r package git_revision mod_git_revision repository; do generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" done < "${work_dir}/build-list.new" -{ - # shellcheck disable=SC2016 - { - printf 'SELECT DISTINCT ' - printf 'replace(to_base64(`package_sources`.`%s`),"\\n",""),' \ - 'pkgbase' 'git_revision' 'mod_git_revision' - printf 'replace(to_base64(`upstream_repositories`.`name`),"\\n","")' - printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' - printf ' WHERE `repositories`.`name` in ("build-list","deletion-list")' - } | \ - ${mysql_command} --raw --batch | \ - sed ' - 1d - y/\t/ / - ' | \ - while read -r a b c d; do - printf '%s ' \ - "$(printf '%s' "${a}" | base64 -d)" \ - "$(printf '%s' "${b}" | base64 -d)" \ - "$(printf '%s' "${c}" | base64 -d)" \ - "$(printf '%s' "${d}" | base64 -d)" | \ - sed 's/ /\n/' - done | \ - grep -vxF '' | \ - sed 'p' - cat "${work_dir}/build-list.new" -} | \ - sort | \ - uniq -u | \ - while read -r package git_revision mod_git_revision repository; do - mysql_generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" - done - echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date diff --git a/bin/mysql-functions b/bin/mysql-functions index 64f1915..19fb773 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -190,15 +190,16 @@ mysql_add_install_target() { ${mysql_command} } -# mysql_generate_package_metadata $package $git_revision $mod_git_revision $repository +# mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository # or -# mysql_generate_package_metadata $package.$git_revision.$mod_git_revision.$repository -# if sub_pkgrel should be determined automatically and the package is on the build-list +# mysql_generate_package_metadata $current_repository $package.$git_revision.$mod_git_revision.$repository +# if sub_pkgrel should be determined automatically # and # mysql_generate_package_metadata $sub_pkgrel $current_repository $package $git_revision $mod_git_revision $repository # or # mysql_generate_package_metadata $sub_pkgrel $current_repository $package.$git_revision.$mod_git_revision.$repository -# if $sub_pkgrel should be forced and the package is currently in $current_repository +# if $sub_pkgrel should be forced + # generate the meta data of a package (dependencies, built packages, ...) in the database mysql_generate_package_metadata() { @@ -207,10 +208,6 @@ mysql_generate_package_metadata() { case "$1" in ''|*[!0-9]*) unset forced_sub_pkgrel - current_repository=$( - printf 'build-list' | \ - base64 -w0 - ) ;; *) forced_sub_pkgrel=$( @@ -218,21 +215,12 @@ mysql_generate_package_metadata() { base64 -w0 ) shift - current_repository=$( - printf '%s' "$1" | \ - base64 -w0 - ) - shift ;; esac - package="$1" - git_revision="$2" - mod_git_revision="$3" - repository="$4" - temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) - trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + current_repository="$1" + package="$2" - if [ $# -eq 1 ]; then + if [ $# -eq 2 ]; then # second form repository="${package##*.}" package="${package%.*}" @@ -240,8 +228,15 @@ mysql_generate_package_metadata() { package="${package%.*}" git_revision="${package##*.}" package="${package%.*}" + else + git_revision="$3" + mod_git_revision="$4" + repository="$5" fi + temp_dir=$(mktemp -d 'tmp.mysql-functions.mysql_generate_package_metadata.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + printf '.' >&2 if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO"; then printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${temp_dir}/SRCINFO" @@ -276,6 +271,10 @@ mysql_generate_package_metadata() { printf '.' >&2 # now we encode everything in base64 + current_repository=$( + printf '%s' "${current_repository}" | \ + base64 -w0 + ) pkgbase=$( printf '%s' "${pkgbase}" | \ base64 -w0 diff --git a/bin/seed-build-list b/bin/seed-build-list index e17242c..b049560 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -305,7 +305,7 @@ if ${update}; then tee -a "${work_dir}/build-list" while read -r git_revision mod_git_revision repository pkgbase; do - mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" + mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}" done < \ "${tmp_dir}/build-list.new" -- cgit v1.2.3-54-g00ecf From e7b14b22c38ed83e5d0d8c473c1c65423e11c9bd Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Feb 2018 13:13:57 +0100 Subject: bin/bootstrap-mysql: create `repository_stability_relations` --- bin/bootstrap-mysql | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 1a50496..c965319 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -133,7 +133,10 @@ if [ ! "$1" = 'slim' ]; then ' description VARCHAR(512)' 'todo_links MEDIUMINT' \ ' dependent MEDIUMINT' \ - ' depending_on MEDIUMINT' + ' depending_on MEDIUMINT' \ + 'repository_stability_relations MEDIUMINT' \ + ' more_stable MEDIUMINT :repository_stabilities' \ + ' less_stable MEDIUMINT :repository_stabilities' ) { @@ -396,6 +399,30 @@ fi } | \ sed 's|,;|;|' + # shellcheck disable=SC2016 + { + printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)' + printf ' SELECT `ms`.`id`,`ls`.`id`' + printf ' FROM `repository_stabilities` AS `ms` JOIN `repository_stabilities` AS `ls`' + printf ' WHERE ' + printf '(`ms`.`name`="%s" AND `ls`.`name`="%s") OR ' \ + 'stable' 'stable' \ + 'stable' 'testing' \ + 'stable' 'staging' \ + 'stable' 'unbuilt' \ + 'stable' 'forbidden' \ + 'testing' 'testing' \ + 'testing' 'staging' \ + 'testing' 'unbuilt' \ + 'testing' 'forbidden' \ + 'staging' 'staging' \ + 'staging' 'unbuilt' \ + 'staging' 'forbidden' \ + 'unbuilt' 'forbidden' + printf ';\n' + } | \ + sed 's| OR ;|;|' + for repo in \ 'core:stable:AQ==' \ 'extra:stable:AQ==' \ -- cgit v1.2.3-54-g00ecf From 33322fbda73861befb4b56299a23915b7d72bbe5 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Feb 2018 14:57:35 +0100 Subject: bin/mysql-functions: separate checkdependencies from makedependencies --- bin/bootstrap-mysql | 2 +- bin/mysql-functions | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 11 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index c965319..322e0a1 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -458,7 +458,7 @@ fi printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES' { printf '\n ("%s",%s,%s),' \ - 'build' '1' '0' \ + 'make' '1' '0' \ 'check' '0' '0' \ 'link' '0' '1' \ 'run' '1' '1' diff --git a/bin/mysql-functions b/bin/mysql-functions index 78728c0..0a8e3d3 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -407,8 +407,14 @@ mysql_generate_package_metadata() { sed 's/[<>=].*$//' | \ base64_encode_each ) - builddepends=$( - grep '^\(checkdepends\|makedepends\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + makedepends=$( + grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ + cut -d' ' -f3 | \ + sed 's/[<>=].*$//' | \ + base64_encode_each + ) + checkdepends=$( + grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \ cut -d' ' -f3 | \ sed 's/[<>=].*$//' | \ base64_encode_each @@ -514,7 +520,8 @@ mysql_generate_package_metadata() { { printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \ 'provides' \ - 'builddepends' \ + 'makedepends' \ + 'checkdepends' \ 'rundepends' printf 'INSERT INTO `provides` VALUES\n' @@ -538,8 +545,19 @@ mysql_generate_package_metadata() { ' printf ' ("base");\n' - printf 'INSERT INTO `builddepends` VALUES\n' - echo "${builddepends}" | \ + echo "${checkdepends}" | \ + sort -u | \ + grep -vxF '' | \ + sed ' + 1 s/^/INSERT INTO `checkdepends` VALUES \n/ + s|^| (from_base64("| + s|$|")),| + $ s/,$/;/ + ' + printf ' ("base-devel");\n' + + printf 'INSERT INTO `makedepends` VALUES\n' + echo "${makedepends}" | \ sort -u | \ grep -vxF '' | \ sed ' @@ -551,17 +569,18 @@ mysql_generate_package_metadata() { printf 'INSERT IGNORE INTO `install_targets` (`name`)' printf ' SELECT (`name`) FROM `%s` UNION' \ 'provides' \ - 'builddepends' \ + 'makedepends' \ + 'checkdepends' \ 'rundepends' | \ sed 's| UNION$|;\n|' - for link in 'provides' 'builddepends' 'rundepends'; do + for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do case "${link}" in 'provides') printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT' printf ' `binary_packages`.`id`,`install_targets`.`id` FROM' ;; - 'builddepends'|'rundepends') + 'makedepends'|'checkdepends'|'rundepends') printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT' printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM' printf ' `dependency_types` JOIN' @@ -574,7 +593,8 @@ mysql_generate_package_metadata() { printf ' JOIN `%s`' "${link}" printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" printf ' WHERE' - if [ "${link}" = 'builddepends' ] || \ + if [ "${link}" = 'makedepends' ] || \ + [ "${link}" = 'checkdepends' ] || \ [ "${link}" = 'rundepends' ]; then printf ' `dependency_types`.`name` = "%s" AND' \ "${link%depends}" @@ -598,7 +618,8 @@ mysql_generate_package_metadata() { printf 'DROP TABLE `%s`;\n' \ 'provides' \ - 'builddepends' \ + 'makedepends' \ + 'checkdepends' \ 'rundepends' } >> \ "${temp_dir}/add-install-targets-command" -- cgit v1.2.3-54-g00ecf From d2e0ef2a558e27e07150f239d91e4c29ce822426 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 16 Feb 2018 11:14:03 +0100 Subject: bin/mysql-functions: mysql_join_*_* new --- bin/bootstrap-mysql | 16 +--- bin/build-master-status-from-mysql | 65 ++++++-------- bin/check-bugtracker | 2 +- bin/common-functions | 5 +- bin/copy-to-build-support | 6 +- bin/db-update | 4 +- bin/get-assignment | 34 ++++---- bin/get-package-updates | 43 ++++----- bin/modify-package-state | 21 ++--- bin/mysql-functions | 174 +++++++++++++++++++++++++------------ bin/prioritize-build-list | 7 +- bin/return-assignment | 40 ++++----- bin/seed-build-list | 15 ++-- bin/show-dependencies | 7 +- bin/why-dont-you | 23 +++-- 15 files changed, 243 insertions(+), 219 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 322e0a1..30c260a 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -512,9 +512,8 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ ' | \ while read -r state_file pkgbase git_revision mod_git_revision repository; do printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ "$( tr -d '\n' < \ @@ -563,10 +562,9 @@ grep '^\('"$( "${tmp_dir}/new-stable-packages" if [ -s "${tmp_dir}/new-stable-packages" ]; then + # shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'CREATE TEMPORARY TABLE `stable_packages` (' - # shellcheck disable=SC2016 printf '`%s` %s,' \ 'pkgname' 'VARCHAR(64)' \ 'epoch' 'MEDIUMINT' \ @@ -576,9 +574,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then 'architecture' 'VARCHAR(16)' \ 'repository' 'VARCHAR(64)' \ 'build_assignment' 'BIGINT NOT NULL AUTO_INCREMENT' - # shellcheck disable=SC2016 printf 'PRIMARY KEY (`build_assignment`));\n' - # shellcheck disable=SC2016 sed ' 1~10 ! b not_start s/^/INSERT IGNORE INTO `stable_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`) VALUES \n/ @@ -589,28 +585,22 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then :end s/,$/;/ ' "${tmp_dir}/new-stable-packages" - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `binary_packages` (`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architecture`,`repository`,`build_assignment`,`has_issues`,`is_tested`,`is_to_be_deleted`)' printf ' SELECT ' - # shellcheck disable=SC2016 printf '`stable_packages`.`%s`,' \ 'pkgname' \ 'epoch' \ 'pkgver' \ 'pkgrel' \ 'sub_pkgrel' - # shellcheck disable=SC2016 printf '`%s`.`id`,' \ 'architectures' \ 'repositories' - # shellcheck disable=SC2016 printf -- '-`build_assignment`,0,1,0 FROM `stable_packages`' - # shellcheck disable=SC2016 printf ' JOIN `%s` ON `stable_packages`.`%s`=`%s`.`name`' \ 'repositories' 'repository' 'repositories' \ 'architectures' 'architecture' 'architectures' printf ';\n' - # shellcheck disable=SC2016 printf 'DROP TABLE `stable_packages`;\n' } | \ ${mysql_command} diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index 7a6605a..ea8d0f1 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -21,11 +21,10 @@ fi { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`package_sources`.`git_revision`,`package_sources`.`mod_git_revision`,`upstream_repositories`.`name`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'upstream_repositories' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_upstream_repositories + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ ${mysql_command} --batch | \ @@ -45,10 +44,9 @@ sort "${work_dir}/build-list" > \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_slaves_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } | \ ${mysql_command} --raw --batch | \ sed ' @@ -68,10 +66,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin { printf 'SELECT DISTINCT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ @@ -89,9 +86,8 @@ sort "${work_dir}/deletion-list" > \ { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'build_dependency_loops' 'build_dependency_loops' 'build_assignment' 'build_assignments' + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ ${mysql_command} --batch | \ @@ -118,10 +114,9 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -154,10 +149,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'sub_pkgrel' printf '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures + mysql_join_repositories_repository_stabilities printf ' WHERE NOT `binary_packages`.`is_tested`' printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' @@ -188,15 +182,13 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'mod_git_revision' printf '`upstream_repositories`.`name`,`install_targets`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'dependencies' 'binary_packages' 'dependencies' 'dependent' \ - 'dependency_types' 'dependency_types' 'dependencies' 'dependency_type' - printf ' AND `dependency_types`.`relevant_for_building`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'install_targets' 'install_targets' 'dependencies' 'depending_on' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + mysql_join_dependencies_install_targets + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ ${mysql_command} --raw --batch | \ sed ' @@ -214,10 +206,9 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages } | \ ${mysql_command} --raw --batch | \ sed ' diff --git a/bin/check-bugtracker b/bin/check-bugtracker index 72ab133..d6e92e5 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -31,7 +31,7 @@ bug_list=$( while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_binary_packages_repositories printf ' SET `has_issues`=%s' \ "${has_issues%:*}" printf ' WHERE `repositories`.`stability`=%s' \ diff --git a/bin/common-functions b/bin/common-functions index d7f3e41..a84271b 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -478,9 +478,8 @@ remove_old_package_versions() { printf '`binary_packages`.`sub_pkgrel`' printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'architectures' 'architectures' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`' printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`' # name must match diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 4710012..808d041 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -31,8 +31,8 @@ sed -n ' { printf 'SELECT `binary_packages`.`id`,`repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \ @@ -89,7 +89,7 @@ sed -n ' # shellcheck disable=SC2086 printf ',`binary_packages`.`%s`' ${to_copy} printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`id`=%s;\n' \ "${id}" diff --git a/bin/db-update b/bin/db-update index e5ea193..b830981 100755 --- a/bin/db-update +++ b/bin/db-update @@ -325,8 +325,8 @@ move_packages() { while read -r package; do while read -r part; do printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo` ON `binary_packages`.`repository`=`from_repo`.`id`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' + mysql_join_binary_packages_architectures printf ' SET `binary_packages`.`repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` as `to_repo`' diff --git a/bin/get-assignment b/bin/get-assignment index a7eec2d..1edcffb 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -45,11 +45,10 @@ hand_out_assignment() { 'pkgbase' 'git_revision' 'mod_git_revision' printf '`upstream_repositories`.`name`,`binary_packages`.`sub_pkgrel`' printf ' FROM `upstream_repositories`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'package_sources' 'upstream_package_repository' 'upstream_repositories' \ - 'build_assignments' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_upstream_repositories_package_sources + mysql_join_package_sources_build_assignments + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `package_sources`.`%s`=from_base64("%s")' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -80,8 +79,8 @@ hand_out_assignment() { printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' printf ' FROM `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ 'pkgbase' "$(printf '%s' "$1" | base64 -w0)" \ @@ -94,8 +93,8 @@ hand_out_assignment() { "$(printf '%s' "${slave}" | base64 -w0)" printf 'UPDATE `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' - printf ' JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository`=`upstream_repositories`.`id`' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' SET `build_assignments`.`priority`=0' printf ' WHERE' printf ' `package_sources`.`%s` = from_base64("%s") AND' \ @@ -150,10 +149,9 @@ currently_building=$( { printf 'SELECT ' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_slaves' 'build_slaves' 'currently_building' 'build_assignments' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' @@ -189,9 +187,8 @@ next_building=$( mysql_query_is_part_of_loop '`build_assignments`.`id`' printf ' AS `part_of_loop`,' mysql_query_select_pkgbase_and_revision - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND NOT EXISTS (' printf ' SELECT *' @@ -231,9 +228,8 @@ count_pending=$( { printf 'SELECT count(*)' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' diff --git a/bin/get-package-updates b/bin/get-package-updates index 89b09df..e81d128 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,22 +110,18 @@ delete_package() { { # packages from the build-list/to-be-decided go straight to the deletion-list printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`repository`=(SELECT `repositories`.`id` FROM `repositories` WHERE `repositories`.`name`="deletion-list")' printf ' WHERE `repositories`.`name` in ("build-list","to-be-decided")' printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$(printf '%s' "$1" | base64 -w0)" # other packages are marked as `is_to_be_deleted` printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`binary_packages`.`%s`' \ - 'repositories' 'repositories' 'repository' \ - 'build_assignments' 'build_assignments' 'build_assignment' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' SET `binary_packages`.`is_to_be_deleted`=1' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" @@ -341,10 +337,9 @@ done | \ # which are not yet built or on the deletion list printf 'DELETE `build_assignments`,`binary_packages`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$( printf '%s' "${package}" | \ @@ -353,10 +348,9 @@ done | \ printf ' AND (`repositories`.`name`="build-list" OR `repositories`.`name`="deletion-list");\n' # remove is-to-be-deleted marker from old binary packages printf 'UPDATE `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_binary_packages_repositories printf ' SET `is_to_be_deleted`=0' printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \ "$( @@ -431,9 +425,9 @@ while [ -n "${black_listed_new}" ]; do ' printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' printf ' FROM `package_sources` AS `a_ps`' - printf ' JOIN `build_assignments` AS `a_ba` ON `a_ba`.`package_source`=`a_ps`.`id`' - printf ' JOIN `binary_packages` AS `a_bp` ON `a_bp`.`build_assignment`=`a_ba`.`id`' - printf ' JOIN `dependencies` ON `dependencies`.`dependent`=`a_bp`.`id`' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_binary_packages_dependencies 'a_bp' printf ' WHERE NOT EXISTS (' printf ' SELECT *' printf ' FROM `install_target_providers`' @@ -441,8 +435,8 @@ while [ -n "${black_listed_new}" ]; do printf ' SELECT *' printf ' FROM `bl`' printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' - printf ' JOIN `build_assignments` AS `b_ba` ON `b_ba`.`package_source`=`b_ps`.`id`' - printf ' JOIN `binary_packages` AS `b_bp` ON `b_bp`.`build_assignment`=`b_ba`.`id`' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' printf ')' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' @@ -607,8 +601,7 @@ fi done # move binary_packages from "to-be-decided" to "build-list" printf 'UPDATE `binary_packages`' - printf ' JOIN `repositories` AS `from_repo`' - printf ' ON `binary_packages`.`repository`=`from_repo`.`id`' + mysql_join_binary_packages_repositories '' 'from_repo' printf ' SET `repository`=(' printf 'SELECT `to_repo`.`id`' printf ' FROM `repositories` AS `to_repo`' diff --git a/bin/modify-package-state b/bin/modify-package-state index b2d3133..b0ba53e 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -136,10 +136,9 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' \ - 'architectures' 'architectures' 'binary_packages' 'architecture' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_architectures ) # shellcheck disable=SC2016,SC2031 selector=$( @@ -161,12 +160,11 @@ while read -r package reason; do # shellcheck disable=SC2016 combiner=$( printf '`build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'repository_stabilities' 'repository_stabilities' 'repositories' 'stability' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities ) # shellcheck disable=SC2016 selector=$( @@ -238,8 +236,7 @@ while read -r package reason; do printf 'UPDATE `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b`' printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories`' - printf ' ON `b`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'b' printf ' SET `b`.`is_tested`=1' printf ' WHERE `a`.`is_tested`' printf ' AND NOT `b`.`is_tested`' diff --git a/bin/mysql-functions b/bin/mysql-functions index 7f2cdab..6950ee8 100755 --- a/bin/mysql-functions +++ b/bin/mysql-functions @@ -131,7 +131,6 @@ mysql_show_binary_package() { shift done - local joint { printf 'SELECT' printf ' `%s`.`%s`,' \ @@ -146,17 +145,11 @@ mysql_show_binary_package() { 'package_sources' 'mod_git_revision' \ 'upstream_repositories' 'name' printf ' FROM `binary_packages`' - for joint in \ - 'architectures:binary_packages:architecture' \ - 'package_sources:binary_packages:package_source' \ - 'repositories:binary_packages:repository' \ - 'upstream_repositories:package_sources:upstream_package_repository'; do - printf ' JOIN `%s` ON `%s`.`id` =' \ - "${joint%%:*}" "${joint%%:*}" - joint="${joint#*:}" - printf ' `%s`.`%s`' \ - "${joint%:*}" "${joint#*:}" - done + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'binary_packages' 'pkgname' "${pkgname}" \ @@ -434,8 +427,8 @@ mysql_generate_package_metadata() { # build-list or in $current_repository (beware of split # packages!) printf '(SELECT `sub_pkgrel` FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -448,7 +441,7 @@ mysql_generate_package_metadata() { "${current_repository}" # max(sub_pkgrel)+1 printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE' printf ' `binary_packages`.`%s`=from_base64("%s") AND' \ 'epoch' "${epoch}" \ @@ -496,11 +489,9 @@ mysql_generate_package_metadata() { printf ' `%s` JOIN' \ 'repositories' \ 'architectures' \ - 'build_assignments' \ - 'package_sources' - printf ' ON `build_assignments`.`package_source` = `package_sources`.`id`' - printf ' JOIN `upstream_repositories`' - printf ' ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id`' + 'build_assignments' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE' printf ' `%s`.`%s` = from_base64("%s") AND' \ 'repositories' 'name' "${current_repository}" \ @@ -513,7 +504,6 @@ mysql_generate_package_metadata() { } | \ sed ' s|,)|)|g - s|JOIN ON|ON|g s| AND;$|;| ' >> \ "${temp_dir}/add-binary-packages-command" @@ -586,8 +576,7 @@ mysql_generate_package_metadata() { ;; esac printf ' `binary_packages`' - printf ' JOIN `architectures`' - printf ' ON `binary_packages`.`architecture` = `architectures`.`id`' + mysql_join_binary_packages_architectures printf ' JOIN `install_targets`' printf ' JOIN `%s`' "${link}" printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}" @@ -649,9 +638,9 @@ mysql_sanity_check() { { printf 'SELECT CONCAT("\\"any\\" build-assignment building \\"",`bp_arch`.`name`,"\\" binary package: ",`binary_packages`.`pkgname`)' printf ' FROM `binary_packages`' - printf ' JOIN `build_assignments` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `architectures` as `bp_arch` ON `binary_packages`.`architecture`=`bp_arch`.`id`' - printf ' JOIN `architectures` as `ba_arch` ON `build_assignments`.`architecture`=`ba_arch`.`id`' + mysql_join_binary_packages_build_assignments + mysql_join_binary_packages_architectures '' 'bp_arch' + mysql_join_build_assignments_architectures '' 'ba_arch' printf ' WHERE `bp_arch`.`name`!="any"' printf ' AND `ba_arch`.`name`="any";\n' printf 'SELECT DISTINCT CONCAT("package multiple times on build list: ",`a`.`pkgname`)' @@ -660,14 +649,13 @@ mysql_sanity_check() { printf ' ON `a`.`pkgname`=`b`.`pkgname`' printf ' AND `a`.`repository`=`b`.`repository`' printf ' AND `a`.`id`!=`b`.`id`' - printf ' JOIN `repositories`' - printf ' ON `a`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories 'a' printf ' WHERE `repositories`.`name`="build-list";\n' printf 'SELECT DISTINCT CONCAT("\\"split-package with differing sub_pkgrels on the build-list: ",`a`.`pkgname`)' printf ' FROM `binary_packages` AS `a`' printf ' JOIN `binary_packages` AS `b` ON `a`.`build_assignment`=`b`.`build_assignment`' - printf ' JOIN `repositories` AS `arep` ON `a`.`repository`=`arep`.`id`' - printf ' JOIN `repositories` AS `brep` ON `b`.`repository`=`brep`.`id`' + mysql_join_binary_packages_repositories 'a' 'arep' + mysql_join_binary_packages_repositories 'b' 'brep' printf ' WHERE `a`.`sub_pkgrel`!=`b`.`sub_pkgrel`' printf ' AND `%srep`.`name`="build-list"' \ 'a' 'b' @@ -700,8 +688,8 @@ mysql_sanity_check() { { printf 'SELECT `repositories`.`name`,`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`,`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ ${mysql_command} --batch | \ @@ -728,12 +716,11 @@ mysql_find_build_assignment_loops() { { printf 'SELECT DISTINCT `packages_dependency`.`build_assignment`,`packages_dependent`.`build_assignment`' printf ' FROM `dependencies`' - printf ' JOIN `install_target_providers` ON `dependencies`.`depending_on`=`install_target_providers`.`install_target`' - printf ' JOIN `binary_packages` `packages_dependent` ON `dependencies`.`dependent`=`packages_dependent`.`id`' - printf ' JOIN `binary_packages` `packages_dependency` ON `install_target_providers`.`package`=`packages_dependency`.`id`' - printf ' JOIN `repositories` `repositories_%s` ON `packages_%s`.`repository`=`repositories_%s`.`id`' \ - 'dependent' 'dependent' 'dependent' \ - 'dependency' 'dependency' 'dependency' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'packages_dependency' + mysql_join_dependencies_binary_packages '' 'packages_dependent' + mysql_join_binary_packages_repositories 'packages_dependency' 'repositories_dependency' + mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ ${mysql_command} --raw --batch | \ @@ -836,8 +823,8 @@ mysql_cleanup() { printf 'WHERE `build_assignments`.`package_source`=`package_sources`.`id`' printf ');\n' printf 'UPDATE `build_slaves`' - printf ' JOIN `binary_packages` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`' + mysql_join_build_slaves_binary_packages + mysql_join_binary_packages_repositories printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ @@ -849,12 +836,11 @@ mysql_cleanup() { mysql_query_has_pending_dependencies() { printf 'EXISTS (' printf 'SELECT * FROM `binary_packages` as `to_dos`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`%s`' \ - 'repositories` AS `to_do_repos' 'to_do_repos' 'id' 'to_dos' 'repository' \ - 'dependencies' 'to_dos' 'id' 'dependencies' 'dependent' \ - 'install_target_providers' 'dependencies' 'depending_on' 'install_target_providers' 'install_target' \ - 'binary_packages` AS `bin_deps' 'bin_deps' 'id' 'install_target_providers' 'package' \ - 'repositories` AS `dep_repos' 'dep_repos' 'id' 'bin_deps' 'repository' + mysql_join_binary_packages_repositories 'to_dos' 'to_do_repos' + mysql_join_binary_packages_dependencies 'to_dos' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'bin_deps' + mysql_join_binary_packages_repositories 'bin_deps' 'dep_repos' printf ' WHERE' printf ' `%s`.`name`="build-list" AND' \ 'dep_repos' 'to_do_repos' @@ -884,9 +870,8 @@ mysql_query_select_pkgbase_and_revision() { 'mod_git_revision' printf '`upstream_repositories`.`name`' printf ' FROM `build_assignments`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories } # mysql_repair_binary_packages_without_build_assignment @@ -900,7 +885,7 @@ mysql_repair_binary_packages_without_build_assignment() { '`binary_packages`.`pkgname`' \ '`architectures`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ ${mysql_command} --raw --batch | \ @@ -954,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' LIMIT 1;\n' printf 'UPDATE `binary_packages`' printf ' JOIN `build_assignments`' - printf ' JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_binary_packages_build_assignments printf ' SET `binary_packages`.`build_assignment`=`build_assignments`.`id`' printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" @@ -980,9 +965,8 @@ mysql_remove_duplicate_build_order() { 'pkgver' 'pkgver' \ 'pkgrel' 'pkgrel' printf ' `old`.`sub_pkgrel`+1=`new`.`sub_pkgrel`' - printf ' JOIN `repositories` AS `%s` ON `%s`.`id`=`%s`.`repository`' \ - 'orep' 'orep' 'old' \ - 'nrep' 'nrep' 'new' + mysql_join_binary_packages_repositories 'old' 'orep' + mysql_join_binary_packages_repositories 'new' 'nrep' printf ' WHERE `orep`.`name`!="to-be-decided"' printf ' AND `nrep`.`name`="to-be-decided";\n' printf 'UPDATE IGNORE `dependencies`' @@ -1004,6 +988,9 @@ mysql_remove_duplicate_build_order() { ${mysql_command} } +# mysql_package_name_query +# print a mysql query of the full name of a package file + mysql_package_name_query() { printf 'CONCAT(' printf '`binary_packages`.`pkgname`,"-",' @@ -1014,3 +1001,82 @@ mysql_package_name_query() { printf '`architectures`.`name`,".pkg.tar.xz"' printf ')' } + +# mysql_join_*_* +# print 'JOIN' part of mysql query to connect the respective tables +# these functions take 2 optional arguments, acting as aliases for +# the tables + +# mysql_join__generic $table_a $column_a $table_b $column_b +# create mysql_join_${table_a}_${table_b}() function + +mysql_join__generic() { + eval "$( + printf 'mysql_join_%s_%s() {\n' "$1" "$3" + printf ' printf '"'"' JOIN `%s`'"'"'\n' "$3" + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"' AS `%%s`'"'"' "$2"\n' + printf ' fi\n' + printf ' if [ -n "$1" ]; then\n' + printf ' printf '"'"' ON `%%s`.`%s`='"'"' "$1"\n' "$2" + printf ' else\n' + printf ' printf '"'"' ON `%s`.`%s`='"'"'\n' "$1" "$2" + printf ' fi\n' + printf ' if [ -n "$2" ]; then\n' + printf ' printf '"'"'`%%s`.`%s`'"'"' "$2"\n' "$4" + printf ' else\n' + printf ' printf '"'"'`%s`.`%s`'"'"'\n' "$3" "$4" + printf ' fi\n' + printf '}\n' + )" +} + +for link in \ + 'binary_packages:architecture:architectures' \ + 'binary_packages:repository:repositories' \ + 'binary_packages:build_assignment:build_assignments' \ + \ + 'build_assignments:architecture:architectures' \ + 'build_assignments:package_source:package_sources' \ + \ + 'build_dependency_loops:build_assignment:build_assignments' \ + 'build_dependency_loops:build_assignment build_assignment:binary_packages' \ + \ + 'build_slaves:currently_building:build_assignments' \ + 'build_slaves:currently_building build_assignment:binary_packages' \ + \ + 'dependencies:depending_on:install_targets' \ + 'dependencies:dependent:binary_packages' \ + 'dependencies:dependency_type:dependency_types' \ + \ + 'failed_builds:reason:fail_reason' \ + 'failed_builds:build_assignment:build_assignments' \ + 'failed_builds:build_slave:build_slaves' \ + \ + 'install_target_providers:package:binary_packages' \ + 'install_target_providers:install_target:install_targets' \ + 'install_target_providers:install_target depending_on:dependencies' \ + \ + 'package_sources:upstream_package_repository:upstream_repositories' \ + \ + 'repositories:stability:repository_stabilities' \ + \ + 'upstream_repositories:git_repository:git_repositories'; do +# A join for these cannot be done, because it's not clear on what to join: +# 'repository_stability_relations:more_stable:repository_stabilities' +# 'repository_stability_relations:less_stable:repository_stabilities' + + table_b="${link##*:}" + table_a="${link%:*}" + column_b="${table_a##*:}" + table_a="${table_a%:*}" + column_a="${column_b% *}" + if [ "${column_a}" = "${column_b}" ]; then + column_b='id' + else + column_b="${column_b##* }" + fi + + mysql_join__generic "${table_a}" "${column_a}" "${table_b}" "${column_b}" + mysql_join__generic "${table_b}" "${column_b}" "${table_a}" "${column_a}" +done diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index d611e79..e8dcf0a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -32,10 +32,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'package_sources' 'build_assignments' 'package_source' 'package_sources' \ - 'binary_packages' 'binary_packages' 'build_assignment' 'build_assignments' \ - 'repositories' 'binary_packages' 'repository' 'repositories' + mysql_join_build_assignments_package_sources + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' SET `build_assignments`.`priority`=(' printf 'SELECT COALESCE(MAX(`all_priorities`.`priority`),0)+1' printf ' FROM (' diff --git a/bin/return-assignment b/bin/return-assignment index 7b2dec3..995eae4 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -51,12 +51,11 @@ if [ "$5" = 'ERROR' ]; then infos=$( { printf 'SELECT DISTINCT `build_assignments`.`id`,IF(`build_assignments`.`is_broken`,"true","false") FROM `build_slaves`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'build_slaves' 'currently_building' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'binary_packages' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'repositories' 'repositories' 'binary_packages' 'repository' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( # shellcheck disable=SC2154 @@ -157,7 +156,7 @@ if [ "$5" = 'ERROR' ]; then printf 'DROP TABLE `failures`;\n' fi printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' + mysql_join_build_assignments_build_slaves printf ' SET `build_assignments`.`is_broken`=1, `build_slaves`.`currently_building`=NULL' printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$( @@ -267,12 +266,11 @@ clean_up_lock_file() { if ! { printf 'SELECT count(*)' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'build_slaves' 'build_assignments' 'build_slaves' 'currently_building' \ - 'repositories' 'repositories' 'binary_packages' 'repository' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_build_slaves + mysql_join_binary_packages_repositories + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories printf ' WHERE `repositories`.`name`="build-list"' printf ' AND `build_slaves`.`name`=from_base64("%s")' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -396,9 +394,9 @@ package_errors=$( mysql_package_name_query printf ')' printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`binary_packages`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_architectures + mysql_join_binary_packages_build_slaves + mysql_join_binary_packages_repositories printf ' WHERE `build_slaves`.`name`=from_base64("%s")' \ "$( printf '%s' "${slave}" | \ @@ -476,9 +474,9 @@ fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' - printf ' JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_assignments`.`id`' - printf ' JOIN `repositories` AS `old_repo` ON `binary_packages`.`repository`=`old_repo`.`id`' + mysql_join_build_assignments_build_slaves + mysql_join_build_assignments_binary_packages + mysql_join_binary_packages_repositories '' 'old_repo' printf ' SET' printf ' `build_assignments`.`is_broken`=0,' printf ' `build_assignments`.`priority`=0,' @@ -507,8 +505,8 @@ fi printf 'CREATE TEMPORARY TABLE `loops_to_delete` (`loop` MEDIUMINT);\n' printf 'INSERT INTO `loops_to_delete`' printf ' SELECT `build_dependency_loops`.`loop` FROM `build_dependency_loops`' - printf ' JOIN `binary_packages` ON `binary_packages`.`build_assignment`=`build_dependency_loops`.`build_assignment`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_build_dependency_loops_binary_packages + mysql_join_binary_packages_repositories printf ' WHERE NOT `repositories`.`name` = "build-list";\n' printf 'DELETE FROM `build_dependency_loops` WHERE EXISTS (' printf 'SELECT * FROM `loops_to_delete` WHERE `loops_to_delete`.`loop`=`build_dependency_loops`.`loop`' diff --git a/bin/seed-build-list b/bin/seed-build-list index bfe975e..5ce9037 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -123,7 +123,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf 'SELECT ' mysql_package_name_query printf ' FROM `binary_packages`' - printf ' JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`' + mysql_join_binary_packages_architectures } | \ ${mysql_command} --raw --batch | \ sed ' @@ -161,7 +161,7 @@ fi printf 'INSERT IGNORE INTO `ignore_packages` (`pkgname`)' printf ' SELECT DISTINCT `binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`' + mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name` IN ("build-list","deletion-list");\n' } | \ sponge "${tmp_dir}/ignore-packages" @@ -253,7 +253,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '`pkgbases`.`repository`' printf ' FROM `pkgbases`' printf ' JOIN `upstream_repositories` ON `upstream_repositories`.`name`=`pkgbases`.`repository`' - printf ' JOIN `git_repositories` ON `git_repositories`.`id`=`upstream_repositories`.`git_repository`' + mysql_join_upstream_repositories_git_repositories printf ';\n' printf 'SELECT ' printf '`package_sources`.`pkgbase`,' @@ -264,11 +264,10 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '),' printf '`upstream_repositories`.`name`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' \ - 'upstream_repositories' 'upstream_repositories' 'package_sources' 'upstream_package_repository' \ - 'git_repositories' 'git_repositories' 'upstream_repositories' 'git_repository' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_git_repositories printf ' WHERE (' if [ -s "${tmp_dir}/package-regexes" ]; then grep -vxF '' "${tmp_dir}/package-regexes" | \ diff --git a/bin/show-dependencies b/bin/show-dependencies index 0a7a1fd..bb1ae37 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -13,7 +13,7 @@ if [ $# -eq 0 ]; then { printf 'SELECT `package_sources`.`pkgbase`' printf ' FROM `package_sources`' - printf ' JOIN `build_assignments` ON `build_assignments`.`package_source`=`package_sources`.`id`' + mysql_join_package_sources_build_assignments printf ' WHERE `build_assignments`.`is_broken`' } | \ ${mysql_command} --raw --batch | \ @@ -28,9 +28,8 @@ if [ $# -eq 0 ]; then { printf 'SELECT DISTINCT `package_sources`.`pkgbase`,`binary_packages`.`pkgname`' printf ' FROM `binary_packages`' - printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \ - 'build_assignments' 'build_assignments' 'binary_packages' 'build_assignment' \ - 'package_sources' 'package_sources' 'build_assignments' 'package_source' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources printf ' ORDER BY `binary_packages`.`pkgname`' # Why can't mysql order in the same way "sort" does! } | \ diff --git a/bin/why-dont-you b/bin/why-dont-you index e9922b7..d95f2a7 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -57,10 +57,9 @@ case "${action}" in # package_source, build_assignment, binary_package, repostory printf 'SELECT DISTINCT `tb_ps`.`pkgbase`,`tb_bin`.`id` AS `bin_id`,`tb_ba`.`id` AS `ba_id`,`tb_ba`.`is_blocked`,`tb_ba`.`is_broken`' printf ' FROM `package_sources` AS `tb_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `tb_ba' 'tb_ba' 'package_source' 'tb_ps' \ - 'binary_packages` AS `tb_bin' 'tb_bin' 'build_assignment' 'tb_ba' \ - 'repositories` AS `tb_rep' 'tb_bin' 'repository' 'tb_rep' + mysql_join_package_sources_build_assignments 'tb_ps' 'tb_ba' + mysql_join_build_assignments_binary_packages 'tb_ba' 'tb_bin' + mysql_join_binary_packages_repositories 'tb_bin' 'tb_rep' printf ' WHERE `tb_rep`.`name`="build-list"' printf ') AS `to_build`' printf ' ON `to_build`.`pkgbase`=`pkgbases`.`pkgbase`' @@ -70,16 +69,14 @@ case "${action}" in # potential dependencies printf 'SELECT DISTINCT `dep_ps`.`pkgbase`,`dependencies`.`dependent`' printf ' FROM `package_sources` AS `dep_ps`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'build_assignments` AS `dep_ba' 'dep_ba' 'package_source' 'dep_ps' \ - 'binary_packages` AS `dep_bin' 'dep_bin' 'build_assignment' 'dep_ba' \ - 'repositories` AS `dep_rep' 'dep_bin' 'repository' 'dep_rep' \ - 'install_target_providers' 'install_target_providers' 'package' 'dep_bin' - # starting from the line above, we have some additional joins, + mysql_join_package_sources_build_assignments 'dep_ps' 'dep_ba' + mysql_join_build_assignments_binary_packages 'dep_ba' 'dep_bin' + mysql_join_binary_packages_repositories 'dep_bin' 'dep_rep' + # now we have some (=3) additional joins, # because we are interested in dependency relations to `to_build` - printf ' JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' JOIN `%s` ON `%s`.`%s`=`%s`.`id`' \ - 'dependency_types' 'dependencies' 'dependency_type' 'dependency_types' + mysql_join_binary_packages_install_target_providers 'dep_bin' + mysql_join_install_target_providers_dependencies + mysql_join_dependencies_dependency_types printf ' WHERE `dep_rep`.`name`="build-list"' printf ' AND `dependency_types`.`relevant_for_building`' printf ') AS `deps`' -- cgit v1.2.3-54-g00ecf From a9bb4512da90d21fde9a6f6095e304f461fe2dc7 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 16 Feb 2018 11:33:29 +0100 Subject: bin/bootstrap-mysql: build-support should superseed build-support, too --- bin/bootstrap-mysql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 30c260a..4dee0f8 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -418,7 +418,8 @@ fi 'staging' 'staging' \ 'staging' 'unbuilt' \ 'staging' 'forbidden' \ - 'unbuilt' 'forbidden' + 'unbuilt' 'forbidden' \ + 'build-support' 'build-support' printf ';\n' } | \ sed 's| OR ;|;|' -- cgit v1.2.3-54-g00ecf From 8fae4bb4650ea62def1c6f90bfa8fd89b2caabde Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 09:50:56 +0100 Subject: bin/bootstrap-mysql: simplify --- bin/bootstrap-mysql | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4dee0f8..48f2850 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -139,6 +139,7 @@ if [ ! "$1" = 'slim' ]; then ' less_stable MEDIUMINT :repository_stabilities' ) + # shellcheck disable=SC2016 { printf '%s\n' \ '/*!40014 SET UNIQUE_CHECKS=0 */;' \ @@ -151,14 +152,11 @@ if [ ! "$1" = 'slim' ]; then while read -r table size; do case "${turn}" in 'drop') - # shellcheck disable=SC2016 printf 'DROP TABLE IF EXISTS `%s`;\n' "${table}" ;; 'create'|'link') if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf 'CREATE TABLE `%s` (\n' "${table}" - # shellcheck disable=SC2016 printf ' `id` %s NOT NULL AUTO_INCREMENT,\n' "${size}" fi echo "${tables}" | \ @@ -181,16 +179,14 @@ if [ ! "$1" = 'slim' ]; then if [ -z "${rest}" ]; then rest="${type}" fi - # shellcheck disable=SC2016 printf ' UNIQUE KEY `%s` (' "${type}" - # shellcheck disable=SC2016,SC2086 + # shellcheck disable=SC2086 printf '`%s`,' ${rest} | \ sed 's|,$||' printf ')' ;; *) if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf ' `%s` %s' "${column}" "${type}" if ! echo "${rest}" | \ grep -qwF 'NULL'; then @@ -201,7 +197,6 @@ if [ ! "$1" = 'slim' ]; then tr ' ' '\n' | \ sed -n 's/^://;T;p' | \ while read -r link; do - # shellcheck disable=SC2016 printf 'ALTER TABLE `%s` ADD FOREIGN KEY (`%s`) REFERENCES `%s` (`id`);\n' \ "${table}" \ "${column}" \ @@ -215,7 +210,6 @@ if [ ! "$1" = 'slim' ]; then fi done if [ "${turn}" = 'create' ]; then - # shellcheck disable=SC2016 printf ' PRIMARY KEY (`id`));\n' fi ;; @@ -230,26 +224,26 @@ if [ ! "$1" = 'slim' ]; then '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;' printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ - 'SHOW VIEW' 'buildmaster.*' \ + 'EXECUTE' 'buildmaster.*' \ + 'RELOAD' '*.*' \ 'SELECT' 'buildmaster.*' \ - 'UPDATE' 'buildmaster.*' \ - 'RELOAD' '*.*' - printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \ 'SHOW VIEW' 'buildmaster.*' \ - 'SELECT' 'buildmaster.*' + 'UPDATE' 'buildmaster.*' + printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \ + 'SELECT' 'buildmaster.*' \ + 'SHOW VIEW' 'buildmaster.*' printf 'FLUSH PRIVILEGES;\n' } | \ mysql -u root -p buildmaster fi +# shellcheck disable=SC2016 { - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `architectures` (`name`) VALUES ' printf '("%s"),' \ 'any' 'i686' | \ sed 's|,$||' printf ';\n' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `fail_reasons` (`identifier`,`name`,`severity`) VALUES ' printf '(from_base64("%s"),"%s",%s),' \ "$( @@ -290,7 +284,6 @@ fi )" 'unknown' 100 | \ sed 's|,$||' printf ';\n' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `git_repositories` (`name`,`url`,`directory`,`head`) VALUES' { for repo in ${repo_names}; do @@ -328,9 +321,7 @@ fi 'extra:packages' \ 'multilib:packages' \ 'community:community'; do - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `upstream_repositories` (`name`,`git_repository`) SELECT\n' - # shellcheck disable=SC2016 printf ' from_base64("%s"),`id` FROM `git_repositories` WHERE `name` = from_base64("%s");\n' \ "$( printf '%s' "${repo%:*}" | \ @@ -342,7 +333,6 @@ fi )" done - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `build_slaves` (`name`,`ssh_key`,`operator`,`last_connection`) VALUES' { sed -n ' @@ -385,7 +375,6 @@ fi } | \ sed 's|,;|;|' - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `repository_stabilities` (`name`,`bugtracker_category`) VALUES' { printf '\n ("%s",%s),' \ @@ -399,7 +388,6 @@ fi } | \ sed 's|,;|;|' - # shellcheck disable=SC2016 { printf 'INSERT IGNORE INTO `repository_stability_relations` (`more_stable`,`less_stable`)' printf ' SELECT `ms`.`id`,`ls`.`id`' @@ -435,9 +423,7 @@ fi 'community-staging:staging:AQ==' \ 'build-list:unbuilt:AA==' \ 'deletion-list:forbidden:AA=='; do - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `repositories` (`name`,`stability`,`is_on_master_mirror`) SELECT' - # shellcheck disable=SC2016 printf ' from_base64("%s"),`id`,from_base64("%s") FROM `repository_stabilities` WHERE `name`=from_base64("%s");\n' \ "$( printf '%s' "${repo}" | \ @@ -455,7 +441,6 @@ fi )" done - # shellcheck disable=SC2016 printf 'INSERT IGNORE INTO `dependency_types` (`name`,`relevant_for_building`,`relevant_for_binary_packages`) VALUES' { printf '\n ("%s",%s,%s),' \ -- cgit v1.2.3-54-g00ecf From 2e3532e6a9e9b69b722b1a8da51fd2d27e895887 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 09:54:40 +0100 Subject: bin/bootstrap-mysql: show_broken_packages_and_dependencies new --- bin/bootstrap-mysql | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 48f2850..fd670c7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -222,7 +222,45 @@ if [ ! "$1" = 'slim' ]; then printf '%s\n' \ '/*!40014 SET UNIQUE_CHECKS=1 */;' \ '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;' + + printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n' + printf 'BEGIN\n' + printf 'CREATE TEMPORARY TABLE `%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ + 'broken_packages_and_dependencies' 'broken_packages_and_dependencies_old' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `build_assignments`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_build_assignments + printf ' AND `build_assignments`.`is_broken`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list";\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies_old` (`id`)' + printf ' SELECT `broken_packages_and_dependencies`.`id` FROM `broken_packages_and_dependencies`;\n' + printf 'INSERT IGNORE INTO `broken_packages_and_dependencies` (`id`)' + printf ' SELECT `new_bp`.`build_assignment`' + printf ' FROM `broken_packages_and_dependencies_old`' + mysql_join_build_assignments_binary_packages 'broken_packages_and_dependencies_old' 'old_bp' + mysql_join_binary_packages_dependencies 'old_bp' + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + mysql_join_install_target_providers_binary_packages '' 'new_bp' + mysql_join_binary_packages_repositories 'new_bp' 'new_repo' + printf ' AND `new_repo`.`name` IN ("build-list","deletion-list");\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'SELECT ' + mysql_query_select_pkgbase_and_revision + printf ' JOIN `broken_packages_and_dependencies` ON `broken_packages_and_dependencies`.`id`=`build_assignments`.`id`;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE ROUTINE' 'buildmaster.*' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ 'EXECUTE' 'buildmaster.*' \ 'RELOAD' '*.*' \ -- cgit v1.2.3-54-g00ecf From b2a175e38818f4bdf088d2695e243572fe5eb2e8 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 10:26:33 +0100 Subject: bin/bootstrap-mysql: add comment --- bin/bootstrap-mysql | 3 +++ 1 file changed, 3 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index fd670c7..275764b 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -223,6 +223,9 @@ if [ ! "$1" = 'slim' ]; then '/*!40014 SET UNIQUE_CHECKS=1 */;' \ '/*!40014 SET FOREIGN_KEY_CHECKS=1 */;' + # show_broken_packages_and_dependencies + # gives a list of broken packages and their dependencies if they're + # still on the build- or deletion-list printf 'DROP PROCEDURE IF EXISTS show_broken_packages_and_dependencies;\n' printf 'DELIMITER //\n' printf 'CREATE PROCEDURE show_broken_packages_and_dependencies()\n' -- cgit v1.2.3-54-g00ecf From 5a72e06a2ef65e8cf2991938896bc520ae2b5c4a Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 26 Feb 2018 15:20:54 +0100 Subject: bin/bootstrap-mysql: calculate_dependencies_of_package_upto_first_built_one new --- bin/bootstrap-mysql | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 275764b..a172d82 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -262,6 +262,40 @@ if [ ! "$1" = 'slim' ]; then printf '//\n' printf 'DELIMITER ;\n' + # calculate_dependencies_of_package_upto_first_built_one + # save binary packages (only their `id`) in `relevant_binary_packages` and `relevant_binary_packages_copy` + printf 'DROP PROCEDURE IF EXISTS calculate_dependencies_of_package_upto_first_built_one;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_dependencies_of_package_upto_first_built_one(IN `target_pkgbase` VARCHAR(64))\n' + printf 'BEGIN\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `binary_packages`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + printf ' WHERE `package_sources`.`pkgbase`=`target_pkgbase`;\n' + printf 'REPEAT\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages_copy` (`id`)' + printf ' SELECT `relevant_binary_packages`.`id` FROM `relevant_binary_packages`;\n' + printf 'INSERT IGNORE INTO `relevant_binary_packages` (`id`)' + printf ' SELECT `install_target_providers`.`package`' + printf ' FROM `relevant_binary_packages_copy`' + printf ' JOIN `binary_packages` ON `relevant_binary_packages_copy`.`id`=`binary_packages`.`id`' + mysql_join_binary_packages_repositories + printf ' AND `repositories`.`name`="build-list"' + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_building`' + mysql_join_dependencies_install_target_providers + printf ';\n' + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ 'CREATE ROUTINE' 'buildmaster.*' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ -- cgit v1.2.3-54-g00ecf From 3567c64454903149b78fe35952e91c18c35af264 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 28 Feb 2018 20:41:35 +0100 Subject: bin/bootstrap-mysql: grant create temporary tables to webserver@localhost --- bin/bootstrap-mysql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index a172d82..b23bba4 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -304,7 +304,8 @@ if [ ! "$1" = 'slim' ]; then 'SELECT' 'buildmaster.*' \ 'SHOW VIEW' 'buildmaster.*' \ 'UPDATE' 'buildmaster.*' - printf 'GRANT %s ON %s TO '"'"'http'"'"'@'"'"'localhost'"'"';\n' \ + printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \ + 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ 'SELECT' 'buildmaster.*' \ 'SHOW VIEW' 'buildmaster.*' printf 'FLUSH PRIVILEGES;\n' -- cgit v1.2.3-54-g00ecf From 6047f2686cfe623e08fbaf4eae0a552ded731eb1 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 08:37:27 +0100 Subject: switch from calling ${mysql_command} to calling the wrapper function mysql_run_query() --- bin/bootstrap-mysql | 4 ++-- bin/build-master-status | 6 +++--- bin/build-master-status-from-mysql | 16 ++++++++-------- bin/check-bugtracker | 4 ++-- bin/copy-to-build-support | 4 ++-- bin/db-update | 6 +++--- bin/get-assignment | 10 +++++----- bin/get-package-updates | 12 ++++++------ bin/modify-package-state | 4 ++-- bin/prioritize-build-list | 2 +- bin/return-assignment | 14 +++++++------- bin/seed-build-list | 6 +++--- bin/show-dependencies | 4 ++-- bin/slave-build-connect | 2 +- bin/why-dont-you | 2 +- lib/common-functions | 4 ++-- lib/mysql-functions | 34 ++++++++++++++++++++-------------- 17 files changed, 70 insertions(+), 64 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b23bba4..236b0b7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -590,7 +590,7 @@ find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${repository}" | base64 -w0)" done | \ - ${mysql_command} + mysql_run_query grep '^\('"$( # shellcheck disable=SC2086 @@ -665,7 +665,7 @@ if [ -s "${tmp_dir}/new-stable-packages" ]; then printf ';\n' printf 'DROP TABLE `stable_packages`;\n' } | \ - ${mysql_command} + mysql_run_query fi mysql_repair_binary_packages_without_build_assignment diff --git a/bin/build-master-status b/bin/build-master-status index d74ff8a..9be7823 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -557,12 +557,12 @@ if ${web}; then "${line}" done < \ "${tmp_dir}/todos" | \ - ${mysql_command} + mysql_run_query # insert unfound todos # shellcheck disable=SC2016 { printf 'SHOW CREATE TABLE `todos`' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d 2s/^\S\+\s\+CREATE TABLE `todos` /CREATE TEMPORARY TABLE `td` / @@ -611,7 +611,7 @@ if ${web}; then printf 'WHERE `todos`.`id`=`todo_links`.`dependent`' printf ');\n' } | \ - ${mysql_command} + mysql_run_query rm -f "${tmp_dir}/todos" fi diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql index b55ca65..fa54177 100755 --- a/bin/build-master-status-from-mysql +++ b/bin/build-master-status-from-mysql @@ -27,7 +27,7 @@ fi mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`name`="build-list"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -48,7 +48,7 @@ sort "${work_dir}/build-list" > \ mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -72,7 +72,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.locked' -prin printf ' WHERE `repositories`.`name`="deletion-list"' printf ' OR `binary_packages`.`is_to_be_deleted`;\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -90,7 +90,7 @@ sort "${work_dir}/deletion-list" > \ mysql_join_build_assignments_build_dependency_loops printf ';\n' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d y/\t/ / @@ -121,7 +121,7 @@ find "${work_dir}/build-list.loops" -regextype grep -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -156,7 +156,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ printf ' AND NOT `binary_packages`.`has_issues`' printf ' AND `repository_stabilities`.`name`="testing"' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s/^\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)\s\(\S\+\)$/\1-\2:\3-\4.\5-\6.pkg.tar.xz/ @@ -190,7 +190,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_upstream_repositories printf ' WHERE `dependency_types`.`relevant_for_building`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -210,7 +210,7 @@ find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/bin/check-bugtracker b/bin/check-bugtracker index d6e92e5..f15757b 100755 --- a/bin/check-bugtracker +++ b/bin/check-bugtracker @@ -26,7 +26,7 @@ bug_list=$( printf ' FROM `repository_stabilities`' printf ' WHERE NOT `repository_stabilities`.`bugtracker_category` IS NULL' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r stability_id category; do for has_issues in '1:' '0:NOT '; do @@ -62,4 +62,4 @@ bug_list=$( printf '"");\n' done done | tee /dev/stderr | \ - ${mysql_command} + mysql_run_query diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support index 808d041..0b87107 100755 --- a/bin/copy-to-build-support +++ b/bin/copy-to-build-support @@ -44,7 +44,7 @@ sed -n ' "$(printf '%s' "${architecture}" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -123,7 +123,7 @@ ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/build-support/" if [ -s "${tmp_dir}/mysql-command" ]; then - ${mysql_command} < \ + mysql_run_query < \ "${tmp_dir}/mysql-command" fi diff --git a/bin/db-update b/bin/db-update index de263be..534c245 100755 --- a/bin/db-update +++ b/bin/db-update @@ -370,7 +370,7 @@ move_packages() { if ${no_action}; then sed 's|^|mysql: |' else - ${mysql_command} + mysql_run_query fi ${dummynator} trigger_mirror_refreshs @@ -457,7 +457,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "testing"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p @@ -496,7 +496,7 @@ errors=$( printf ' JOIN `repository_stabilities` ON `repository_stability_relations`.`more_stable`=`repository_stabilities`.`id`' printf ' WHERE `repository_stabilities`.`name` = "staging"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d p diff --git a/bin/get-assignment b/bin/get-assignment index e3299f6..bc67fab 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -58,7 +58,7 @@ hand_out_assignment() { "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -104,7 +104,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query # lock every loop this package breaks find "${work_dir}/build-list.loops" -maxdepth 1 -regextype grep \ @@ -157,7 +157,7 @@ currently_building=$( printf ' AND `repositories`.`name`="build-list"' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) @@ -214,7 +214,7 @@ next_building=$( printf ' ORDER BY `requested` DESC, `priority` DESC, `last_trial`, `part_of_loop`, `build_assignments`.`id`' printf ' LIMIT 1;\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed ' 1d y/\t/ / @@ -238,7 +238,7 @@ count_pending=$( printf ' AND `build_assignments`.`is_blocked` IS NULL' printf ';\n' } | \ - ${mysql_command} --batch --raw | \ + mysql_run_query --batch --raw | \ sed '1d' ) diff --git a/bin/get-package-updates b/bin/get-package-updates index 2fd0993..11f384c 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -129,7 +129,7 @@ delete_package() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");' \ "$(printf '%s' "$1" | base64 -w0)" } | \ - ${mysql_command} + mysql_run_query fi } @@ -144,7 +144,7 @@ clean_up() { printf ' WHERE `repositories`.`name`="to-be-decided"' printf ');\n' } | \ - ${mysql_command} + mysql_run_query } tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) trap 'clean_up' EXIT @@ -353,7 +353,7 @@ done | \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue @@ -391,7 +391,7 @@ black_listed_new=$( printf ' FROM `package_sources`' printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' } | \ sort -u @@ -443,7 +443,7 @@ while [ -n "${black_listed_new}" ]; do printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ sort -u } | \ @@ -604,7 +604,7 @@ fi printf ')' printf ' WHERE `from_repo`.`name`="to-be-decided";\n' } | \ - ${mysql_command} + mysql_run_query # update loop list in database (beware, the packages are expected to be in "build-list", # not "to-be-decided", so we need to run this after moving the packages from "to-be-decided" to the "build-list". diff --git a/bin/modify-package-state b/bin/modify-package-state index b0ba53e..8c5fac7 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -225,7 +225,7 @@ while read -r package reason; do ;; esac if printf 'SELECT 1 FROM %s %s AND %s LIMIT 1' "${combiner}" "${selector}" "${tester}" | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qxF '1'; then # shellcheck disable=SC2016 @@ -242,7 +242,7 @@ while read -r package reason; do printf ' AND NOT `b`.`is_tested`' printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ - ${mysql_command} + mysql_run_query case "${action}" in 'block') printf '%s' "${reason}" > \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index e8dcf0a..1ff3d5a 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -46,7 +46,7 @@ fi "$(printf '%s' "$1" | base64 -w0)" printf ' AND `repositories`.`name`="build-list";\n' } | \ - ${mysql_command} + mysql_run_query # Remove the lock file diff --git a/bin/return-assignment b/bin/return-assignment index db433ee..ed29f34 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -70,7 +70,7 @@ if [ "$5" = 'ERROR' ]; then "$(printf '%s' "$4" | base64 -w0)" printf ' AND `repositories`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -111,7 +111,7 @@ if [ "$5" = 'ERROR' ]; then printf 'SELECT `fail_reasons`.`id`,replace(to_base64(`fail_reasons`.`identifier`),"\\n","")' printf ' FROM `fail_reasons` ORDER BY `fail_reasons`.`severity`' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' ) for saved_build_log in ${saved_build_logs}; do @@ -164,7 +164,7 @@ if [ "$5" = 'ERROR' ]; then base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query # shellcheck disable=SC2154 sed -i ' @@ -283,7 +283,7 @@ if ! { printf ' AND `binary_packages`.`sub_pkgrel`=from_base64("%s");\n' \ "$(printf '%s' "$5" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -vxF '0'; then >&2 echo 'Sorry, the sent package is outdated.' @@ -441,7 +441,7 @@ package_errors=$( printf ' AND `repositories`.`name`="build-list"' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d ' @@ -513,7 +513,7 @@ find . -maxdepth 1 -name '*.pkg.tar.xz-namcap.log.gz' -execdir mv '{}' "${build_ printf ';\n' done } | \ - ${mysql_command} + mysql_run_query # move packages destination=$(official_or_community "$1.$2.$3.$4" 'staging') @@ -605,7 +605,7 @@ fi printf ');\n' printf 'DROP TABLE `loops_to_delete`;\n' } | \ - ${mysql_command} + mysql_run_query rm -f \ "${work_dir}/package-states/$1.$2.$3.$4.locked" \ "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/seed-build-list b/bin/seed-build-list index 465ef5d..b9ae193 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -125,7 +125,7 @@ if [ -s "${tmp_dir}/mirrors" ]; then printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)/ours \2 \3 \1/ @@ -206,7 +206,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`must_haves`.`pkgname`' printf ') AND NOT `must_haves`.`pkgname` LIKE "lib32-%%";\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r pkgname; do content=$( @@ -294,7 +294,7 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf ' WHERE `ignore_packages`.`pkgname`=`binary_packages`.`pkgname`' printf ');\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sort -u | \ sed ' y/\t/ / diff --git a/bin/show-dependencies b/bin/show-dependencies index 4e90258..9e94fe3 100755 --- a/bin/show-dependencies +++ b/bin/show-dependencies @@ -10,7 +10,7 @@ if [ $# -eq 0 ]; then broken=$( printf 'CALL show_broken_packages_and_dependencies;\n' | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d s/\s.*$// @@ -178,7 +178,7 @@ for target_package in "$@"; do mysql_join_dependencies_install_targets printf ';\n' } | \ - ${mysql_command} -N --raw --batch | \ + mysql_run_query -N --raw --batch | \ sed ' y/\t/ / /^knot /{ diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 119592e..f8ee9ad 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -29,7 +29,7 @@ if [ "x${SSH_ORIGINAL_COMMAND%% *}" = 'xget-assignment' ] || \ base64 -w0 )" } | \ - ${mysql_command} + mysql_run_query slave="$1" /bin/sh -c "${base_dir}/bin/${SSH_ORIGINAL_COMMAND}" diff --git a/bin/why-dont-you b/bin/why-dont-you index d95f2a7..b5d73c5 100755 --- a/bin/why-dont-you +++ b/bin/why-dont-you @@ -85,7 +85,7 @@ case "${action}" in printf ' LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`to_build`.`ba_id`' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / diff --git a/lib/common-functions b/lib/common-functions index 26151d1..740c266 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -501,7 +501,7 @@ remove_old_package_versions() { "$(printf '%s' "${repository}" | base64 -w0)" printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^\S\+\sCONCAT(/d y/\t/ / @@ -680,7 +680,7 @@ remove_old_package_versions() { sed 's/ AND$//' printf ';\n' done | \ - ${mysql_command} + mysql_run_query sed ' s| \(\S\+\)$|-\1| diff --git a/lib/mysql-functions b/lib/mysql-functions index 185d9a9..6cbc502 100755 --- a/lib/mysql-functions +++ b/lib/mysql-functions @@ -22,6 +22,12 @@ base64_encode_each() { done } +# mysql_run_query +# wrapper function to query mysql +mysql_run_query() { + ${mysql_command} "$@" +} + # mysql_add_package_source $pkgbase $git_revision $mod_git_revision $upstream_package_repository # shellcheck disable=SC2086 @@ -69,7 +75,7 @@ mysql_add_package_source() { printf ' WHERE `upstream_repositories`.`name` = from_base64("%s");' \ "${values##* }" } | \ - ${mysql_command} + mysql_run_query } # mysql_add_binary_package $pkgbase $git_revision $mod_git_revision $upstream_package_repository $pkgname $sub_pkgrel $architecture $repository @@ -113,7 +119,7 @@ mysql_add_binary_package() { s|, )|)|g s|, FROM| FROM|g ' | \ - ${mysql_command} + mysql_run_query } # mysql_show_binary_package $pkgname $pkgver $pkgrel $sub_pkgrel @@ -162,7 +168,7 @@ mysql_show_binary_package() { s|, FROM| FROM|g s|AND;|;|g ' | \ - ${mysql_command} --html + mysql_run_query --html } # mysql_add_install_target $install_target @@ -180,7 +186,7 @@ mysql_add_install_target() { printf ' VALUES (from_base64("%s"))' \ "${install_target}" } | \ - ${mysql_command} + mysql_run_query } # mysql_generate_package_metadata $current_repository $package $git_revision $mod_git_revision $repository @@ -626,7 +632,7 @@ mysql_generate_package_metadata() { cat "${temp_dir}/add-install-targets-command" fi } | \ - ${mysql_command} + mysql_run_query printf '.' >&2 ) @@ -662,7 +668,7 @@ mysql_sanity_check() { 'a' 'b' printf ';\n' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' /^CONCAT("/d s,^,, @@ -693,7 +699,7 @@ mysql_sanity_check() { mysql_join_binary_packages_repositories printf ' WHERE `repositories`.`is_on_master_mirror`' } | \ - ${mysql_command} --batch | \ + mysql_run_query --batch | \ sed ' 1d s,\t,/, @@ -724,7 +730,7 @@ mysql_find_build_assignment_loops() { mysql_join_binary_packages_repositories 'packages_dependent' 'repositories_dependent' printf ' WHERE `repositories_dependent`.`name`="build-list" AND `repositories_dependency`.`name`="build-list"' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed ' 1d y/\t/ / @@ -754,7 +760,7 @@ mysql_find_build_assignment_loops() { "${new_loops}" fi } | \ - ${mysql_command} + mysql_run_query } # mysql_cleanup [dry] @@ -844,7 +850,7 @@ mysql_cleanup() { printf ' SET `currently_building`=NULL' printf ' WHERE `repositories`.`name`!="build-list";\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_query_has_pending_dependencies `build_assignment`.`id` @@ -904,7 +910,7 @@ mysql_repair_binary_packages_without_build_assignment() { mysql_join_binary_packages_architectures printf ' WHERE `binary_packages`.`build_assignment`<0' } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ while read -r id pkgname arch; do pkgname=$( @@ -933,7 +939,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \ "$(printf '%s' "${pkgname}" | base64 -w0)" } | \ - ${mysql_command} --raw --batch | \ + mysql_run_query --raw --batch | \ sed '1d' | \ grep -qvxF '0'; then pkgbase="${pkgname}" @@ -960,7 +966,7 @@ mysql_repair_binary_packages_without_build_assignment() { printf ' WHERE `binary_packages`.`id`=%s' "${id}" printf ' AND `package_sources`.`pkgbase`=from_base64("%s");\n' "${pkgbase}" done | \ - ${mysql_command} + mysql_run_query } # mysql_remove_duplicate_binary_packages @@ -1001,7 +1007,7 @@ mysql_remove_duplicate_build_order() { printf ' SET `binary_packages`.`repository`=`ren`.`repo`,' printf ' `binary_packages`.`sub_pkgrel`=`ren`.`sub_pkgrel`;\n' } | \ - ${mysql_command} + mysql_run_query } # mysql_package_name_query -- cgit v1.2.3-54-g00ecf From 48c7a60e79436c320627740c1cf5a7bc85a8fe96 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 1 Mar 2018 14:04:31 +0100 Subject: bin/bootstrap-mysql: repository_moves new --- bin/bootstrap-mysql | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 236b0b7..c828809 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -136,7 +136,12 @@ if [ ! "$1" = 'slim' ]; then ' depending_on MEDIUMINT' \ 'repository_stability_relations MEDIUMINT' \ ' more_stable MEDIUMINT :repository_stabilities' \ - ' less_stable MEDIUMINT :repository_stabilities' + ' less_stable MEDIUMINT :repository_stabilities' \ + 'repository_moves MEDIUMINT' \ + ' from_repository MEDIUMINT :repositories' \ + ' to_repository MEDIUMINT :repositories' \ + ' upstream_package_repository SMALLINT :upstream_repositories' \ + ' UNIQUE source from_repository upstream_package_repository' ) # shellcheck disable=SC2016 @@ -488,6 +493,29 @@ fi } | \ sed 's| OR ;|;|' + { + printf 'INSERT IGNORE INTO `repository_moves` (`from_repository`,`to_repository`,`upstream_package_repository`)' + printf ' SELECT `f`.`id`,`t`.`id`,`u`.`id`' + printf ' FROM' + printf ' `repositories` AS `%s` JOIN' \ + 'f' 't' + printf ' `upstream_repositories` AS `u`' + printf ' WHERE ' + printf '(`f`.`name`="%s" AND `t`.`name`="%s" AND `u`.`name`="%s") OR ' \ + 'staging' 'testing' 'core' \ + 'staging' 'testing' 'extra' \ + 'staging' 'testing' 'multilib' \ + 'community-staging' 'community-testing' 'community' \ + 'community-staging' 'community-testing' 'multilib' \ + 'testing' 'core' 'core' \ + 'testing' 'extra' 'extra' \ + 'testing' 'extra' 'multilib' \ + 'community-testing' 'community' 'community' \ + 'community-testing' 'community' 'multilib' + printf ';\n' + } | \ + sed 's| OR ;|;|' + for repo in \ 'core:stable:AQ==' \ 'extra:stable:AQ==' \ -- cgit v1.2.3-54-g00ecf From 390561a3f13eedec674259bb5b2e491886bb56d0 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 5 Mar 2018 16:15:09 +0100 Subject: bin/bootstrap-mysql: implement stored function with new condition from db-update (which is not yet used, but looks ok) --- bin/bootstrap-mysql | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/db-update | 6 +++ 2 files changed, 126 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index c828809..1e5f45f 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -301,6 +301,126 @@ if [ ! "$1" = 'slim' ]; then printf '//\n' printf 'DELIMITER ;\n' + # calculate_maximal_moveable_set + # stores results in `moveable_binary_packages` and `replaced_binary_packages` + # Give a maximal list of packages to be moved, while implementing the + # condition from db-update: + # Every package which is replaced, must have its provided install_targets: + # a) provided by another moved or not-replaced package or + # b) not required by any not-replaced package. + printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' + printf 'DELIMITER //\n' + printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' + printf 'BEGIN\n' + for table in 'moveable' 'replaced'; do + for copy in '' '_copy'; do + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ + "${table}" "${copy}" + printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ + "${table}" "${copy}" + done + done + # these packages are considered for moving: + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`)' + printf ' SELECT `binary_packages`.`id`' + printf ' FROM `binary_packages`' + mysql_join_binary_packages_repositories + mysql_join_repositories_repository_stabilities + # correct stability: "testing"/"staging" - as chosen + printf ' WHERE `repository_stabilities`.`name` = `from_stability`' + printf ' AND (' + printf '`from_stability`="staging"' + # "testing" packages must be tested + printf ' OR `binary_packages`.`is_tested`' + printf ')' + # no open issues + printf ' AND NOT `binary_packages`.`has_issues`' + printf ';\n' + # these packages are considered for being replaced: + # for each moved package + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`)' + printf ' SELECT `r_bp`.`id`' + printf ' FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories 'm_bp' 'm_r' + mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' + mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' + mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' + mysql_join_upstream_repositories_repository_moves 'm_ur' + printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' + # in its target repository + printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' + mysql_join_repositories_binary_packages 'r_r' 'r_bp' + # all packages with identical names + printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`;\n' + # now we delete all unreplaceable and unmoveable packages from the respective + # list until no further changes are required + printf 'REPEAT\n' + for s in 'replaced' 'moveable'; do + printf 'DELETE FROM `%s_binary_packages_copy`;\n' "$s" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy` (`id`)' "$s" + printf ' SELECT `%s_binary_packages`.`id`' "$s" + printf ' FROM `%s_binary_packages`;\n' "$s" + done + # packages which should not be replaced: + printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`' + printf ' JOIN `binary_packages` AS `d_bp`' + mysql_join_binary_packages_dependencies 'd_bp' + mysql_join_binary_packages_repositories 'd_bp' 'd_r' + # consider only dependencies which can be met at all + printf ' AND EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ')' + mysql_join_dependencies_dependency_types + # only consider runtime dependencies + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + # not provided by a not-replaced, not-less-stable package + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'itp_bp' + mysql_join_binary_packages_repositories 'itp_bp' 'itp_r' + printf ' JOIN `repository_stability_relations` ON `itp_r`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' WHERE `d_r`.`stability`=`repository_stability_relations`.`less_stable`' + printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`itp_bp`.`id`' + printf ')' + printf ')' + # not provided by a moved package + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `moveable_binary_packages`' + mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' + printf ' WHERE `moveable_binary_packages`.`id`=`install_target_providers`.`package`' + printf ');\n' + # remove packages from the moveable-list which replace packages no longer on the replaced-list (e.g un-replaceable packages) + printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`' + printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories 'm_bp' 'm_r' + mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' + mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' + mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' + mysql_join_upstream_repositories_repository_moves 'm_ur' + printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' + printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' + mysql_join_repositories_binary_packages 'r_r' 'r_bp' + printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`' + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `replaced_binary_packages`' + printf ' WHERE `replaced_binary_packages`.`id`=`r_bp`.`id`' + printf ');\n' + # actually, this only checks the count of deletions from the moveable-list, but that should be ok + printf 'UNTIL ROW_COUNT()=0\n' + printf 'END REPEAT;\n' + for table in 'moveable' 'replaced'; do + printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy`;\n' \ + "${table}" + done + printf 'END\n' + printf '//\n' + printf 'DELIMITER ;\n' + printf 'GRANT %s ON %s TO '"'"'buildmaster'"'"'@'"'"'localhost'"'"';\n' \ 'CREATE ROUTINE' 'buildmaster.*' \ 'CREATE TEMPORARY TABLES' 'buildmaster.*' \ diff --git a/bin/db-update b/bin/db-update index 78ff622..06754bb 100755 --- a/bin/db-update +++ b/bin/db-update @@ -10,6 +10,12 @@ # b) no done package B (in a not-more stable repository) which is # not being moved run-depends on A +# The new condition will be shorter, cleaner and less error prone: +# (_Moving_ is without risk. It is _deleting_, that may break stuff.) +# Every package which is replaced, must have its provided install_targets: +# a) provided by another new or not-replaced package or +# b) not required by any not-replaced package. + # TODO: separate locks for staging, testing (and stable) # TODO: handle deletion of parts of a split package -- cgit v1.2.3-54-g00ecf From 7e0a82e87099700c990c64e75ffef6e6792153a4 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 08:38:20 +0100 Subject: bin/bootstrap-mysql: stored function works, but far too slow --- bin/bootstrap-mysql | 124 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 38 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 1e5f45f..41d2c73 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -313,7 +313,11 @@ if [ ! "$1" = 'slim' ]; then printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' printf 'BEGIN\n' for table in 'moveable' 'replaced'; do - for copy in '' '_copy'; do + for copy in '' '_copy' '_copy2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '_copy2' ]; then + continue + fi printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ "${table}" "${copy}" printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ @@ -356,55 +360,92 @@ if [ ! "$1" = 'slim' ]; then # now we delete all unreplaceable and unmoveable packages from the respective # list until no further changes are required printf 'REPEAT\n' - for s in 'replaced' 'moveable'; do - printf 'DELETE FROM `%s_binary_packages_copy`;\n' "$s" - printf 'INSERT IGNORE INTO `%s_binary_packages_copy` (`id`)' "$s" - printf ' SELECT `%s_binary_packages`.`id`' "$s" - printf ' FROM `%s_binary_packages`;\n' "$s" + for table in 'replaced' 'moveable'; do +printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}" + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s` (`id`)' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.`id`' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" + done done - # packages which should not be replaced: + # packages which should not be replaced: ones providing something that is: + # a) still needed by a not-replaced package x "less stable" than the target repository and + # b) not provided by: + # 1) a moved package or + # 2) any current, not-replaced package in a repository more stable than x's repository + # + # Note, that this is not 100% clean from an academic point of view: + # It might require _downgrading_ of a package to keep fulfilling a dependency + # and it might require to do so _without_ any chance for the user to notice, + # because there may be more "dependencies" in the database than in the package files. + # + # However, in practice both should not happen. + # printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`' - printf ' JOIN `binary_packages` AS `d_bp`' - mysql_join_binary_packages_dependencies 'd_bp' - mysql_join_binary_packages_repositories 'd_bp' 'd_r' - # consider only dependencies which can be met at all - printf ' AND EXISTS (' - printf 'SELECT * FROM `install_target_providers`' - printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`' + mysql_join_binary_packages_install_target_providers 'repl_bp' + mysql_join_install_target_providers_dependencies + printf ' AND NOT EXISTS (' + # dependencies of replaced packages don't matter + printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`dependencies`.`dependent`' printf ')' mysql_join_dependencies_dependency_types - # only consider runtime dependencies + # consider only runtime dependencies printf ' AND `dependency_types`.`relevant_for_binary_packages`' - # not provided by a not-replaced, not-less-stable package - printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `install_target_providers`' - mysql_join_install_target_providers_binary_packages '' 'itp_bp' - mysql_join_binary_packages_repositories 'itp_bp' 'itp_r' - printf ' JOIN `repository_stability_relations` ON `itp_r`.`stability`=`repository_stability_relations`.`more_stable`' - printf ' WHERE `d_r`.`stability`=`repository_stability_relations`.`less_stable`' - printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + mysql_join_dependencies_binary_packages '' 'req_bp' + mysql_join_binary_packages_repositories 'repl_bp' 'repl_r' + mysql_join_binary_packages_repositories 'req_bp' 'req_r' + # dependent package is "less stable" than dependency + printf ' JOIN `repository_stability_relations` AS `repl_rr`' + printf ' ON `repl_rr`.`more_stable`=`repl_r`.`stability`' + printf ' AND `repl_rr`.`less_stable`=`req_r`.`stability`' + # a) ^ + printf ' WHERE NOT EXISTS (' + # no moved package ... + printf 'SELECT *' + printf ' FROM `moveable_binary_packages`' + mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' 'subst_itp' + # ... provides the same + printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 1) ^ + printf ') AND NOT EXISTS (' + # no current package ... + printf 'SELECT *' + printf ' FROM `binary_packages` AS `subst_bp`' + mysql_join_binary_packages_install_target_providers 'subst_bp' 'subst_itp' + # ... in a repository ... + mysql_join_binary_packages_repositories 'subst_bp' 'subst_r' + # ... more stable ... + printf ' JOIN `repository_stability_relations` AS `subst_rr`' + printf ' ON `subst_rr`.`more_stable`=`subst_r`.`stability`' + # ... than x's repository ... + printf ' WHERE `subst_rr`.`less_stable`=`repl_r`.`stability`' printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `replaced_binary_packages_copy`' - printf ' WHERE `replaced_binary_packages_copy`.`id`=`itp_bp`.`id`' + # ... and which is not replaced ... + printf 'SELECT * FROM `replaced_binary_packages_copy2`' + printf ' WHERE `replaced_binary_packages_copy2`.`id`=`subst_bp`.`id`' printf ')' - printf ')' - # not provided by a moved package - printf ' AND NOT EXISTS (' - printf 'SELECT * FROM `moveable_binary_packages`' - mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' - printf ' WHERE `moveable_binary_packages`.`id`=`install_target_providers`.`package`' + # ... and provides the same + printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' + # b) 2) ^ printf ');\n' - # remove packages from the moveable-list which replace packages no longer on the replaced-list (e.g un-replaceable packages) + # packages which should not be moved: ones which replace packages no + # longer on the replaced-list (e.g un-replaceable packages) printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`' printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' - mysql_join_binary_packages_repositories 'm_bp' 'm_r' mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' mysql_join_upstream_repositories_repository_moves 'm_ur' - printf ' AND `repository_moves`.`from_repository`=`m_r`.`id`' - printf ' JOIN `repositories` AS `r_r` ON `repository_moves`.`to_repository`=`r_r`.`id`' - mysql_join_repositories_binary_packages 'r_r' 'r_bp' + printf ' AND `repository_moves`.`from_repository`=`m_bp`.`repository`' + printf ' JOIN `binary_packages` AS `r_bp`' + printf ' ON `repository_moves`.`to_repository`=`r_bp`.`repository`' printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`' printf ' WHERE NOT EXISTS (' printf 'SELECT * FROM `replaced_binary_packages`' @@ -414,8 +455,14 @@ if [ ! "$1" = 'slim' ]; then printf 'UNTIL ROW_COUNT()=0\n' printf 'END REPEAT;\n' for table in 'moveable' 'replaced'; do - printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy`;\n' \ - "${table}" + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DROP TEMPORARY TABLE `%s_binary_packages_copy%s`;\n' \ + "${table}" "${copy}" + done done printf 'END\n' printf '//\n' @@ -427,6 +474,7 @@ if [ ! "$1" = 'slim' ]; then 'EXECUTE' 'buildmaster.*' \ 'RELOAD' '*.*' \ 'SELECT' 'buildmaster.*' \ + 'SELECT' 'mysql.proc' \ 'SHOW VIEW' 'buildmaster.*' \ 'UPDATE' 'buildmaster.*' printf 'GRANT %s ON %s TO '"'"'webserver'"'"'@'"'"'localhost'"'"';\n' \ -- cgit v1.2.3-54-g00ecf From aa97f3e03393b01bf9df9a8e4f6e7f3b09c7905c Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 13:11:02 +0100 Subject: bin/bootstrap-mysql: calculate_maximal_moveable_set: have more columns in temporary tables to avoid some querys/joins --- bin/bootstrap-mysql | 62 ++++++++++++++++++++++------------------------------- 1 file changed, 26 insertions(+), 36 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 41d2c73..4e118ee 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -312,24 +312,29 @@ if [ ! "$1" = 'slim' ]; then printf 'DELIMITER //\n' printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' printf 'BEGIN\n' - for table in 'moveable' 'replaced'; do - for copy in '' '_copy' '_copy2'; do - if [ "${table}" = 'moveable' ] && \ - [ "${copy}" = '_copy2' ]; then - continue - fi - printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ - "${table}" "${copy}" - printf 'CREATE TEMPORARY TABLE `%s_binary_packages%s` (`id` BIGINT, UNIQUE KEY (`id`));\n' \ - "${table}" "${copy}" - done + for copy in '' '_copy' '_copy2'; do + printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ + 'moveable' "${copy}" \ + 'replaced' "${copy}" + printf 'CREATE TEMPORARY TABLE `replaced_binary_packages%s` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n' \ + "${copy}" + if [ "${copy}" = '_copy2' ]; then + continue + fi + printf 'CREATE TEMPORARY TABLE `moveable_binary_packages%s` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n' \ + "${copy}" done # these packages are considered for moving: - printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`)' - printf ' SELECT `binary_packages`.`id`' + printf 'INSERT IGNORE INTO `moveable_binary_packages` (`id`,`to_repository`)' + printf ' SELECT `binary_packages`.`id`,`repository_moves`.`to_repository`' printf ' FROM `binary_packages`' mysql_join_binary_packages_repositories mysql_join_repositories_repository_stabilities + mysql_join_binary_packages_build_assignments + mysql_join_build_assignments_package_sources + mysql_join_package_sources_upstream_repositories + mysql_join_upstream_repositories_repository_moves + printf ' AND `repository_moves`.`from_repository`=`binary_packages`.`repository`' # correct stability: "testing"/"staging" - as chosen printf ' WHERE `repository_stabilities`.`name` = `from_stability`' printf ' AND (' @@ -342,8 +347,8 @@ if [ ! "$1" = 'slim' ]; then printf ';\n' # these packages are considered for being replaced: # for each moved package - printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`)' - printf ' SELECT `r_bp`.`id`' + printf 'INSERT IGNORE INTO `replaced_binary_packages` (`id`,`replaced_by`)' + printf ' SELECT `r_bp`.`id`,`m_bp`.`id`' printf ' FROM `moveable_binary_packages`' printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' mysql_join_binary_packages_repositories 'm_bp' 'm_r' @@ -361,15 +366,15 @@ if [ ! "$1" = 'slim' ]; then # list until no further changes are required printf 'REPEAT\n' for table in 'replaced' 'moveable'; do -printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}" +printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table}" "${table}" for copy in '' '2'; do if [ "${table}" = 'moveable' ] && \ [ "${copy}" = '2' ]; then continue fi printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" - printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s` (`id`)' "${table}" "${copy}" - printf ' SELECT `%s_binary_packages`.`id`' "${table}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" printf ' FROM `%s_binary_packages`;\n' "${table}" done done @@ -386,7 +391,8 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}" # # However, in practice both should not happen. # - printf 'DELETE `replaced_binary_packages` FROM `replaced_binary_packages`' + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages` FROM `replaced_binary_packages`' + printf ' JOIN `moveable_binary_packages` ON `replaced_binary_packages`.`replaced_by`=`moveable_binary_packages`.`id`' printf ' JOIN `binary_packages` AS `repl_bp` ON `repl_bp`.`id`=`replaced_binary_packages`.`id`' mysql_join_binary_packages_install_target_providers 'repl_bp' mysql_join_install_target_providers_dependencies @@ -399,6 +405,7 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}" # consider only runtime dependencies printf ' AND `dependency_types`.`relevant_for_binary_packages`' mysql_join_dependencies_binary_packages '' 'req_bp' + # we need to check wether req_bp's dependency is (un)critical mysql_join_binary_packages_repositories 'repl_bp' 'repl_r' mysql_join_binary_packages_repositories 'req_bp' 'req_r' # dependent package is "less stable" than dependency @@ -435,23 +442,6 @@ printf 'SELECT count(*) FROM `%s_binary_packages`;\n' "${table}" printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' # b) 2) ^ printf ');\n' - # packages which should not be moved: ones which replace packages no - # longer on the replaced-list (e.g un-replaceable packages) - printf 'DELETE `moveable_binary_packages` FROM `moveable_binary_packages`' - printf ' JOIN `binary_packages` AS `m_bp` ON `m_bp`.`id`=`moveable_binary_packages`.`id`' - mysql_join_binary_packages_build_assignments 'm_bp' 'm_ba' - mysql_join_build_assignments_package_sources 'm_ba' 'm_ps' - mysql_join_package_sources_upstream_repositories 'm_ps' 'm_ur' - mysql_join_upstream_repositories_repository_moves 'm_ur' - printf ' AND `repository_moves`.`from_repository`=`m_bp`.`repository`' - printf ' JOIN `binary_packages` AS `r_bp`' - printf ' ON `repository_moves`.`to_repository`=`r_bp`.`repository`' - printf ' AND `r_bp`.`pkgname`=`m_bp`.`pkgname`' - printf ' WHERE NOT EXISTS (' - printf 'SELECT * FROM `replaced_binary_packages`' - printf ' WHERE `replaced_binary_packages`.`id`=`r_bp`.`id`' - printf ');\n' - # actually, this only checks the count of deletions from the moveable-list, but that should be ok printf 'UNTIL ROW_COUNT()=0\n' printf 'END REPEAT;\n' for table in 'moveable' 'replaced'; do -- cgit v1.2.3-54-g00ecf From eddb9d9e1d01c15f48366e5210f90f51a4d38c32 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 13:15:23 +0100 Subject: bin/bootstrap-mysql: deduplicate names --- bin/bootstrap-mysql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4e118ee..8b1db28 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -416,8 +416,8 @@ printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table printf ' WHERE NOT EXISTS (' # no moved package ... printf 'SELECT *' - printf ' FROM `moveable_binary_packages`' - mysql_join_binary_packages_install_target_providers 'moveable_binary_packages' 'subst_itp' + printf ' FROM `moveable_binary_packages_copy`' + mysql_join_binary_packages_install_target_providers 'moveable_binary_packages_copy' 'subst_itp' # ... provides the same printf ' WHERE `subst_itp`.`install_target`=`install_target_providers`.`install_target`' # b) 1) ^ -- cgit v1.2.3-54-g00ecf From acfd4fd009f2dc080688c44cb0e1070083a62f50 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 15:34:00 +0100 Subject: bin/bootstrap-mysql: remove debug output --- bin/bootstrap-mysql | 1 - 1 file changed, 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 8b1db28..8b01c38 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -366,7 +366,6 @@ if [ ! "$1" = 'slim' ]; then # list until no further changes are required printf 'REPEAT\n' for table in 'replaced' 'moveable'; do -printf 'SELECT NOW(),"count(%s)",count(*) FROM `%s_binary_packages`;\n' "${table}" "${table}" for copy in '' '2'; do if [ "${table}" = 'moveable' ] && \ [ "${copy}" = '2' ]; then -- cgit v1.2.3-54-g00ecf From f86e8f204c54c5354c5db44c62ae3517a765d202 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 8 Mar 2018 18:18:57 +0100 Subject: bin/bootstrap-mysql: add todo --- bin/bootstrap-mysql | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 1e5f45f..519b493 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -32,6 +32,8 @@ done | \ sort -k2,2 > \ "${tmp_dir}/master-mirror-listing" +# TODO: include hash of binary package in mysql database + if [ ! "$1" = 'slim' ]; then tables=$( printf '%s\n' \ -- cgit v1.2.3-54-g00ecf From 83effa9743fda2a0ca064a1dc0fc4d515088a858 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 9 Mar 2018 22:56:29 +0100 Subject: bin/bootstrap-mysql: add TODO --- bin/bootstrap-mysql | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index afe1850..5f938fc 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -310,6 +310,8 @@ if [ ! "$1" = 'slim' ]; then # Every package which is replaced, must have its provided install_targets: # a) provided by another moved or not-replaced package or # b) not required by any not-replaced package. +# TODO: additionally, every package being moved needs to have all dependencies +# installable in the target repository ! printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' printf 'DELIMITER //\n' printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' -- cgit v1.2.3-54-g00ecf From 5807cfc618e5f92b4323110546885ba267d09a38 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Sat, 10 Mar 2018 13:45:43 +0100 Subject: bin/bootstrap-mysql: calculate_maximal_moveable_set: packages are not moveable if they cannot be installed in the target repository --- bin/bootstrap-mysql | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 5f938fc..b7e08ba 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -307,15 +307,20 @@ if [ ! "$1" = 'slim' ]; then # stores results in `moveable_binary_packages` and `replaced_binary_packages` # Give a maximal list of packages to be moved, while implementing the # condition from db-update: + # Every package which is replaced, must have its provided install_targets: # a) provided by another moved or not-replaced package or # b) not required by any not-replaced package. -# TODO: additionally, every package being moved needs to have all dependencies -# installable in the target repository ! + + # Every package being moved needs to have all dependencies + # installable in the target repository. + printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' printf 'DELIMITER //\n' printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' printf 'BEGIN\n' + # variables to store count of changed rows + printf 'DECLARE row_count_saved INT DEFAULT 0;\n' for copy in '' '_copy' '_copy2'; do printf 'DROP TEMPORARY TABLE IF EXISTS `%s_binary_packages%s`;\n' \ 'moveable' "${copy}" \ @@ -369,6 +374,7 @@ if [ ! "$1" = 'slim' ]; then # now we delete all unreplaceable and unmoveable packages from the respective # list until no further changes are required printf 'REPEAT\n' + # create copies of our temporary tables *yuck* for table in 'replaced' 'moveable'; do for copy in '' '2'; do if [ "${table}" = 'moveable' ] && \ @@ -381,6 +387,50 @@ if [ ! "$1" = 'slim' ]; then printf ' FROM `%s_binary_packages`;\n' "${table}" done done + # a package is not moveable if its dependencies are not provided ... + printf 'DELETE `replaced_binary_packages`,`moveable_binary_packages`' + printf ' FROM `replaced_binary_packages`' + printf ' RIGHT JOIN `moveable_binary_packages`' + printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' + printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' + mysql_join_binary_packages_repositories + mysql_join_binary_packages_dependencies + mysql_join_dependencies_dependency_types + printf ' AND `dependency_types`.`relevant_for_binary_packages`' + # ... by a not-deleted, "more stable" package already in place or ... + printf ' WHERE NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + mysql_join_install_target_providers_binary_packages '' 'prov_bp' + mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' + printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ' AND `repositories`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' AND NOT EXISTS (' + printf 'SELECT * FROM `replaced_binary_packages_copy`' + printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' + printf ')' + # ... by a moved package + printf ') AND NOT EXISTS (' + printf 'SELECT * FROM `install_target_providers`' + printf ' JOIN `moveable_binary_packages_copy` ON `moveable_binary_packages_copy`.`id`=`install_target_providers`.`package`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' + printf 'SET @row_count_saved = ROW_COUNT();\n' + + # refresh copies of our temporary tables *yuck* + for table in 'replaced' 'moveable'; do + for copy in '' '2'; do + if [ "${table}" = 'moveable' ] && \ + [ "${copy}" = '2' ]; then + continue + fi + printf 'DELETE FROM `%s_binary_packages_copy%s`;\n' "${table}" "${copy}" + printf 'INSERT IGNORE INTO `%s_binary_packages_copy%s`' "${table}" "${copy}" + printf ' SELECT `%s_binary_packages`.*' "${table}" + printf ' FROM `%s_binary_packages`;\n' "${table}" + done + done + # packages which should not be replaced: ones providing something that is: # a) still needed by a not-replaced package x "less stable" than the target repository and # b) not provided by: @@ -445,7 +495,7 @@ if [ ! "$1" = 'slim' ]; then printf ' AND `subst_itp`.`install_target`=`install_target_providers`.`install_target`' # b) 2) ^ printf ');\n' - printf 'UNTIL ROW_COUNT()=0\n' + printf 'UNTIL row_count_saved=0 AND ROW_COUNT()=0\n' printf 'END REPEAT;\n' for table in 'moveable' 'replaced'; do for copy in '' '2'; do -- cgit v1.2.3-54-g00ecf From 3137ac0fe134369bc1f4805b00df09af01965022 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Mon, 12 Mar 2018 15:15:58 +0100 Subject: bin/bootstrap-mysql: "build-support" should be "standalone" and also the least stable repository on the master mirror --- bin/bootstrap-mysql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index b7e08ba..06bbd7f 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -689,17 +689,20 @@ fi 'stable' 'stable' \ 'stable' 'testing' \ 'stable' 'staging' \ + 'stable' 'standalone' \ 'stable' 'unbuilt' \ 'stable' 'forbidden' \ 'testing' 'testing' \ 'testing' 'staging' \ + 'testing' 'standalone' \ 'testing' 'unbuilt' \ 'testing' 'forbidden' \ 'staging' 'staging' \ + 'staging' 'standalone' \ 'staging' 'unbuilt' \ 'staging' 'forbidden' \ 'unbuilt' 'forbidden' \ - 'build-support' 'build-support' + 'standalone' 'standalone' printf ';\n' } | \ sed 's| OR ;|;|' -- cgit v1.2.3-54-g00ecf From 73409ac0d623289da164be28c78286434ab4706e Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Wed, 14 Mar 2018 10:43:01 +0100 Subject: bin/bootstrap-mysql: use mysql_run_query, too --- bin/bootstrap-mysql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 06bbd7f..d18247e 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -526,7 +526,7 @@ if [ ! "$1" = 'slim' ]; then 'SHOW VIEW' 'buildmaster.*' printf 'FLUSH PRIVILEGES;\n' } | \ - mysql -u root -p buildmaster + mysql_run_query -u root -p fi # shellcheck disable=SC2016 @@ -770,7 +770,7 @@ fi } | \ sed 's|,;|;|' } | \ - mysql buildmaster + mysql_run_query find "${work_dir}/package-states" \ -maxdepth 1 \ -- cgit v1.2.3-54-g00ecf From 3595bae3abb8514bb150e11094fd1ece763c4c46 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Mar 2018 11:49:27 +0100 Subject: bin/bootstrap-mysql: bugfix: stability of source repository was used instead of target repository --- bin/bootstrap-mysql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index d18247e..22603fc 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -393,7 +393,7 @@ if [ ! "$1" = 'slim' ]; then printf ' RIGHT JOIN `moveable_binary_packages`' printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' - mysql_join_binary_packages_repositories + printf ' JOIN `target_repositories` ON `moveable_binary_packages`.`to_repository`=`repositories`.`id`' mysql_join_binary_packages_dependencies mysql_join_dependencies_dependency_types printf ' AND `dependency_types`.`relevant_for_binary_packages`' @@ -404,7 +404,7 @@ if [ ! "$1" = 'slim' ]; then mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' AND `repositories`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`more_stable`' printf ' AND NOT EXISTS (' printf 'SELECT * FROM `replaced_binary_packages_copy`' printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' -- cgit v1.2.3-54-g00ecf From fcf305cd0df5cb0ef5e13d0867b1e57ea8eff1fb Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Mar 2018 12:03:00 +0100 Subject: bin/bootstrap-mysql: fix typo --- bin/bootstrap-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 22603fc..87ac313 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -393,7 +393,7 @@ if [ ! "$1" = 'slim' ]; then printf ' RIGHT JOIN `moveable_binary_packages`' printf ' ON `moveable_binary_packages`.`id`=`replaced_binary_packages`.`replaced_by`' printf ' JOIN `binary_packages` ON `binary_packages`.`id`=`moveable_binary_packages`.`id`' - printf ' JOIN `target_repositories` ON `moveable_binary_packages`.`to_repository`=`repositories`.`id`' + printf ' JOIN `repositories` AS `target_repositories` ON `moveable_binary_packages`.`to_repository`=`target_repositories`.`id`' mysql_join_binary_packages_dependencies mysql_join_dependencies_dependency_types printf ' AND `dependency_types`.`relevant_for_binary_packages`' -- cgit v1.2.3-54-g00ecf From 96193bd8db538d68d2324c1d97e1118c63d0582f Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 15 Mar 2018 14:07:05 +0100 Subject: bin/bootstrap-mysql: bugfix and new TODO --- bin/bootstrap-mysql | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 87ac313..bd6df9c 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -308,13 +308,18 @@ if [ ! "$1" = 'slim' ]; then # Give a maximal list of packages to be moved, while implementing the # condition from db-update: - # Every package which is replaced, must have its provided install_targets: + # Every package which is replaced[1], must have its provided install_targets: # a) provided by another moved or not-replaced package or # b) not required by any not-replaced package. # Every package being moved needs to have all dependencies # installable in the target repository. + # TODO: [1] A "replaced" package may also be in a different repository + # e.g. if a-2 is moved from [staging] to [testing] and there is only + # a-1 in [core], then this will be "replaced" by a-2 on a system + # running on [testing] repositories. + printf 'DROP PROCEDURE IF EXISTS calculate_maximal_moveable_set;\n' printf 'DELIMITER //\n' printf 'CREATE PROCEDURE calculate_maximal_moveable_set(IN `from_stability` VARCHAR(32))\n' @@ -402,9 +407,9 @@ if [ ! "$1" = 'slim' ]; then printf 'SELECT * FROM `install_target_providers`' mysql_join_install_target_providers_binary_packages '' 'prov_bp' mysql_join_binary_packages_repositories 'prov_bp' 'prov_r' - printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`less_stable`' + printf ' JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`more_stable`' + printf ' AND `target_repositories`.`stability`=`repository_stability_relations`.`less_stable`' printf ' AND NOT EXISTS (' printf 'SELECT * FROM `replaced_binary_packages_copy`' printf ' WHERE `replaced_binary_packages_copy`.`id`=`prov_bp`.`id`' -- cgit v1.2.3-54-g00ecf From 816a6c66f443c8dfdb3892e3c77b76cbbf7125ab Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 20 Mar 2018 10:06:47 +0100 Subject: bin/bootstrap-mysql: table `statistics` new --- bin/bootstrap-mysql | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index bd6df9c..4c1938f 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -143,7 +143,22 @@ if [ ! "$1" = 'slim' ]; then ' from_repository MEDIUMINT :repositories' \ ' to_repository MEDIUMINT :repositories' \ ' upstream_package_repository SMALLINT :upstream_repositories' \ - ' UNIQUE source from_repository upstream_package_repository' + ' UNIQUE source from_repository upstream_package_repository' \ + 'statistics BIGINT' \ + ' date DATETIME' \ + ' stable_packages_count MEDIUMINT' \ + ' pending_tasks_count MEDIUMINT' \ + ' pending_packages_count MEDIUMINT' \ + ' staging_packages_count MEDIUMINT' \ + ' testing_packages_count MEDIUMINT' \ + ' tested_packages_count MEDIUMINT' \ + ' broken_tasks_count MEDIUMINT' \ + ' dependency_loops_count MEDIUMINT' \ + ' dependency_looped_tasks_count MEDIUMINT' \ + ' locked_tasks_count MEDIUMINT' \ + ' blocked_tasks_count MEDIUMINT' \ + ' next_tasks_count MEDIUMINT' \ + ' UNIQUE date' ) # shellcheck disable=SC2016 -- cgit v1.2.3-54-g00ecf From c772be57eca3a2176ac3931c02fb582f85c0d575 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:19:23 +0100 Subject: decommission package-states directory - we rely on the database now --- bin/bootstrap-mysql | 60 ++++------------------------------------------ bin/cleanup | 20 ---------------- bin/db-update | 62 ------------------------------------------------ bin/get-assignment | 27 +++------------------ bin/get-package-updates | 6 ----- bin/modify-package-state | 38 ----------------------------- bin/return-assignment | 38 ----------------------------- lib/common-functions | 4 ---- 8 files changed, 7 insertions(+), 248 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4c1938f..fc347d7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: this should become some sort of mysql dump to bootstrap the +# tables and stored functions only - when the database is gone, we're +# pretty much screwed anyway :-D + tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT @@ -792,68 +796,12 @@ fi } | \ mysql_run_query -find "${work_dir}/package-states" \ - -maxdepth 1 \ - -mindepth 1 \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed ' - 1!d - s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@ - s@^\S*/@@ - s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 / - s/ \([0-9]\+\) / \1.0 / - s/ [0-9]\+\.\([0-9]\+\) / \1 / - ' {} \; | \ - while read -r state_file sub_pkgrel arch package_file; do - printf '%s ' "${state_file}" - repository=$( - grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \ - cut -d' ' -f1 - ) - if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then - printf 'not found exactly once on the master mirror.\n' - >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}" - continue - fi - mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}" - printf ' ok\n' - done - tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do mysql_generate_package_metadata 'build-list' "${state_file}" done -# shellcheck disable=SC2016 -find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ - sed ' - s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3| - ' | \ - while read -r state_file pkgbase git_revision mod_git_revision repository; do - printf 'UPDATE `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ - "$( - tr -d '\n' < \ - "${state_file}" | \ - base64 -w0 - )" - printf ' WHERE' - printf ' `package_sources`.`%s`=from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" - printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${repository}" | base64 -w0)" - done | \ - mysql_run_query - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/cleanup b/bin/cleanup index a0bf790..7aaa39f 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# remove blocked/broken/locked markes of packages not on the buildlist anymore - -{ - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5| - ' - awk '{ - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked" - }' "${work_dir}/build-list" -} | \ - sort -k3 | \ - uniq -uf2 | \ - grep '^state ' | \ - awk '{print $2}' | \ - sed "s|^|${work_dir}/package-states/|" | \ - xargs -rn1 rm - # remove logs where package is not broken/locked anymore { diff --git a/bin/db-update b/bin/db-update index 6ecf9c9..6722f24 100755 --- a/bin/db-update +++ b/bin/db-update @@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do mysql_join_repositories_repository_stabilities printf ' AND `repository_stabilities`.`name`="%s"' \ "${source_stability}" - # TODO: we may relax this condition, if we a) got rid of state files - # or b) improved keeping them in sync (*.testing -> /dev/null is not - # implemented, only .tested -> /dev/null) - if [ ! "${source_stability}" = 'staging' ]; then - printf ' AND `binary_packages`.`is_tested`' - fi mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories @@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query - - sed_apply_rm=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/rm" - ) - - sed_apply_remove_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/mv" - ) - - sed_apply_move_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/{\n' "$(str_to_regex "${pkg}")" - printf 'w %%s\n' - printf 'd\n' - printf '}\n' - done < \ - "${tmp_dir}/mv" - ) - - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed -i "${sed_apply_rm}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.tested' \ - -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.done' \ - -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \ - -printf '%p\n' | \ - while read -r sf; do - sed -i "$( - printf '%s\n' "${sed_apply_move_part_of_mv}" | \ - sed ' - s,%s,'"${sf%.done}.testing"',g - ' - )" "${sf}" - done - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -type f \ - -not -exec test -s '{}' \; \ - -delete done trigger_mirror_refreshs diff --git a/bin/get-assignment b/bin/get-assignment index 3357fc6..a82e1f8 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -18,17 +18,8 @@ # TODO: honor manual build order of tool-chain: # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc -mkdir -p "${work_dir}/package-states" - hand_out_assignment() { - # we don't care anymore if an older version of this package was - # "locked" or "broken" - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \ - -delete - # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { @@ -57,21 +48,7 @@ hand_out_assignment() { printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' - } | \ - mysql_run_query | \ - tr '\t' ' ' - { - # shellcheck disable=SC2154 - echo "${slave}" - if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - cat "${work_dir}/package-states/$1.$2.$3.$4.locked" - fi - } | \ - sort -u | \ - sponge "${work_dir}/package-states/$1.$2.$3.$4.locked" - # shellcheck disable=SC2016 - { printf 'UPDATE `build_slaves`' printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' @@ -86,6 +63,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ')' + # shellcheck disable=SC2154 printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -101,7 +79,8 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - mysql_run_query + mysql_run_query | \ + tr '\t' ' ' exit 0 diff --git a/bin/get-package-updates b/bin/get-package-updates index a36cb8a..3e9edf9 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -111,9 +111,6 @@ delete_package() { mysql_run_query >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \ - -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -317,9 +314,6 @@ done | \ if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -regextype grep \ - -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \ - -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" diff --git a/bin/modify-package-state b/bin/modify-package-state index 76b8c1e..4a2f5ac 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -243,46 +243,8 @@ while read -r package reason; do { printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" printf ';\n' - # TODO: remove the below part, once the database if fully functional - printf 'UPDATE `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b`' - printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - mysql_join_binary_packages_repositories 'b' - printf ' SET `b`.`is_tested`=1' - printf ' WHERE `a`.`is_tested`' - printf ' AND NOT `b`.`is_tested`' - printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ mysql_run_query - case "${action}" in - 'block') - printf '%s' "${reason}" > \ - "${work_dir}/package-states/${package}.blocked" - ;; - 'unblock') - rm -f "${work_dir}/package-states/${package}.blocked" - ;; - 'faulty') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.tested' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ed}ing" - done - ;; - 'tested') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.testing' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ing}ed" - done - ;; - *) - >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" - exit 42 - ;; - esac printf '%s %s\n' "${action}" "${package}" fi done \ diff --git a/bin/return-assignment b/bin/return-assignment index 02bd491..da9e762 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then '*.build-log.gz' ) - # shellcheck disable=SC2154 - echo "${slave}" >> \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then @@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then } | \ mysql_run_query - # shellcheck disable=SC2154 - sed -i ' - /^'"$(str_to_regex "${slave}")"'$/d - ' "${work_dir}/package-states/$1.$2.$3.$4.locked" - if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - fi - # release lock on build-list - otherwise seed-build-list won't run flock -u 9 @@ -517,24 +497,9 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove old state files (these should be only "done" markers, but -# actually we don't care what it is) - as long as it's not "testing" or "tested" -find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -not -name '*.testing' \ - -not -name '*.tested' \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ - -exec rm '{}' \; - # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" -# remove package lock file -if ! [ "${destination}" = 'build-support' ]; then - # shellcheck disable=SC2086 - printf '%s\n' ${packages} > \ - "${work_dir}/package-states/$1.$2.$3.$4.done" -fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' @@ -578,6 +543,3 @@ fi printf 'DROP TABLE `loops_to_delete`;\n' } | \ mysql_run_query -rm -f \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/lib/common-functions b/lib/common-functions index 0a5ef1c..f66ccd1 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -4,10 +4,6 @@ # shellcheck disable=SC2039 -# TODO: have full information (currently in files) in database - -# TODO: remove state files - if [ -z "${base_dir}" ]; then # just to make shellcheck happy . 'conf/default.conf' -- cgit v1.2.3-54-g00ecf From b3023cfe9c2d4e8319885e22a575bd1a522b3f6c Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Thu, 22 Mar 2018 15:42:11 +0100 Subject: decommission build-list file --- bin/bootstrap-mysql | 6 ------ bin/get-assignment | 9 --------- bin/get-package-updates | 33 ++++----------------------------- bin/prioritize-build-list | 6 ------ bin/return-assignment | 3 --- bin/seed-build-list | 2 -- conf/default.conf | 1 - 7 files changed, 4 insertions(+), 56 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index fc347d7..817989d 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -796,12 +796,6 @@ fi } | \ mysql_run_query -tr ' ' '.' < \ - "${work_dir}/build-list" | \ - while read -r state_file; do - mysql_generate_package_metadata 'build-list' "${state_file}" - done - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/get-assignment b/bin/get-assignment index a82e1f8..7ac749f 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -20,15 +20,6 @@ hand_out_assignment() { - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'SELECT ' diff --git a/bin/get-package-updates b/bin/get-package-updates index 3e9edf9..dcc9f3b 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -110,7 +110,6 @@ delete_package() { } | \ mysql_run_query >> \ "${work_dir}/deletion-list.new" - sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -201,9 +200,6 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -cp \ - "${work_dir}/build-list" \ - "${work_dir}/build-list.new" cp \ "${work_dir}/deletion-list" \ "${work_dir}/deletion-list.new" @@ -223,26 +219,9 @@ for repo in ${repo_names}; do git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}" fi } | \ - # Packages which are already on the build list should receive a git_revision bump if _any_ file changed. - # Thus, we rename any file "PKGBUILD" to trigger the successive logic. - if [ "${repo}" = 'archlinux32' ]; then - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t[^/]\\+/| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - else - sed "$( - sed ' - s/ .*$// - s|^|\\@^.\\t| - s|$|/@ s@/[^/]*$@/PKGBUILD@| - ' "${work_dir}/build-list" - )" - fi | \ # only track changes in PKGBUILDs + # TODO: However, packages which are already on the build list should + # receive a git_revision bump if _any_ file changed. grep '/PKGBUILD$' | \ if [ "${repo}" = "archlinux32" ]; then # modify the directory structure from the modifiaction-repository @@ -312,11 +291,6 @@ done | \ fi fi if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then - # new or modified PKGBUILD - sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - # shellcheck disable=SC2154 - echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ - "${work_dir}/build-list.new" sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new" # shellcheck disable=SC2016 { @@ -346,6 +320,7 @@ done | \ )" } | \ mysql_run_query + # shellcheck disable=SC2154 mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" continue fi @@ -530,7 +505,7 @@ mysql_find_build_assignment_loops # Move the .new-files to the actual files { - printf '%s\n' "build-list" "deletion-list" + printf '%s\n' "deletion-list" # shellcheck disable=SC2086 printf '%s.revision\n' ${repo_names} } | \ diff --git a/bin/prioritize-build-list b/bin/prioritize-build-list index 1ff3d5a..db828ae 100755 --- a/bin/prioritize-build-list +++ b/bin/prioritize-build-list @@ -23,12 +23,6 @@ if ! flock -s -n 8; then exit 1 fi -{ - sed -n "/^$1/p" "${work_dir}/build-list" - sed "/^$1/d" "${work_dir}/build-list" -} | \ - sponge "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' diff --git a/bin/return-assignment b/bin/return-assignment index da9e762..a2669f6 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -497,9 +497,6 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove package from build list -sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" - # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' diff --git a/bin/seed-build-list b/bin/seed-build-list index 41707dd..ae08d57 100755 --- a/bin/seed-build-list +++ b/bin/seed-build-list @@ -374,8 +374,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ sed 's/ $/\n/' mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" - printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \ - "${work_dir}/build-list" printf '\n' >&2 done diff --git a/conf/default.conf b/conf/default.conf index b508deb..e8e0169 100755 --- a/conf/default.conf +++ b/conf/default.conf @@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then fi mkdir -p "${work_dir}" -touch "${work_dir}/build-list" touch "${work_dir}/deletion-list" for repo in ${repo_names}; do -- cgit v1.2.3-54-g00ecf From 68eb1424fa9d08c1c60fc3b1807a67d266fb66e5 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 10:07:52 +0100 Subject: bin/bootstrap-mysql: new column `is_black_listed` in `build_assignments` --- bin/bootstrap-mysql | 1 + 1 file changed, 1 insertion(+) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 817989d..acf2170 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -88,6 +88,7 @@ if [ ! "$1" = 'slim' ]; then ' package_source BIGINT :package_sources' \ ' architecture SMALLINT :architectures' \ ' is_blocked VARCHAR(128) NULL' \ + ' is_black_listed VARCHAR(128) NULL' \ ' is_broken BIT' \ ' priority SMALLINT' \ ' UNIQUE content package_source architecture' \ -- cgit v1.2.3-54-g00ecf From 3682408492801e1325f742267394db7cca7f2339 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 10:27:23 +0100 Subject: bin/bootstrap-mysql: build_assignments.is_black_listed should be TEXT --- bin/bootstrap-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index acf2170..98f8efb 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -88,7 +88,7 @@ if [ ! "$1" = 'slim' ]; then ' package_source BIGINT :package_sources' \ ' architecture SMALLINT :architectures' \ ' is_blocked VARCHAR(128) NULL' \ - ' is_black_listed VARCHAR(128) NULL' \ + ' is_black_listed TEXT NULL' \ ' is_broken BIT' \ ' priority SMALLINT' \ ' UNIQUE content package_source architecture' \ -- cgit v1.2.3-54-g00ecf From 5474aa6f94f4d83c4f3b03e00357b7fd47a8f2b8 Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Fri, 23 Mar 2018 10:59:53 +0100 Subject: bin/bootstrap-mysql: set head on bootstrapping to HEAD --- bin/bootstrap-mysql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bootstrap-mysql') diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 98f8efb..86fa645 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -622,8 +622,8 @@ fi base64 -w0 )" \ "$( - base64_encode_each < \ - "${work_dir}/${repo}.revision" + git -C "${repo_path}" rev-parse HEAD | \ + base64_encode_each )" | \ sed 's|,$|),|' done -- cgit v1.2.3-54-g00ecf