summaryrefslogtreecommitdiff
path: root/bin/return-assignment
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2020-03-22 17:17:09 +0100
committerErich Eckner <git@eckner.net>2020-03-22 17:17:16 +0100
commit6fe7d1d139f9b01092fac2baeb15a7f7ba9b3c30 (patch)
tree2d21b56d172977e763d00f712af8a059a9bf5fa5 /bin/return-assignment
parentd557887db8c168cb637edc2a4cada866700abcd3 (diff)
downloadbuilder-6fe7d1d139f9b01092fac2baeb15a7f7ba9b3c30.tar.xz
bin/return-assignment: reschedule haskell stuff not only on the first error
Diffstat (limited to 'bin/return-assignment')
-rwxr-xr-xbin/return-assignment389
1 files changed, 194 insertions, 195 deletions
diff --git a/bin/return-assignment b/bin/return-assignment
index f55fd79..b1a5ef6 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -286,210 +286,209 @@ if [ "$6" = 'ERROR' ]; then
} | \
mysql_run_query
- if ! ${was_broken_before}; then
-
- mysql_load_min_and_max_versions
-
- # this will hold a list of "$build_time $haskell_package $version"
- # meaning that the given $haskell_package with version $version did
- # not work at $build_time
- # note, that $build_time is only considered when $architecture or
- # $version are unavailable
- find "${build_log_directory}/error" -type f \
- -name "$1.$2.$3.$4.${build_assignment_architecture}.*.build-log.gz" \
- -exec zgrep -qF "$(
- printf '%s\n' \
- 'The following packages are broken because other packages they depend on are missing. These broken packages must be rebuilt before they can be used.' \
- 'mismatched interface file versions (wanted ' \
- )" {} \; \
- -printf '%p\n' | \
- sed '
- s/^.*\.\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\)\.build-log\.gz$/\1 \0/
- ' | \
- while read -r build_time build_log; do
- build_time=$(
- date +%s -d"${build_time}"
- )
- zcat "${build_log}" | \
- sed -n '
- /^Packages ([0-9]\+) /,/^\s*$/ p
- ' | \
- tr ' ' '\n' | \
- sed '
- 1,2 d
- /^\s*$/d
- p
- s/^haskell-//
- t
- d
- ' | \
- sort -u | \
- sed '
- s/-\(\([0-9]\+\):\)\?\([^-:]\+\)-\([^-.]\+\)\(\.\([^-.]\+\)\)\?$/ \2 \3 \4 \6/
- s/ $/ 0/
- s/ / 0 /g
- ' > \
- "${tmp_dir}/installed-versions"
- zcat "${build_log}" | \
- sed '
- s/^installed package \(.*\) is broken due to missing package .*$/\1/
- t
- s@^.*Bad interface file: /usr/lib/ghc-[0-9.]\+/site-local/\([^/]\+\)-[0-9.]\+/.*$@\1@
- t
- d
- ' | \
- tr ' ' '\n' | \
- tr '[:upper:]' '[:lower:]' | \
- sed '
- s/^/'"${build_time}"' /
- s/-[0-9.]\+$//
- ' | \
- sort -u | \
- sort -k2,2 > \
- "${tmp_dir}/broken-packages"
- {
- cut -d' ' -f1 < \
- "${tmp_dir}/installed-versions" | \
- sed 'p'
- cut -d' ' -f2 < \
- "${tmp_dir}/broken-packages"
- } | \
- sort | \
- uniq -u | \
- sed '
- s/$/ '"${max_version%%:*}"' '"${max_version#*:}"' 0 0/
- ' | \
- sponge -a "${tmp_dir}/installed-versions"
- sort -k1,1 < \
- "${tmp_dir}/installed-versions" | \
- sponge "${tmp_dir}/installed-versions"
- join -1 2 -2 1 \
- "${tmp_dir}/broken-packages" \
- "${tmp_dir}/installed-versions"
- done | \
- sort -k2 -k1nr,1 | \
- uniq -f1 | \
- tr ' ' '\t' > \
- "${tmp_dir}/broken-packages-with-version"
-
- # now we look if the broken packages have been rebuilt in the meantime
- haskell_rebuild_packages=$(
- # shellcheck disable=SC2016
+ mysql_load_min_and_max_versions
+
+ # this will hold a list of "$build_time $haskell_package $version"
+ # meaning that the given $haskell_package with version $version did
+ # not work at $build_time
+ # note, that $build_time is only considered when $architecture or
+ # $version are unavailable
+ find "${build_log_directory}/error" -type f \
+ -name "$1.$2.$3.$4.${build_assignment_architecture}.*.build-log.gz" \
+ -exec zgrep -qF "$(
+ printf '%s\n' \
+ 'The following packages are broken because other packages they depend on are missing. These broken packages must be rebuilt before they can be used.' \
+ 'mismatched interface file versions (wanted ' \
+ )" {} \; \
+ -printf '%p\n' | \
+ sed '
+ s/^.*\.\([0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\)\.build-log\.gz$/\1 \0/
+ ' | \
+ while read -r build_time build_log; do
+ build_time=$(
+ date +%s -d"${build_time}"
+ )
+ zcat "${build_log}" | \
+ sed -n '
+ /^Packages ([0-9]\+) /,/^\s*$/ p
+ ' | \
+ tr ' ' '\n' | \
+ sed '
+ 1,2 d
+ /^\s*$/d
+ p
+ s/^haskell-//
+ t
+ d
+ ' | \
+ sort -u | \
+ sed '
+ s/-\(\([0-9]\+\):\)\?\([^-:]\+\)-\([^-.]\+\)\(\.\([^-.]\+\)\)\?$/ \2 \3 \4 \6/
+ s/ $/ 0/
+ s/ / 0 /g
+ ' > \
+ "${tmp_dir}/installed-versions"
+ zcat "${build_log}" | \
+ sed '
+ s/^installed package \(.*\) is broken due to missing package .*$/\1/
+ t
+ s@^.*Bad interface file: /usr/lib/ghc-[0-9.]\+/site-local/\([^/]\+\)-[0-9.]\+/.*$@\1@
+ t
+ d
+ ' | \
+ tr ' ' '\n' | \
+ tr '[:upper:]' '[:lower:]' | \
+ sed '
+ s/^/'"${build_time}"' /
+ s/-[0-9.]\+$//
+ ' | \
+ sort -u | \
+ sort -k2,2 > \
+ "${tmp_dir}/broken-packages"
{
- printf 'CREATE TEMPORARY TABLE `broken`('
- printf '`time_stamp` BIGINT,'
- printf '`pkgname` VARCHAR(64),'
- printf '`epoch` MEDIUMINT,'
- printf '`pkgver` VARCHAR(64),'
- printf '`pkgrel` MEDIUMINT,'
- printf '`sub_pkgrel` MEDIUMINT,'
- printf 'KEY `time_stamp`(`time_stamp`),'
- printf 'KEY `pkgname`(`pkgname`),'
- printf 'KEY `epoch`(`epoch`),'
- printf 'KEY `pkgver`(`pkgver`),'
- printf 'KEY `pkgrel`(`pkgrel`),'
- printf 'KEY `sub_pkgrel`(`sub_pkgrel`),'
- printf 'UNIQUE KEY `pkgfile`(`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`)'
- printf ');\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `broken`(' \
- "${tmp_dir}/broken-packages-with-version"
- printf '`%s`,' \
- 'pkgname' \
- 'time_stamp' \
- 'epoch' \
- 'pkgver' \
- 'pkgrel' \
- 'sub_pkgrel' | \
- sed 's/,$//'
- printf ');\n'
-
- printf 'SELECT DISTINCT `broken`.`pkgname`'
- printf ' FROM `broken`'
- printf ' WHERE NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `binary_packages`'
- mysql_join_binary_packages_binary_packages_in_repositories
- mysql_join_binary_packages_in_repositories_repositories
- printf ' AND ('
- printf '`repositories`.`is_on_master_mirror`'
- printf ' OR `repositories`.`id`=%s' \
- "${repository_ids__any_build_list}"
- printf ')'
- mysql_join_binary_packages_build_assignments
- printf ' JOIN `architecture_compatibilities`'
- printf ' ON `architecture_compatibilities`.`fully_compatible`'
- printf ' AND `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
- printf ' JOIN `architectures`'
- printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
- printf ' AND `architectures`.`name`=from_base64("%s")' \
- "$(
- printf '%s' "${build_assignment_architecture}" | \
- base64 -w0
- )"
- printf ' WHERE ('
- printf '`binary_packages`.`pkgname`=`broken`.`pkgname`'
- printf ' OR `binary_packages`.`pkgname`=CONCAT("haskell-",`broken`.`pkgname`)'
- printf ') AND ('
- printf '`build_assignments`.`return_date`>FROM_UNIXTIME(`broken`.`time_stamp`)'
- printf ' OR ('
- printf '`binary_packages`.`epoch`>`broken`.`epoch`'
- printf ' OR ('
- printf '`binary_packages`.`epoch`=`broken`.`epoch`'
- printf ' AND `binary_packages`.`pkgver`>`broken`.`pkgver`'
- printf ')'
- printf ' OR ('
- printf '`binary_packages`.`epoch`=`broken`.`epoch`'
- printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
- printf ' AND `binary_packages`.`pkgrel`>`broken`.`pkgrel`'
- printf ')'
- printf ' OR ('
- printf '`binary_packages`.`epoch`=`broken`.`epoch`'
- printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
- printf ' AND `binary_packages`.`pkgrel`=`broken`.`pkgrel`'
- printf ' AND `binary_packages`.`sub_pkgrel`>`broken`.`sub_pkgrel`'
- printf ')'
- printf ')'
- printf ' OR `repositories`.`id`=%s' \
- "${repository_ids__any_build_list}"
- printf ')'
- printf ');\n'
+ cut -d' ' -f1 < \
+ "${tmp_dir}/installed-versions" | \
+ sed 'p'
+ cut -d' ' -f2 < \
+ "${tmp_dir}/broken-packages"
} | \
- mysql_run_query | \
+ sort | \
+ uniq -u | \
sed '
- s/^/-p ^(haskell-)?/
- s/$/$/
- '
- )
-
- # release lock on build-list - otherwise seed-build-list won't run
- flock -u 9
-
- rescheduled_packages=$(
- if [ -n "${haskell_rebuild_packages}" ]; then
- # shellcheck disable=SC2086
- "${base_dir}/bin/seed-build-list" ${haskell_rebuild_packages} | \
- sed 's/ .*$//'
- fi
- )
+ s/$/ '"${max_version%%:*}"' '"${max_version#*:}"' 0 0/
+ ' | \
+ sponge -a "${tmp_dir}/installed-versions"
+ sort -k1,1 < \
+ "${tmp_dir}/installed-versions" | \
+ sponge "${tmp_dir}/installed-versions"
+ join -1 2 -2 1 \
+ "${tmp_dir}/broken-packages" \
+ "${tmp_dir}/installed-versions"
+ done | \
+ sort -k2 -k1nr,1 | \
+ uniq -f1 | \
+ tr ' ' '\t' > \
+ "${tmp_dir}/broken-packages-with-version"
+
+ # now we look if the broken packages have been rebuilt in the meantime
+ haskell_rebuild_packages=$(
+ # shellcheck disable=SC2016
+ {
+ printf 'CREATE TEMPORARY TABLE `broken`('
+ printf '`time_stamp` BIGINT,'
+ printf '`pkgname` VARCHAR(64),'
+ printf '`epoch` MEDIUMINT,'
+ printf '`pkgver` VARCHAR(64),'
+ printf '`pkgrel` MEDIUMINT,'
+ printf '`sub_pkgrel` MEDIUMINT,'
+ printf 'KEY `time_stamp`(`time_stamp`),'
+ printf 'KEY `pkgname`(`pkgname`),'
+ printf 'KEY `epoch`(`epoch`),'
+ printf 'KEY `pkgver`(`pkgver`),'
+ printf 'KEY `pkgrel`(`pkgrel`),'
+ printf 'KEY `sub_pkgrel`(`sub_pkgrel`),'
+ printf 'UNIQUE KEY `pkgfile`(`pkgname`,`epoch`,`pkgver`,`pkgrel`,`sub_pkgrel`)'
+ printf ');\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `broken`(' \
+ "${tmp_dir}/broken-packages-with-version"
+ printf '`%s`,' \
+ 'pkgname' \
+ 'time_stamp' \
+ 'epoch' \
+ 'pkgver' \
+ 'pkgrel' \
+ 'sub_pkgrel' | \
+ sed 's/,$//'
+ printf ');\n'
- # prioritize _this_ build assignment iff we rescheduled any broken dependencies
- if [ -n "${rescheduled_packages}" ]; then
- # shellcheck disable=SC2016
- {
- printf 'UPDATE `build_assignments`'
- printf ' SET `build_assignments`.`priority`=('
- printf 'SELECT MAX(`build_assignments`.`priority`)'
- printf ' FROM `build_assignments`'
- printf ')+1'
- printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \
+ printf 'SELECT DISTINCT `broken`.`pkgname`'
+ printf ' FROM `broken`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT 1'
+ printf ' FROM `binary_packages`'
+ mysql_join_binary_packages_binary_packages_in_repositories
+ mysql_join_binary_packages_in_repositories_repositories
+ printf ' AND ('
+ printf '`repositories`.`is_on_master_mirror`'
+ printf ' OR `repositories`.`id`=%s' \
+ "${repository_ids__any_build_list}"
+ printf ')'
+ mysql_join_binary_packages_build_assignments
+ printf ' JOIN `architecture_compatibilities`'
+ printf ' ON `architecture_compatibilities`.`fully_compatible`'
+ printf ' AND `architecture_compatibilities`.`built_for`=`build_assignments`.`architecture`'
+ printf ' JOIN `architectures`'
+ printf ' ON `architecture_compatibilities`.`runs_on`=`architectures`.`id`'
+ printf ' AND `architectures`.`name`=from_base64("%s")' \
"$(
- printf '%s' "${build_assignment_id}" | \
+ printf '%s' "${build_assignment_architecture}" | \
base64 -w0
)"
- } | \
- mysql_run_query
+ printf ' WHERE ('
+ printf '`binary_packages`.`pkgname`=`broken`.`pkgname`'
+ printf ' OR `binary_packages`.`pkgname`=CONCAT("haskell-",`broken`.`pkgname`)'
+ printf ') AND ('
+ printf '`build_assignments`.`return_date`>FROM_UNIXTIME(`broken`.`time_stamp`)'
+ printf ' OR ('
+ printf '`binary_packages`.`epoch`>`broken`.`epoch`'
+ printf ' OR ('
+ printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+ printf ' AND `binary_packages`.`pkgver`>`broken`.`pkgver`'
+ printf ')'
+ printf ' OR ('
+ printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+ printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
+ printf ' AND `binary_packages`.`pkgrel`>`broken`.`pkgrel`'
+ printf ')'
+ printf ' OR ('
+ printf '`binary_packages`.`epoch`=`broken`.`epoch`'
+ printf ' AND `binary_packages`.`pkgver`=`broken`.`pkgver`'
+ printf ' AND `binary_packages`.`pkgrel`=`broken`.`pkgrel`'
+ printf ' AND `binary_packages`.`sub_pkgrel`>`broken`.`sub_pkgrel`'
+ printf ')'
+ printf ')'
+ printf ' OR `repositories`.`id`=%s' \
+ "${repository_ids__any_build_list}"
+ printf ')'
+ printf ');\n'
+ } | \
+ mysql_run_query | \
+ sed '
+ s/^/-p ^(haskell-)?/
+ s/$/$/
+ '
+ )
+
+ # release lock on build-list - otherwise seed-build-list won't run
+ flock -u 9
+
+ rescheduled_packages=$(
+ if [ -n "${haskell_rebuild_packages}" ]; then
+ # shellcheck disable=SC2086
+ "${base_dir}/bin/seed-build-list" ${haskell_rebuild_packages} | \
+ sed 's/ .*$//'
fi
+ )
+ # prioritize _this_ build assignment iff we rescheduled any broken dependencies
+ if [ -n "${rescheduled_packages}" ]; then
+ # shellcheck disable=SC2016
+ {
+ printf 'UPDATE `build_assignments`'
+ printf ' SET `build_assignments`.`priority`=('
+ printf 'SELECT MAX(`build_assignments`.`priority`)'
+ printf ' FROM `build_assignments`'
+ printf ')+1'
+ printf ' WHERE `build_assignments`.`id`=from_base64("%s");\n' \
+ "$(
+ printf '%s' "${build_assignment_id}" | \
+ base64 -w0
+ )"
+ } | \
+ mysql_run_query
+ fi
+
+ if ! ${was_broken_before}; then
# shellcheck disable=SC2119
{
printf '%s/%s ' \