summaryrefslogtreecommitdiff
path: root/bin
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2017-06-20 09:01:35 +0200
committerErich Eckner <git@eckner.net>2017-06-20 09:01:35 +0200
commit05e3561b3df4078a33865913d8135dbfe4ad5809 (patch)
tree9afde5a0c4d87187cc6e193c3666eb77a6a9a4cd /bin
parent797adfe1c1f26c57d31bc7b1b67e54d36257806f (diff)
downloadbuilder-05e3561b3df4078a33865913d8135dbfe4ad5809.tar.xz
hand out broken packages multiple times
Diffstat (limited to 'bin')
-rwxr-xr-xbin/build-master-status4
-rwxr-xr-xbin/common-functions7
-rwxr-xr-xbin/db-update12
-rwxr-xr-xbin/get-assignment67
-rwxr-xr-xbin/get-package-updates11
-rwxr-xr-xbin/return-assignment28
-rwxr-xr-xbin/why_dont_you13
7 files changed, 103 insertions, 39 deletions
diff --git a/bin/build-master-status b/bin/build-master-status
index 15ac1a5..54bf3de 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -88,9 +88,9 @@ stable="$(
grep -c '\.pkg\.tar\.xz$'
)"
tasks="$(
- wc -l < \
+ grep -c '^\S\+ \S\+ \S\+ \S\+$' \
"${work_dir}/build-list"
- )"
+ )" || true
pending_packages="$(
tr ' ' '.' < \
"${work_dir}"/build-list | \
diff --git a/bin/common-functions b/bin/common-functions
index cc59e94..7f82042 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -142,12 +142,11 @@ find_git_repository_to_package_repository() {
}
-# package_locked_broken_or_blocked package git_revision mod_git_revision repository
-# return if package - of given repository and revisions - is [locked or broken or blocked]
+# package_locked_or_blocked package git_revision mod_git_revision repository
+# return if package - of given repository and revisions - is [locked or blocked]
-package_locked_broken_or_blocked() {
+package_locked_or_blocked() {
[ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] || \
- [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ] || \
[ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ]
}
diff --git a/bin/db-update b/bin/db-update
index 0542f5b..d5e6009 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -234,11 +234,11 @@ fi
# packages which can't be un-staged because they're still dependencies
# of any job on the build-list
-while read -r pkg pkg_rev mod_rev repo; do
- generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
- cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
-done < \
- "${work_dir}/build-list" | \
+grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ while read -r pkg pkg_rev mod_rev repo; do
+ generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
+ cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
+ done | \
sort -u > \
"${tmp_dir}/keep_packages"
@@ -309,7 +309,7 @@ done
done_packages="$(cat "${tmp_dir}/done_packages")"
# if build list is empty, remember all entries of 'deletion-list'
-if [ -s "${work_dir}/build-list" ]; then
+if grep -qvxF 'break_loops' "${work_dir}/build-list"; then
delete_packages=''
else
delete_packages="$(
diff --git a/bin/get-assignment b/bin/get-assignment
index 73aeb1d..66e8971 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -17,16 +17,14 @@
# respect build-manually-list ("blocked")
-# possibly hand out "broken" packages to different build slave
-
. "${0%/*}/../conf/default.conf"
mkdir -p "${work_dir}/package-states"
hand_out_assignment() {
- # locked, broken and blocked packages won't be handed out
- if package_locked_broken_or_blocked "$1" "$2" "$3" "$4"; then
+ # locked and blocked packages won't be handed out
+ if package_locked_or_blocked "$1" "$2" "$3" "$4"; then
return 0
fi
@@ -34,6 +32,7 @@ hand_out_assignment() {
# "locked" or "broken" (we keep only marker for older "done" packages)
ls "${work_dir}/package-states" | \
grep "^$(str_to_regex "${1}")\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \
+ grep -v "^$(str_to_regex "$1.$2.$3.$4.")[^.]\+\$" | \
sed "s|^|${work_dir}/package-states/|" | \
xargs -rn1 rm -f
@@ -73,8 +72,12 @@ pending_packages=false
while read -r package git_revision mod_git_revision repository; do
- if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ] ||
- [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
+ if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \
+ [ "${package}" = 'break_loops' ]; then
+ continue
+ fi
+
+ if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
continue
fi
@@ -103,14 +106,25 @@ fi
while read -r package git_revision mod_git_revision repository; do
- if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
+ if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \
+ [ "${package}" = 'break_loops' ]; then
+ sed -i \
+ '/^break_loops$/d' \
+ "${work_dir}/build-list"
+ echo 'break_loops' >> \
+ "${work_dir}/build-list"
+ break
+ fi
+
+ if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
continue
fi
[ -z "$(
(
cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
- awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+ grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ awk '{print $1 "." $2 "." $3}' | \
sed "
s|^|${work_dir}/package-infos/|
s|\$|\.builds|
@@ -126,7 +140,7 @@ while read -r package git_revision mod_git_revision repository; do
done < "${work_dir}/build-list"
-# Find package (of all packages which are not locked)
+# Find package (of all packages which are not locked and have been broken the least times)
# which breaks the most unlocked loops
locked_packages="$(
@@ -135,6 +149,10 @@ locked_packages="$(
sed 's|\(\.[0-9a-f]\{40\}\)\{2\}\.[^\.\]\+\.locked$||'
)"
+grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ sort -k1,1 > \
+ "${work_dir}/build-list.sorted-by-package"
+
for package in $(
ls "${work_dir}/build-list.loops/" | \
grep '^loop_[0-9]\+$' | \
@@ -152,12 +170,33 @@ for package in $(
done | \
sort | \
uniq -c | \
- sort -k1nr,1 | \
- awk '{print $2}'
+ join -1 2 -2 1 -o 1.1,2.1,2.2,2.3,2.4 \
+ - \
+ "${work_dir}/build-list.sorted-by-package" | \
+ while read -r count package git_revision git_mod_revision repository; do
+ if [ -f "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" ]; then
+ versuche="$(
+ wc -l < \
+ "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken"
+ )"
+ else
+ versuche='0'
+ fi
+ printf '%s %s %s.%s.%s.%s\n' \
+ "${versuche}" \
+ "${count}" \
+ "${package}" \
+ "${git_revision}" \
+ "${git_mod_revision}" \
+ "${repository}"
+ done | \
+ sort -k1n,1 -k2nr,2 | \
+ cut -d' ' -f3
); do
- if assignment="$(grep "^$(str_to_regex "${package}") " "${work_dir}/build-list")"; then
- hand_out_assignment ${assignment}
- fi
+ hand_out_assignment $(
+ echo "${package}" | \
+ sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|'
+ )
done
# Remove the lock file
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 35afbe1..d1ceaaa 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -123,9 +123,10 @@ echo 'Check modified packages from the last update, and put them to the build li
# If a package is deleted, remove from the rebuild list, and add it to the deletion list.
# If a new package is added, then ensure that it's not on the deletion list.
-cp \
- "${work_dir}/build-list" \
- "${work_dir}/build-list.new"
+grep -vxF 'break_loops' \
+ "${work_dir}/build-list" > \
+ "${work_dir}/build-list.new" || \
+ true
cp \
"${work_dir}/deletion-list" \
"${work_dir}/deletion-list.new"
@@ -326,6 +327,10 @@ if [ -s "${work_dir}/tsort.error" ]; then
"${work_dir}/build-list.loops.new/${loop}"
done
+ # add order to break loops to build list
+ echo 'break_loops' >>
+ "${work_dir}/build-list.new.new"
+
else
rm "${work_dir}/tsort.error"
fi
diff --git a/bin/return-assignment b/bin/return-assignment
index ec7db45..213d4f0 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -48,10 +48,12 @@ if [ "$5" = 'ERROR' ]; then
exit 0
fi
- mv \
- "${work_dir}/package-states/$1.$2.$3.$4.locked" \
+ cat \
+ "${work_dir}/package-states/$1.$2.$3.$4.locked" >> \
"${work_dir}/package-states/$1.$2.$3.$4.broken"
+ rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
+
# unlock every loop this package would have broken and which is not
# broken by another locked package
locked_packages="$(
@@ -74,6 +76,14 @@ if [ "$5" = 'ERROR' ]; then
fi
done
+ # move build order to end of build list
+
+ sed -i \
+ "/^$(str_to_regex "$1 $2 $3 $4")\$/d" \
+ "${work_dir}/build-list"
+ echo "$1 $2 $3 $4" >> \
+ "${work_dir}/build-list"
+
exit 0
fi
@@ -152,8 +162,8 @@ repo-add "${destination}.db.tar.gz" ${packages}
${master_mirror_command} \
"${destination}.db."* \
- *".pkg.tar.xz" \
- *".pkg.tar.xz.sig" \
+ ./*".pkg.tar.xz" \
+ ./*".pkg.tar.xz.sig" \
"${master_mirror_directory}/i686/${destination}/"
for package in ${packages}; do
@@ -175,10 +185,18 @@ grep -xF "$1" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \
sed 'p;s|$|.locked|' | \
xargs -rn1 rm -f
+if ! ls "${work_dir}/build-list.loops" | \
+ grep -q '^loop_[0-9]\+$'; then
+ # no loops left
+ sed -i '/^break_loops$/d' "${work_dir}/build-list"
+fi
+
# remove package from build list
sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
# remove package lock file
printf '%s\n' ${packages} > \
"${work_dir}/package-states/$1.$2.$3.$4.done"
-rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
+rm -f \
+ "${work_dir}/package-states/$1.$2.$3.$4.locked" \
+ "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/bin/why_dont_you b/bin/why_dont_you
index aaa7df5..2c4d43e 100755
--- a/bin/why_dont_you
+++ b/bin/why_dont_you
@@ -23,15 +23,16 @@ case "${action}" in
continue
fi
- if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
- echo 'is locked, broken or blocked'
+ if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
+ echo 'is locked or blocked'
continue
fi
if [ -n "$(
(
cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
- awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+ grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ awk '{print $1 "." $2 "." $3}' | \
sed "
s|^|${work_dir}/package-infos/|
s|\$|\.builds|
@@ -45,7 +46,8 @@ case "${action}" in
echo 'has unmet dependencies:'
(
cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
- awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+ grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ awk '{print $1 "." $2 "." $3}' | \
sed "
s|^|${work_dir}/package-infos/|
s|\$|\.builds|
@@ -95,7 +97,8 @@ case "${action}" in
build_list_items_file="$(mktemp)"
dependent_packages_file="$(mktemp)"
- awk '{print $1 "." $2 "." $3 " " $4}' "${work_dir}/build-list" | \
+ grep -vxF 'break_loops' "${work_dir}/build-list" | \
+ awk '{print $1 "." $2 "." $3 " " $4}' | \
sort -u > \
"${build_list_items_file}"