diff options
| -rwxr-xr-x | bin/build-master-status | 4 | ||||
| -rwxr-xr-x | bin/common-functions | 7 | ||||
| -rwxr-xr-x | bin/db-update | 12 | ||||
| -rwxr-xr-x | bin/get-assignment | 67 | ||||
| -rwxr-xr-x | bin/get-package-updates | 11 | ||||
| -rwxr-xr-x | bin/return-assignment | 28 | ||||
| -rwxr-xr-x | bin/why_dont_you | 13 |
7 files changed, 103 insertions, 39 deletions
diff --git a/bin/build-master-status b/bin/build-master-status index 15ac1a5..54bf3de 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -88,9 +88,9 @@ stable="$( grep -c '\.pkg\.tar\.xz$' )" tasks="$( - wc -l < \ + grep -c '^\S\+ \S\+ \S\+ \S\+$' \ "${work_dir}/build-list" - )" + )" || true pending_packages="$( tr ' ' '.' < \ "${work_dir}"/build-list | \ diff --git a/bin/common-functions b/bin/common-functions index cc59e94..7f82042 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -142,12 +142,11 @@ find_git_repository_to_package_repository() { } -# package_locked_broken_or_blocked package git_revision mod_git_revision repository -# return if package - of given repository and revisions - is [locked or broken or blocked] +# package_locked_or_blocked package git_revision mod_git_revision repository +# return if package - of given repository and revisions - is [locked or blocked] -package_locked_broken_or_blocked() { +package_locked_or_blocked() { [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] || \ - [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ] || \ [ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ] } diff --git a/bin/db-update b/bin/db-update index 0542f5b..d5e6009 100755 --- a/bin/db-update +++ b/bin/db-update @@ -234,11 +234,11 @@ fi # packages which can't be un-staged because they're still dependencies # of any job on the build-list -while read -r pkg pkg_rev mod_rev repo; do - generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}" - cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends" -done < \ - "${work_dir}/build-list" | \ +grep -vxF 'break_loops' "${work_dir}/build-list" | \ + while read -r pkg pkg_rev mod_rev repo; do + generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}" + cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends" + done | \ sort -u > \ "${tmp_dir}/keep_packages" @@ -309,7 +309,7 @@ done done_packages="$(cat "${tmp_dir}/done_packages")" # if build list is empty, remember all entries of 'deletion-list' -if [ -s "${work_dir}/build-list" ]; then +if grep -qvxF 'break_loops' "${work_dir}/build-list"; then delete_packages='' else delete_packages="$( diff --git a/bin/get-assignment b/bin/get-assignment index 73aeb1d..66e8971 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -17,16 +17,14 @@ # respect build-manually-list ("blocked") -# possibly hand out "broken" packages to different build slave - . "${0%/*}/../conf/default.conf" mkdir -p "${work_dir}/package-states" hand_out_assignment() { - # locked, broken and blocked packages won't be handed out - if package_locked_broken_or_blocked "$1" "$2" "$3" "$4"; then + # locked and blocked packages won't be handed out + if package_locked_or_blocked "$1" "$2" "$3" "$4"; then return 0 fi @@ -34,6 +32,7 @@ hand_out_assignment() { # "locked" or "broken" (we keep only marker for older "done" packages) ls "${work_dir}/package-states" | \ grep "^$(str_to_regex "${1}")\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \ + grep -v "^$(str_to_regex "$1.$2.$3.$4.")[^.]\+\$" | \ sed "s|^|${work_dir}/package-states/|" | \ xargs -rn1 rm -f @@ -73,8 +72,12 @@ pending_packages=false while read -r package git_revision mod_git_revision repository; do - if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ] || - [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then + if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \ + [ "${package}" = 'break_loops' ]; then + continue + fi + + if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then continue fi @@ -103,14 +106,25 @@ fi while read -r package git_revision mod_git_revision repository; do - if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then + if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \ + [ "${package}" = 'break_loops' ]; then + sed -i \ + '/^break_loops$/d' \ + "${work_dir}/build-list" + echo 'break_loops' >> \ + "${work_dir}/build-list" + break + fi + + if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then continue fi [ -z "$( ( cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs" - awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \ + grep -vxF 'break_loops' "${work_dir}/build-list" | \ + awk '{print $1 "." $2 "." $3}' | \ sed " s|^|${work_dir}/package-infos/| s|\$|\.builds| @@ -126,7 +140,7 @@ while read -r package git_revision mod_git_revision repository; do done < "${work_dir}/build-list" -# Find package (of all packages which are not locked) +# Find package (of all packages which are not locked and have been broken the least times) # which breaks the most unlocked loops locked_packages="$( @@ -135,6 +149,10 @@ locked_packages="$( sed 's|\(\.[0-9a-f]\{40\}\)\{2\}\.[^\.\]\+\.locked$||' )" +grep -vxF 'break_loops' "${work_dir}/build-list" | \ + sort -k1,1 > \ + "${work_dir}/build-list.sorted-by-package" + for package in $( ls "${work_dir}/build-list.loops/" | \ grep '^loop_[0-9]\+$' | \ @@ -152,12 +170,33 @@ for package in $( done | \ sort | \ uniq -c | \ - sort -k1nr,1 | \ - awk '{print $2}' + join -1 2 -2 1 -o 1.1,2.1,2.2,2.3,2.4 \ + - \ + "${work_dir}/build-list.sorted-by-package" | \ + while read -r count package git_revision git_mod_revision repository; do + if [ -f "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" ]; then + versuche="$( + wc -l < \ + "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" + )" + else + versuche='0' + fi + printf '%s %s %s.%s.%s.%s\n' \ + "${versuche}" \ + "${count}" \ + "${package}" \ + "${git_revision}" \ + "${git_mod_revision}" \ + "${repository}" + done | \ + sort -k1n,1 -k2nr,2 | \ + cut -d' ' -f3 ); do - if assignment="$(grep "^$(str_to_regex "${package}") " "${work_dir}/build-list")"; then - hand_out_assignment ${assignment} - fi + hand_out_assignment $( + echo "${package}" | \ + sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|' + ) done # Remove the lock file diff --git a/bin/get-package-updates b/bin/get-package-updates index 35afbe1..d1ceaaa 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -123,9 +123,10 @@ echo 'Check modified packages from the last update, and put them to the build li # If a package is deleted, remove from the rebuild list, and add it to the deletion list. # If a new package is added, then ensure that it's not on the deletion list. -cp \ - "${work_dir}/build-list" \ - "${work_dir}/build-list.new" +grep -vxF 'break_loops' \ + "${work_dir}/build-list" > \ + "${work_dir}/build-list.new" || \ + true cp \ "${work_dir}/deletion-list" \ "${work_dir}/deletion-list.new" @@ -326,6 +327,10 @@ if [ -s "${work_dir}/tsort.error" ]; then "${work_dir}/build-list.loops.new/${loop}" done + # add order to break loops to build list + echo 'break_loops' >> + "${work_dir}/build-list.new.new" + else rm "${work_dir}/tsort.error" fi diff --git a/bin/return-assignment b/bin/return-assignment index ec7db45..213d4f0 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -48,10 +48,12 @@ if [ "$5" = 'ERROR' ]; then exit 0 fi - mv \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ + cat \ + "${work_dir}/package-states/$1.$2.$3.$4.locked" >> \ "${work_dir}/package-states/$1.$2.$3.$4.broken" + rm "${work_dir}/package-states/$1.$2.$3.$4.locked" + # unlock every loop this package would have broken and which is not # broken by another locked package locked_packages="$( @@ -74,6 +76,14 @@ if [ "$5" = 'ERROR' ]; then fi done + # move build order to end of build list + + sed -i \ + "/^$(str_to_regex "$1 $2 $3 $4")\$/d" \ + "${work_dir}/build-list" + echo "$1 $2 $3 $4" >> \ + "${work_dir}/build-list" + exit 0 fi @@ -152,8 +162,8 @@ repo-add "${destination}.db.tar.gz" ${packages} ${master_mirror_command} \ "${destination}.db."* \ - *".pkg.tar.xz" \ - *".pkg.tar.xz.sig" \ + ./*".pkg.tar.xz" \ + ./*".pkg.tar.xz.sig" \ "${master_mirror_directory}/i686/${destination}/" for package in ${packages}; do @@ -175,10 +185,18 @@ grep -xF "$1" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \ sed 'p;s|$|.locked|' | \ xargs -rn1 rm -f +if ! ls "${work_dir}/build-list.loops" | \ + grep -q '^loop_[0-9]\+$'; then + # no loops left + sed -i '/^break_loops$/d' "${work_dir}/build-list" +fi + # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" # remove package lock file printf '%s\n' ${packages} > \ "${work_dir}/package-states/$1.$2.$3.$4.done" -rm "${work_dir}/package-states/$1.$2.$3.$4.locked" +rm -f \ + "${work_dir}/package-states/$1.$2.$3.$4.locked" \ + "${work_dir}/package-states/$1.$2.$3.$4.broken" diff --git a/bin/why_dont_you b/bin/why_dont_you index aaa7df5..2c4d43e 100755 --- a/bin/why_dont_you +++ b/bin/why_dont_you @@ -23,15 +23,16 @@ case "${action}" in continue fi - if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then - echo 'is locked, broken or blocked' + if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then + echo 'is locked or blocked' continue fi if [ -n "$( ( cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs" - awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \ + grep -vxF 'break_loops' "${work_dir}/build-list" | \ + awk '{print $1 "." $2 "." $3}' | \ sed " s|^|${work_dir}/package-infos/| s|\$|\.builds| @@ -45,7 +46,8 @@ case "${action}" in echo 'has unmet dependencies:' ( cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs" - awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \ + grep -vxF 'break_loops' "${work_dir}/build-list" | \ + awk '{print $1 "." $2 "." $3}' | \ sed " s|^|${work_dir}/package-infos/| s|\$|\.builds| @@ -95,7 +97,8 @@ case "${action}" in build_list_items_file="$(mktemp)" dependent_packages_file="$(mktemp)" - awk '{print $1 "." $2 "." $3 " " $4}' "${work_dir}/build-list" | \ + grep -vxF 'break_loops' "${work_dir}/build-list" | \ + awk '{print $1 "." $2 "." $3 " " $4}' | \ sort -u > \ "${build_list_items_file}" |