From 05e3561b3df4078a33865913d8135dbfe4ad5809 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Tue, 20 Jun 2017 09:01:35 +0200
Subject: hand out broken packages multiple times

---
 bin/build-master-status |  4 +--
 bin/common-functions    |  7 +++---
 bin/db-update           | 12 ++++-----
 bin/get-assignment      | 67 ++++++++++++++++++++++++++++++++++++++-----------
 bin/get-package-updates | 11 +++++---
 bin/return-assignment   | 28 +++++++++++++++++----
 bin/why_dont_you        | 13 ++++++----
 7 files changed, 103 insertions(+), 39 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index 15ac1a5..54bf3de 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -88,9 +88,9 @@ stable="$(
     grep -c '\.pkg\.tar\.xz$'
   )"
 tasks="$(
-  wc -l < \
+  grep -c '^\S\+ \S\+ \S\+ \S\+$' \
     "${work_dir}/build-list"
-  )"
+  )" || true
 pending_packages="$(
   tr ' ' '.' < \
     "${work_dir}"/build-list | \
diff --git a/bin/common-functions b/bin/common-functions
index cc59e94..7f82042 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -142,12 +142,11 @@ find_git_repository_to_package_repository() {
 
 }
 
-# package_locked_broken_or_blocked package git_revision mod_git_revision repository
-# return if package - of given repository and revisions - is [locked or broken or blocked]
+# package_locked_or_blocked package git_revision mod_git_revision repository
+# return if package - of given repository and revisions - is [locked or blocked]
 
-package_locked_broken_or_blocked() {
+package_locked_or_blocked() {
   [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ] || \
-    [ -f "${work_dir}/package-states/$1.$2.$3.$4.broken" ] || \
     [ -f "${work_dir}/package-states/$1.$2.$3.$4.blocked" ]
 }
 
diff --git a/bin/db-update b/bin/db-update
index 0542f5b..d5e6009 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -234,11 +234,11 @@ fi
 # packages which can't be un-staged because they're still dependencies
 # of any job on the build-list
 
-while read -r pkg pkg_rev mod_rev repo; do
-  generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
-  cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
-done < \
-  "${work_dir}/build-list" | \
+grep -vxF 'break_loops' "${work_dir}/build-list" | \
+  while read -r pkg pkg_rev mod_rev repo; do
+    generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
+    cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
+  done | \
   sort -u > \
   "${tmp_dir}/keep_packages"
 
@@ -309,7 +309,7 @@ done
 done_packages="$(cat "${tmp_dir}/done_packages")"
 
 # if build list is empty, remember all entries of 'deletion-list'
-if [ -s "${work_dir}/build-list" ]; then
+if grep -qvxF 'break_loops' "${work_dir}/build-list"; then
   delete_packages=''
 else
   delete_packages="$(
diff --git a/bin/get-assignment b/bin/get-assignment
index 73aeb1d..66e8971 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -17,16 +17,14 @@
 
 #  respect build-manually-list ("blocked")
 
-#  possibly hand out "broken" packages to different build slave
-
 . "${0%/*}/../conf/default.conf"
 
 mkdir -p "${work_dir}/package-states"
 
 hand_out_assignment() {
 
-  # locked, broken and blocked packages won't be handed out
-  if package_locked_broken_or_blocked "$1" "$2" "$3" "$4"; then
+  # locked and blocked packages won't be handed out
+  if package_locked_or_blocked "$1" "$2" "$3" "$4"; then
     return 0
   fi
 
@@ -34,6 +32,7 @@ hand_out_assignment() {
   # "locked" or "broken" (we keep only marker for older "done" packages)
   ls "${work_dir}/package-states" | \
     grep "^$(str_to_regex "${1}")\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \
+    grep -v "^$(str_to_regex "$1.$2.$3.$4.")[^.]\+\$" | \
     sed "s|^|${work_dir}/package-states/|" | \
     xargs -rn1 rm -f
 
@@ -73,8 +72,12 @@ pending_packages=false
 
 while read -r package git_revision mod_git_revision repository; do
 
-  if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.broken" ] ||
-    [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
+  if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \
+    [ "${package}" = 'break_loops' ]; then
+    continue
+  fi
+
+  if [ -f "${work_dir}/package-states/${package}.${git_revision}.${mod_git_revision}.${repository}.blocked" ]; then
     continue
   fi
 
@@ -103,14 +106,25 @@ fi
 
 while read -r package git_revision mod_git_revision repository; do
 
-  if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
+  if [ -z "${git_revision}${mod_git_revision}${repository}" ] && \
+    [ "${package}" = 'break_loops' ]; then
+    sed -i \
+      '/^break_loops$/d' \
+      "${work_dir}/build-list"
+    echo 'break_loops' >> \
+      "${work_dir}/build-list"
+    break
+  fi
+
+  if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
     continue
   fi
 
   [ -z "$(
     (
       cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
-      awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+      grep -vxF 'break_loops' "${work_dir}/build-list" | \
+        awk '{print $1 "." $2 "." $3}' | \
         sed "
           s|^|${work_dir}/package-infos/|
           s|\$|\.builds|
@@ -126,7 +140,7 @@ while read -r package git_revision mod_git_revision repository; do
 
 done < "${work_dir}/build-list"
 
-# Find package (of all packages which are not locked)
+# Find package (of all packages which are not locked and have been broken the least times)
 # which breaks the most unlocked loops
 
 locked_packages="$(
@@ -135,6 +149,10 @@ locked_packages="$(
     sed 's|\(\.[0-9a-f]\{40\}\)\{2\}\.[^\.\]\+\.locked$||'
 )"
 
+grep -vxF 'break_loops' "${work_dir}/build-list" | \
+  sort -k1,1 > \
+  "${work_dir}/build-list.sorted-by-package"
+
 for package in $(
   ls "${work_dir}/build-list.loops/" | \
     grep '^loop_[0-9]\+$' | \
@@ -152,12 +170,33 @@ for package in $(
     done | \
     sort | \
     uniq -c | \
-    sort -k1nr,1 | \
-    awk '{print $2}'
+    join -1 2 -2 1 -o 1.1,2.1,2.2,2.3,2.4 \
+      - \
+      "${work_dir}/build-list.sorted-by-package" | \
+    while read -r count package git_revision git_mod_revision repository; do
+      if [ -f "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken" ]; then
+        versuche="$(
+          wc -l < \
+            "${work_dir}/package-states/${package}.${git_revision}.${git_mod_revision}.${repository}.broken"
+        )"
+      else
+        versuche='0'
+      fi
+      printf '%s %s %s.%s.%s.%s\n' \
+        "${versuche}" \
+        "${count}" \
+        "${package}" \
+        "${git_revision}" \
+        "${git_mod_revision}" \
+        "${repository}"
+    done | \
+    sort -k1n,1 -k2nr,2 | \
+    cut -d' ' -f3
 ); do
-  if assignment="$(grep "^$(str_to_regex "${package}") " "${work_dir}/build-list")"; then
-    hand_out_assignment ${assignment}
-  fi
+  hand_out_assignment $(
+    echo "${package}" | \
+      sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|'
+  )
 done
 
 # Remove the lock file
diff --git a/bin/get-package-updates b/bin/get-package-updates
index 35afbe1..d1ceaaa 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -123,9 +123,10 @@ echo 'Check modified packages from the last update, and put them to the build li
 # If a package is deleted, remove from the rebuild list, and add it to the deletion list.
 # If a new package is added, then ensure that it's not on the deletion list.
 
-cp \
-  "${work_dir}/build-list" \
-  "${work_dir}/build-list.new"
+grep -vxF 'break_loops' \
+  "${work_dir}/build-list" > \
+  "${work_dir}/build-list.new" || \
+  true
 cp \
   "${work_dir}/deletion-list" \
   "${work_dir}/deletion-list.new"
@@ -326,6 +327,10 @@ if [ -s "${work_dir}/tsort.error" ]; then
         "${work_dir}/build-list.loops.new/${loop}"
     done
 
+  # add order to break loops to build list
+  echo 'break_loops' >>
+    "${work_dir}/build-list.new.new"
+
 else
   rm "${work_dir}/tsort.error"
 fi
diff --git a/bin/return-assignment b/bin/return-assignment
index ec7db45..213d4f0 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -48,10 +48,12 @@ if [ "$5" = 'ERROR' ]; then
     exit 0
   fi
 
-  mv \
-    "${work_dir}/package-states/$1.$2.$3.$4.locked" \
+  cat \
+    "${work_dir}/package-states/$1.$2.$3.$4.locked" >> \
     "${work_dir}/package-states/$1.$2.$3.$4.broken"
 
+  rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
+
   # unlock every loop this package would have broken and which is not
   # broken by another locked package
   locked_packages="$(
@@ -74,6 +76,14 @@ if [ "$5" = 'ERROR' ]; then
       fi
     done
 
+  # move build order to end of build list
+
+  sed -i \
+    "/^$(str_to_regex "$1 $2 $3 $4")\$/d" \
+    "${work_dir}/build-list"
+  echo "$1 $2 $3 $4" >> \
+    "${work_dir}/build-list"
+
   exit 0
 
 fi
@@ -152,8 +162,8 @@ repo-add "${destination}.db.tar.gz" ${packages}
 
 ${master_mirror_command} \
   "${destination}.db."* \
-  *".pkg.tar.xz" \
-  *".pkg.tar.xz.sig" \
+  ./*".pkg.tar.xz" \
+  ./*".pkg.tar.xz.sig" \
   "${master_mirror_directory}/i686/${destination}/"
 
 for package in ${packages}; do
@@ -175,10 +185,18 @@ grep -xF "$1" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \
   sed 'p;s|$|.locked|' | \
   xargs -rn1 rm -f
 
+if ! ls "${work_dir}/build-list.loops" | \
+  grep -q '^loop_[0-9]\+$'; then
+  # no loops left
+  sed -i '/^break_loops$/d' "${work_dir}/build-list"
+fi
+
 # remove package from build list
 sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
 
 # remove package lock file
 printf '%s\n' ${packages} > \
   "${work_dir}/package-states/$1.$2.$3.$4.done"
-rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
+rm -f \
+  "${work_dir}/package-states/$1.$2.$3.$4.locked" \
+  "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/bin/why_dont_you b/bin/why_dont_you
index aaa7df5..2c4d43e 100755
--- a/bin/why_dont_you
+++ b/bin/why_dont_you
@@ -23,15 +23,16 @@ case "${action}" in
             continue
           fi
 
-          if package_locked_broken_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
-            echo 'is locked, broken or blocked'
+          if package_locked_or_blocked "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"; then
+            echo 'is locked or blocked'
             continue
           fi
 
           if [ -n "$(
             (
               cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
-              awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+              grep -vxF 'break_loops' "${work_dir}/build-list" | \
+                awk '{print $1 "." $2 "." $3}' | \
                 sed "
                   s|^|${work_dir}/package-infos/|
                   s|\$|\.builds|
@@ -45,7 +46,8 @@ case "${action}" in
             echo 'has unmet dependencies:'
             (
               cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
-              awk '{print $1 "." $2 "." $3}' "${work_dir}/build-list" | \
+              grep -vxF 'break_loops' "${work_dir}/build-list" | \
+                awk '{print $1 "." $2 "." $3}' | \
                 sed "
                   s|^|${work_dir}/package-infos/|
                   s|\$|\.builds|
@@ -95,7 +97,8 @@ case "${action}" in
       build_list_items_file="$(mktemp)"
       dependent_packages_file="$(mktemp)"
 
-      awk '{print $1 "." $2 "." $3 " " $4}' "${work_dir}/build-list" | \
+      grep -vxF 'break_loops' "${work_dir}/build-list" | \
+        awk '{print $1 "." $2 "." $3 " " $4}' | \
         sort -u > \
         "${build_list_items_file}"
 
-- 
cgit v1.2.3-54-g00ecf